def _compute_default_cs(dim, itemsize, logger=None):
# obtain a human readable memory size from the config, convert it to bytes and calc maximum chunksize.
from pyemma import config
from pyemma.util.units import string_to_bytes
max_bytes = string_to_bytes(config.default_chunksize)
# TODO: consider rounding this to some cache size of CPU? e.g py-cpuinfo can obtain it.
# if one time step is already bigger than max_memory, we set the chunksize to 1.
max_elements = max(1, int(np.floor(max_bytes / (itemsize * dim))))
assert max_elements * dim * itemsize <= max_bytes or max_elements == 1
result = max(1, max_elements // dim)
assert result > 0
if logger is not None:
logger.debug(
'computed default chunksize to %s'
' to limit memory per chunk to %s', result,
config.default_chunksize)
return result
def default_chunksize(self, val):
from pyemma.util.units import string_to_bytes
# check for parsing exceptions
string_to_bytes(val)
self._conf_values.set('pyemma', 'default_chunksize', str(val))
def max_chunksize_from_config(itemsize):
from pyemma import config
from pyemma.util.units import string_to_bytes
max_bytes = string_to_bytes(config.default_chunksize)
max_frames = max(1, int(np.floor(max_bytes / itemsize)))
return max_frames
