def _compute_default_cs(dim, itemsize, logger=None): # obtain a human readable memory size from the config, convert it to bytes and calc maximum chunksize. from pyemma import config from pyemma.util.units import string_to_bytes max_bytes = string_to_bytes(config.default_chunksize) # TODO: consider rounding this to some cache size of CPU? e.g py-cpuinfo can obtain it. # if one time step is already bigger than max_memory, we set the chunksize to 1. max_elements = max(1, int(np.floor(max_bytes / (itemsize * dim)))) assert max_elements * dim * itemsize <= max_bytes or max_elements == 1 result = max(1, max_elements // dim) assert result > 0 if logger is not None: logger.debug( 'computed default chunksize to %s' ' to limit memory per chunk to %s', result, config.default_chunksize) return result
def default_chunksize(self, val): from pyemma.util.units import string_to_bytes # check for parsing exceptions string_to_bytes(val) self._conf_values.set('pyemma', 'default_chunksize', str(val))
def max_chunksize_from_config(itemsize): from pyemma import config from pyemma.util.units import string_to_bytes max_bytes = string_to_bytes(config.default_chunksize) max_frames = max(1, int(np.floor(max_bytes / itemsize))) return max_frames