def __init__(self, *args, **kwargs): self.max_workers = kwargs.pop('max_workers', None) super(AutoscalePool, self).__init__(*args, **kwargs) if self.max_workers is None: settings_absmem = getattr(settings, 'SYSTEM_TASK_ABS_MEM', None) if settings_absmem is not None: # There are 1073741824 bytes in a gigabyte. Convert bytes to gigabytes by dividing by 2**30 total_memory_gb = convert_mem_str_to_bytes(settings_absmem) // 2**30 else: total_memory_gb = (psutil.virtual_memory().total >> 30) + 1 # noqa: round up # Get same number as max forks based on memory, this function takes memory as bytes self.max_workers = get_mem_effective_capacity(total_memory_gb * 2**30) # add magic prime number of extra workers to ensure # we have a few extra workers to run the heartbeat self.max_workers += 7 # max workers can't be less than min_workers self.max_workers = max(self.min_workers, self.max_workers) # the task manager enforces settings.TASK_MANAGER_TIMEOUT on its own # but if the task takes longer than the time defined here, we will force it to stop here self.task_manager_timeout = settings.TASK_MANAGER_TIMEOUT + settings.TASK_MANAGER_TIMEOUT_GRACE_PERIOD
def test_SYSTEM_TASK_ABS_MEM_conversion(value, converted_value, mem_capacity): with mock.patch('django.conf.settings') as mock_settings: mock_settings.SYSTEM_TASK_ABS_MEM = value mock_settings.SYSTEM_TASK_FORKS_MEM = 100 mock_settings.IS_K8S = True assert convert_mem_str_to_bytes(value) == converted_value assert get_corrected_memory(-1) == converted_value assert get_mem_effective_capacity(-1) == mem_capacity
def refresh_capacity_fields(self): """Update derived capacity fields from cpu and memory (no save)""" if self.node_type == 'hop': self.cpu_capacity = 0 self.mem_capacity = 0 # formula has a non-zero offset, so we make sure it is 0 for hop nodes else: self.cpu_capacity = get_cpu_effective_capacity(self.cpu) self.mem_capacity = get_mem_effective_capacity(self.memory) self.set_capacity_value()
def __init__(self, *args, **kwargs): self.max_workers = kwargs.pop('max_workers', None) super(AutoscalePool, self).__init__(*args, **kwargs) if self.max_workers is None: settings_absmem = getattr(settings, 'SYSTEM_TASK_ABS_MEM', None) if settings_absmem is not None: # There are 1073741824 bytes in a gigabyte. Convert bytes to gigabytes by dividing by 2**30 total_memory_gb = convert_mem_str_to_bytes( settings_absmem) // 2**30 else: total_memory_gb = ( psutil.virtual_memory().total >> 30) + 1 # noqa: round up # Get same number as max forks based on memory, this function takes memory as bytes self.max_workers = get_mem_effective_capacity(total_memory_gb * 2**30) # max workers can't be less than min_workers self.max_workers = max(self.min_workers, self.max_workers)
def refresh_capacity_fields(self): """Update derived capacity fields from cpu and memory (no save)""" self.cpu_capacity = get_cpu_effective_capacity(self.cpu) self.mem_capacity = get_mem_effective_capacity(self.memory) self.set_capacity_value()