def decrement(self, n=1): n = validate_int(n) r = self._broadcast('prefetch_decrement', arguments={'n':n}, reply=True) r = util._merge_broadcast_result(r) if not r: raise ApiError('Unable to decrement prefetch.') # check that the value doesn't indicate an error for worker_ret in r: if isinstance(worker_ret, dict) and 'error' in worker_ret: raise ApiError('Error occurred while decrementing prefetch.')
def broadcast(self, name, *args, **kwargs): if 'reply' not in kwargs: kwargs['reply'] = True result = broadcast(name, *args, **kwargs) result = util._merge_broadcast_result(result) # turn it into a single dict result = util._condense_broadcast_result(result) # remove worker key for k,v in result.iteritems(): if isinstance(v, dict) and 'error' in v: raise RuntimeError('Found error in broadcast()') return result
def decrement(self, n=1): n = validate_int(n) r = self._broadcast('pool_shrink', arguments={'n':n}, reply=True) r = util._merge_broadcast_result(r) if not r: raise ApiError('Unable to decrement worker subprocess pool.') # check that the value doesn't indicate an error for worker_ret in r: if isinstance(worker_ret, dict) and 'error' in worker_ret: msg = 'Error occurred while decrementing ' msg += 'worker subprocess pool.' raise ApiError(msg)
def get(self): if len(self.names) != 1: msg = 'Cannot retrieve this attribute for multiple workers.' raise ApiError(msg) r = self._broadcast('stats', destination=self.names, reply=True) r = util._merge_broadcast_result(r) if not r: raise ApiError('Unable to retrieve worker attribute: prefetch.') r = util._condense_broadcast_result(r) if isinstance(r, dict) and 'error' in r: raise ApiError('Error occurred while retrieving worker prefetch.') return r['consumer']['prefetch_count']
def get_all_task_settings(): """ Like get_task_settings(), but it always returns the settings for all tasks in all workers. """ # don't do anything if there are no workers if len(util.get_all_worker_names()) == 0: return {} settings = util.broadcast('get_task_settings', arguments={'tasknames': None, 'setting_names': _setting_names}, reply=True) settings = util._merge_broadcast_result(settings) return util._condense_broadcast_result(settings) or {}
def get(self): if len(self.names) != 1: msg = 'Cannot retrieve this attribute for multiple workers.' raise ApiError(msg) r = self._broadcast('stats', destination=self.names, reply=True) r = util._merge_broadcast_result(r) if not r: msg = 'Unable to retrieve worker attribute: subprocess pool.' raise ApiError(msg) r = util._condense_broadcast_result(r) if isinstance(r, dict) and 'error' in r: msg = 'Error occurred while retrieving worker subprocess pool.' raise ApiError(msg) return len(r['pool']['processes'])
def __set__(self, taskapi, value): value = self.validator(value) arguments = {'tasknames': taskapi.names, 'attrname':self.attrname, 'value': value} r = taskapi._broadcast('set_task_attribute', arguments=arguments, reply=True) r = util._merge_broadcast_result(r) if not r: msg = 'Unable to set task attribute: {0}.'.format(self.attrname) raise ApiError(msg) # check that all 'values' are equal r = util._condense_broadcast_result(r) # check that the value doesn't indicate an error if isinstance(r, dict) and 'error' in r: tmpl = 'Error occurred while setting task attribute: {0}.' msg = tmpl.format(self.attrname) raise ApiError(msg) taskapi._on_task_modified(self.attrname, value)
def __get__(self, taskapi, owner): assert taskapi is not None if len(taskapi.names) != 1: raise ApiError('Cannot retrieve this attribute for multiple tasks.') arguments = {'taskname': taskapi.names[0], 'attrname': self.attrname} r = taskapi._broadcast('get_task_attribute', arguments=arguments, reply=True) r = util._merge_broadcast_result(r) if not r: tmpl = 'Unable to retrieve task attribute: {0}.' msg = tmpl.format(self.attrname) raise ApiError(msg) # check that all 'values' are equal r = util._condense_broadcast_result(r) # check that the value doesn't indicate an error if isinstance(r, dict) and 'error' in r: tmpl = 'Error occurred while retrieving task attribute: {0}.' msg = tmpl.format(self.attrname) raise ApiError(msg) return r
def get_task_settings(workername, tasknames): """ Get the settings for one or more tasks. :param workername: The name of a worker to get the task settings from, or None to examine all task settings. :param tasknames: The name or names of tasks to get settings from. It can either be a single string, a or a tuple or list of strings. :returns: A dict whose format varies depending on whether a workername is supplied or not. If the workername is None, the dict has three levels: name of the worker, name of the task, name of the setting, and finally the setting's value. If a workername is supplied, the top level of the dict is omitted. """ destination = [workername] if workername else None settings = util.broadcast('get_task_settings', destination=destination, arguments={'tasknames': tasknames, 'setting_names': _setting_names}, reply=True) settings = util._merge_broadcast_result(settings) if workername: return settings.get(workername) else: return settings