def decrement(self, n=1):
n = validate_int(n)
r = self._broadcast('prefetch_decrement', arguments={'n':n}, reply=True)
r = util._merge_broadcast_result(r)
if not r:
raise ApiError('Unable to decrement prefetch.')
# check that the value doesn't indicate an error
for worker_ret in r:
if isinstance(worker_ret, dict) and 'error' in worker_ret:
raise ApiError('Error occurred while decrementing prefetch.')
def broadcast(self, name, *args, **kwargs):
if 'reply' not in kwargs:
kwargs['reply'] = True
result = broadcast(name, *args, **kwargs)
result = util._merge_broadcast_result(result) # turn it into a single dict
result = util._condense_broadcast_result(result) # remove worker key
for k,v in result.iteritems():
if isinstance(v, dict) and 'error' in v:
raise RuntimeError('Found error in broadcast()')
return result
def decrement(self, n=1):
n = validate_int(n)
r = self._broadcast('pool_shrink', arguments={'n':n}, reply=True)
r = util._merge_broadcast_result(r)
if not r:
raise ApiError('Unable to decrement worker subprocess pool.')
# check that the value doesn't indicate an error
for worker_ret in r:
if isinstance(worker_ret, dict) and 'error' in worker_ret:
msg = 'Error occurred while decrementing '
msg += 'worker subprocess pool.'
raise ApiError(msg)
def get(self):
if len(self.names) != 1:
msg = 'Cannot retrieve this attribute for multiple workers.'
raise ApiError(msg)
r = self._broadcast('stats', destination=self.names, reply=True)
r = util._merge_broadcast_result(r)
if not r:
raise ApiError('Unable to retrieve worker attribute: prefetch.')
r = util._condense_broadcast_result(r)
if isinstance(r, dict) and 'error' in r:
raise ApiError('Error occurred while retrieving worker prefetch.')
return r['consumer']['prefetch_count']
def get_all_task_settings():
""" Like get_task_settings(), but it always returns the settings for all
tasks in all workers.
"""
# don't do anything if there are no workers
if len(util.get_all_worker_names()) == 0:
return {}
settings = util.broadcast('get_task_settings',
arguments={'tasknames': None,
'setting_names': _setting_names},
reply=True)
settings = util._merge_broadcast_result(settings)
return util._condense_broadcast_result(settings) or {}
def get(self):
if len(self.names) != 1:
msg = 'Cannot retrieve this attribute for multiple workers.'
raise ApiError(msg)
r = self._broadcast('stats', destination=self.names, reply=True)
r = util._merge_broadcast_result(r)
if not r:
msg = 'Unable to retrieve worker attribute: subprocess pool.'
raise ApiError(msg)
r = util._condense_broadcast_result(r)
if isinstance(r, dict) and 'error' in r:
msg = 'Error occurred while retrieving worker subprocess pool.'
raise ApiError(msg)
return len(r['pool']['processes'])
def __set__(self, taskapi, value):
value = self.validator(value)
arguments = {'tasknames': taskapi.names, 'attrname':self.attrname,
'value': value}
r = taskapi._broadcast('set_task_attribute', arguments=arguments,
reply=True)
r = util._merge_broadcast_result(r)
if not r:
msg = 'Unable to set task attribute: {0}.'.format(self.attrname)
raise ApiError(msg)
# check that all 'values' are equal
r = util._condense_broadcast_result(r)
# check that the value doesn't indicate an error
if isinstance(r, dict) and 'error' in r:
tmpl = 'Error occurred while setting task attribute: {0}.'
msg = tmpl.format(self.attrname)
raise ApiError(msg)
taskapi._on_task_modified(self.attrname, value)
def __get__(self, taskapi, owner):
assert taskapi is not None
if len(taskapi.names) != 1:
raise ApiError('Cannot retrieve this attribute for multiple tasks.')
arguments = {'taskname': taskapi.names[0], 'attrname': self.attrname}
r = taskapi._broadcast('get_task_attribute', arguments=arguments,
reply=True)
r = util._merge_broadcast_result(r)
if not r:
tmpl = 'Unable to retrieve task attribute: {0}.'
msg = tmpl.format(self.attrname)
raise ApiError(msg)
# check that all 'values' are equal
r = util._condense_broadcast_result(r)
# check that the value doesn't indicate an error
if isinstance(r, dict) and 'error' in r:
tmpl = 'Error occurred while retrieving task attribute: {0}.'
msg = tmpl.format(self.attrname)
raise ApiError(msg)
return r
def get_task_settings(workername, tasknames):
""" Get the settings for one or more tasks.
:param workername: The name of a worker to get the task settings from,
or None to examine all task settings.
:param tasknames: The name or names of tasks to get settings from. It
can either be a single string, a or a tuple or list
of strings.
:returns: A dict whose format varies depending on whether a workername
is supplied or not. If the workername is None, the dict has
three levels: name of the worker, name of the task, name of
the setting, and finally the setting's value. If a workername
is supplied, the top level of the dict is omitted.
"""
destination = [workername] if workername else None
settings = util.broadcast('get_task_settings', destination=destination,
arguments={'tasknames': tasknames,
'setting_names': _setting_names}, reply=True)
settings = util._merge_broadcast_result(settings)
if workername:
return settings.get(workername)
else:
return settings