Example #1
0
 def decrement(self, n=1):
     n = validate_int(n)
     r = self._broadcast('prefetch_decrement', arguments={'n':n}, reply=True)
     r = util._merge_broadcast_result(r)
     if not r:
         raise ApiError('Unable to decrement prefetch.')
     # check that the value doesn't indicate an error
     for worker_ret in r:
         if isinstance(worker_ret, dict) and 'error' in worker_ret:
             raise ApiError('Error occurred while decrementing prefetch.')
Example #2
0
 def broadcast(self, name, *args, **kwargs):
     if 'reply' not in kwargs:
         kwargs['reply'] = True
     result = broadcast(name, *args, **kwargs)
     result = util._merge_broadcast_result(result)  # turn it into a single dict
     result = util._condense_broadcast_result(result)  # remove worker key
     for k,v in result.iteritems():
         if isinstance(v, dict) and 'error' in v:
             raise RuntimeError('Found error in broadcast()')
     return result
Example #3
0
 def decrement(self, n=1):
     n = validate_int(n)
     r = self._broadcast('pool_shrink', arguments={'n':n}, reply=True)
     r = util._merge_broadcast_result(r)
     if not r:
         raise ApiError('Unable to decrement worker subprocess pool.')
     # check that the value doesn't indicate an error
     for worker_ret in r:
         if isinstance(worker_ret, dict) and 'error' in worker_ret:
             msg =  'Error occurred while decrementing '
             msg += 'worker subprocess pool.'
             raise ApiError(msg)
Example #4
0
 def get(self):
     if len(self.names) != 1:
         msg = 'Cannot retrieve this attribute for multiple workers.'
         raise ApiError(msg)
     r = self._broadcast('stats', destination=self.names, reply=True)
     r = util._merge_broadcast_result(r)
     if not r:
         raise ApiError('Unable to retrieve worker attribute: prefetch.')
     r = util._condense_broadcast_result(r)
     if isinstance(r, dict) and 'error' in r:
         raise ApiError('Error occurred while retrieving worker prefetch.')
     return r['consumer']['prefetch_count']
Example #5
0
def get_all_task_settings():
    """ Like get_task_settings(), but it always returns the settings for all 
        tasks in all workers. 
    """
    # don't do anything if there are no workers
    if len(util.get_all_worker_names()) == 0:
        return {}
    settings = util.broadcast('get_task_settings', 
                         arguments={'tasknames': None, 
                         'setting_names': _setting_names}, 
                         reply=True)
    settings = util._merge_broadcast_result(settings)
    return util._condense_broadcast_result(settings) or {}
Example #6
0
 def get(self):
     if len(self.names) != 1:
         msg = 'Cannot retrieve this attribute for multiple workers.'
         raise ApiError(msg)
     r = self._broadcast('stats', destination=self.names, reply=True)
     r = util._merge_broadcast_result(r)
     if not r:
         msg = 'Unable to retrieve worker attribute: subprocess pool.'
         raise ApiError(msg)
     r = util._condense_broadcast_result(r)
     if isinstance(r, dict) and 'error' in r:
         msg = 'Error occurred while retrieving worker subprocess pool.'
         raise ApiError(msg)
     return len(r['pool']['processes'])
Example #7
0
 def __set__(self, taskapi, value):
     value = self.validator(value)
     arguments = {'tasknames': taskapi.names, 'attrname':self.attrname, 
                  'value': value}
     r = taskapi._broadcast('set_task_attribute', arguments=arguments, 
                            reply=True)
     r = util._merge_broadcast_result(r)
     if not r:
         msg = 'Unable to set task attribute: {0}.'.format(self.attrname)
         raise ApiError(msg)
     # check that all 'values' are equal
     r = util._condense_broadcast_result(r)
     # check that the value doesn't indicate an error
     if isinstance(r, dict) and 'error' in r:
         tmpl = 'Error occurred while setting task attribute: {0}.'
         msg = tmpl.format(self.attrname)
         raise ApiError(msg)
     taskapi._on_task_modified(self.attrname, value)
Example #8
0
 def __get__(self, taskapi, owner):
     assert taskapi is not None
     if len(taskapi.names) != 1:
         raise ApiError('Cannot retrieve this attribute for multiple tasks.')
     arguments = {'taskname': taskapi.names[0], 'attrname': self.attrname}
     r = taskapi._broadcast('get_task_attribute', arguments=arguments, 
                            reply=True)
     r = util._merge_broadcast_result(r)
     if not r:
         tmpl = 'Unable to retrieve task attribute: {0}.'
         msg = tmpl.format(self.attrname)
         raise ApiError(msg)
     # check that all 'values' are equal
     r = util._condense_broadcast_result(r)
     # check that the value doesn't indicate an error
     if isinstance(r, dict) and 'error' in r:
         tmpl = 'Error occurred while retrieving task attribute: {0}.'
         msg = tmpl.format(self.attrname)
         raise ApiError(msg)
     return r
Example #9
0
def get_task_settings(workername, tasknames):
    """ Get the settings for one or more tasks. 
    
        :param workername: The name of a worker to get the task settings from, 
                           or None to examine all task settings.
        :param tasknames: The name or names of tasks to get settings from.  It 
                          can either be a single string, a or a tuple or list 
                          of strings.
        :returns: A dict whose format varies depending on whether a workername 
                  is supplied or not. If the workername is None, the dict has 
                  three levels: name of the worker, name of the task, name of 
                  the setting, and finally the setting's value. If a workername 
                  is supplied, the top level of the dict is omitted.
    """
    destination = [workername] if workername else None
    settings = util.broadcast('get_task_settings', destination=destination, 
                         arguments={'tasknames': tasknames, 
                         'setting_names': _setting_names}, reply=True)
    settings = util._merge_broadcast_result(settings)
    if workername:
        return settings.get(workername)
    else:
        return settings