def _set_task_finished_no_blocking(redis,
queue,
task_id,
task_hash_key,
result=True,
result_value=None,
error_reason=None,
error_exception=None,
logger=None):
results_channel_key = get_results_channel_key()
executing_list_key = get_executing_list_key(queue)
task = {TASK_HKEY_STATE: STATE_FINISHED,
TASK_HKEY_RESULT: result,
TASK_HKEY_FINISHED_CREATED: strftime(datetime.utcnow())}
if result_value:
task[TASK_HKEY_RESULT_VALUE] = result_value
if error_reason:
task[TASK_HKEY_ERROR_REASON] = error_reason
if error_exception:
task[TASK_HKEY_ERROR_EXCEPTION] = error_exception
pipe = redis.pipeline(transaction=True)
pipe.hmset(task_hash_key, task)
pipe.lrem(executing_list_key, task_id, 0)
pipe.execute()
print 'pub ' + results_channel_key + ' ' + task_hash_key
redis.publish(results_channel_key, task_hash_key)
if logger:
logger.info('result:{0}\nresult-value:{1}\nerror-reason:{2}\n'.format(result, result_value, error_reason))
def exec_async(self, *args, **kwargs):
if self.task_id:
task_id = self.task_id
else:
task_id = uuid1()
print self.func
args_json = json.dumps(getcallargs(self.func, *args, **kwargs), indent=2)
redis = create_redis_connection(self.redis_connection)
task_hash_key = get_task_hash_key(task_id)
pending_list_key = get_pending_list_key(self.queue)
results_channel_key = get_results_channel_key()
pubsub = redis.pubsub(ignore_subscribe_messages=True)
pubsub.subscribe(results_channel_key)
pipe = redis.pipeline(transaction=True)
pipe.lpush(pending_list_key, task_id)
print self.virtual_memory_limit
pipe.hmset(task_hash_key,
{TASK_HKEY_FUNCTION: self.func.__name__,
TASK_HKEY_ARGS: args_json,
TASK_HKEY_STATE: STATE_PENDING,
TASK_HKEY_USERNAME: self.username,
TASK_HKEY_PENDING_CREATED: json.strftime(datetime.utcnow()),
TASK_HKEY_VIRTUAL_MEMORY_LIMIT: self.virtual_memory_limit})
pipe.execute()
return AsyncResult(redis, pubsub, task_hash_key, task_id)
def _register_worker(redis, worker_uuid):
redis.hmset(get_worker_hash_key(worker_uuid),
{
'hostname': gethostname(),
'pid': os.getpid(),
'cpu_cores': psutil.cpu_count(logical=True),
'cpu_threads': psutil.cpu_count(logical=False),
'virtual-mem': psutil.virtual_memory(),
'swap-mem': psutil.swap_memory(),
'registered': strftime(datetime.utcnow())
})
def _set_task_exexcuting(redis, task_hash_key):
redis.hmset(task_hash_key,
{TASK_HKEY_STATE: STATE_EXECUTING,
TASK_HKEY_EXECUTING_CREATED: strftime(datetime.utcnow())})