def exec_async(self, *args, **kwargs):
if self.task_id:
task_id = self.task_id
else:
task_id = uuid1()
print self.func
args_json = json.dumps(getcallargs(self.func, *args, **kwargs), indent=2)
redis = create_redis_connection(self.redis_connection)
task_hash_key = get_task_hash_key(task_id)
pending_list_key = get_pending_list_key(self.queue)
results_channel_key = get_results_channel_key()
pubsub = redis.pubsub(ignore_subscribe_messages=True)
pubsub.subscribe(results_channel_key)
pipe = redis.pipeline(transaction=True)
pipe.lpush(pending_list_key, task_id)
print self.virtual_memory_limit
pipe.hmset(task_hash_key,
{TASK_HKEY_FUNCTION: self.func.__name__,
TASK_HKEY_ARGS: args_json,
TASK_HKEY_STATE: STATE_PENDING,
TASK_HKEY_USERNAME: self.username,
TASK_HKEY_PENDING_CREATED: json.strftime(datetime.utcnow()),
TASK_HKEY_VIRTUAL_MEMORY_LIMIT: self.virtual_memory_limit})
pipe.execute()
return AsyncResult(redis, pubsub, task_hash_key, task_id)
def _exexcute_task(module, queue, task_id, task_hash_key):
redis = create_redis_connection()
logger = _create_logger()
try:
task = redis.hmget(task_hash_key,
(TASK_HKEY_FUNCTION,
TASK_HKEY_STATE,
TASK_HKEY_ARGS,
TASK_HKEY_VIRTUAL_MEMORY_LIMIT))
print task
function = task[0]
state = task[1]
args = task[2]
virtual_memory_limit = task[3]
if function is None or len(function) == 0:
_set_task_finished(redis, queue, task_id, task_hash_key,
RESULT_FAILED,
error_reason='for function to execute was not supplied')
return
elif state != STATE_PENDING:
_set_task_finished(redis, queue, task_id, task_hash_key,
RESULT_FAILED,
error_reason='to execute a task its state must be "{0}" not "{1}"'.format(STATE_PENDING,
state))
return
_set_task_exexcuting(redis, task_hash_key)
logger.info('executing task_id:{0}\nfunction: {1}\nargs: {2}\n virtual_memory_limit:{3}\n'.format(task_id,
function,
args,
virtual_memory_limit))
logger.info('--------------------------------------')
if virtual_memory_limit:
if isinstance(virtual_memory_limit, basestring):
virtual_memory_limit = int(virtual_memory_limit)
if virtual_memory_limit > 0:
set_virtual_memory_limit(virtual_memory_limit)
func = getattr(module, function)
setattr(func, 'task_id', task_id)
if args:
result_value = func(**json.loads(args))
else:
result_value = func()
if result_value:
result_value = json.dumps(result_value, indent=2)
_set_task_finished(redis,
queue,
task_id,
task_hash_key,
RESULT_SUCCESSFUL,
result_value,
logger=logger)
except KeyboardInterrupt:
logger.info('KeyboardInterrupt _exexcute_task')
except Exception as ex:
logger.info(traceback.format_exc())
_set_task_finished(redis, queue, task_id, task_hash_key,
RESULT_FAILED,
logger=logger,
error_reason= '{0}: {1}'.format(type(ex), ex.message),
error_exception=traceback.format_exc())
def dumps(obj):
json = json_serializer.dumps(obj)
res = _json_to_toml(json)
return res