def exec_async(self, *args, **kwargs):
if self.task_id:
task_id = self.task_id
else:
task_id = uuid1()
print self.func
args_json = json.dumps(getcallargs(self.func, *args, **kwargs), indent=2)
redis = create_redis_connection(self.redis_connection)
task_hash_key = get_task_hash_key(task_id)
pending_list_key = get_pending_list_key(self.queue)
results_channel_key = get_results_channel_key()
pubsub = redis.pubsub(ignore_subscribe_messages=True)
pubsub.subscribe(results_channel_key)
pipe = redis.pipeline(transaction=True)
pipe.lpush(pending_list_key, task_id)
print self.virtual_memory_limit
pipe.hmset(task_hash_key,
{TASK_HKEY_FUNCTION: self.func.__name__,
TASK_HKEY_ARGS: args_json,
TASK_HKEY_STATE: STATE_PENDING,
TASK_HKEY_USERNAME: self.username,
TASK_HKEY_PENDING_CREATED: json.strftime(datetime.utcnow()),
TASK_HKEY_VIRTUAL_MEMORY_LIMIT: self.virtual_memory_limit})
pipe.execute()
return AsyncResult(redis, pubsub, task_hash_key, task_id)
def worker_server(module, queue):
worker_uuid = uuid4()
redis = create_redis_connection()
logger = _create_logger()
logger.info('worker:{0} pid: {1} started'.format(worker_uuid, os.getpid()))
cmd_pubsub = _subscribe_to_cmd_pubsub(redis)
_register_worker(redis, worker_uuid)
task_id = None
task_hash_key = None
process = None
try:
while not _terminate.is_set():
task_id = _get_next_task_id(redis, queue, timeout=1)
# if task_id is None we got a timeout
if task_id is None:
continue
task_hash_key = get_task_hash_key(task_id)
print task_hash_key
logger.info('wtf')
process = Process(target=_exexcute_task,
args=(module, queue, task_id, task_hash_key))
terminated = False
killed = False
process.start()
while process.is_alive() and not _terminate.is_set():
process.join(0.2)
if _poll_kill_task(cmd_pubsub, task_hash_key):
killed = True
break
if _terminate.is_set():
break
if process.is_alive():
terminated = True
_kill_process(process, logger)
if terminated:
_set_task_finished(redis, queue, task_id, task_hash_key,
RESULT_FAILED,
logger=logger,
error_reason= "task was pubsub killed" if killed else "task was terminated")
elif (not hasattr(process, '_popen')
or not hasattr(process._popen, 'returncode')):
_set_task_finished(redis, queue, task_id, task_hash_key,
RESULT_FAILED,
logger=logger,
error_reason= 'task was never executed for some reason')
elif process._popen.returncode != 0:
_set_task_finished(redis, queue, task_id, task_hash_key,
RESULT_FAILED,
logger=logger,
error_reason= "task's process returncode was {0}".format(process._popen.returncode))
if _terminate.is_set():
break
task_id = None
task_hash_key = None
process = None
except KeyboardInterrupt:
logger.info('KeyboardInterrupt worker')
_terminate.set()
if _terminate.is_set():
if process is not None:
_kill_process(process, logger)
if (task_id is not None and task_hash_key is not None):
_set_task_finished_with_timeout(queue, task_id, task_hash_key,
RESULT_FAILED,
error_reason= 'task / process was killed before completion',
error_exception=traceback.format_exc(),
timeout=6)
_deregister_worker_with_timeout(worker_uuid, timeout=2)
cmd_pubsub.close()
def canel_task(redis, task_id):
redis.publish(get_command_channel_key(),
'task:kill:{0}'.format(get_task_hash_key(task_id)))
