def _set_task_finished_with_timeout(queue,
task_id,
task_hash_key,
result=True,
result_value=None,
error_reason=None,
error_exception=None,
logger=None,
timeout=6):
redis = create_redis_connection(timeout=timeout)
_set_task_finished_no_blocking(redis,
queue,
task_id,
task_hash_key,
result,
result_value,
error_reason,
error_exception,
logger=logger)
def exec_async(self, *args, **kwargs):
if self.task_id:
task_id = self.task_id
else:
task_id = str(uuid1())
args_json = json.dumps(getcallargs(self.func, *args, **kwargs), indent=2)
redis = create_redis_connection(self.redis_connection)
task_hash_key = get_task_hash_key(task_id)
pending_list_key = get_pending_list_key(self.queue)
results_channel_key = get_results_channel_key()
pubsub = redis.pubsub(ignore_subscribe_messages=True)
pubsub.subscribe(results_channel_key)
pipe = redis.pipeline(transaction=True)
pipe.lpush(pending_list_key, task_id)
t = {
TASK_HKEY_FUNCTION: self.func.__name__,
TASK_HKEY_ARGS: args_json,
TASK_HKEY_STATE: STATE_PENDING,
TASK_HKEY_PENDING_CREATED: json.strftime(datetime.utcnow()),
}
if self.username:
t[TASK_HKEY_USERNAME] = self.username
if self.virtual_memory_limit > 0:
t[TASK_HKEY_USERNAME] = self.virtual_memory_limit
pipe.hmset(task_hash_key,t)
pipe.execute()
return AsyncResult(redis, pubsub, task_hash_key, task_id)
def _deregister_worker_with_timeout(worker_uuid, timeout=2):
redis = create_redis_connection(timeout=timeout)
redis.delete(get_worker_hash_key(worker_uuid))
def worker_server(module, queue):
worker_uuid = uuid4()
redis = create_redis_connection()
logger = _create_logger()
logger.info('worker:{0} pid: {1} started'.format(worker_uuid, os.getpid()))
cmd_pubsub = _subscribe_to_cmd_pubsub(redis)
_register_worker(redis, worker_uuid)
task_id = None
task_hash_key = None
process = None
try:
while not _terminate.is_set():
task_id = _get_next_task_id(redis, queue, timeout=1)
# if task_id is None we got a timeout
if task_id is None:
continue
task_hash_key = get_task_hash_key(task_id)
print(task_hash_key)
logger.info('wtf')
process = Process(target=_exexcute_task,
args=(module, queue, task_id, task_hash_key))
terminated = False
killed = False
process.start()
while process.is_alive() and not _terminate.is_set():
process.join(0.2)
if _poll_kill_task(cmd_pubsub, task_hash_key):
killed = True
break
if _terminate.is_set():
break
if process.is_alive():
terminated = True
_kill_process(process, logger)
if terminated:
_set_task_finished(redis,
queue,
task_id,
task_hash_key,
RESULT_FAILED,
logger=logger,
error_reason="task was pubsub killed"
if killed else "task was terminated")
elif (not hasattr(process, '_popen')
or not hasattr(process._popen, 'returncode')):
_set_task_finished(
redis,
queue,
task_id,
task_hash_key,
RESULT_FAILED,
logger=logger,
error_reason='task was never executed for some reason')
elif process._popen.returncode != 0:
_set_task_finished(
redis,
queue,
task_id,
task_hash_key,
RESULT_FAILED,
logger=logger,
error_reason="task's process returncode was {0}".format(
process._popen.returncode))
if _terminate.is_set():
break
task_id = None
task_hash_key = None
process = None
except KeyboardInterrupt:
logger.info('KeyboardInterrupt worker')
_terminate.set()
if _terminate.is_set():
if process is not None:
_kill_process(process, logger)
if (task_id is not None and task_hash_key is not None):
_set_task_finished_with_timeout(
queue,
task_id,
task_hash_key,
RESULT_FAILED,
error_reason='task / process was killed before completion',
error_exception=traceback.format_exc(),
timeout=6)
_deregister_worker_with_timeout(worker_uuid, timeout=2)
cmd_pubsub.close()
def _exexcute_task(module, queue, task_id, task_hash_key):
redis = create_redis_connection()
logger = _create_logger()
try:
task = redis.hmget(task_hash_key,
(TASK_HKEY_FUNCTION, TASK_HKEY_STATE,
TASK_HKEY_ARGS, TASK_HKEY_VIRTUAL_MEMORY_LIMIT))
print(task)
function = task[0]
state = task[1]
args = task[2]
virtual_memory_limit = task[3]
if function is None or len(function) == 0:
_set_task_finished(
redis,
queue,
task_id,
task_hash_key,
RESULT_FAILED,
error_reason='for function to execute was not supplied')
return
elif state != STATE_PENDING:
_set_task_finished(
redis,
queue,
task_id,
task_hash_key,
RESULT_FAILED,
error_reason=
'to execute a task its state must be "{0}" not "{1}"'.format(
STATE_PENDING, state))
return
_set_task_exexcuting(redis, task_hash_key)
logger.info(
'executing task_id:{0}\nfunction: {1}\nargs: {2}\n virtual_memory_limit:{3}\n'
.format(task_id, function, args, virtual_memory_limit))
logger.info('--------------------------------------')
if virtual_memory_limit:
if isinstance(virtual_memory_limit, str):
virtual_memory_limit = int(virtual_memory_limit)
if virtual_memory_limit > 0:
set_virtual_memory_limit(virtual_memory_limit)
func = getattr(module, function)
setattr(func, 'task_id', task_id)
if args:
result_value = func(**json.loads(args))
else:
result_value = func()
if result_value:
result_value = json.dumps(result_value, indent=2)
_set_task_finished(redis,
queue,
task_id,
task_hash_key,
RESULT_SUCCESSFUL,
result_value,
logger=logger)
except KeyboardInterrupt:
logger.info('KeyboardInterrupt _exexcute_task')
except Exception as ex:
logger.info(traceback.format_exc())
_set_task_finished(redis,
queue,
task_id,
task_hash_key,
RESULT_FAILED,
logger=logger,
error_reason='{0}: {1}'.format(
type(ex), ex.message),
error_exception=traceback.format_exc())