def exec_async(self, *args, **kwargs): if self.task_id: task_id = self.task_id else: task_id = uuid1() print self.func args_json = json.dumps(getcallargs(self.func, *args, **kwargs), indent=2) redis = create_redis_connection(self.redis_connection) task_hash_key = get_task_hash_key(task_id) pending_list_key = get_pending_list_key(self.queue) results_channel_key = get_results_channel_key() pubsub = redis.pubsub(ignore_subscribe_messages=True) pubsub.subscribe(results_channel_key) pipe = redis.pipeline(transaction=True) pipe.lpush(pending_list_key, task_id) print self.virtual_memory_limit pipe.hmset(task_hash_key, {TASK_HKEY_FUNCTION: self.func.__name__, TASK_HKEY_ARGS: args_json, TASK_HKEY_STATE: STATE_PENDING, TASK_HKEY_USERNAME: self.username, TASK_HKEY_PENDING_CREATED: json.strftime(datetime.utcnow()), TASK_HKEY_VIRTUAL_MEMORY_LIMIT: self.virtual_memory_limit}) pipe.execute() return AsyncResult(redis, pubsub, task_hash_key, task_id)
def worker_server(module, queue): worker_uuid = uuid4() redis = create_redis_connection() logger = _create_logger() logger.info('worker:{0} pid: {1} started'.format(worker_uuid, os.getpid())) cmd_pubsub = _subscribe_to_cmd_pubsub(redis) _register_worker(redis, worker_uuid) task_id = None task_hash_key = None process = None try: while not _terminate.is_set(): task_id = _get_next_task_id(redis, queue, timeout=1) # if task_id is None we got a timeout if task_id is None: continue task_hash_key = get_task_hash_key(task_id) print task_hash_key logger.info('wtf') process = Process(target=_exexcute_task, args=(module, queue, task_id, task_hash_key)) terminated = False killed = False process.start() while process.is_alive() and not _terminate.is_set(): process.join(0.2) if _poll_kill_task(cmd_pubsub, task_hash_key): killed = True break if _terminate.is_set(): break if process.is_alive(): terminated = True _kill_process(process, logger) if terminated: _set_task_finished(redis, queue, task_id, task_hash_key, RESULT_FAILED, logger=logger, error_reason= "task was pubsub killed" if killed else "task was terminated") elif (not hasattr(process, '_popen') or not hasattr(process._popen, 'returncode')): _set_task_finished(redis, queue, task_id, task_hash_key, RESULT_FAILED, logger=logger, error_reason= 'task was never executed for some reason') elif process._popen.returncode != 0: _set_task_finished(redis, queue, task_id, task_hash_key, RESULT_FAILED, logger=logger, error_reason= "task's process returncode was {0}".format(process._popen.returncode)) if _terminate.is_set(): break task_id = None task_hash_key = None process = None except KeyboardInterrupt: logger.info('KeyboardInterrupt worker') _terminate.set() if _terminate.is_set(): if process is not None: _kill_process(process, logger) if (task_id is not None and task_hash_key is not None): _set_task_finished_with_timeout(queue, task_id, task_hash_key, RESULT_FAILED, error_reason= 'task / process was killed before completion', error_exception=traceback.format_exc(), timeout=6) _deregister_worker_with_timeout(worker_uuid, timeout=2) cmd_pubsub.close()
def canel_task(redis, task_id): redis.publish(get_command_channel_key(), 'task:kill:{0}'.format(get_task_hash_key(task_id)))