Example #1
0
def _set_task_finished_no_blocking(redis,
                  queue,
                  task_id,
                  task_hash_key,
                  result=True,
                  result_value=None,
                  error_reason=None,
                  error_exception=None,
                  logger=None):
    results_channel_key = get_results_channel_key()
    executing_list_key = get_executing_list_key(queue)

    task = {TASK_HKEY_STATE: STATE_FINISHED,
            TASK_HKEY_RESULT: result,
            TASK_HKEY_FINISHED_CREATED: strftime(datetime.utcnow())}

    if result_value:
        task[TASK_HKEY_RESULT_VALUE] = result_value

    if error_reason:
        task[TASK_HKEY_ERROR_REASON] = error_reason

    if error_exception:
        task[TASK_HKEY_ERROR_EXCEPTION] = error_exception

    pipe = redis.pipeline(transaction=True)
    pipe.hmset(task_hash_key, task)
    pipe.lrem(executing_list_key, task_id, 0)
    pipe.execute()

    print 'pub ' + results_channel_key + ' ' + task_hash_key
    redis.publish(results_channel_key, task_hash_key)

    if logger:
        logger.info('result:{0}\nresult-value:{1}\nerror-reason:{2}\n'.format(result, result_value, error_reason))
Example #2
0
    def exec_async(self, *args, **kwargs):
        if self.task_id:
            task_id = self.task_id
        else:
            task_id = uuid1()
        print self.func
        args_json = json.dumps(getcallargs(self.func, *args, **kwargs), indent=2)

        redis = create_redis_connection(self.redis_connection)

        task_hash_key = get_task_hash_key(task_id)
        pending_list_key = get_pending_list_key(self.queue)
        results_channel_key = get_results_channel_key()

        pubsub = redis.pubsub(ignore_subscribe_messages=True)
        pubsub.subscribe(results_channel_key)

        pipe = redis.pipeline(transaction=True)

        pipe.lpush(pending_list_key, task_id)

        print self.virtual_memory_limit

        pipe.hmset(task_hash_key,
                   {TASK_HKEY_FUNCTION: self.func.__name__,
                   TASK_HKEY_ARGS: args_json,
                   TASK_HKEY_STATE: STATE_PENDING,
                   TASK_HKEY_USERNAME: self.username,
                   TASK_HKEY_PENDING_CREATED: json.strftime(datetime.utcnow()),
                   TASK_HKEY_VIRTUAL_MEMORY_LIMIT: self.virtual_memory_limit})

        pipe.execute()

        return AsyncResult(redis, pubsub, task_hash_key, task_id)
Example #3
0
def _register_worker(redis, worker_uuid):
    redis.hmset(get_worker_hash_key(worker_uuid),
                {
                    'hostname': gethostname(),
                    'pid': os.getpid(),
                    'cpu_cores': psutil.cpu_count(logical=True),
                    'cpu_threads': psutil.cpu_count(logical=False),
                    'virtual-mem': psutil.virtual_memory(),
                    'swap-mem': psutil.swap_memory(),
                    'registered': strftime(datetime.utcnow())
                })
Example #4
0
def _set_task_exexcuting(redis, task_hash_key):
    redis.hmset(task_hash_key,
                {TASK_HKEY_STATE: STATE_EXECUTING,
                TASK_HKEY_EXECUTING_CREATED: strftime(datetime.utcnow())})