Beispiel #1
0
    def exec_async(self, *args, **kwargs):
        if self.task_id:
            task_id = self.task_id
        else:
            task_id = uuid1()
        print self.func
        args_json = json.dumps(getcallargs(self.func, *args, **kwargs), indent=2)

        redis = create_redis_connection(self.redis_connection)

        task_hash_key = get_task_hash_key(task_id)
        pending_list_key = get_pending_list_key(self.queue)
        results_channel_key = get_results_channel_key()

        pubsub = redis.pubsub(ignore_subscribe_messages=True)
        pubsub.subscribe(results_channel_key)

        pipe = redis.pipeline(transaction=True)

        pipe.lpush(pending_list_key, task_id)

        print self.virtual_memory_limit

        pipe.hmset(task_hash_key,
                   {TASK_HKEY_FUNCTION: self.func.__name__,
                   TASK_HKEY_ARGS: args_json,
                   TASK_HKEY_STATE: STATE_PENDING,
                   TASK_HKEY_USERNAME: self.username,
                   TASK_HKEY_PENDING_CREATED: json.strftime(datetime.utcnow()),
                   TASK_HKEY_VIRTUAL_MEMORY_LIMIT: self.virtual_memory_limit})

        pipe.execute()

        return AsyncResult(redis, pubsub, task_hash_key, task_id)
Beispiel #2
0
def _exexcute_task(module, queue, task_id, task_hash_key):

    redis = create_redis_connection()
    logger = _create_logger()
    try:
        task = redis.hmget(task_hash_key,
                            (TASK_HKEY_FUNCTION,
                            TASK_HKEY_STATE,
                            TASK_HKEY_ARGS,
                            TASK_HKEY_VIRTUAL_MEMORY_LIMIT))
        print task

        function = task[0]
        state = task[1]
        args = task[2]
        virtual_memory_limit = task[3]

        if function is None or len(function) == 0:
            _set_task_finished(redis, queue, task_id, task_hash_key,
                          RESULT_FAILED,
                          error_reason='for function to execute was not supplied')
            return

        elif state != STATE_PENDING:
            _set_task_finished(redis, queue, task_id, task_hash_key,
                          RESULT_FAILED,
                          error_reason='to execute a task its state must be "{0}" not "{1}"'.format(STATE_PENDING,
                                                                                                    state))
            return

        _set_task_exexcuting(redis, task_hash_key)

        logger.info('executing task_id:{0}\nfunction: {1}\nargs: {2}\n virtual_memory_limit:{3}\n'.format(task_id,
                                                                         function,
                                                                         args,
                                                                         virtual_memory_limit))
        logger.info('--------------------------------------')

        if virtual_memory_limit:
            if isinstance(virtual_memory_limit, basestring):
                virtual_memory_limit = int(virtual_memory_limit)
            if virtual_memory_limit > 0:
                set_virtual_memory_limit(virtual_memory_limit)

        func = getattr(module, function)
        setattr(func, 'task_id', task_id)

        if args:
            result_value = func(**json.loads(args))
        else:
            result_value = func()

        if result_value:
            result_value = json.dumps(result_value, indent=2)

        _set_task_finished(redis,
                           queue,
                           task_id,
                           task_hash_key,
                           RESULT_SUCCESSFUL,
                           result_value,
                           logger=logger)

    except KeyboardInterrupt:
        logger.info('KeyboardInterrupt _exexcute_task')

    except Exception as ex:
        logger.info(traceback.format_exc())
        _set_task_finished(redis, queue, task_id, task_hash_key,
                      RESULT_FAILED,
                      logger=logger,
                      error_reason= '{0}: {1}'.format(type(ex), ex.message),
                      error_exception=traceback.format_exc())
Beispiel #3
0
def dumps(obj):
    json = json_serializer.dumps(obj)
    res = _json_to_toml(json)
    return res