コード例 #1
0
ファイル: backend.py プロジェクト: robgil/pulsar
 def _execute_task(self, worker, task):
     # Asynchronous execution of a Task. This method is called
     # on a separate thread of execution from the worker event loop thread.
     logger = worker.logger
     pubsub = self._pubsub
     task_id = task.id
     lock_id = task.get('lock_id')
     time_ended = time.time()
     job = self.registry.get(task.get('name'))
     consumer = TaskConsumer(self, worker, task_id, job)
     task_info = task.lazy_info()
     try:
         if not consumer.job:
             raise RuntimeError('%s not in registry' % task_info)
         if task['status'] > states.STARTED:
             expiry = task.get('expiry')
             if expiry and time_ended > expiry:
                 raise TaskTimeout
             else:
                 logger.info('starting %s', task_info)
                 kwargs = task.get('kwargs') or {}
                 self.models.task.update(task,
                                         status=states.STARTED,
                                         time_started=time_ended,
                                         worker=worker.aid)
                 pubsub.publish(self.channel('task_started'), task_id)
                 # This may block for a while
                 result = yield job(consumer, **kwargs)
                 status = states.SUCCESS
         else:
             logger.error('invalid status for %s', task_info)
             self.concurrent_tasks.discard(task_id)
             coroutine_return(task_id)
     except TaskTimeout:
         logger.warning('%s timed-out', task_info)
         result = None
         status = states.REVOKED
     except Exception as exc:
         logger.exception('failure in %s', task_info)
         result = str(exc)
         status = states.FAILURE
     #
     try:
         yield self.models.task.update(task,
                                       time_ended=time.time(),
                                       status=status,
                                       result=result)
     finally:
         self.concurrent_tasks.discard(task_id)
         self.finish_task(task_id, lock_id)
     #
     logger.info('finished %s', task_info)
     # publish into the task_done channel
     pubsub.publish(self.channel('task_done'), task_id)
     coroutine_return(task_id)
コード例 #2
0
ファイル: backend.py プロジェクト: huobao36/pulsar
 def _execute_task(self, worker, task):
     # Asynchronous execution of a Task. This method is called
     # on a separate thread of execution from the worker event loop thread.
     logger = get_logger(worker.logger)
     pubsub = self._pubsub
     task_id = task['id']
     lock_id = task.get('lock_id')
     time_ended = time.time()
     job = self.registry.get(task.get('name'))
     consumer = TaskConsumer(self, worker, task_id, job)
     task_info = task.lazy_info()
     try:
         if not consumer.job:
             raise RuntimeError('%s not in registry' % task_info)
         if task['status'] > states.STARTED:
             expiry = task.get('expiry')
             if expiry and time_ended > expiry:
                 raise TaskTimeout
             else:
                 logger.info('starting %s', task_info)
                 kwargs = task.get('kwargs') or {}
                 self.models.task.update(id=task_id, status=states.STARTED,
                                         time_started=time_ended,
                                         worker=worker.aid)
                 pubsub.publish(self.channel('task_started'), task_id)
                 # This may block for a while
                 result = yield job(consumer, **kwargs)
                 status = states.SUCCESS
         else:
             logger.error('invalid status for %s', task_info)
             self.concurrent_tasks.discard(task_id)
             coroutine_return(task_id)
     except TaskTimeout:
         logger.info('%s timed-out', task_info)
         result = None
         status = states.REVOKED
     except Exception as exc:
         logger.exception('failure in %s', task_info)
         result = str(exc)
         status = states.FAILURE
     #
     try:
         yield self.models.task.update(id=task_id, time_ended=time.time(),
                                       status=status, result=result)
     finally:
         self.concurrent_tasks.discard(task_id)
         self.finish_task(task_id, lock_id)
     #
     logger.info('finished %s', task_info)
     # publish into the task_done channel
     pubsub.publish(self.channel('task_done'), task_id)
     coroutine_return(task_id)
コード例 #3
0
ファイル: backend.py プロジェクト: robgil/pulsar
    def queue_task(self, jobname, meta_params=None, expiry=None, **kwargs):
        '''Try to queue a new :ref:`Task`.

        This method returns a :class:`.Future` which results in the
        task ``id`` created. If ``jobname`` is not a valid
        :attr:`.Job.name`, a ``TaskNotAvailable`` exception occurs.

        :param jobname: the name of a :class:`.Job`
            registered with the :class:`.TaskQueue` application.
        :param meta_params: Additional parameters to be passed to the
            :class:`Task` constructor (not its callable function).
        :param expiry: optional expiry timestamp to override the default
            expiry of a task.
        :param kwargs: optional dictionary used for the key-valued arguments
            in the task callable.
        :return: a :class:`.Future` resulting in a task id on success.
        '''
        pubsub = self._pubsub
        if jobname in self.registry:
            job = self.registry[jobname]
            task_id, lock_id = self.generate_task_ids(job, kwargs)
            queued = time.time()
            if expiry is not None:
                expiry = get_time(expiry, queued)
            elif job.timeout:
                expiry = get_time(job.timeout, queued)
            meta_params = meta_params or {}
            task = self.models.task(id=task_id,
                                    lock_id=lock_id,
                                    name=job.name,
                                    time_queued=queued,
                                    expiry=expiry,
                                    kwargs=kwargs,
                                    status=states.QUEUED)
            if meta_params:
                task.update(meta_params)
            task = yield self.maybe_queue_task(task)
            if task:
                pubsub.publish(self.channel('task_queued'), task['id'])
                scheduled = self.entries.get(job.name)
                if scheduled:
                    scheduled.next()
                self.logger.debug('queued %s', task.lazy_info())
                coroutine_return(task['id'])
            else:
                self.logger.debug('%s cannot queue new task. Locked', jobname)
                coroutine_return()
        else:
            raise TaskNotAvailable(jobname)
コード例 #4
0
ファイル: backend.py プロジェクト: huobao36/pulsar
    def queue_task(self, jobname, meta_params=None, expiry=None, **kwargs):
        '''Try to queue a new :ref:`Task`.

        This method returns a :class:`.Future` which results in the
        task ``id`` created. If ``jobname`` is not a valid
        :attr:`.Job.name`, a ``TaskNotAvailable`` exception occurs.

        :param jobname: the name of a :class:`.Job`
            registered with the :class:`.TaskQueue` application.
        :param meta_params: Additional parameters to be passed to the
            :class:`Task` constructor (not its callable function).
        :param expiry: optional expiry timestamp to override the default
            expiry of a task.
        :param kwargs: optional dictionary used for the key-valued arguments
            in the task callable.
        :return: a :class:`.Future` resulting in a task id on success.
        '''
        pubsub = self._pubsub
        if jobname in self.registry:
            job = self.registry[jobname]
            task_id, lock_id = self.generate_task_ids(job, kwargs)
            queued = time.time()
            if expiry is not None:
                expiry = get_time(expiry, queued)
            elif job.timeout:
                expiry = get_time(job.timeout, queued)
            meta_params = meta_params or {}
            task = self.models.task(id=task_id, lock_id=lock_id, name=job.name,
                                    time_queued=queued, expiry=expiry,
                                    kwargs=kwargs, status=states.QUEUED)
            if meta_params:
                task.update(meta_params)
            task = yield self.maybe_queue_task(task)
            if task:
                pubsub.publish(self.channel('task_queued'), task['id'])
                scheduled = self.entries.get(job.name)
                if scheduled:
                    scheduled.next()
                self.logger.debug('queued %s', task.lazy_info())
                coroutine_return(task['id'])
            else:
                self.logger.debug('%s cannot queue new task. Locked', jobname)
                coroutine_return()
        else:
            raise TaskNotAvailable(jobname)