def create_task(self, jobname, targs=None, tkwargs=None, expiry=None, **params): '''Create a new :class:`Task` from ``jobname``, positional arguments ``targs``, key-valued arguments ``tkwargs`` and :class:`Task` meta parameters ``params``. This method can be called by any process domain. :param jobname: the name of the :class:`Job` which create the task. :param targs: task positional arguments (a ``tuple`` or ``None``). :param tkwargs: task key-valued arguments (a ``dict`` or ``None``). :return: a :ref:`coroutine <coroutine>` resulting in a :attr:`Task.id` or ``None`` if no task was created. ''' if jobname in self.registry: pubsub = self.pubsub job = self.registry[jobname] targs = targs or EMPTY_TUPLE tkwargs = tkwargs or EMPTY_DICT task_id, overlap_id = job.generate_task_ids(targs, tkwargs) task = None if overlap_id: tasks = yield self.get_tasks(overlap_id=overlap_id) # Tasks with overlap id already available for task in tasks: if task.done(): yield self.save_task(task.id, overlap_id='') task = None else: break if task: LOGGER.debug('Task %s cannot run.', task) yield None else: if self.entries and job.name in self.entries: self.entries[job.name].next() time_executed = datetime.now() if expiry is not None: expiry = get_datetime(expiry, time_executed) elif job.timeout: expiry = get_datetime(job.timeout, time_executed) LOGGER.debug('Queue new task %s (%s).', job.name, task_id) yield self.save_task(task_id, overlap_id=overlap_id, name=job.name, time_executed=time_executed, expiry=expiry, args=targs, kwargs=tkwargs, status=states.PENDING, **params) pubsub.publish(self.channel('task_created'), task_id) else: raise TaskNotAvailable(jobname)
def _execute_task(self, worker, task): #Asynchronous execution of a Task. This method is called #on a separate thread of execution from the worker event loop thread. pubsub = self.pubsub task_id = task.id result = None status = None consumer = None time_ended = datetime.now() try: job = self.registry.get(task.name) consumer = TaskConsumer(self, worker, task_id, job) if not consumer.job: raise RuntimeError('Task "%s" not in registry %s' % (task.name, self.registry)) if task.status_code > states.PRECEDENCE_MAPPING[states.STARTED]: if task.expiry and time_ended > task.expiry: raise TaskTimeout else: worker.logger.info('starting task %s', task) yield self.save_task(task_id, status=states.STARTED, time_started=time_ended) pubsub.publish(self.channel('task_start'), task_id) result = yield job(consumer, *task.args, **task.kwargs) time_ended = datetime.now() else: consumer = None except TaskTimeout: worker.logger.info('Task %s timed-out', task) status = states.REVOKED except Exception: result = Failure(sys.exc_info()) if isinstance(result, Failure): result.log(msg='Failure in task %s' % task, log=worker.logger) status = states.FAILURE result = str(result) elif not status: status = states.SUCCESS if consumer: yield self.save_task(task_id, time_ended=time_ended, status=status, result=result) worker.logger.info('Finished task %s', task) # PUBLISH task_done pubsub.publish(self.channel('task_done'), task.id) self.concurrent_tasks.discard(task.id) yield task_id