def create_task(self,
jobname,
targs=None,
tkwargs=None,
expiry=None,
**params):
'''Create a new :class:`Task` from ``jobname``, positional arguments
``targs``, key-valued arguments ``tkwargs`` and :class:`Task` meta parameters
``params``. This method can be called by any process domain.
:param jobname: the name of the :class:`Job` which create the task.
:param targs: task positional arguments (a ``tuple`` or ``None``).
:param tkwargs: task key-valued arguments (a ``dict`` or ``None``).
:return: a :ref:`coroutine <coroutine>` resulting in a :attr:`Task.id`
or ``None`` if no task was created.
'''
if jobname in self.registry:
pubsub = self.pubsub
job = self.registry[jobname]
targs = targs or EMPTY_TUPLE
tkwargs = tkwargs or EMPTY_DICT
task_id, overlap_id = job.generate_task_ids(targs, tkwargs)
task = None
if overlap_id:
tasks = yield self.get_tasks(overlap_id=overlap_id)
# Tasks with overlap id already available
for task in tasks:
if task.done():
yield self.save_task(task.id, overlap_id='')
task = None
else:
break
if task:
LOGGER.debug('Task %s cannot run.', task)
yield None
else:
if self.entries and job.name in self.entries:
self.entries[job.name].next()
time_executed = datetime.now()
if expiry is not None:
expiry = get_datetime(expiry, time_executed)
elif job.timeout:
expiry = get_datetime(job.timeout, time_executed)
LOGGER.debug('Queue new task %s (%s).', job.name, task_id)
yield self.save_task(task_id,
overlap_id=overlap_id,
name=job.name,
time_executed=time_executed,
expiry=expiry,
args=targs,
kwargs=tkwargs,
status=states.PENDING,
**params)
pubsub.publish(self.channel('task_created'), task_id)
else:
raise TaskNotAvailable(jobname)
def create_task(self, jobname, targs=None, tkwargs=None, expiry=None,
**params):
'''Create a new :class:`Task` from ``jobname``, positional arguments
``targs``, key-valued arguments ``tkwargs`` and :class:`Task` meta parameters
``params``. This method can be called by any process domain.
:param jobname: the name of the :class:`Job` which create the task.
:param targs: task positional arguments (a ``tuple`` or ``None``).
:param tkwargs: task key-valued arguments (a ``dict`` or ``None``).
:return: a :ref:`coroutine <coroutine>` resulting in a :attr:`Task.id`
or ``None`` if no task was created.
'''
if jobname in self.registry:
pubsub = self.pubsub
job = self.registry[jobname]
targs = targs or EMPTY_TUPLE
tkwargs = tkwargs or EMPTY_DICT
task_id, overlap_id = job.generate_task_ids(targs, tkwargs)
task = None
if overlap_id:
tasks = yield self.get_tasks(overlap_id=overlap_id)
# Tasks with overlap id already available
for task in tasks:
if task.done():
yield self.save_task(task.id, overlap_id='')
task = None
else:
break
if task:
LOGGER.debug('Task %s cannot run.', task)
yield None
else:
if self.entries and job.name in self.entries:
self.entries[job.name].next()
time_executed = datetime.now()
if expiry is not None:
expiry = get_datetime(expiry, time_executed)
elif job.timeout:
expiry = get_datetime(job.timeout, time_executed)
LOGGER.debug('Queue new task %s (%s).', job.name, task_id)
yield self.save_task(task_id, overlap_id=overlap_id,
name=job.name,
time_executed=time_executed,
expiry=expiry, args=targs,
kwargs=tkwargs,
status=states.PENDING, **params)
pubsub.publish(self.channel('task_created'), task_id)
else:
raise TaskNotAvailable(jobname)
def _execute_task(self, worker, task):
#Asynchronous execution of a Task. This method is called
#on a separate thread of execution from the worker event loop thread.
pubsub = self.pubsub
task_id = task.id
result = None
status = None
consumer = None
time_ended = datetime.now()
try:
job = self.registry.get(task.name)
consumer = TaskConsumer(self, worker, task_id, job)
if not consumer.job:
raise RuntimeError('Task "%s" not in registry %s' %
(task.name, self.registry))
if task.status_code > states.PRECEDENCE_MAPPING[states.STARTED]:
if task.expiry and time_ended > task.expiry:
raise TaskTimeout
else:
worker.logger.info('starting task %s', task)
yield self.save_task(task_id,
status=states.STARTED,
time_started=time_ended)
pubsub.publish(self.channel('task_start'), task_id)
result = yield job(consumer, *task.args, **task.kwargs)
time_ended = datetime.now()
else:
consumer = None
except TaskTimeout:
worker.logger.info('Task %s timed-out', task)
status = states.REVOKED
except Exception:
result = Failure(sys.exc_info())
if isinstance(result, Failure):
result.log(msg='Failure in task %s' % task, log=worker.logger)
status = states.FAILURE
result = str(result)
elif not status:
status = states.SUCCESS
if consumer:
yield self.save_task(task_id,
time_ended=time_ended,
status=status,
result=result)
worker.logger.info('Finished task %s', task)
# PUBLISH task_done
pubsub.publish(self.channel('task_done'), task.id)
self.concurrent_tasks.discard(task.id)
yield task_id
def _execute_task(self, worker, task):
#Asynchronous execution of a Task. This method is called
#on a separate thread of execution from the worker event loop thread.
pubsub = self.pubsub
task_id = task.id
result = None
status = None
consumer = None
time_ended = datetime.now()
try:
job = self.registry.get(task.name)
consumer = TaskConsumer(self, worker, task_id, job)
if not consumer.job:
raise RuntimeError('Task "%s" not in registry %s' %
(task.name, self.registry))
if task.status_code > states.PRECEDENCE_MAPPING[states.STARTED]:
if task.expiry and time_ended > task.expiry:
raise TaskTimeout
else:
worker.logger.info('starting task %s', task)
yield self.save_task(task_id, status=states.STARTED,
time_started=time_ended)
pubsub.publish(self.channel('task_start'), task_id)
result = yield job(consumer, *task.args, **task.kwargs)
time_ended = datetime.now()
else:
consumer = None
except TaskTimeout:
worker.logger.info('Task %s timed-out', task)
status = states.REVOKED
except Exception:
result = Failure(sys.exc_info())
if isinstance(result, Failure):
result.log(msg='Failure in task %s' % task, log=worker.logger)
status = states.FAILURE
result = str(result)
elif not status:
status = states.SUCCESS
if consumer:
yield self.save_task(task_id, time_ended=time_ended,
status=status, result=result)
worker.logger.info('Finished task %s', task)
# PUBLISH task_done
pubsub.publish(self.channel('task_done'), task.id)
self.concurrent_tasks.discard(task.id)
yield task_id
