def _execute_task(self, worker, task):
# Asynchronous execution of a Task. This method is called
# on a separate thread of execution from the worker event loop thread.
logger = worker.logger
pubsub = self._pubsub
task_id = task.id
lock_id = task.get('lock_id')
time_ended = time.time()
job = self.registry.get(task.get('name'))
consumer = TaskConsumer(self, worker, task_id, job)
task_info = task.lazy_info()
try:
if not consumer.job:
raise RuntimeError('%s not in registry' % task_info)
if task['status'] > states.STARTED:
expiry = task.get('expiry')
if expiry and time_ended > expiry:
raise TaskTimeout
else:
logger.info('starting %s', task_info)
kwargs = task.get('kwargs') or {}
self.models.task.update(task,
status=states.STARTED,
time_started=time_ended,
worker=worker.aid)
pubsub.publish(self.channel('task_started'), task_id)
# This may block for a while
result = yield job(consumer, **kwargs)
status = states.SUCCESS
else:
logger.error('invalid status for %s', task_info)
self.concurrent_tasks.discard(task_id)
coroutine_return(task_id)
except TaskTimeout:
logger.warning('%s timed-out', task_info)
result = None
status = states.REVOKED
except Exception as exc:
logger.exception('failure in %s', task_info)
result = str(exc)
status = states.FAILURE
#
try:
yield self.models.task.update(task,
time_ended=time.time(),
status=status,
result=result)
finally:
self.concurrent_tasks.discard(task_id)
self.finish_task(task_id, lock_id)
#
logger.info('finished %s', task_info)
# publish into the task_done channel
pubsub.publish(self.channel('task_done'), task_id)
coroutine_return(task_id)
def _execute_task(self, worker, task):
# Asynchronous execution of a Task. This method is called
# on a separate thread of execution from the worker event loop thread.
logger = get_logger(worker.logger)
pubsub = self._pubsub
task_id = task['id']
lock_id = task.get('lock_id')
time_ended = time.time()
job = self.registry.get(task.get('name'))
consumer = TaskConsumer(self, worker, task_id, job)
task_info = task.lazy_info()
try:
if not consumer.job:
raise RuntimeError('%s not in registry' % task_info)
if task['status'] > states.STARTED:
expiry = task.get('expiry')
if expiry and time_ended > expiry:
raise TaskTimeout
else:
logger.info('starting %s', task_info)
kwargs = task.get('kwargs') or {}
self.models.task.update(id=task_id, status=states.STARTED,
time_started=time_ended,
worker=worker.aid)
pubsub.publish(self.channel('task_started'), task_id)
# This may block for a while
result = yield job(consumer, **kwargs)
status = states.SUCCESS
else:
logger.error('invalid status for %s', task_info)
self.concurrent_tasks.discard(task_id)
coroutine_return(task_id)
except TaskTimeout:
logger.info('%s timed-out', task_info)
result = None
status = states.REVOKED
except Exception as exc:
logger.exception('failure in %s', task_info)
result = str(exc)
status = states.FAILURE
#
try:
yield self.models.task.update(id=task_id, time_ended=time.time(),
status=status, result=result)
finally:
self.concurrent_tasks.discard(task_id)
self.finish_task(task_id, lock_id)
#
logger.info('finished %s', task_info)
# publish into the task_done channel
pubsub.publish(self.channel('task_done'), task_id)
coroutine_return(task_id)
def queue_task(self, jobname, meta_params=None, expiry=None, **kwargs):
'''Try to queue a new :ref:`Task`.
This method returns a :class:`.Future` which results in the
task ``id`` created. If ``jobname`` is not a valid
:attr:`.Job.name`, a ``TaskNotAvailable`` exception occurs.
:param jobname: the name of a :class:`.Job`
registered with the :class:`.TaskQueue` application.
:param meta_params: Additional parameters to be passed to the
:class:`Task` constructor (not its callable function).
:param expiry: optional expiry timestamp to override the default
expiry of a task.
:param kwargs: optional dictionary used for the key-valued arguments
in the task callable.
:return: a :class:`.Future` resulting in a task id on success.
'''
pubsub = self._pubsub
if jobname in self.registry:
job = self.registry[jobname]
task_id, lock_id = self.generate_task_ids(job, kwargs)
queued = time.time()
if expiry is not None:
expiry = get_time(expiry, queued)
elif job.timeout:
expiry = get_time(job.timeout, queued)
meta_params = meta_params or {}
task = self.models.task(id=task_id,
lock_id=lock_id,
name=job.name,
time_queued=queued,
expiry=expiry,
kwargs=kwargs,
status=states.QUEUED)
if meta_params:
task.update(meta_params)
task = yield self.maybe_queue_task(task)
if task:
pubsub.publish(self.channel('task_queued'), task['id'])
scheduled = self.entries.get(job.name)
if scheduled:
scheduled.next()
self.logger.debug('queued %s', task.lazy_info())
coroutine_return(task['id'])
else:
self.logger.debug('%s cannot queue new task. Locked', jobname)
coroutine_return()
else:
raise TaskNotAvailable(jobname)
def queue_task(self, jobname, meta_params=None, expiry=None, **kwargs):
'''Try to queue a new :ref:`Task`.
This method returns a :class:`.Future` which results in the
task ``id`` created. If ``jobname`` is not a valid
:attr:`.Job.name`, a ``TaskNotAvailable`` exception occurs.
:param jobname: the name of a :class:`.Job`
registered with the :class:`.TaskQueue` application.
:param meta_params: Additional parameters to be passed to the
:class:`Task` constructor (not its callable function).
:param expiry: optional expiry timestamp to override the default
expiry of a task.
:param kwargs: optional dictionary used for the key-valued arguments
in the task callable.
:return: a :class:`.Future` resulting in a task id on success.
'''
pubsub = self._pubsub
if jobname in self.registry:
job = self.registry[jobname]
task_id, lock_id = self.generate_task_ids(job, kwargs)
queued = time.time()
if expiry is not None:
expiry = get_time(expiry, queued)
elif job.timeout:
expiry = get_time(job.timeout, queued)
meta_params = meta_params or {}
task = self.models.task(id=task_id, lock_id=lock_id, name=job.name,
time_queued=queued, expiry=expiry,
kwargs=kwargs, status=states.QUEUED)
if meta_params:
task.update(meta_params)
task = yield self.maybe_queue_task(task)
if task:
pubsub.publish(self.channel('task_queued'), task['id'])
scheduled = self.entries.get(job.name)
if scheduled:
scheduled.next()
self.logger.debug('queued %s', task.lazy_info())
coroutine_return(task['id'])
else:
self.logger.debug('%s cannot queue new task. Locked', jobname)
coroutine_return()
else:
raise TaskNotAvailable(jobname)