def unlock_chord(setid, callback, interval=1, propagate=False, max_retries=None, result=None): result = _res.TaskSetResult(setid, map(_res.AsyncResult, result)) j = result.join_native if result.supports_native_join else result.join if result.ready(): subtask(callback).delay(j(propagate=propagate)) else: unlock_chord.retry(countdown=interval, max_retries=max_retries)
def unlock_chord(group_id, callback, interval=1, propagate=False, max_retries=None, result=None): result = _res.GroupResult(group_id, map(_res.AsyncResult, result)) j = result.join_native if result.supports_native_join else result.join if result.ready(): subtask(callback).delay(j(propagate=propagate)) else: unlock_chord.retry(countdown=interval, max_retries=max_retries)
def unlock_chord(group_id, callback, interval=1, propagate=False, max_retries=None, result=None): AR = _res.AsyncResult result = _res.GroupResult(group_id, [AR(r) for r in result]) j = result.join_native if result.supports_native_join else result.join if result.ready(): subtask(callback).delay(j(propagate=propagate)) else: unlock_chord.retry(countdown=interval, max_retries=max_retries)
def unlock_chord(group_id, callback, interval=None, propagate=False, max_retries=None, result=None, Result=_res.AsyncResult): if interval is None: interval = unlock_chord.default_retry_delay result = _res.GroupResult(group_id, [Result(r) for r in result]) j = result.join_native if result.supports_native_join else result.join if result.ready(): subtask(callback).delay(j(propagate=propagate)) else: unlock_chord.retry(countdown=interval, max_retries=max_retries)
def unlock_chord(group_id, callback, interval=None, propagate=False, max_retries=None, result=None): if interval is None: interval = unlock_chord.default_retry_delay result = _res.GroupResult(group_id, map(_res.AsyncResult, result)) j = result.join_native if result.supports_native_join else result.join if result.ready(): subtask(callback).delay(j(propagate=propagate)) else: unlock_chord.retry(countdown=interval, max_retries=max_retries)
def run(self, tasks, result, setid): app = self.app result = from_serializable(result) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.TaskSetResult(result.id, [subtask(task).apply(taskset_id=setid) for task in tasks]) with app.default_producer() as pub: [subtask(task).apply_async(taskset_id=setid, publisher=pub) for task in tasks] parent = get_current_task() if parent: parent.request.children.append(result) return result
def run(self, tasks, result, group_id): app = self.app result = from_serializable(result) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.GroupResult(result.id, [subtask(task).apply(group_id=group_id) for task in tasks]) with app.default_producer() as pub: [subtask(task).apply_async(group_id=group_id, publisher=pub, add_to_parent=False) for task in tasks] parent = get_current_worker_task() if parent: parent.request.children.append(result) return result
def unlock_chord(group_id, callback, interval=None, propagate=None, max_retries=None, result=None, Result=app.AsyncResult, GroupResult=app.GroupResult, from_serializable=from_serializable): # if propagate is disabled exceptions raised by chord tasks # will be sent as part of the result list to the chord callback. # Since 3.1 propagate will be enabled by default, and instead # the chord callback changes state to FAILURE with the # exception set to ChordError. propagate = default_propagate if propagate is None else propagate # check if the task group is ready, and if so apply the callback. deps = GroupResult( group_id, [from_serializable(r, Result=Result) for r in result], ) j = deps.join_native if deps.supports_native_join else deps.join if deps.ready(): callback = subtask(callback) try: ret = j(propagate=propagate) except Exception, exc: culprit = deps._failed_join_report().next() app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError('Dependency %s raised %r' % ( culprit.id, exc)), ) else: try: callback.delay(ret) except Exception, exc: app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError('Call callback error: %r' % (exc, )))
def subtask(self, *args, **kwargs): """Returns :class:`~celery.subtask` object for this task, wrapping arguments and execution options for a single task invocation.""" from celery.canvas import subtask return subtask(self, *args, **kwargs)
def unlock_chord(group_id, callback, interval=None, propagate=True, max_retries=None, result=None, Result=app.AsyncResult, GroupResult=app.GroupResult, from_serializable=from_serializable): if interval is None: interval = unlock_chord.default_retry_delay deps = GroupResult( group_id, [from_serializable(r, Result=Result) for r in result], ) j = deps.join_native if deps.supports_native_join else deps.join if deps.ready(): callback = subtask(callback) try: ret = j(propagate=propagate) except Exception as exc: culprit = next(deps._failed_join_report()) app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError('Dependency %s raised %r' % ( culprit.id, exc)), ) else: callback.delay(ret) else: return unlock_chord.retry(countdown=interval, max_retries=max_retries)
def run(self, tasks, result): app = self.app result = from_serializable(result) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.TaskSetResult(result.id, [subtask(task).apply(taskset_id=self.request.taskset) for task in tasks]) with app.pool.acquire(block=True) as conn: with app.amqp.TaskPublisher(conn) as publisher: [subtask(task).apply_async( taskset_id=self.request.taskset, publisher=publisher) for task in tasks] parent = get_current_task() if parent: parent.request.children.append(result) return result
def run(self, tasks, result): app = self.app result = from_serializable(result) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.TaskSetResult(result.id, [ subtask(task).apply(taskset_id=self.request.taskset) for task in tasks ]) with app.pool.acquire(block=True) as conn: with app.amqp.TaskPublisher(conn) as publisher: [ subtask(task).apply_async( taskset_id=self.request.taskset, publisher=publisher) for task in tasks ] parent = get_current_task() if parent: parent.request.children.append(result) return result
def unlock_chord(group_id, callback, interval=None, propagate=None, max_retries=None, result=None, Result=app.AsyncResult, GroupResult=app.GroupResult, from_serializable=from_serializable): # if propagate is disabled exceptions raised by chord tasks # will be sent as part of the result list to the chord callback. # Since 3.1 propagate will be enabled by default, and instead # the chord callback changes state to FAILURE with the # exception set to ChordError. propagate = default_propagate if propagate is None else propagate if interval is None: interval = unlock_chord.default_retry_delay # check if the task group is ready, and if so apply the callback. deps = GroupResult( group_id, [from_serializable(r, app=app) for r in result], ) j = deps.join_native if deps.supports_native_join else deps.join if deps.ready(): callback = subtask(callback) try: ret = j(propagate=propagate) except Exception as exc: try: culprit = next(deps._failed_join_report()) reason = 'Dependency {0.id} raised {1!r}'.format( culprit, exc, ) except StopIteration: reason = repr(exc) app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError(reason), ) else: try: callback.delay(ret) except Exception as exc: app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError('Callback error: {0!r}'.format(exc)), ) else: return unlock_chord.retry(countdown=interval, max_retries=max_retries)
def run(self, tasks, result, group_id, partial_args): app = self.app result = from_serializable(result, app) # any partial args are added to all tasks in the group taskit = (subtask(task).clone(partial_args) for i, task in enumerate(tasks)) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.GroupResult(result.id, [stask.apply(group_id=group_id) for stask in taskit]) with app.producer_or_acquire() as pub: [stask.apply_async(group_id=group_id, publisher=pub, add_to_parent=False) for stask in taskit] parent = get_current_worker_task() if parent: parent.request.children.append(result) return result
def run(self, tasks, result, group_id, partial_args): app = self.app result = from_serializable(result) # any partial args are added to all tasks in the group taskit = (subtask(task).clone(partial_args) for i, task in enumerate(tasks)) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.GroupResult(result.id, [task.apply(group_id=group_id) for task in taskit]) with app.producer_or_acquire() as pub: [task.apply_async(group_id=group_id, publisher=pub, add_to_parent=False) for task in taskit] parent = get_current_worker_task() if parent: parent.request.children.append(result) return result
def xstarmap(task, it): task = subtask(task).type return [task(*args) for args in it]
def xmap(task, it): task = subtask(task).type return [task(value) for value in it]
def apply(self, args=(), kwargs={}, subtask=maybe_subtask, **options): last, fargs = None, args # fargs passed to first task only for task in kwargs['tasks']: res = subtask(task).clone(fargs).apply(last and (last.get(), )) res.parent, last, fargs = last, res, None return last
def xmap(task, it): task = subtask(task).type return list(imap(task, it))
def test_reverse(self): x = chord([add.s(2, 2), add.s(4, 4)], body=mul.s(4)) self.assertIsInstance(subtask(x), chord) self.assertIsInstance(subtask(dict(x)), chord)
def xmap(task, it): task = subtask(task).type return [task(item) for item in it]
def subtask(self, args=None, *starargs, **starkwargs): """Returns :class:`~celery.subtask` object for this task, wrapping arguments and execution options for a single task invocation.""" return subtask(self, args, *starargs, **starkwargs)
def test_reverse(self): x = add.s(2, 2) | add.s(2) self.assertIsInstance(subtask(x), chain) self.assertIsInstance(subtask(dict(x)), chain)
def test_link(self): x = subtask(SIG) x.link(SIG) x.link(SIG) self.assertIn(SIG, x.options['link']) self.assertEqual(len(x.options['link']), 1)
def test_apply_async_when_not_registered(self): s = subtask('xxx.not.registered') self.assertTrue(s._apply_async)
def xmap(task, it): task = subtask(task).type return list(map(task, it))
def test_AsyncResult_when_not_registerd(self): s = subtask('xxx.not.registered') self.assertTrue(s.AsyncResult)
def xstarmap(task, it): task = subtask(task).type return list(starmap(task, it))
def xstarmap(task, it): task = subtask(task).type return [task(*item) for item in it]
def test_reverse(self): x = group([add.s(2, 2), add.s(4, 4)]) self.assertIsInstance(subtask(x), group) self.assertIsInstance(subtask(dict(x)), group)
def test_link_error(self): x = subtask(SIG) x.link_error(SIG) x.link_error(SIG) self.assertIn(SIG, x.options["link_error"]) self.assertEqual(len(x.options["link_error"]), 1)
def call_task(self, task): try: X = subtask(task) X.apply_async() except Exception as exc: error('Could not call task: %r', exc, exc_info=1)