def test_GroupResult(self): x = self.app.GroupResult( uuid(), [self.app.AsyncResult(uuid()) for _ in range(10)], ) self.assertEqual(x, from_serializable(x.serializable(), self.app)) self.assertEqual(x, from_serializable(x, self.app))
def unlock_chord(group_id, callback, interval=None, propagate=True, max_retries=None, result=None, Result=app.AsyncResult, GroupResult=app.GroupResult, from_serializable=from_serializable): if interval is None: interval = unlock_chord.default_retry_delay deps = GroupResult( group_id, [from_serializable(r, Result=Result) for r in result], ) j = deps.join_native if deps.supports_native_join else deps.join if deps.ready(): callback = subtask(callback) try: ret = j(propagate=propagate) except Exception as exc: culprit = next(deps._failed_join_report()) app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError('Dependency %s raised %r' % ( culprit.id, exc)), ) else: callback.delay(ret) else: return unlock_chord.retry(countdown=interval, max_retries=max_retries)
def test_with_parent(self): x = self.app.AsyncResult(uuid()) x.parent = self.app.AsyncResult(uuid()) y = from_serializable(x.serializable(), self.app) self.assertEqual(y, x) self.assertEqual(y.parent, x.parent) self.assertIsInstance(y.parent, AsyncResult)
def unlock_chord(group_id, callback, interval=None, propagate=None, max_retries=None, result=None, Result=app.AsyncResult, GroupResult=app.GroupResult, from_serializable=from_serializable): # if propagate is disabled exceptions raised by chord tasks # will be sent as part of the result list to the chord callback. # Since 3.1 propagate will be enabled by default, and instead # the chord callback changes state to FAILURE with the # exception set to ChordError. propagate = default_propagate if propagate is None else propagate # check if the task group is ready, and if so apply the callback. deps = GroupResult( group_id, [from_serializable(r, Result=Result) for r in result], ) j = deps.join_native if deps.supports_native_join else deps.join if deps.ready(): callback = subtask(callback) try: ret = j(propagate=propagate) except Exception, exc: culprit = deps._failed_join_report().next() app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError('Dependency %s raised %r' % ( culprit.id, exc)), ) else: try: callback.delay(ret) except Exception, exc: app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError('Call callback error: %r' % (exc, )))
def _restore_group(self, group_id): """Get task metadata for a task by id.""" meta = self.get(self.get_key_for_group(group_id)) # previously this was always pickled, but later this # was extended to support other serializers, so the # structure is kind of weird. if meta: meta = self.decode(meta) result = meta["result"] meta["result"] = from_serializable(result, self.app) return meta
def unlock_chord(group_id, callback, interval=None, propagate=None, max_retries=None, result=None, Result=app.AsyncResult, GroupResult=app.GroupResult, from_serializable=from_serializable): # if propagate is disabled exceptions raised by chord tasks # will be sent as part of the result list to the chord callback. # Since 3.1 propagate will be enabled by default, and instead # the chord callback changes state to FAILURE with the # exception set to ChordError. propagate = default_propagate if propagate is None else propagate if interval is None: interval = unlock_chord.default_retry_delay # check if the task group is ready, and if so apply the callback. deps = GroupResult( group_id, [from_serializable(r, app=app) for r in result], ) j = deps.join_native if deps.supports_native_join else deps.join if deps.ready(): callback = subtask(callback) try: ret = j(propagate=propagate) except Exception as exc: try: culprit = next(deps._failed_join_report()) reason = 'Dependency {0.id} raised {1!r}'.format( culprit, exc, ) except StopIteration: reason = repr(exc) app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError(reason), ) else: try: callback.delay(ret) except Exception as exc: app._tasks[callback.task].backend.fail_from_current_stack( callback.id, exc=ChordError('Callback error: {0!r}'.format(exc)), ) else: return unlock_chord.retry(countdown=interval, max_retries=max_retries)
def _restore_group(self, group_id): """Get task metadata for a task by id.""" meta = self.get(self.get_key_for_group(group_id)) # previously this was always pickled, but later this # was extended to support other serializers, so the # structure is kind of weird. if meta: meta = self.decode(meta) result = meta['result'] meta['result'] = from_serializable(result, self.app) return meta
def _restore_taskset(self, taskset_id): """Get task metadata for a task by id.""" meta = self.get(self.get_key_for_taskset(taskset_id)) # previously this was always pickled, but later this # was extended to support other serializers, so the # structure is kind of weird. if meta: meta = self.decode(meta) result = meta["result"] if isinstance(result, (list, tuple)): return {"result": from_serializable(result)} return meta
def _restore_group(self, group_id): """Get task metadata for a task by id.""" meta = self.get(self.get_key_for_group(group_id)) # previously this was always pickled, but later this # was extended to support other serializers, so the # structure is kind of weird. if meta: meta = self.decode(meta) result = meta['result'] if isinstance(result, (list, tuple)): return {'result': from_serializable(result)} return meta
def run(self, tasks, result, group_id, partial_args): app = self.app result = from_serializable(result, app) # any partial args are added to all tasks in the group taskit = (subtask(task).clone(partial_args) for i, task in enumerate(tasks)) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.GroupResult(result.id, [stask.apply(group_id=group_id) for stask in taskit]) with app.producer_or_acquire() as pub: [stask.apply_async(group_id=group_id, publisher=pub, add_to_parent=False) for stask in taskit] parent = get_current_worker_task() if parent: parent.request.children.append(result) return result
def run(self, tasks, result, setid): app = self.app result = from_serializable(result) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.TaskSetResult(result.id, [subtask(task).apply(taskset_id=setid) for task in tasks]) with app.default_producer() as pub: [subtask(task).apply_async(taskset_id=setid, publisher=pub) for task in tasks] parent = get_current_task() if parent: parent.request.children.append(result) return result
def run(self, tasks, result, group_id): app = self.app result = from_serializable(result) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.GroupResult(result.id, [subtask(task).apply(group_id=group_id) for task in tasks]) with app.default_producer() as pub: [subtask(task).apply_async(group_id=group_id, publisher=pub, add_to_parent=False) for task in tasks] parent = get_current_worker_task() if parent: parent.request.children.append(result) return result
def run(self, tasks, result): app = self.app result = from_serializable(result) with app.pool.acquire(block=True) as conn: with app.amqp.TaskPublisher(conn) as publisher: res_ = [subtask(task).apply_async( taskset_id=self.request.taskset, publisher=publisher) for task in tasks] parent = get_current_task() if parent: parent.request.children.append(result) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.TaskSetResult(result.id, res_) return result
def run(self, tasks, result, group_id, partial_args): app = self.app result = from_serializable(result) # any partial args are added to all tasks in the group taskit = (subtask(task).clone(partial_args) for i, task in enumerate(tasks)) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.GroupResult(result.id, [task.apply(group_id=group_id) for task in taskit]) with app.producer_or_acquire() as pub: [task.apply_async(group_id=group_id, publisher=pub, add_to_parent=False) for task in taskit] parent = get_current_worker_task() if parent: parent.request.children.append(result) return result
def run(self, tasks, result): app = self.app result = from_serializable(result) if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER: return app.TaskSetResult(result.id, [ subtask(task).apply(taskset_id=self.request.taskset) for task in tasks ]) with app.pool.acquire(block=True) as conn: with app.amqp.TaskPublisher(conn) as publisher: [ subtask(task).apply_async( taskset_id=self.request.taskset, publisher=publisher) for task in tasks ] parent = get_current_task() if parent: parent.request.children.append(result) return result
def test_compat(self): uid = uuid() x = from_serializable([uid, []], app=self.app) self.assertEqual(x.id, uid)
def test_GroupResult(self): x = GroupResult(uuid(), [AsyncResult(uuid()) for _ in range(10)]) self.assertEqual(x, from_serializable(x.serializable())) self.assertEqual(x, from_serializable(x))
def test_AsyncResult(self): x = AsyncResult(uuid()) self.assertEqual(x, from_serializable(x.serializable())) self.assertEqual(x, from_serializable(x))
def test_compat(self): uid = uuid() x = from_serializable([uid, []]) self.assertEqual(x.id, uid)