def apply_async(self, args=(), kwargs={}, task_id=None, group_id=None, chord=None, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) header = kwargs.pop('header') body = kwargs.pop('body') header, body = (list(maybe_subtask(header)), maybe_subtask(body)) # forward certain options to body if chord is not None: body.options['chord'] = chord if group_id is not None: body.options['group_id'] = group_id [body.link(s) for s in options.pop('link', [])] [body.link_error(s) for s in options.pop('link_error', [])] callback_id = body.options.setdefault('task_id', task_id or uuid()) parent = super(Chord, self).apply_async((header, body, args), kwargs, **options) body_result = self.AsyncResult(callback_id) body_result.parent = parent return body_result
def prepare_steps(self, args, tasks): steps = deque(tasks) next_step = prev_task = prev_res = None tasks, results = [], [] i = 0 while steps: # First task get partial args from chain. task = maybe_subtask(steps.popleft()) task = task.clone() if i else task.clone(args) res = task._freeze() i += 1 if isinstance(task, group): # automatically upgrade group(..) | s to chord(group, s) try: next_step = steps.popleft() # for chords we freeze by pretending it's a normal # task instead of a group. res = Signature._freeze(task) task = chord(task, body=next_step, task_id=res.task_id) except IndexError: pass if prev_task: # link previous task to this task. prev_task.link(task) # set the results parent attribute. res.parent = prev_res if not isinstance(prev_task, chord): results.append(res) tasks.append(task) prev_task, prev_res = task, res return tasks, results
def apply_async(self, args=(), kwargs={}, task_id=None, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) group_id = options.pop('group_id', None) chord = options.pop('chord', None) header, body = (list(maybe_subtask(kwargs['header'])), maybe_subtask(kwargs['body'])) if group_id: body.set(group_id=group_id) if chord: body.set(chord=chord) callback_id = body.options.setdefault('task_id', task_id or uuid()) parent = super(Chord, self).apply_async((header, body), **options) body_result = self.AsyncResult(callback_id) body_result.parent = parent return body_result
def run(self, header, body, partial_args=(), interval=1, countdown=1, max_retries=None, propagate=None, eager=False, **kwargs): propagate = default_propagate if propagate is None else propagate group_id = uuid() AsyncResult = self.app.AsyncResult prepare_member = self._prepare_member # - convert back to group if serialized tasks = header.tasks if isinstance(header, group) else header header = group([maybe_subtask(s).clone() for s in tasks]) # - eager applies the group inline if eager: return header.apply(args=partial_args, task_id=group_id) results = [AsyncResult(prepare_member(task, body, group_id)) for task in header.tasks] # - fallback implementations schedules the chord_unlock task here app.backend.on_chord_apply(group_id, body, interval=interval, countdown=countdown, max_retries=max_retries, propagate=propagate, result=results) # - call the header group, returning the GroupResult. # XXX Python 2.5 doesn't allow kwargs after star-args. return header(*partial_args, **{'task_id': group_id})
def prepare_steps(self, args, tasks): steps = deque(tasks) next_step = prev_task = prev_res = None tasks, results = [], [] i = 0 while steps: # First task get partial args from chain. task = maybe_subtask(steps.popleft()) task = task.clone() if i else task.clone(args) res = task.freeze() i += 1 if isinstance(task, group): # automatically upgrade group(..) | s to chord(group, s) try: next_step = steps.popleft() # for chords we freeze by pretending it's a normal # task instead of a group. res = Signature.freeze(task) task = chord(task, body=next_step, task_id=res.task_id) except IndexError: pass # no callback, so keep as group if prev_task: # link previous task to this task. prev_task.link(task) # set the results parent attribute. res.parent = prev_res if not isinstance(prev_task, chord): results.append(res) tasks.append(task) prev_task, prev_res = task, res return tasks, results
def prepare_steps(self, args, tasks): steps = deque(tasks) next_step = prev_task = prev_res = None tasks, results = [], [] i = 0 while steps: # First task get partial args from chain. task = maybe_subtask(steps.popleft()) task = task.clone() if i else task.clone(args) i += 1 tid = task.options.get('task_id') if tid is None: tid = task.options['task_id'] = uuid() res = task.type.AsyncResult(tid) # automatically upgrade group(..) | s to chord(group, s) if isinstance(task, group): try: next_step = steps.popleft() except IndexError: next_step = None if next_step is not None: task = chord(task, body=next_step, task_id=tid) if prev_task: # link previous task to this task. prev_task.link(task) # set the results parent attribute. res.parent = prev_res if not isinstance(prev_task, chord): results.append(res) tasks.append(task) prev_task, prev_res = task, res return tasks, results
def apply(self, args=(), kwargs={}, **options): tasks = [maybe_subtask(task).clone() for task in kwargs['tasks']] res = prev = None for task in tasks: res = task.apply((prev.get(), ) if prev else ()) res.parent, prev = prev, res return res
def run(self, header, body, partial_args=(), interval=None, countdown=1, max_retries=None, propagate=None, eager=False, **kwargs): propagate = default_propagate if propagate is None else propagate group_id = uuid() AsyncResult = self.app.AsyncResult prepare_member = self._prepare_member # - convert back to group if serialized tasks = header.tasks if isinstance(header, group) else header header = group([maybe_subtask(s).clone() for s in tasks]) # - eager applies the group inline if eager: return header.apply(args=partial_args, task_id=group_id) results = [AsyncResult(prepare_member(task, body, group_id)) for task in header.tasks] # - fallback implementations schedules the chord_unlock task here app.backend.on_chord_apply(group_id, body, interval=interval, countdown=countdown, max_retries=max_retries, propagate=propagate, result=results) # - call the header group, returning the GroupResult. return header(*partial_args, task_id=group_id)
def prepare_steps(self, args, tasks): steps = deque(tasks) next_step = prev_task = prev_res = None tasks, results = [], [] i = 0 while steps: # First task get partial args from chain. task = maybe_subtask(steps.popleft()) task = task.clone() if i else task.clone(args) i += 1 tid = task.options.get('task_id') if tid is None: tid = task.options['task_id'] = uuid() res = task.type.AsyncResult(tid) # automatically upgrade group(..) | s to chord(group, s) if isinstance(task, group): try: next_step = steps.popleft() except IndexError: next_step = None if next_step is not None: task = chord(task, body=next_step, task_id=tid) if prev_task: # link previous task to this task. prev_task.link(task) # set the results parent attribute. res.parent = prev_res results.append(res) tasks.append(task) prev_task, prev_res = task, res return tasks, results
def apply(self, args=(), kwargs={}, **options): tasks = [maybe_subtask(task).clone() for task in kwargs["tasks"]] res = prev = None for task in tasks: res = task.apply((prev.get(), ) if prev else ()) res.parent, prev = prev, res return res
def apply_async(self, args=(), kwargs={}, task_id=None, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) body = maybe_subtask(kwargs["body"]) callback_id = body.options.setdefault("task_id", task_id or uuid()) super(Chord, self).apply_async(args, kwargs, **options) return self.AsyncResult(callback_id)
def apply_async(self, args=(), kwargs={}, task_id=None, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) group_id = options.pop("group_id", None) chord = options.pop("chord", None) header = kwargs.pop("header") body = kwargs.pop("body") header, body = (list(maybe_subtask(header)), maybe_subtask(body)) if group_id: body.set(group_id=group_id) if chord: body.set(chord=chord) callback_id = body.options.setdefault("task_id", task_id or uuid()) parent = super(Chord, self).apply_async((header, body, args), kwargs, **options) body_result = self.AsyncResult(callback_id) body_result.parent = parent return body_result
def apply_async(self, args=(), kwargs={}, task_id=None, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) group_id = options.pop('group_id', None) chord = options.pop('chord', None) header = kwargs.pop('header') body = kwargs.pop('body') header, body = (list(maybe_subtask(header)), maybe_subtask(body)) if group_id: body.set(group_id=group_id) if chord: body.set(chord=chord) callback_id = body.options.setdefault('task_id', task_id or uuid()) parent = super(Chord, self).apply_async((header, body, args), kwargs, **options) body_result = self.AsyncResult(callback_id) body_result.parent = parent return body_result
def prepare_member(task): task = maybe_subtask(task) opts = task.options opts['group_id'] = group_id try: tid = opts['task_id'] except KeyError: tid = opts['task_id'] = uuid() return task, AsyncResult(tid)
def apply_async(self, args=(), kwargs={}, task_id=None, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) header = kwargs.pop("header") body = kwargs.pop("body") header, body = (list(maybe_subtask(header)), maybe_subtask(body)) # forward certain options to body for opt_name in ["group_id", "chord"]: opt_value = options.pop(opt_name, None) if opt_value: body.set(**{opt_name: opt_value}) map(body.link, options.pop("link", [])) map(body.link_error, options.pop("link_error", [])) callback_id = body.options.setdefault("task_id", task_id or uuid()) parent = super(Chord, self).apply_async((header, body, args), kwargs, **options) body_result = self.AsyncResult(callback_id) body_result.parent = parent return body_result
def apply_async(self, args=(), kwargs={}, task_id=None, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) body = maybe_subtask(kwargs["body"]) callback_id = body.options.setdefault("task_id", task_id or uuid()) parent = super(Chord, self).apply_async(args, kwargs, **options) body_result = self.AsyncResult(callback_id) body_result.parent = parent return body_result
def apply_async(self, args=(), kwargs={}, task_id=None, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) header = kwargs.pop('header') body = kwargs.pop('body') header, body = (list(maybe_subtask(header)), maybe_subtask(body)) # forward certain options to body for opt_name in ['group_id', 'chord']: opt_value = options.pop(opt_name, None) if opt_value: body.set(**{opt_name: opt_value}) map(body.link, options.pop('link', [])) map(body.link_error, options.pop('link_error', [])) callback_id = body.options.setdefault('task_id', task_id or uuid()) parent = super(Chord, self).apply_async((header, body, args), kwargs, **options) body_result = self.AsyncResult(callback_id) body_result.parent = parent return body_result
def apply_async(self, args=(), kwargs={}, task_id=None, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) header = kwargs.pop('header') body = kwargs.pop('body') header, body = (list(maybe_subtask(header)), maybe_subtask(body)) # forward certain options to body for opt_name in ['group_id', 'chord']: opt_value = options.pop(opt_name, None) if opt_value: body.set(**{opt_name: opt_value}) [body.link(s) for s in options.pop('link', [])] [body.link_error(s) for s in options.pop('link_error', [])] callback_id = body.options.setdefault('task_id', task_id or uuid()) parent = super(Chord, self).apply_async((header, body, args), kwargs, **options) body_result = self.AsyncResult(callback_id) body_result.parent = parent return body_result
def apply_async(self, args=(), kwargs={}, task_id=None, group_id=None, chord=None, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) header = kwargs.pop('header') body = kwargs.pop('body') header, body = (list(maybe_subtask(header)), maybe_subtask(body)) # forward certain options to body if chord is not None: body.options['chord'] = chord if group_id is not None: body.options['group_id'] = group_id [body.link(s) for s in options.pop('link', [])] [body.link_error(s) for s in options.pop('link_error', [])] body_result = body.freeze(task_id) parent = super(Chord, self).apply_async((header, body, args), kwargs, **options) body_result.parent = parent return body_result
def apply_async(self, args=(), kwargs={}, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) tasks = [maybe_subtask(task).clone(task_id=uuid(), **kwargs) for task in kwargs["tasks"]] reduce(lambda a, b: a.link(b), tasks) tasks[0].apply_async() results = [task.type.AsyncResult(task.options["task_id"]) for task in tasks] reduce(lambda a, b: a.set_parent(b), reversed(results)) return results[-1]
def apply_async(self, args=(), kwargs={}, task_id=None, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) header, body = (list(kwargs["header"]), maybe_subtask(kwargs["body"])) callback_id = body.options.setdefault("task_id", task_id or uuid()) parent = super(Chord, self).apply_async((header, body), **options) body_result = self.AsyncResult(callback_id) body_result.parent = parent return body_result
def apply_async(self, args=(), kwargs={}, task_id=None, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) header, body = (list(kwargs['header']), maybe_subtask(kwargs['body'])) callback_id = body.options.setdefault('task_id', task_id or uuid()) parent = super(Chord, self).apply_async((header, body), **options) body_result = self.AsyncResult(callback_id) body_result.parent = parent return body_result
def apply_async(self, args=(), kwargs={}, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) tasks = kwargs["tasks"] tasks = [ maybe_subtask(task).clone(task_id=uuid(), **kwargs) for task in kwargs["tasks"] ] reduce(lambda a, b: a.link(b), tasks) tasks[0].apply_async() results = [ task.type.AsyncResult(task.options["task_id"]) for task in tasks ] reduce(lambda a, b: a.set_parent(b), reversed(results)) return results[-1]
def apply_async(self, args=(), kwargs={}, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) options.pop('publisher', None) group_id = options.pop('group_id', None) chord = options.pop('chord', None) tasks = [maybe_subtask(t).clone( task_id=options.pop('task_id', uuid()), **options ) for t in kwargs['tasks']] reduce(lambda a, b: a.link(b), tasks) if group_id: tasks[-1].set(group_id=group_id) if chord: tasks[-1].set(chord=chord) tasks[0].apply_async() results = [task.type.AsyncResult(task.options['task_id']) for task in tasks] reduce(lambda a, b: a.set_parent(b), reversed(results)) return results[-1]
def apply_async(self, args=(), kwargs={}, **options): if self.app.conf.CELERY_ALWAYS_EAGER: return self.apply(args, kwargs, **options) options.pop("publisher", None) taskset_id = options.pop("taskset_id", None) chord = options.pop("chord", None) tasks = [maybe_subtask(t).clone( task_id=options.pop("task_id", uuid()), **options ) for t in kwargs["tasks"]] reduce(lambda a, b: a.link(b), tasks) if taskset_id: tasks[-1].set(taskset_id=taskset_id) if chord: tasks[-1].set(chord=chord) tasks[0].apply_async() results = [task.type.AsyncResult(task.options["task_id"]) for task in tasks] reduce(lambda a, b: a.set_parent(b), reversed(results)) return results[-1]
def apply(self, args=(), kwargs={}, propagate=True, **options): body = kwargs['body'] res = super(Chord, self).apply(args, dict(kwargs, eager=True), **options) return maybe_subtask(body).apply( args=(res.get(propagate=propagate).get(), ))
def apply(self, args=(), kwargs={}, **options): body = kwargs["body"] res = super(Chord, self).apply(args, kwargs, **options) return maybe_subtask(body).apply(args=(res.get().join(), ))
def __init__(self, tasks=None, app=None, Publisher=None): self.app = app_or_default(app or self.app) self.data = [maybe_subtask(t) for t in tasks or []] self.total = len(self.tasks) self.Publisher = Publisher or self.app.amqp.TaskProducer
def __init__(self, tasks=None, app=None, Publisher=None): self._app = app or self._app self.data = [maybe_subtask(t) for t in tasks or []] self._Publisher = Publisher
def test_is_None(self): self.assertIsNone(maybe_subtask(None))
def test_is_dict(self): self.assertIsInstance(maybe_subtask(dict(add.s())), Signature)
def test_when_sig(self): s = add.s() self.assertIs(maybe_subtask(s), s)
def __init__(self, tasks=None, app=None, Publisher=None): super(TaskSet, self).__init__(maybe_subtask(t) for t in tasks or []) self.app = app_or_default(app or self.app) self.Publisher = Publisher or self.app.amqp.TaskProducer self.total = len(self) # XXX compat
def apply(self, args=(), kwargs={}, propagate=True, **options): body = kwargs['body'] res = super(Chord, self).apply(args, dict(kwargs, eager=True), **options) return maybe_subtask(body).apply(args=(res.get( propagate=propagate).get(), ))