def apply_async(self,
args=(),
kwargs={},
task_id=None,
group_id=None,
chord=None,
**options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
header = kwargs.pop('header')
body = kwargs.pop('body')
header, body = (list(maybe_subtask(header)), maybe_subtask(body))
# forward certain options to body
if chord is not None:
body.options['chord'] = chord
if group_id is not None:
body.options['group_id'] = group_id
[body.link(s) for s in options.pop('link', [])]
[body.link_error(s) for s in options.pop('link_error', [])]
callback_id = body.options.setdefault('task_id', task_id or uuid())
parent = super(Chord, self).apply_async((header, body, args),
kwargs, **options)
body_result = self.AsyncResult(callback_id)
body_result.parent = parent
return body_result
def prepare_steps(self, args, tasks):
steps = deque(tasks)
next_step = prev_task = prev_res = None
tasks, results = [], []
i = 0
while steps:
# First task get partial args from chain.
task = maybe_subtask(steps.popleft())
task = task.clone() if i else task.clone(args)
res = task._freeze()
i += 1
if isinstance(task, group):
# automatically upgrade group(..) | s to chord(group, s)
try:
next_step = steps.popleft()
# for chords we freeze by pretending it's a normal
# task instead of a group.
res = Signature._freeze(task)
task = chord(task, body=next_step, task_id=res.task_id)
except IndexError:
pass
if prev_task:
# link previous task to this task.
prev_task.link(task)
# set the results parent attribute.
res.parent = prev_res
if not isinstance(prev_task, chord):
results.append(res)
tasks.append(task)
prev_task, prev_res = task, res
return tasks, results
def apply_async(self, args=(), kwargs={}, task_id=None, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
group_id = options.pop('group_id', None)
chord = options.pop('chord', None)
header, body = (list(maybe_subtask(kwargs['header'])),
maybe_subtask(kwargs['body']))
if group_id:
body.set(group_id=group_id)
if chord:
body.set(chord=chord)
callback_id = body.options.setdefault('task_id', task_id or uuid())
parent = super(Chord, self).apply_async((header, body), **options)
body_result = self.AsyncResult(callback_id)
body_result.parent = parent
return body_result
def run(self, header, body, partial_args=(), interval=1, countdown=1,
max_retries=None, propagate=None, eager=False, **kwargs):
propagate = default_propagate if propagate is None else propagate
group_id = uuid()
AsyncResult = self.app.AsyncResult
prepare_member = self._prepare_member
# - convert back to group if serialized
tasks = header.tasks if isinstance(header, group) else header
header = group([maybe_subtask(s).clone() for s in tasks])
# - eager applies the group inline
if eager:
return header.apply(args=partial_args, task_id=group_id)
results = [AsyncResult(prepare_member(task, body, group_id))
for task in header.tasks]
# - fallback implementations schedules the chord_unlock task here
app.backend.on_chord_apply(group_id, body,
interval=interval,
countdown=countdown,
max_retries=max_retries,
propagate=propagate,
result=results)
# - call the header group, returning the GroupResult.
# XXX Python 2.5 doesn't allow kwargs after star-args.
return header(*partial_args, **{'task_id': group_id})
def prepare_steps(self, args, tasks):
steps = deque(tasks)
next_step = prev_task = prev_res = None
tasks, results = [], []
i = 0
while steps:
# First task get partial args from chain.
task = maybe_subtask(steps.popleft())
task = task.clone() if i else task.clone(args)
res = task.freeze()
i += 1
if isinstance(task, group):
# automatically upgrade group(..) | s to chord(group, s)
try:
next_step = steps.popleft()
# for chords we freeze by pretending it's a normal
# task instead of a group.
res = Signature.freeze(task)
task = chord(task, body=next_step, task_id=res.task_id)
except IndexError:
pass # no callback, so keep as group
if prev_task:
# link previous task to this task.
prev_task.link(task)
# set the results parent attribute.
res.parent = prev_res
if not isinstance(prev_task, chord):
results.append(res)
tasks.append(task)
prev_task, prev_res = task, res
return tasks, results
def prepare_steps(self, args, tasks):
steps = deque(tasks)
next_step = prev_task = prev_res = None
tasks, results = [], []
i = 0
while steps:
# First task get partial args from chain.
task = maybe_subtask(steps.popleft())
task = task.clone() if i else task.clone(args)
i += 1
tid = task.options.get('task_id')
if tid is None:
tid = task.options['task_id'] = uuid()
res = task.type.AsyncResult(tid)
# automatically upgrade group(..) | s to chord(group, s)
if isinstance(task, group):
try:
next_step = steps.popleft()
except IndexError:
next_step = None
if next_step is not None:
task = chord(task, body=next_step, task_id=tid)
if prev_task:
# link previous task to this task.
prev_task.link(task)
# set the results parent attribute.
res.parent = prev_res
if not isinstance(prev_task, chord):
results.append(res)
tasks.append(task)
prev_task, prev_res = task, res
return tasks, results
def apply(self, args=(), kwargs={}, **options):
tasks = [maybe_subtask(task).clone() for task in kwargs['tasks']]
res = prev = None
for task in tasks:
res = task.apply((prev.get(), ) if prev else ())
res.parent, prev = prev, res
return res
def run(self, header, body, partial_args=(), interval=None, countdown=1,
max_retries=None, propagate=None, eager=False, **kwargs):
propagate = default_propagate if propagate is None else propagate
group_id = uuid()
AsyncResult = self.app.AsyncResult
prepare_member = self._prepare_member
# - convert back to group if serialized
tasks = header.tasks if isinstance(header, group) else header
header = group([maybe_subtask(s).clone() for s in tasks])
# - eager applies the group inline
if eager:
return header.apply(args=partial_args, task_id=group_id)
results = [AsyncResult(prepare_member(task, body, group_id))
for task in header.tasks]
# - fallback implementations schedules the chord_unlock task here
app.backend.on_chord_apply(group_id, body,
interval=interval,
countdown=countdown,
max_retries=max_retries,
propagate=propagate,
result=results)
# - call the header group, returning the GroupResult.
return header(*partial_args, task_id=group_id)
def prepare_steps(self, args, tasks):
steps = deque(tasks)
next_step = prev_task = prev_res = None
tasks, results = [], []
i = 0
while steps:
# First task get partial args from chain.
task = maybe_subtask(steps.popleft())
task = task.clone() if i else task.clone(args)
i += 1
tid = task.options.get('task_id')
if tid is None:
tid = task.options['task_id'] = uuid()
res = task.type.AsyncResult(tid)
# automatically upgrade group(..) | s to chord(group, s)
if isinstance(task, group):
try:
next_step = steps.popleft()
except IndexError:
next_step = None
if next_step is not None:
task = chord(task, body=next_step, task_id=tid)
if prev_task:
# link previous task to this task.
prev_task.link(task)
# set the results parent attribute.
res.parent = prev_res
results.append(res)
tasks.append(task)
prev_task, prev_res = task, res
return tasks, results
def apply(self, args=(), kwargs={}, **options):
tasks = [maybe_subtask(task).clone() for task in kwargs["tasks"]]
res = prev = None
for task in tasks:
res = task.apply((prev.get(), ) if prev else ())
res.parent, prev = prev, res
return res
def apply_async(self, args=(), kwargs={}, task_id=None, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
body = maybe_subtask(kwargs["body"])
callback_id = body.options.setdefault("task_id", task_id or uuid())
super(Chord, self).apply_async(args, kwargs, **options)
return self.AsyncResult(callback_id)
def apply_async(self, args=(), kwargs={}, task_id=None, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
group_id = options.pop("group_id", None)
chord = options.pop("chord", None)
header = kwargs.pop("header")
body = kwargs.pop("body")
header, body = (list(maybe_subtask(header)), maybe_subtask(body))
if group_id:
body.set(group_id=group_id)
if chord:
body.set(chord=chord)
callback_id = body.options.setdefault("task_id", task_id or uuid())
parent = super(Chord, self).apply_async((header, body, args), kwargs, **options)
body_result = self.AsyncResult(callback_id)
body_result.parent = parent
return body_result
def apply_async(self, args=(), kwargs={}, task_id=None, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
group_id = options.pop('group_id', None)
chord = options.pop('chord', None)
header = kwargs.pop('header')
body = kwargs.pop('body')
header, body = (list(maybe_subtask(header)), maybe_subtask(body))
if group_id:
body.set(group_id=group_id)
if chord:
body.set(chord=chord)
callback_id = body.options.setdefault('task_id', task_id or uuid())
parent = super(Chord, self).apply_async((header, body, args),
kwargs, **options)
body_result = self.AsyncResult(callback_id)
body_result.parent = parent
return body_result
def prepare_member(task):
task = maybe_subtask(task)
opts = task.options
opts['group_id'] = group_id
try:
tid = opts['task_id']
except KeyError:
tid = opts['task_id'] = uuid()
return task, AsyncResult(tid)
def apply_async(self, args=(), kwargs={}, task_id=None, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
header = kwargs.pop("header")
body = kwargs.pop("body")
header, body = (list(maybe_subtask(header)), maybe_subtask(body))
# forward certain options to body
for opt_name in ["group_id", "chord"]:
opt_value = options.pop(opt_name, None)
if opt_value:
body.set(**{opt_name: opt_value})
map(body.link, options.pop("link", []))
map(body.link_error, options.pop("link_error", []))
callback_id = body.options.setdefault("task_id", task_id or uuid())
parent = super(Chord, self).apply_async((header, body, args), kwargs, **options)
body_result = self.AsyncResult(callback_id)
body_result.parent = parent
return body_result
def prepare_member(task):
task = maybe_subtask(task)
opts = task.options
opts['group_id'] = group_id
try:
tid = opts['task_id']
except KeyError:
tid = opts['task_id'] = uuid()
return task, AsyncResult(tid)
def apply_async(self, args=(), kwargs={}, task_id=None, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
body = maybe_subtask(kwargs["body"])
callback_id = body.options.setdefault("task_id", task_id or uuid())
parent = super(Chord, self).apply_async(args, kwargs, **options)
body_result = self.AsyncResult(callback_id)
body_result.parent = parent
return body_result
def apply_async(self, args=(), kwargs={}, task_id=None, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
header = kwargs.pop('header')
body = kwargs.pop('body')
header, body = (list(maybe_subtask(header)),
maybe_subtask(body))
# forward certain options to body
for opt_name in ['group_id', 'chord']:
opt_value = options.pop(opt_name, None)
if opt_value:
body.set(**{opt_name: opt_value})
map(body.link, options.pop('link', []))
map(body.link_error, options.pop('link_error', []))
callback_id = body.options.setdefault('task_id', task_id or uuid())
parent = super(Chord, self).apply_async((header, body, args),
kwargs, **options)
body_result = self.AsyncResult(callback_id)
body_result.parent = parent
return body_result
def apply_async(self, args=(), kwargs={}, task_id=None, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
header = kwargs.pop('header')
body = kwargs.pop('body')
header, body = (list(maybe_subtask(header)),
maybe_subtask(body))
# forward certain options to body
for opt_name in ['group_id', 'chord']:
opt_value = options.pop(opt_name, None)
if opt_value:
body.set(**{opt_name: opt_value})
[body.link(s) for s in options.pop('link', [])]
[body.link_error(s) for s in options.pop('link_error', [])]
callback_id = body.options.setdefault('task_id', task_id or uuid())
parent = super(Chord, self).apply_async((header, body, args),
kwargs, **options)
body_result = self.AsyncResult(callback_id)
body_result.parent = parent
return body_result
def apply_async(self, args=(), kwargs={}, task_id=None,
group_id=None, chord=None, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
header = kwargs.pop('header')
body = kwargs.pop('body')
header, body = (list(maybe_subtask(header)),
maybe_subtask(body))
# forward certain options to body
if chord is not None:
body.options['chord'] = chord
if group_id is not None:
body.options['group_id'] = group_id
[body.link(s) for s in options.pop('link', [])]
[body.link_error(s) for s in options.pop('link_error', [])]
body_result = body.freeze(task_id)
parent = super(Chord, self).apply_async((header, body, args),
kwargs, **options)
body_result.parent = parent
return body_result
def apply_async(self, args=(), kwargs={}, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
tasks = [maybe_subtask(task).clone(task_id=uuid(), **kwargs)
for task in kwargs["tasks"]]
reduce(lambda a, b: a.link(b), tasks)
tasks[0].apply_async()
results = [task.type.AsyncResult(task.options["task_id"])
for task in tasks]
reduce(lambda a, b: a.set_parent(b), reversed(results))
return results[-1]
def apply_async(self, args=(), kwargs={}, task_id=None, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
header, body = (list(kwargs["header"]),
maybe_subtask(kwargs["body"]))
callback_id = body.options.setdefault("task_id", task_id or uuid())
parent = super(Chord, self).apply_async((header, body), **options)
body_result = self.AsyncResult(callback_id)
body_result.parent = parent
return body_result
def apply_async(self, args=(), kwargs={}, task_id=None, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
header, body = (list(kwargs['header']),
maybe_subtask(kwargs['body']))
callback_id = body.options.setdefault('task_id', task_id or uuid())
parent = super(Chord, self).apply_async((header, body), **options)
body_result = self.AsyncResult(callback_id)
body_result.parent = parent
return body_result
def apply_async(self, args=(), kwargs={}, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
tasks = kwargs["tasks"]
tasks = [
maybe_subtask(task).clone(task_id=uuid(), **kwargs)
for task in kwargs["tasks"]
]
reduce(lambda a, b: a.link(b), tasks)
tasks[0].apply_async()
results = [
task.type.AsyncResult(task.options["task_id"])
for task in tasks
]
reduce(lambda a, b: a.set_parent(b), reversed(results))
return results[-1]
def apply_async(self, args=(), kwargs={}, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
options.pop('publisher', None)
group_id = options.pop('group_id', None)
chord = options.pop('chord', None)
tasks = [maybe_subtask(t).clone(
task_id=options.pop('task_id', uuid()),
**options
)
for t in kwargs['tasks']]
reduce(lambda a, b: a.link(b), tasks)
if group_id:
tasks[-1].set(group_id=group_id)
if chord:
tasks[-1].set(chord=chord)
tasks[0].apply_async()
results = [task.type.AsyncResult(task.options['task_id'])
for task in tasks]
reduce(lambda a, b: a.set_parent(b), reversed(results))
return results[-1]
def apply_async(self, args=(), kwargs={}, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
options.pop("publisher", None)
taskset_id = options.pop("taskset_id", None)
chord = options.pop("chord", None)
tasks = [maybe_subtask(t).clone(
task_id=options.pop("task_id", uuid()),
**options
)
for t in kwargs["tasks"]]
reduce(lambda a, b: a.link(b), tasks)
if taskset_id:
tasks[-1].set(taskset_id=taskset_id)
if chord:
tasks[-1].set(chord=chord)
tasks[0].apply_async()
results = [task.type.AsyncResult(task.options["task_id"])
for task in tasks]
reduce(lambda a, b: a.set_parent(b), reversed(results))
return results[-1]
def apply(self, args=(), kwargs={}, propagate=True, **options):
body = kwargs['body']
res = super(Chord, self).apply(args, dict(kwargs, eager=True),
**options)
return maybe_subtask(body).apply(
args=(res.get(propagate=propagate).get(), ))
def apply(self, args=(), kwargs={}, **options):
body = kwargs["body"]
res = super(Chord, self).apply(args, kwargs, **options)
return maybe_subtask(body).apply(args=(res.get().join(), ))
def __init__(self, tasks=None, app=None, Publisher=None):
self.app = app_or_default(app or self.app)
self.data = [maybe_subtask(t) for t in tasks or []]
self.total = len(self.tasks)
self.Publisher = Publisher or self.app.amqp.TaskProducer
def __init__(self, tasks=None, app=None, Publisher=None):
self._app = app or self._app
self.data = [maybe_subtask(t) for t in tasks or []]
self._Publisher = Publisher
def test_is_None(self):
self.assertIsNone(maybe_subtask(None))
def apply(self, args=(), kwargs={}, **options):
body = kwargs["body"]
res = super(Chord, self).apply(args, kwargs, **options)
return maybe_subtask(body).apply(args=(res.get().join(), ))
def test_is_dict(self):
self.assertIsInstance(maybe_subtask(dict(add.s())), Signature)
def test_when_sig(self):
s = add.s()
self.assertIs(maybe_subtask(s), s)
def __init__(self, tasks=None, app=None, Publisher=None):
super(TaskSet, self).__init__(maybe_subtask(t) for t in tasks or [])
self.app = app_or_default(app or self.app)
self.Publisher = Publisher or self.app.amqp.TaskProducer
self.total = len(self) # XXX compat
def apply(self, args=(), kwargs={}, propagate=True, **options):
body = kwargs['body']
res = super(Chord, self).apply(args, dict(kwargs, eager=True),
**options)
return maybe_subtask(body).apply(args=(res.get(
propagate=propagate).get(), ))
def __init__(self, tasks=None, app=None, Publisher=None):
super(TaskSet, self).__init__(maybe_subtask(t) for t in tasks or [])
self.app = app_or_default(app or self.app)
self.Publisher = Publisher or self.app.amqp.TaskProducer
self.total = len(self) # XXX compat
def test_is_None(self):
self.assertIsNone(maybe_subtask(None))
def test_when_sig(self):
s = add.s()
self.assertIs(maybe_subtask(s), s)
def test_is_dict(self):
self.assertIsInstance(maybe_subtask(dict(add.s())), Signature)
