def unlock_chord(setid, callback, interval=1, propagate=False,
max_retries=None, result=None):
result = _res.TaskSetResult(setid, map(_res.AsyncResult, result))
j = result.join_native if result.supports_native_join else result.join
if result.ready():
subtask(callback).delay(j(propagate=propagate))
else:
unlock_chord.retry(countdown=interval, max_retries=max_retries)
def unlock_chord(group_id, callback, interval=1, propagate=False,
max_retries=None, result=None):
result = _res.GroupResult(group_id, map(_res.AsyncResult, result))
j = result.join_native if result.supports_native_join else result.join
if result.ready():
subtask(callback).delay(j(propagate=propagate))
else:
unlock_chord.retry(countdown=interval, max_retries=max_retries)
def unlock_chord(group_id, callback, interval=1, propagate=False,
max_retries=None, result=None):
AR = _res.AsyncResult
result = _res.GroupResult(group_id, [AR(r) for r in result])
j = result.join_native if result.supports_native_join else result.join
if result.ready():
subtask(callback).delay(j(propagate=propagate))
else:
unlock_chord.retry(countdown=interval, max_retries=max_retries)
def unlock_chord(group_id, callback, interval=None, propagate=False,
max_retries=None, result=None, Result=_res.AsyncResult):
if interval is None:
interval = unlock_chord.default_retry_delay
result = _res.GroupResult(group_id, [Result(r) for r in result])
j = result.join_native if result.supports_native_join else result.join
if result.ready():
subtask(callback).delay(j(propagate=propagate))
else:
unlock_chord.retry(countdown=interval, max_retries=max_retries)
def unlock_chord(group_id, callback, interval=None, propagate=False,
max_retries=None, result=None):
if interval is None:
interval = unlock_chord.default_retry_delay
result = _res.GroupResult(group_id, map(_res.AsyncResult, result))
j = result.join_native if result.supports_native_join else result.join
if result.ready():
subtask(callback).delay(j(propagate=propagate))
else:
unlock_chord.retry(countdown=interval, max_retries=max_retries)
def run(self, tasks, result, setid):
app = self.app
result = from_serializable(result)
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.TaskSetResult(result.id,
[subtask(task).apply(taskset_id=setid)
for task in tasks])
with app.default_producer() as pub:
[subtask(task).apply_async(taskset_id=setid, publisher=pub)
for task in tasks]
parent = get_current_task()
if parent:
parent.request.children.append(result)
return result
def run(self, tasks, result, group_id):
app = self.app
result = from_serializable(result)
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.GroupResult(result.id,
[subtask(task).apply(group_id=group_id)
for task in tasks])
with app.default_producer() as pub:
[subtask(task).apply_async(group_id=group_id, publisher=pub,
add_to_parent=False)
for task in tasks]
parent = get_current_worker_task()
if parent:
parent.request.children.append(result)
return result
def run(self, tasks, result, group_id):
app = self.app
result = from_serializable(result)
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.GroupResult(result.id,
[subtask(task).apply(group_id=group_id)
for task in tasks])
with app.default_producer() as pub:
[subtask(task).apply_async(group_id=group_id, publisher=pub,
add_to_parent=False)
for task in tasks]
parent = get_current_worker_task()
if parent:
parent.request.children.append(result)
return result
def unlock_chord(group_id, callback, interval=None, propagate=None,
max_retries=None, result=None,
Result=app.AsyncResult, GroupResult=app.GroupResult,
from_serializable=from_serializable):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
# check if the task group is ready, and if so apply the callback.
deps = GroupResult(
group_id,
[from_serializable(r, Result=Result) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = subtask(callback)
try:
ret = j(propagate=propagate)
except Exception, exc:
culprit = deps._failed_join_report().next()
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=ChordError('Dependency %s raised %r' % (
culprit.id, exc)),
)
else:
try:
callback.delay(ret)
except Exception, exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Call callback error: %r' % (exc, )))
def subtask(self, *args, **kwargs):
"""Returns :class:`~celery.subtask` object for
this task, wrapping arguments and execution options
for a single task invocation."""
from celery.canvas import subtask
return subtask(self, *args, **kwargs)
def unlock_chord(group_id, callback, interval=None, propagate=True,
max_retries=None, result=None,
Result=app.AsyncResult, GroupResult=app.GroupResult,
from_serializable=from_serializable):
if interval is None:
interval = unlock_chord.default_retry_delay
deps = GroupResult(
group_id,
[from_serializable(r, Result=Result) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = subtask(callback)
try:
ret = j(propagate=propagate)
except Exception as exc:
culprit = next(deps._failed_join_report())
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=ChordError('Dependency %s raised %r' % (
culprit.id, exc)),
)
else:
callback.delay(ret)
else:
return unlock_chord.retry(countdown=interval,
max_retries=max_retries)
def run(self, tasks, result):
app = self.app
result = from_serializable(result)
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.TaskSetResult(result.id,
[subtask(task).apply(taskset_id=self.request.taskset)
for task in tasks])
with app.pool.acquire(block=True) as conn:
with app.amqp.TaskPublisher(conn) as publisher:
[subtask(task).apply_async(
taskset_id=self.request.taskset,
publisher=publisher)
for task in tasks]
parent = get_current_task()
if parent:
parent.request.children.append(result)
return result
def run(self, tasks, result):
app = self.app
result = from_serializable(result)
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.TaskSetResult(result.id, [
subtask(task).apply(taskset_id=self.request.taskset)
for task in tasks
])
with app.pool.acquire(block=True) as conn:
with app.amqp.TaskPublisher(conn) as publisher:
[
subtask(task).apply_async(
taskset_id=self.request.taskset,
publisher=publisher) for task in tasks
]
parent = get_current_task()
if parent:
parent.request.children.append(result)
return result
def unlock_chord(group_id,
callback,
interval=None,
propagate=None,
max_retries=None,
result=None,
Result=app.AsyncResult,
GroupResult=app.GroupResult,
from_serializable=from_serializable):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
if interval is None:
interval = unlock_chord.default_retry_delay
# check if the task group is ready, and if so apply the callback.
deps = GroupResult(
group_id,
[from_serializable(r, app=app) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = subtask(callback)
try:
ret = j(propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError(reason),
)
else:
try:
callback.delay(ret)
except Exception as exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
else:
return unlock_chord.retry(countdown=interval,
max_retries=max_retries)
def run(self, tasks, result, group_id, partial_args):
app = self.app
result = from_serializable(result, app)
# any partial args are added to all tasks in the group
taskit = (subtask(task).clone(partial_args) for i, task in enumerate(tasks))
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.GroupResult(result.id, [stask.apply(group_id=group_id) for stask in taskit])
with app.producer_or_acquire() as pub:
[stask.apply_async(group_id=group_id, publisher=pub, add_to_parent=False) for stask in taskit]
parent = get_current_worker_task()
if parent:
parent.request.children.append(result)
return result
def run(self, tasks, result, group_id, partial_args):
app = self.app
result = from_serializable(result)
# any partial args are added to all tasks in the group
taskit = (subtask(task).clone(partial_args)
for i, task in enumerate(tasks))
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.GroupResult(result.id,
[task.apply(group_id=group_id) for task in taskit])
with app.producer_or_acquire() as pub:
[task.apply_async(group_id=group_id, publisher=pub,
add_to_parent=False) for task in taskit]
parent = get_current_worker_task()
if parent:
parent.request.children.append(result)
return result
def unlock_chord(group_id, callback, interval=None, propagate=None,
max_retries=None, result=None,
Result=app.AsyncResult, GroupResult=app.GroupResult,
from_serializable=from_serializable):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
if interval is None:
interval = unlock_chord.default_retry_delay
# check if the task group is ready, and if so apply the callback.
deps = GroupResult(
group_id,
[from_serializable(r, app=app) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = subtask(callback)
try:
ret = j(propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit, exc,
)
except StopIteration:
reason = repr(exc)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=ChordError(reason),
)
else:
try:
callback.delay(ret)
except Exception as exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
else:
return unlock_chord.retry(countdown=interval,
max_retries=max_retries)
def xstarmap(task, it):
task = subtask(task).type
return [task(*args) for args in it]
def xmap(task, it):
task = subtask(task).type
return [task(value) for value in it]
def apply(self, args=(), kwargs={}, subtask=maybe_subtask, **options):
last, fargs = None, args # fargs passed to first task only
for task in kwargs['tasks']:
res = subtask(task).clone(fargs).apply(last and (last.get(), ))
res.parent, last, fargs = last, res, None
return last
def xmap(task, it):
task = subtask(task).type
return list(imap(task, it))
def test_reverse(self):
x = chord([add.s(2, 2), add.s(4, 4)], body=mul.s(4))
self.assertIsInstance(subtask(x), chord)
self.assertIsInstance(subtask(dict(x)), chord)
def xmap(task, it):
task = subtask(task).type
return [task(item) for item in it]
def subtask(self, args=None, *starargs, **starkwargs):
"""Returns :class:`~celery.subtask` object for
this task, wrapping arguments and execution options
for a single task invocation."""
return subtask(self, args, *starargs, **starkwargs)
def test_reverse(self):
x = add.s(2, 2) | add.s(2)
self.assertIsInstance(subtask(x), chain)
self.assertIsInstance(subtask(dict(x)), chain)
def test_reverse(self):
x = add.s(2, 2) | add.s(2)
self.assertIsInstance(subtask(x), chain)
self.assertIsInstance(subtask(dict(x)), chain)
def test_reverse(self):
x = chord([add.s(2, 2), add.s(4, 4)], body=mul.s(4))
self.assertIsInstance(subtask(x), chord)
self.assertIsInstance(subtask(dict(x)), chord)
def subtask(self, *args, **kwargs):
"""Returns :class:`~celery.subtask` object for
this task, wrapping arguments and execution options
for a single task invocation."""
from celery.canvas import subtask
return subtask(self, *args, **kwargs)
def test_link(self):
x = subtask(SIG)
x.link(SIG)
x.link(SIG)
self.assertIn(SIG, x.options['link'])
self.assertEqual(len(x.options['link']), 1)
def test_apply_async_when_not_registered(self):
s = subtask('xxx.not.registered')
self.assertTrue(s._apply_async)
def xmap(task, it):
task = subtask(task).type
return list(map(task, it))
def test_AsyncResult_when_not_registerd(self):
s = subtask('xxx.not.registered')
self.assertTrue(s.AsyncResult)
def xstarmap(task, it):
task = subtask(task).type
return list(starmap(task, it))
def xstarmap(task, it):
task = subtask(task).type
return [task(*item) for item in it]
def test_reverse(self):
x = group([add.s(2, 2), add.s(4, 4)])
self.assertIsInstance(subtask(x), group)
self.assertIsInstance(subtask(dict(x)), group)
def xstarmap(task, it):
task = subtask(task).type
return [task(*args) for args in it]
def test_link_error(self):
x = subtask(SIG)
x.link_error(SIG)
x.link_error(SIG)
self.assertIn(SIG, x.options["link_error"])
self.assertEqual(len(x.options["link_error"]), 1)
def xstarmap(task, it):
task = subtask(task).type
return list(starmap(task, it))
def xstarmap(task, it):
task = subtask(task).type
return [task(*item) for item in it]
def subtask(self, args=None, *starargs, **starkwargs):
"""Returns :class:`~celery.subtask` object for
this task, wrapping arguments and execution options
for a single task invocation."""
return subtask(self, args, *starargs, **starkwargs)
def apply(self, args=(), kwargs={}, subtask=maybe_subtask, **options):
last, fargs = None, args # fargs passed to first task only
for task in kwargs['tasks']:
res = subtask(task).clone(fargs).apply(last and (last.get(), ))
res.parent, last, fargs = last, res, None
return last
def xmap(task, it):
task = subtask(task).type
return [task(value) for value in it]
def call_task(self, task):
try:
X = subtask(task)
X.apply_async()
except Exception as exc:
error('Could not call task: %r', exc, exc_info=1)
def test_AsyncResult_when_not_registerd(self):
s = subtask('xxx.not.registered')
self.assertTrue(s.AsyncResult)
def test_reverse(self):
x = group([add.s(2, 2), add.s(4, 4)])
self.assertIsInstance(subtask(x), group)
self.assertIsInstance(subtask(dict(x)), group)
def test_apply_async_when_not_registered(self):
s = subtask('xxx.not.registered')
self.assertTrue(s._apply_async)
def test_link_error(self):
x = subtask(SIG)
x.link_error(SIG)
x.link_error(SIG)
self.assertIn(SIG, x.options["link_error"])
self.assertEqual(len(x.options["link_error"]), 1)
def test_link(self):
x = subtask(SIG)
x.link(SIG)
x.link(SIG)
self.assertIn(SIG, x.options['link'])
self.assertEqual(len(x.options['link']), 1)
def call_task(self, task):
try:
X = subtask(task)
X.apply_async()
except Exception as exc:
error('Could not call task: %r', exc, exc_info=1)
def xmap(task, it):
task = subtask(task).type
return [task(item) for item in it]
