def test_GroupResult(self):
x = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(10)],
)
self.assertEqual(x, from_serializable(x.serializable(), self.app))
self.assertEqual(x, from_serializable(x, self.app))
def unlock_chord(group_id, callback, interval=None, propagate=True,
max_retries=None, result=None,
Result=app.AsyncResult, GroupResult=app.GroupResult,
from_serializable=from_serializable):
if interval is None:
interval = unlock_chord.default_retry_delay
deps = GroupResult(
group_id,
[from_serializable(r, Result=Result) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = subtask(callback)
try:
ret = j(propagate=propagate)
except Exception as exc:
culprit = next(deps._failed_join_report())
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=ChordError('Dependency %s raised %r' % (
culprit.id, exc)),
)
else:
callback.delay(ret)
else:
return unlock_chord.retry(countdown=interval,
max_retries=max_retries)
def test_with_parent(self):
x = self.app.AsyncResult(uuid())
x.parent = self.app.AsyncResult(uuid())
y = from_serializable(x.serializable(), self.app)
self.assertEqual(y, x)
self.assertEqual(y.parent, x.parent)
self.assertIsInstance(y.parent, AsyncResult)
def test_with_parent(self):
x = self.app.AsyncResult(uuid())
x.parent = self.app.AsyncResult(uuid())
y = from_serializable(x.serializable(), self.app)
self.assertEqual(y, x)
self.assertEqual(y.parent, x.parent)
self.assertIsInstance(y.parent, AsyncResult)
def unlock_chord(group_id, callback, interval=None, propagate=None,
max_retries=None, result=None,
Result=app.AsyncResult, GroupResult=app.GroupResult,
from_serializable=from_serializable):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
# check if the task group is ready, and if so apply the callback.
deps = GroupResult(
group_id,
[from_serializable(r, Result=Result) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = subtask(callback)
try:
ret = j(propagate=propagate)
except Exception, exc:
culprit = deps._failed_join_report().next()
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=ChordError('Dependency %s raised %r' % (
culprit.id, exc)),
)
else:
try:
callback.delay(ret)
except Exception, exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Call callback error: %r' % (exc, )))
def _restore_group(self, group_id):
"""Get task metadata for a task by id."""
meta = self.get(self.get_key_for_group(group_id))
# previously this was always pickled, but later this
# was extended to support other serializers, so the
# structure is kind of weird.
if meta:
meta = self.decode(meta)
result = meta["result"]
meta["result"] = from_serializable(result, self.app)
return meta
def unlock_chord(group_id,
callback,
interval=None,
propagate=None,
max_retries=None,
result=None,
Result=app.AsyncResult,
GroupResult=app.GroupResult,
from_serializable=from_serializable):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
if interval is None:
interval = unlock_chord.default_retry_delay
# check if the task group is ready, and if so apply the callback.
deps = GroupResult(
group_id,
[from_serializable(r, app=app) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = subtask(callback)
try:
ret = j(propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError(reason),
)
else:
try:
callback.delay(ret)
except Exception as exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
else:
return unlock_chord.retry(countdown=interval,
max_retries=max_retries)
def _restore_group(self, group_id):
"""Get task metadata for a task by id."""
meta = self.get(self.get_key_for_group(group_id))
# previously this was always pickled, but later this
# was extended to support other serializers, so the
# structure is kind of weird.
if meta:
meta = self.decode(meta)
result = meta['result']
meta['result'] = from_serializable(result, self.app)
return meta
def _restore_taskset(self, taskset_id):
"""Get task metadata for a task by id."""
meta = self.get(self.get_key_for_taskset(taskset_id))
# previously this was always pickled, but later this
# was extended to support other serializers, so the
# structure is kind of weird.
if meta:
meta = self.decode(meta)
result = meta["result"]
if isinstance(result, (list, tuple)):
return {"result": from_serializable(result)}
return meta
def _restore_group(self, group_id):
"""Get task metadata for a task by id."""
meta = self.get(self.get_key_for_group(group_id))
# previously this was always pickled, but later this
# was extended to support other serializers, so the
# structure is kind of weird.
if meta:
meta = self.decode(meta)
result = meta['result']
if isinstance(result, (list, tuple)):
return {'result': from_serializable(result)}
return meta
def run(self, tasks, result, group_id, partial_args):
app = self.app
result = from_serializable(result, app)
# any partial args are added to all tasks in the group
taskit = (subtask(task).clone(partial_args) for i, task in enumerate(tasks))
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.GroupResult(result.id, [stask.apply(group_id=group_id) for stask in taskit])
with app.producer_or_acquire() as pub:
[stask.apply_async(group_id=group_id, publisher=pub, add_to_parent=False) for stask in taskit]
parent = get_current_worker_task()
if parent:
parent.request.children.append(result)
return result
def run(self, tasks, result, setid):
app = self.app
result = from_serializable(result)
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.TaskSetResult(result.id,
[subtask(task).apply(taskset_id=setid)
for task in tasks])
with app.default_producer() as pub:
[subtask(task).apply_async(taskset_id=setid, publisher=pub)
for task in tasks]
parent = get_current_task()
if parent:
parent.request.children.append(result)
return result
def run(self, tasks, result, group_id):
app = self.app
result = from_serializable(result)
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.GroupResult(result.id,
[subtask(task).apply(group_id=group_id)
for task in tasks])
with app.default_producer() as pub:
[subtask(task).apply_async(group_id=group_id, publisher=pub,
add_to_parent=False)
for task in tasks]
parent = get_current_worker_task()
if parent:
parent.request.children.append(result)
return result
def run(self, tasks, result):
app = self.app
result = from_serializable(result)
with app.pool.acquire(block=True) as conn:
with app.amqp.TaskPublisher(conn) as publisher:
res_ = [subtask(task).apply_async(
taskset_id=self.request.taskset,
publisher=publisher)
for task in tasks]
parent = get_current_task()
if parent:
parent.request.children.append(result)
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.TaskSetResult(result.id, res_)
return result
def run(self, tasks, result, group_id):
app = self.app
result = from_serializable(result)
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.GroupResult(result.id,
[subtask(task).apply(group_id=group_id)
for task in tasks])
with app.default_producer() as pub:
[subtask(task).apply_async(group_id=group_id, publisher=pub,
add_to_parent=False)
for task in tasks]
parent = get_current_worker_task()
if parent:
parent.request.children.append(result)
return result
def run(self, tasks, result, group_id, partial_args):
app = self.app
result = from_serializable(result)
# any partial args are added to all tasks in the group
taskit = (subtask(task).clone(partial_args)
for i, task in enumerate(tasks))
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.GroupResult(result.id,
[task.apply(group_id=group_id) for task in taskit])
with app.producer_or_acquire() as pub:
[task.apply_async(group_id=group_id, publisher=pub,
add_to_parent=False) for task in taskit]
parent = get_current_worker_task()
if parent:
parent.request.children.append(result)
return result
def unlock_chord(group_id, callback, interval=None, propagate=None,
max_retries=None, result=None,
Result=app.AsyncResult, GroupResult=app.GroupResult,
from_serializable=from_serializable):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
if interval is None:
interval = unlock_chord.default_retry_delay
# check if the task group is ready, and if so apply the callback.
deps = GroupResult(
group_id,
[from_serializable(r, app=app) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = subtask(callback)
try:
ret = j(propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit, exc,
)
except StopIteration:
reason = repr(exc)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=ChordError(reason),
)
else:
try:
callback.delay(ret)
except Exception as exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
else:
return unlock_chord.retry(countdown=interval,
max_retries=max_retries)
def run(self, tasks, result):
app = self.app
result = from_serializable(result)
if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
return app.TaskSetResult(result.id, [
subtask(task).apply(taskset_id=self.request.taskset)
for task in tasks
])
with app.pool.acquire(block=True) as conn:
with app.amqp.TaskPublisher(conn) as publisher:
[
subtask(task).apply_async(
taskset_id=self.request.taskset,
publisher=publisher) for task in tasks
]
parent = get_current_task()
if parent:
parent.request.children.append(result)
return result
def test_compat(self):
uid = uuid()
x = from_serializable([uid, []], app=self.app)
self.assertEqual(x.id, uid)
def test_GroupResult(self):
x = GroupResult(uuid(), [AsyncResult(uuid()) for _ in range(10)])
self.assertEqual(x, from_serializable(x.serializable()))
self.assertEqual(x, from_serializable(x))
def test_AsyncResult(self):
x = AsyncResult(uuid())
self.assertEqual(x, from_serializable(x.serializable()))
self.assertEqual(x, from_serializable(x))
def test_GroupResult(self):
x = GroupResult(uuid(), [AsyncResult(uuid()) for _ in range(10)])
self.assertEqual(x, from_serializable(x.serializable()))
self.assertEqual(x, from_serializable(x))
def test_AsyncResult(self):
x = AsyncResult(uuid())
self.assertEqual(x, from_serializable(x.serializable()))
self.assertEqual(x, from_serializable(x))
def test_GroupResult(self):
x = self.app.GroupResult(
uuid(), [self.app.AsyncResult(uuid()) for _ in range(10)],
)
self.assertEqual(x, from_serializable(x.serializable(), self.app))
self.assertEqual(x, from_serializable(x, self.app))
def test_compat(self):
uid = uuid()
x = from_serializable([uid, []])
self.assertEqual(x.id, uid)