def on_chord_part_return(self, task, state, result, propagate=None):
if not self.implements_incr:
return
app = self.app
if propagate is None:
propagate = app.conf.CELERY_CHORD_PROPAGATES
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=task.backend)
except Exception as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('Cannot restore group: {0!r}'.format(exc)),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('GroupResult {0} no longer exists'.format(gid)),
)
val = self.incr(key)
if val >= len(deps):
callback = maybe_signature(task.request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def on_chord_part_return(self, task, state, result, propagate=None):
if not self.implements_incr:
return
app = self.app
if propagate is None:
propagate = app.conf.CELERY_CHORD_PROPAGATES
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=task.backend)
except Exception as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('Cannot restore group: {0!r}'.format(exc)),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('GroupResult {0} no longer exists'.format(gid)),
)
val = self.incr(key)
if val >= len(deps):
callback = maybe_signature(task.request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit, exc,
)
except StopIteration:
reason = repr(exc)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def schedule_query(self, source_data, mapper, reducer=None):
mapper = maybe_signature(mapper, self.app)
reducer = maybe_signature(reducer, self.app) if reducer else None
if isinstance(source_data, dict):
sub_tasks = [
mapper.clone(args=((q, ids), )).set(queue=q)
for q, ids in source_data.items()
]
else:
sub_tasks = [mapper.clone(args=(data, )) for data in source_data]
scheduled = chord(sub_tasks)(reducer) if reducer else group(sub_tasks)()
return scheduled
def on_chord_part_return(self, request, state, result, **kwargs):
if not self.implements_incr:
return
app = self.app
gid = request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=self)
except Exception as exc:
callback = maybe_signature(request.chord, app=app)
logger.error("Chord %r raised: %r", gid, exc, exc_info=1)
return self.chord_error_from_stack(callback, ChordError("Cannot restore group: {0!r}".format(exc)))
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(request.chord, app=app)
logger.error("Chord callback %r raised: %r", gid, exc, exc_info=1)
return self.chord_error_from_stack(callback, ChordError("GroupResult {0} no longer exists".format(gid)))
val = self.incr(key)
size = len(deps)
if val > size: # pragma: no cover
logger.warning("Chord counter incremented too many times for %r", gid)
elif val == size:
callback = maybe_signature(request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=True)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = "Dependency {0.id} raised {1!r}".format(culprit, exc)
except StopIteration:
reason = repr(exc)
logger.error("Chord %r raised: %r", gid, reason, exc_info=1)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error("Chord %r raised: %r", gid, exc, exc_info=1)
self.chord_error_from_stack(callback, ChordError("Callback error: {0!r}".format(exc)))
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def on_chord_part_return(self, request, state, result, **kwargs):
"""Called on finishing each part of a Chord header"""
tid, gid = request.id, request.group
if not gid or not tid:
return
call_callback = False
with transaction.atomic():
# We need to know if `count` hits 0.
# wrap the update in a transaction
# with a `select_for_update` lock to prevent race conditions.
# SELECT FOR UPDATE is not supported on all databases
chord_counter = (
ChordCounter.objects.select_for_update()
.get(group_id=gid)
)
chord_counter.count -= 1
if chord_counter.count != 0:
chord_counter.save()
else:
# Last task in the chord header has finished
call_callback = True
chord_counter.delete()
if call_callback:
deps = chord_counter.group_result(app=self.app)
if deps.ready():
callback = maybe_signature(request.chord, app=self.app)
trigger_callback(
app=self.app,
callback=callback,
group_result=deps
)
def get_task_to_run(self, tree):
"""Working from the bottom up, replace each node with a chain to its
descendant, or celery.Group of descendants.
:param tree: Dependancy graph of tasks
:type tree: networkx.DiGraph
:returns: chain to execute
"""
# TODO: This could be more parallel
return chain(*[
maybe_signature(tree.node[name]['task'], self.celery_app)
for name in nx.topological_sort(tree)
])
def get_task_to_run(self, tree):
"""Working from the bottom up, replace each node with a chain to its
descendant, or celery.Group of descendants.
:param tree: Dependancy graph of tasks
:type tree: networkx.DiGraph
:returns: chain to execute
"""
# TODO: This could be more parallel
return chain(*[
maybe_signature(tree.node[name]['task'], self.celery_app)
for name in nx.topological_sort(tree)
])
def on_chord_part_return(self, request, state, result, **kwargs):
if not self.implements_incr:
return
app = self.app
gid = request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=self)
except Exception as exc: # pylint: disable=broad-except
callback = maybe_signature(request.chord, app=app)
logger.exception('Chord %r raised: %r', gid, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Cannot restore group: {exc!r}'),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(request.chord, app=app)
logger.exception('Chord callback %r raised: %r', gid, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'GroupResult {gid} no longer exists'),
)
val = self.incr(key)
# Set the chord size to the value defined in the request, or fall back
# to the number of dependencies we can see from the restored result
size = request.chord.get("chord_size")
if size is None:
size = len(deps)
if val > size: # pragma: no cover
logger.warning('Chord counter incremented too many times for %r',
gid)
elif val == size:
callback = maybe_signature(request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=app.conf.result_chord_join_timeout,
propagate=True)
except Exception as exc: # pylint: disable=broad-except
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
logger.exception('Chord %r raised: %r', gid, reason)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord %r raised: %r', gid, exc)
self.chord_error_from_stack(
callback,
ChordError(f'Callback error: {exc!r}'),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, self.expires)
def on_chord_part_return(self, request, state, result, **kwargs):
if not self.implements_incr:
return
app = self.app
gid = request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=self)
except Exception as exc:
callback = maybe_signature(request.chord, app=app)
logger.error('Chord %r raised: %r', gid, exc, exc_info=1)
return self.chord_error_from_stack(
callback,
ChordError('Cannot restore group: {0!r}'.format(exc)),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(request.chord, app=app)
logger.error('Chord callback %r raised: %r',
gid,
exc,
exc_info=1)
return self.chord_error_from_stack(
callback,
ChordError('GroupResult {0} no longer exists'.format(gid)),
)
val = self.incr(key)
size = len(deps)
if val > size: # pragma: no cover
logger.warning('Chord counter incremented too many times for %r',
gid)
elif val == size:
callback = maybe_signature(request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=True)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', gid, reason, exc_info=1)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r', gid, exc, exc_info=1)
self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def unlock_chord(self, group, callback, interval=1, max_retries=None):
if group.ready():
return maybe_signature(callback).delay(group.join())
raise self.retry(countdown=interval, max_retries=max_retries)