def apply_async(self,
args=(),
kwargs={},
task_id=None,
group_id=None,
chord=None,
**options):
app = self.app
if app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
header = kwargs.pop('header')
body = kwargs.pop('body')
header, body = (maybe_signature(header, app=app),
maybe_signature(body, app=app))
# forward certain options to body
if chord is not None:
body.options['chord'] = chord
if group_id is not None:
body.options['group_id'] = group_id
[body.link(s) for s in options.pop('link', [])]
[body.link_error(s) for s in options.pop('link_error', [])]
body_result = body.freeze(task_id)
parent = super(Chord, self).apply_async((header, body, args),
kwargs, **options)
body_result.parent = parent
return body_result
def unlock_chord(group_id,
callback,
interval=None,
propagate=None,
max_retries=None,
result=None,
Result=app.AsyncResult,
GroupResult=app.GroupResult,
result_from_tuple=result_from_tuple):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
if interval is None:
interval = unlock_chord.default_retry_delay
# check if the task group is ready, and if so apply the callback.
callback = maybe_signature(callback, app)
deps = GroupResult(
group_id,
[result_from_tuple(r, app=app) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = maybe_signature(callback, app=app)
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(callback,
ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r',
group_id,
exc,
exc_info=1)
app.backend.chord_error_from_stack(
callback,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
else:
raise unlock_chord.retry(countdown=interval,
max_retries=max_retries)
def unlock_chord(self,
group_id,
callback,
interval=None,
max_retries=None,
result=None,
Result=app.AsyncResult,
GroupResult=app.GroupResult,
result_from_tuple=result_from_tuple,
**kwargs):
if interval is None:
interval = self.default_retry_delay
# check if the task group is ready, and if so apply the callback.
callback = maybe_signature(callback, app)
deps = GroupResult(
group_id,
[result_from_tuple(r, app=app) for r in result],
app=app,
)
j = deps.join_native if deps.supports_native_join else deps.join
try:
ready = deps.ready()
except Exception as exc:
raise self.retry(
exc=exc,
countdown=interval,
max_retries=max_retries,
)
else:
if not ready:
raise self.retry(countdown=interval, max_retries=max_retries)
callback = maybe_signature(callback, app=app)
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=True)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(
callback,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
def unlock_chord(self, group_id, callback, interval=None, propagate=None,
max_retries=None, result=None,
Result=app.AsyncResult, GroupResult=app.GroupResult,
result_from_tuple=result_from_tuple):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
if interval is None:
interval = self.default_retry_delay
# check if the task group is ready, and if so apply the callback.
callback = maybe_signature(callback, app)
deps = GroupResult(
group_id,
[result_from_tuple(r, app=app) for r in result],
app=app,
)
j = deps.join_native if deps.supports_native_join else deps.join
try:
ready = deps.ready()
except Exception as exc:
raise self.retry(
exc=exc, countdown=interval, max_retries=max_retries,
)
else:
if not ready:
raise self.retry(countdown=interval, max_retries=max_retries)
callback = maybe_signature(callback, app=app)
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit, exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(callback,
ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(
callback,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
def chord(self, header, body, partial_args=(), interval=None,
countdown=1, max_retries=None, eager=False, **kwargs):
app = self.app
# - convert back to group if serialized
tasks = header.tasks if isinstance(header, group) else header
header = group([
maybe_signature(s, app=app) for s in tasks
], app=self.app)
body = maybe_signature(body, app=app)
ch = _chord(header, body)
return ch.run(header, body, partial_args, app, interval,
countdown, max_retries, **kwargs)
def unlock_chord(self, group_id, callback, interval=None,
max_retries=None, result=None,
Result=app.AsyncResult, GroupResult=app.GroupResult,
result_from_tuple=result_from_tuple, **kwargs):
if interval is None:
interval = self.default_retry_delay
# check if the task group is ready, and if so apply the callback.
callback = maybe_signature(callback, app)
deps = GroupResult(
group_id,
[result_from_tuple(r, app=app) for r in result],
app=app,
)
j = deps.join_native if deps.supports_native_join else deps.join
try:
ready = deps.ready()
except Exception as exc:
raise self.retry(
exc=exc, countdown=interval, max_retries=max_retries,
)
else:
if not ready:
raise self.retry(countdown=interval, max_retries=max_retries)
callback = maybe_signature(callback, app=app)
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=True)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit, exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(callback,
ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(
callback,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
def run(self, header, body, partial_args=(), interval=None,
countdown=1, max_retries=None, propagate=None,
eager=False, **kwargs):
app = self.app
propagate = default_propagate if propagate is None else propagate
group_id = uuid()
AsyncResult = app.AsyncResult
prepare_member = self._prepare_member
# - convert back to group if serialized
tasks = header.tasks if isinstance(header, group) else header
header = group([
maybe_signature(s, app=app).clone() for s in tasks
])
# - eager applies the group inline
if eager:
return header.apply(args=partial_args, task_id=group_id)
results = [AsyncResult(prepare_member(task, body, group_id))
for task in header.tasks]
return self.backend.apply_chord(
header, partial_args, group_id,
body, interval=interval, countdown=countdown,
max_retries=max_retries, propagate=propagate, result=results,
)
def run(self, header, body, partial_args=(), interval=None,
countdown=1, max_retries=None, propagate=None,
eager=False, **kwargs):
app = self.app
propagate = default_propagate if propagate is None else propagate
group_id = uuid()
AsyncResult = app.AsyncResult
prepare_member = self._prepare_member
# - convert back to group if serialized
tasks = header.tasks if isinstance(header, group) else header
header = group([
maybe_signature(s, app=app).clone() for s in tasks
])
# - eager applies the group inline
if eager:
return header.apply(args=partial_args, task_id=group_id)
results = [AsyncResult(prepare_member(task, body, group_id))
for task in header.tasks]
# - fallback implementations schedules the chord_unlock task here
app.backend.on_chord_apply(group_id, body,
interval=interval,
countdown=countdown,
max_retries=max_retries,
propagate=propagate,
result=results)
# - call the header group, returning the GroupResult.
final_res = header(*partial_args, task_id=group_id)
return final_res
def run(self, header, body, partial_args=(), interval=None,
countdown=1, max_retries=None, propagate=None,
eager=False, **kwargs):
app = self.app
propagate = default_propagate if propagate is None else propagate
group_id = uuid()
AsyncResult = app.AsyncResult
prepare_member = self._prepare_member
# - convert back to group if serialized
tasks = header.tasks if isinstance(header, group) else header
header = group([
maybe_signature(s, app=app).clone() for s in tasks
])
# - eager applies the group inline
if eager:
return header.apply(args=partial_args, task_id=group_id)
results = [AsyncResult(prepare_member(task, body, group_id))
for task in header.tasks]
return self.backend.apply_chord(
header, partial_args, group_id,
body, interval=interval, countdown=countdown,
max_retries=max_retries, propagate=propagate, result=results,
)
def __init__(self, tasks=None, app=None, Publisher=None):
self.app = app_or_default(app or self.app)
super(TaskSet, self).__init__(
maybe_signature(t, app=self.app) for t in tasks or []
)
self.Publisher = Publisher or self.app.amqp.Producer
self.total = len(self) # XXX compat
def run(self, header, body, partial_args=(), interval=None,
countdown=1, max_retries=None, propagate=None,
eager=False, **kwargs):
app = self.app
propagate = default_propagate if propagate is None else propagate
group_id = uuid()
AsyncResult = app.AsyncResult
prepare_member = self._prepare_member
# - convert back to group if serialized
tasks = header.tasks if isinstance(header, group) else header
header = group([
maybe_signature(s, app=app).clone() for s in tasks
])
# - eager applies the group inline
if eager:
return header.apply(args=partial_args, task_id=group_id)
results = [AsyncResult(prepare_member(task, body, group_id))
for task in header.tasks]
# - fallback implementations schedules the chord_unlock task here
app.backend.on_chord_apply(group_id, body,
interval=interval,
countdown=countdown,
max_retries=max_retries,
propagate=propagate,
result=results)
# - call the header group, returning the GroupResult.
final_res = header(*partial_args, task_id=group_id)
return final_res
def on_chord_part_return(self,
request,
state,
result,
propagate=None,
**kwargs):
app = self.app
tid, gid = request.id, request.group
if not gid or not tid:
return
client = self.client
jkey = self.get_key_for_group(gid, '.j')
tkey = self.get_key_for_group(gid, '.t')
result = self.encode_result(result, state)
with client.pipeline() as pipe:
_, readycount, totaldiff, _, _ = pipe \
.rpush(jkey, self.encode([1, tid, state, result])) \
.llen(jkey) \
.get(tkey) \
.expire(jkey, self.expires) \
.expire(tkey, self.expires) \
.execute()
totaldiff = int(totaldiff or 0)
try:
callback = maybe_signature(request.chord, app=app)
total = callback['chord_size'] + totaldiff
if readycount == total:
decode, unpack = self.decode, self._unpack_chord_result
with client.pipeline() as pipe:
resl, = pipe \
.lrange(jkey, 0, total) \
.execute()
try:
callback.delay([unpack(tup, decode) for tup in resl])
with client.pipeline() as pipe:
_, _ = pipe \
.delete(jkey) \
.delete(tkey) \
.execute()
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord callback for %r raised: %r',
request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
except ChordError as exc:
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(callback, exc)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError('Join error: {0!r}'.format(exc)),
)
def on_chord_part_return(self, request, state, result,
propagate=None, **kwargs):
app = self.app
tid, gid = request.id, request.group
if not gid or not tid:
return
client = self.client
jkey = self.get_key_for_group(gid, '.j')
tkey = self.get_key_for_group(gid, '.t')
result = self.encode_result(result, state)
with client.pipeline() as pipe:
pipeline = pipe \
.rpush(jkey, self.encode([1, tid, state, result])) \
.llen(jkey) \
.get(tkey)
if self.expires is not None:
pipeline = pipeline \
.expire(jkey, self.expires) \
.expire(tkey, self.expires)
_, readycount, totaldiff = pipeline.execute()[:3]
totaldiff = int(totaldiff or 0)
try:
callback = maybe_signature(request.chord, app=app)
total = callback['chord_size'] + totaldiff
if readycount == total:
decode, unpack = self.decode, self._unpack_chord_result
with client.pipeline() as pipe:
resl, = pipe \
.lrange(jkey, 0, total) \
.execute()
try:
callback.delay([unpack(tup, decode) for tup in resl])
with client.pipeline() as pipe:
_, _ = pipe \
.delete(jkey) \
.delete(tkey) \
.execute()
except Exception as exc: # pylint: disable=broad-except
logger.exception(
'Chord callback for %r raised: %r', request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
except ChordError as exc:
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(callback, exc)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError('Join error: {0!r}'.format(exc)),
)
def apply_async(self, args=(), kwargs={}, task_id=None, group_id=None, chord=None, **options):
app = self.app
if app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
header = kwargs.pop("header")
body = kwargs.pop("body")
header, body = (maybe_signature(header, app=app), maybe_signature(body, app=app))
# forward certain options to body
if chord is not None:
body.options["chord"] = chord
if group_id is not None:
body.options["group_id"] = group_id
[body.link(s) for s in options.pop("link", [])]
[body.link_error(s) for s in options.pop("link_error", [])]
body_result = body.freeze(task_id)
parent = super(Chord, self).apply_async((header, body, args), kwargs, **options)
body_result.parent = parent
return body_result
def _new_chord_return(self,
task,
state,
result,
propagate=None,
PROPAGATE_STATES=states.PROPAGATE_STATES):
app = self.app
if propagate is None:
propagate = self.app.conf.CELERY_CHORD_PROPAGATES
request = task.request
tid, gid = request.id, request.group
if not gid or not tid:
return
client = self.client
jkey = self.get_key_for_group(gid, '.j')
result = self.encode_result(result, state)
with client.pipeline() as pipe:
_, readycount, _ = pipe \
.rpush(jkey, self.encode([1, tid, state, result])) \
.llen(jkey) \
.expire(jkey, 86400) \
.execute()
try:
callback = maybe_signature(request.chord, app=app)
total = callback['chord_size']
if readycount == total:
decode, unpack = self.decode, self._unpack_chord_result
with client.pipeline() as pipe:
resl, _, = pipe \
.lrange(jkey, 0, total) \
.delete(jkey) \
.execute()
try:
callback.delay([unpack(tup, decode) for tup in resl])
except Exception as exc:
error('Chord callback for %r raised: %r',
request.group,
exc,
exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
except ChordError as exc:
error('Chord %r raised: %r', request.group, exc, exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=exc,
)
except Exception as exc:
error('Chord %r raised: %r', request.group, exc, exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Join error: {0!r}'.format(exc)),
)
def on_chord_part_return(self, request, state, result, propagate=None):
app = self.app
tid, gid = request.id, request.group
if not gid or not tid:
return
client = self.client
jkey = self.get_key_for_group(gid, '.j')
tkey = self.get_key_for_group(gid, '.t')
result = self.encode_result(result, state)
with client.pipeline() as pipe:
_, readycount, totaldiff, _, _ = pipe \
.rpush(jkey, self.encode([1, tid, state, result])) \
.llen(jkey) \
.get(tkey) \
.expire(jkey, 86400) \
.expire(tkey, 86400) \
.execute()
totaldiff = int(totaldiff or 0)
try:
callback = maybe_signature(request.chord, app=app)
total = callback['chord_size'] + totaldiff
if readycount == total:
decode, unpack = self.decode, self._unpack_chord_result
with client.pipeline() as pipe:
resl, _, _ = pipe \
.lrange(jkey, 0, total) \
.delete(jkey) \
.delete(tkey) \
.execute()
try:
callback.delay([unpack(tup, decode) for tup in resl])
except Exception as exc:
error('Chord callback for %r raised: %r',
request.group,
exc,
exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
except ChordError as exc:
error('Chord %r raised: %r', request.group, exc, exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=exc,
)
except Exception as exc:
error('Chord %r raised: %r', request.group, exc, exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Join error: {0!r}'.format(exc)),
)
def _new_chord_return(self, task, state, result, propagate=None,
PROPAGATE_STATES=states.PROPAGATE_STATES):
app = self.app
if propagate is None:
propagate = self.app.conf.CELERY_CHORD_PROPAGATES
request = task.request
tid, gid = request.id, request.group
if not gid or not tid:
return
client = self.client
jkey = self.get_key_for_group(gid, '.j')
tkey = self.get_key_for_group(gid, '.t')
result = self.encode_result(result, state)
_, readycount, totaldiff, _, _ = client.pipeline() \
.rpush(jkey, self.encode([1, tid, state, result])) \
.llen(jkey) \
.get(tkey) \
.expire(jkey, 86400) \
.expire(tkey, 86400) \
.execute()
totaldiff = int(totaldiff or 0)
try:
callback = maybe_signature(request.chord, app=app)
total = callback['chord_size'] + totaldiff
if readycount >= total:
decode, unpack = self.decode, self._unpack_chord_result
resl, _, _ = client.pipeline() \
.lrange(jkey, 0, total) \
.delete(jkey) \
.delete(tkey) \
.execute()
try:
callback.delay([unpack(tup, decode) for tup in resl])
except Exception as exc:
error('Chord callback for %r raised: %r',
request.group, exc, exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
except ChordError as exc:
error('Chord %r raised: %r', request.group, exc, exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=exc,
)
except Exception as exc:
error('Chord %r raised: %r', request.group, exc, exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=ChordError('Join error: {0!r}'.format(exc)),
)
def _on_complete_lookup(self, completed_task_name):
"""
Private utility method to lookup can callback that may be registered
for a give completed task.
"""
on_complete_map = self.setdefault('_on_complete_map', {})
to_run = on_complete_map.get(completed_task_name)
if to_run:
# Convert back to celery signature object
to_run = maybe_signature(to_run)
return to_run
def group(self, tasks, result, group_id, partial_args, add_to_parent=True):
app = self.app
result = result_from_tuple(result, app)
# any partial args are added to all tasks in the group
taskit = (maybe_signature(task, app=app).clone(partial_args)
for i, task in enumerate(tasks))
with app.producer_or_acquire() as producer:
[stask.apply_async(group_id=group_id, producer=producer,
add_to_parent=False) for stask in taskit]
parent = app.current_worker_task
if add_to_parent and parent:
parent.add_trail(result)
return result
def _on_complete_lookup(self, completed_task_name):
"""
Private utility method to lookup can callback that may be registered
for a give completed task.
"""
on_complete_map = self.setdefault('_on_complete_map', {})
to_run = on_complete_map.get(completed_task_name)
if to_run:
# Convert back to celery signature object
to_run = maybe_signature(to_run)
return to_run
def prepare_steps(self, args, tasks):
app = self.app
steps = deque(tasks)
next_step = prev_task = prev_res = None
tasks, results = [], []
i = 0
while steps:
# First task get partial args from chain.
task = maybe_signature(steps.popleft(), app=app)
task = task.clone() if i else task.clone(args)
res = task.freeze()
i += 1
if isinstance(task, group):
task = maybe_unroll_group(task)
if isinstance(task, chain):
# splice the chain
steps.extendleft(reversed(task.tasks))
continue
elif isinstance(task, group) and steps and \
not isinstance(steps[0], group):
# automatically upgrade group(..) | s to chord(group, s)
try:
next_step = steps.popleft()
# for chords we freeze by pretending it's a normal
# task instead of a group.
res = Signature.freeze(next_step)
task = chord(task, body=next_step, task_id=res.task_id)
except IndexError:
pass # no callback, so keep as group
if prev_task:
# link previous task to this task.
prev_task.link(task)
# set the results parent attribute.
if not res.parent:
res.parent = prev_res
if not isinstance(prev_task, chord):
results.append(res)
tasks.append(task)
prev_task, prev_res = task, res
print(tasks)
return tasks, results
def prepare_steps(self, args, tasks):
app = self.app
steps = deque(tasks)
next_step = prev_task = prev_res = None
tasks, results = [], []
i = 0
while steps:
# First task get partial args from chain.
task = maybe_signature(steps.popleft(), app=app)
task = task.clone() if i else task.clone(args)
res = task.freeze()
i += 1
if isinstance(task, group):
task = maybe_unroll_group(task)
if isinstance(task, chain):
# splice the chain
steps.extendleft(reversed(task.tasks))
continue
elif isinstance(task, group) and steps and \
not isinstance(steps[0], group):
# automatically upgrade group(..) | s to chord(group, s)
try:
next_step = steps.popleft()
# for chords we freeze by pretending it's a normal
# task instead of a group.
res = Signature.freeze(next_step)
task = chord(task, body=next_step, task_id=res.task_id)
except IndexError:
pass # no callback, so keep as group
if prev_task:
# link previous task to this task.
prev_task.link(task)
# set the results parent attribute.
if not res.parent:
res.parent = prev_res
if not isinstance(prev_task, chord):
results.append(res)
tasks.append(task)
prev_task, prev_res = task, res
return tasks, results
def run(self,
header,
body,
partial_args=(),
interval=None,
countdown=1,
max_retries=None,
propagate=None,
eager=False,
**kwargs):
app = self.app
propagate = default_propagate if propagate is None else propagate
group_id = uuid()
# - convert back to group if serialized
tasks = header.tasks if isinstance(header, group) else header
header = group(
[maybe_signature(s, app=app).clone() for s in tasks],
app=self.app)
# - eager applies the group inline
if eager:
return header.apply(args=partial_args, task_id=group_id)
body['chord_size'] = len(header.tasks)
results = header.freeze(group_id=group_id, chord=body).results
return self.backend.apply_chord(
header,
partial_args,
group_id,
body,
interval=interval,
countdown=countdown,
max_retries=max_retries,
propagate=propagate,
result=results,
)
def run(
self,
header,
body,
partial_args=(),
interval=None,
countdown=1,
max_retries=None,
propagate=None,
eager=False,
**kwargs
):
app = self.app
propagate = default_propagate if propagate is None else propagate
group_id = uuid()
# - convert back to group if serialized
tasks = header.tasks if isinstance(header, group) else header
header = group([maybe_signature(s, app=app).clone() for s in tasks], app=self.app)
# - eager applies the group inline
if eager:
return header.apply(args=partial_args, task_id=group_id)
body.setdefault("chord_size", len(header.tasks))
results = header.freeze(group_id=group_id, chord=body).results
return self.backend.apply_chord(
header,
partial_args,
group_id,
body,
interval=interval,
countdown=countdown,
max_retries=max_retries,
propagate=propagate,
result=results,
)
def on_chord_part_return(self,
request,
state,
result,
propagate=None,
**kwargs):
app = self.app
tid, gid, group_index = request.id, request.group, request.group_index
if not gid or not tid:
return
if group_index is None:
group_index = '+inf'
client = self.client
jkey = self.get_key_for_group(gid, '.j')
tkey = self.get_key_for_group(gid, '.t')
skey = self.get_key_for_group(gid, '.s')
result = self.encode_result(result, state)
encoded = self.encode([1, tid, state, result])
with client.pipeline() as pipe:
pipeline = (pipe.zadd(jkey, {
encoded: group_index
}).zcount(jkey, "-inf",
"+inf") if self._chord_zset else pipe.rpush(
jkey, encoded).llen(jkey)).get(tkey).get(skey)
if self.expires:
pipeline = pipeline \
.expire(jkey, self.expires) \
.expire(tkey, self.expires) \
.expire(skey, self.expires)
_, readycount, totaldiff, chord_size_bytes = pipeline.execute()[:4]
totaldiff = int(totaldiff or 0)
if chord_size_bytes:
try:
callback = maybe_signature(request.chord, app=app)
total = int(chord_size_bytes) + totaldiff
if readycount == total:
header_result = GroupResult.restore(gid)
if header_result is not None:
# If we manage to restore a `GroupResult`, then it must
# have been complex and saved by `apply_chord()` earlier.
#
# Before we can join the `GroupResult`, it needs to be
# manually marked as ready to avoid blocking
header_result.on_ready()
# We'll `join()` it to get the results and ensure they are
# structured as intended rather than the flattened version
# we'd construct without any other information.
join_func = (header_result.join_native
if header_result.supports_native_join else
header_result.join)
with allow_join_result():
resl = join_func(
timeout=app.conf.result_chord_join_timeout,
propagate=True)
else:
# Otherwise simply extract and decode the results we
# stashed along the way, which should be faster for large
# numbers of simple results in the chord header.
decode, unpack = self.decode, self._unpack_chord_result
with client.pipeline() as pipe:
if self._chord_zset:
pipeline = pipe.zrange(jkey, 0, -1)
else:
pipeline = pipe.lrange(jkey, 0, total)
resl, = pipeline.execute()
resl = [unpack(tup, decode) for tup in resl]
try:
callback.delay(resl)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord callback for %r raised: %r',
request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Callback error: {exc!r}'),
)
finally:
with client.pipeline() as pipe:
pipe \
.delete(jkey) \
.delete(tkey) \
.delete(skey) \
.execute()
except ChordError as exc:
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(callback, exc)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Join error: {exc!r}'),
)
def test_is_list(self):
sigs = [dict(self.add.s(2, 2)), dict(self.add.s(4, 4))]
sigs = maybe_signature(sigs, app=self.app)
for sig in sigs:
self.assertIsInstance(sig, Signature)
self.assertIs(sig.app, self.app)
def test_is_dict(self):
self.assertIsInstance(
maybe_signature(dict(self.add.s()), app=self.app), Signature,
)
def test_is_None(self):
self.assertIsNone(maybe_signature(None, app=self.app))
def test_is_None(self):
self.assertIsNone(maybe_signature(None, app=self.app))
def test_when_sig(self):
s = self.add.s()
self.assertIs(maybe_signature(s, app=self.app), s)
def test_is_dict(self):
assert isinstance(maybe_signature(dict(self.add.s()), app=self.app),
Signature)
def test_when_sig(self):
s = self.add.s()
assert maybe_signature(s, app=self.app) is s
def test_when_sig(self):
s = self.add.s()
assert maybe_signature(s, app=self.app) is s
def test_is_None(self):
assert maybe_signature(None, app=self.app) is None
def test_is_dict(self):
assert isinstance(maybe_signature(dict(self.add.s()), app=self.app),
Signature)
def test_is_None(self):
assert maybe_signature(None, app=self.app) is None
def on_chord_part_return(self, request, state, result,
propagate=None, **kwargs):
app = self.app
tid, gid, group_index = request.id, request.group, request.group_index
if not gid or not tid:
return
if group_index is None:
group_index = '+inf'
client = self.client
jkey = self.get_key_for_group(gid, '.j')
tkey = self.get_key_for_group(gid, '.t')
result = self.encode_result(result, state)
encoded = self.encode([1, tid, state, result])
with client.pipeline() as pipe:
pipeline = (
pipe.zadd(jkey, {encoded: group_index}).zcount(jkey, "-inf", "+inf")
if self._chord_zset
else pipe.rpush(jkey, encoded).llen(jkey)
).get(tkey)
if self.expires is not None:
pipeline = pipeline \
.expire(jkey, self.expires) \
.expire(tkey, self.expires)
_, readycount, totaldiff = pipeline.execute()[:3]
totaldiff = int(totaldiff or 0)
try:
callback = maybe_signature(request.chord, app=app)
total = callback['chord_size'] + totaldiff
if readycount == total:
decode, unpack = self.decode, self._unpack_chord_result
with client.pipeline() as pipe:
if self._chord_zset:
pipeline = pipe.zrange(jkey, 0, -1)
else:
pipeline = pipe.lrange(jkey, 0, total)
resl, = pipeline.execute()
try:
callback.delay([unpack(tup, decode) for tup in resl])
with client.pipeline() as pipe:
_, _ = pipe \
.delete(jkey) \
.delete(tkey) \
.execute()
except Exception as exc: # pylint: disable=broad-except
logger.exception(
'Chord callback for %r raised: %r', request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Callback error: {exc!r}'),
)
except ChordError as exc:
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(callback, exc)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Join error: {exc!r}'),
)
def apply(self, args=(), kwargs={}, propagate=True, **options):
body = kwargs['body']
res = super(Chord, self).apply(args, dict(kwargs, eager=True),
**options)
return maybe_signature(body, app=self.app).apply(args=(res.get(
propagate=propagate).get(), ))
def test_when_sig(self):
s = self.add.s()
self.assertIs(maybe_signature(s, app=self.app), s)
def test_is_dict(self):
self.assertIsInstance(
maybe_signature(dict(self.add.s()), app=self.app),
Signature,
)
def __init__(self, tasks=None, app=None, Publisher=None):
self.app = app_or_default(app or self.app)
super(TaskSet, self).__init__(
maybe_signature(t, app=self.app) for t in tasks or [])
self.Publisher = Publisher or self.app.amqp.TaskProducer
self.total = len(self) # XXX compat
def prepare_member(task):
task = maybe_signature(task, app=self.app)
task.options['group_id'] = group_id
return task, task.freeze()
def prepare_member(task):
task = maybe_signature(task, app=self.app)
task.options['group_id'] = group_id
return task, task.freeze()
def apply(self, args=(), kwargs={}, propagate=True, **options):
body = kwargs['body']
res = super(Chord, self).apply(args, dict(kwargs, eager=True),
**options)
return maybe_signature(body, app=self.app).apply(
args=(res.get(propagate=propagate).get(), ))
def test_is_list(self):
sigs = [dict(self.add.s(2, 2)), dict(self.add.s(4, 4))]
sigs = maybe_signature(sigs, app=self.app)
for sig in sigs:
self.assertIsInstance(sig, Signature)
self.assertIs(sig.app, self.app)