def unlock_chord(group_id, callback, interval=None, propagate=None,
max_retries=None, result=None,
Result=app.AsyncResult, GroupResult=app.GroupResult,
from_serializable=from_serializable):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
# check if the task group is ready, and if so apply the callback.
deps = GroupResult(
group_id,
[from_serializable(r, Result=Result) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = subtask(callback)
try:
ret = j(propagate=propagate)
except Exception, exc:
culprit = deps._failed_join_report().next()
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=ChordError('Dependency %s raised %r' % (
culprit.id, exc)),
)
else:
try:
callback.delay(ret)
except Exception, exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Call callback error: %r' % (exc, )))
def trigger_callback(app, callback, group_result):
"""Add the callback to the queue or mark the callback as failed
Implementation borrowed from `celery.app.builtins.unlock_chord`
"""
j = (
group_result.join_native
if group_result.supports_native_join
else group_result.join
)
try:
with allow_join_result():
ret = j(timeout=app.conf.result_chord_join_timeout, propagate=True)
except Exception as exc: # pylint: disable=broad-except
try:
culprit = next(group_result._failed_join_report())
reason = "Dependency {0.id} raised {1!r}".format(culprit, exc)
except StopIteration:
reason = repr(exc)
logger.exception("Chord %r raised: %r", group_result.id, exc)
app.backend.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc: # pylint: disable=broad-except
logger.exception("Chord %r raised: %r", group_result.id, exc)
app.backend.chord_error_from_stack(
callback, exc=ChordError("Callback error: {0!r}".format(exc))
)
def on_chord_part_return(self,
request,
state,
result,
propagate=None,
**kwargs):
app = self.app
tid, gid = request.id, request.group
if not gid or not tid:
return
client = self.client
jkey = self.get_key_for_group(gid, '.j')
tkey = self.get_key_for_group(gid, '.t')
result = self.encode_result(result, state)
with client.pipeline() as pipe:
_, readycount, totaldiff, _, _ = pipe \
.rpush(jkey, self.encode([1, tid, state, result])) \
.llen(jkey) \
.get(tkey) \
.expire(jkey, self.expires) \
.expire(tkey, self.expires) \
.execute()
totaldiff = int(totaldiff or 0)
try:
callback = maybe_signature(request.chord, app=app)
total = callback['chord_size'] + totaldiff
if readycount == total:
decode, unpack = self.decode, self._unpack_chord_result
with client.pipeline() as pipe:
resl, = pipe \
.lrange(jkey, 0, total) \
.execute()
try:
callback.delay([unpack(tup, decode) for tup in resl])
with client.pipeline() as pipe:
_, _ = pipe \
.delete(jkey) \
.delete(tkey) \
.execute()
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord callback for %r raised: %r',
request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
except ChordError as exc:
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(callback, exc)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError('Join error: {0!r}'.format(exc)),
)
def unlock_chord(self,
group_id,
callback,
interval=None,
max_retries=None,
result=None,
Result=app.AsyncResult,
GroupResult=app.GroupResult,
result_from_tuple=result_from_tuple,
**kwargs):
if interval is None:
interval = self.default_retry_delay
# check if the task group is ready, and if so apply the callback.
callback = maybe_signature(callback, app)
deps = GroupResult(
group_id,
[result_from_tuple(r, app=app) for r in result],
app=app,
)
j = deps.join_native if deps.supports_native_join else deps.join
try:
ready = deps.ready()
except Exception as exc:
raise self.retry(
exc=exc,
countdown=interval,
max_retries=max_retries,
)
else:
if not ready:
raise self.retry(countdown=interval, max_retries=max_retries)
callback = maybe_signature(callback, app=app)
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=True)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(
callback,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
def unlock_chord(group_id,
callback,
interval=None,
propagate=None,
max_retries=None,
result=None,
Result=app.AsyncResult,
GroupResult=app.GroupResult,
result_from_tuple=result_from_tuple):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
if interval is None:
interval = unlock_chord.default_retry_delay
# check if the task group is ready, and if so apply the callback.
callback = maybe_signature(callback, app)
deps = GroupResult(
group_id,
[result_from_tuple(r, app=app) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = maybe_signature(callback, app=app)
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(callback,
ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r',
group_id,
exc,
exc_info=1)
app.backend.chord_error_from_stack(
callback,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
else:
raise unlock_chord.retry(countdown=interval,
max_retries=max_retries)
def on_chord_part_return(self, task, state, result, propagate=None):
if not self.implements_incr:
return
app = self.app
if propagate is None:
propagate = app.conf.CELERY_CHORD_PROPAGATES
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=task.backend)
except Exception as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('Cannot restore group: {0!r}'.format(exc)),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('GroupResult {0} no longer exists'.format(gid)),
)
val = self.incr(key)
if val >= len(deps):
callback = maybe_signature(task.request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def _new_chord_return(self,
task,
state,
result,
propagate=None,
PROPAGATE_STATES=states.PROPAGATE_STATES):
app = self.app
if propagate is None:
propagate = self.app.conf.CELERY_CHORD_PROPAGATES
request = task.request
tid, gid = request.id, request.group
if not gid or not tid:
return
client = self.client
jkey = self.get_key_for_group(gid, '.j')
result = self.encode_result(result, state)
with client.pipeline() as pipe:
_, readycount, _ = pipe \
.rpush(jkey, self.encode([1, tid, state, result])) \
.llen(jkey) \
.expire(jkey, 86400) \
.execute()
try:
callback = maybe_signature(request.chord, app=app)
total = callback['chord_size']
if readycount == total:
decode, unpack = self.decode, self._unpack_chord_result
with client.pipeline() as pipe:
resl, _, = pipe \
.lrange(jkey, 0, total) \
.delete(jkey) \
.execute()
try:
callback.delay([unpack(tup, decode) for tup in resl])
except Exception as exc:
error('Chord callback for %r raised: %r',
request.group,
exc,
exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
except ChordError as exc:
error('Chord %r raised: %r', request.group, exc, exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=exc,
)
except Exception as exc:
error('Chord %r raised: %r', request.group, exc, exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Join error: {0!r}'.format(exc)),
)
def on_chord_part_return(self, request, state, result, propagate=None):
app = self.app
tid, gid = request.id, request.group
if not gid or not tid:
return
client = self.client
jkey = self.get_key_for_group(gid, '.j')
tkey = self.get_key_for_group(gid, '.t')
result = self.encode_result(result, state)
with client.pipeline() as pipe:
_, readycount, totaldiff, _, _ = pipe \
.rpush(jkey, self.encode([1, tid, state, result])) \
.llen(jkey) \
.get(tkey) \
.expire(jkey, 86400) \
.expire(tkey, 86400) \
.execute()
totaldiff = int(totaldiff or 0)
try:
callback = maybe_signature(request.chord, app=app)
total = callback['chord_size'] + totaldiff
if readycount == total:
decode, unpack = self.decode, self._unpack_chord_result
with client.pipeline() as pipe:
resl, _, _ = pipe \
.lrange(jkey, 0, total) \
.delete(jkey) \
.delete(tkey) \
.execute()
try:
callback.delay([unpack(tup, decode) for tup in resl])
except Exception as exc:
error('Chord callback for %r raised: %r',
request.group,
exc,
exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
except ChordError as exc:
error('Chord %r raised: %r', request.group, exc, exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=exc,
)
except Exception as exc:
error('Chord %r raised: %r', request.group, exc, exc_info=1)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Join error: {0!r}'.format(exc)),
)
def unlock_chord(group_id,
callback,
interval=None,
propagate=None,
max_retries=None,
result=None,
Result=app.AsyncResult,
GroupResult=app.GroupResult,
from_serializable=from_serializable):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
if interval is None:
interval = unlock_chord.default_retry_delay
# check if the task group is ready, and if so apply the callback.
deps = GroupResult(
group_id,
[from_serializable(r, app=app) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = subtask(callback)
try:
ret = j(propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError(reason),
)
else:
try:
callback.delay(ret)
except Exception as exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
else:
return unlock_chord.retry(countdown=interval,
max_retries=max_retries)
def _unpack_chord_result(self,
tup,
decode,
PROPAGATE_STATES=states.PROPAGATE_STATES):
_, tid, state, retval = decode(tup)
if state in PROPAGATE_STATES:
raise ChordError('Dependency {0} raised {1!r}'.format(tid, retval))
return retval
def on_chord_part_return(self, task, propagate=None):
if not self.implements_incr:
return
from celery import subtask
from celery.result import GroupResult
app = self.app
if propagate is None:
propagate = self.app.conf.CELERY_CHORD_PROPAGATES
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
deps = GroupResult.restore(gid, backend=task.backend)
val = self.incr(key)
if val >= len(deps):
j = deps.join_native if deps.supports_native_join else deps.join
callback = subtask(task.request.chord)
try:
ret = j(propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError(reason),
)
else:
try:
callback.delay(ret)
except Exception as exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def _unpack_chord_result(self, tup, decode,
EXCEPTION_STATES=states.EXCEPTION_STATES,
PROPAGATE_STATES=states.PROPAGATE_STATES):
_, tid, state, retval = decode(tup)
if state in EXCEPTION_STATES:
retval = self.exception_to_python(retval)
if state in PROPAGATE_STATES:
raise ChordError('Dependency {0} raised {1!r}'.format(tid, retval))
return retval
def execute_callback(self):
"""
Execute serialized callback. Called via on_chord_part_return.
There are no parameters, as everything we need is already serialized
in the model somewhere.
"""
callback_signature = Signature.from_dict(json.loads(self.serialized_callback))
if any(result.status == FAILURE for result in self.completed_results.all()):
# we either remove the failures and only return results from successful subtasks, or fail the entire chord
if callback_signature.get('options', {}).get('propagate', current_app.conf.CELERY_CHORD_PROPAGATES):
try:
raise ChordError(
"Error in subtasks! Ids: {}".format([
result.task_id
for result in self.completed_results.all()
if result.status == FAILURE
])
)
except ChordError as error:
self.mark_error(error, is_subtask=True)
return
else:
"""
Dev note: this doesn't *quite* match the behavior of the default backend.
According to http://www.pythondoc.com/celery-3.1.11/configuration.html#celery-chord-propagates,
the 2 options are to either propagate through (as done above), or to forward the Exception result
into callback (versus this approach of dropping error results). It seems silly to ask callbacks to
expect exception results as input though, so we drop them.
""" # pylint: disable=pointless-string-statement
for result in self.completed_results.all():
if result.status == FAILURE:
self.completed_results.remove(result) # pylint: disable=no-member
result.delete()
if callback_signature.get('options', {}).get('use_iterator', True):
# If we're using an iterator, it's assumed to be because there are size concerns with the results
# Thus, the callback_result TaskMeta object will have a null 'result' in the database, because you
# stored those results someplace else as part of the callback function, right?
try:
callback_signature(self.completed_results.values_list('result', flat=True).iterator)
except Exception as error: # pylint: disable=broad-except
self.mark_error(error, is_subtask=False)
return
else:
self.callback_result.status = SUCCESS
self.callback_result.date_done = datetime.now()
self.callback_result.save()
else:
results_list = [subtask.result for subtask in self.completed_results.all()]
callback_signature.id = self.callback_result.task_id
callback_signature.apply_async((results_list, ), {})
def test_on_chord_part_return__ChordError(self):
with self.chord_context(1) as (_, request, callback):
self.b.client.pipeline = ContextMock()
raise_on_second_call(self.b.client.pipeline, ChordError())
self.b.client.pipeline.return_value.zadd().zcount().get().expire(
).expire().execute.return_value = (1, 1, 0, 4, 5)
task = self.app._tasks['add'] = Mock(name='add_task')
self.b.on_chord_part_return(request, states.SUCCESS, 10)
task.backend.fail_from_current_stack.assert_called_with(
callback.id, exc=ANY,
)
def test_on_chord_part_return__ChordError__unordered(self):
self.app.conf.result_backend_transport_options = dict(
result_chord_ordered=False, )
with self.chord_context(1) as (_, request, callback):
self.b.client.pipeline = ContextMock()
raise_on_second_call(self.b.client.pipeline, ChordError())
self.b.client.pipeline.return_value.rpush().llen().get().expire(
).expire().execute.return_value = (1, 1, 0, 4, 5)
task = self.app._tasks['add'] = Mock(name='add_task')
self.b.on_chord_part_return(request, states.SUCCESS, 10)
task.backend.fail_from_current_stack.assert_called_with(
callback.id,
exc=ANY,
)
def on_chord_part_return(self, request, state, result, **kwargs):
if not self.implements_incr:
return
app = self.app
gid = request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=self)
except Exception as exc: # pylint: disable=broad-except
callback = maybe_signature(request.chord, app=app)
logger.exception('Chord %r raised: %r', gid, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Cannot restore group: {exc!r}'),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(request.chord, app=app)
logger.exception('Chord callback %r raised: %r', gid, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'GroupResult {gid} no longer exists'),
)
val = self.incr(key)
# Set the chord size to the value defined in the request, or fall back
# to the number of dependencies we can see from the restored result
size = request.chord.get("chord_size")
if size is None:
size = len(deps)
if val > size: # pragma: no cover
logger.warning('Chord counter incremented too many times for %r',
gid)
elif val == size:
callback = maybe_signature(request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=app.conf.result_chord_join_timeout,
propagate=True)
except Exception as exc: # pylint: disable=broad-except
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
logger.exception('Chord %r raised: %r', gid, reason)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord %r raised: %r', gid, exc)
self.chord_error_from_stack(
callback,
ChordError(f'Callback error: {exc!r}'),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, self.expires)
def on_chord_part_return(self, request, state, result, **kwargs):
if not self.implements_incr:
return
app = self.app
gid = request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=self)
except Exception as exc:
callback = maybe_signature(request.chord, app=app)
logger.error('Chord %r raised: %r', gid, exc, exc_info=1)
return self.chord_error_from_stack(
callback,
ChordError('Cannot restore group: {0!r}'.format(exc)),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(request.chord, app=app)
logger.error('Chord callback %r raised: %r',
gid,
exc,
exc_info=1)
return self.chord_error_from_stack(
callback,
ChordError('GroupResult {0} no longer exists'.format(gid)),
)
val = self.incr(key)
size = len(deps)
if val > size: # pragma: no cover
logger.warning('Chord counter incremented too many times for %r',
gid)
elif val == size:
callback = maybe_signature(request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=True)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', gid, reason, exc_info=1)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r', gid, exc, exc_info=1)
self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def on_chord_part_return(self,
request,
state,
result,
propagate=None,
**kwargs):
app = self.app
tid, gid, group_index = request.id, request.group, request.group_index
if not gid or not tid:
return
if group_index is None:
group_index = '+inf'
client = self.client
jkey = self.get_key_for_group(gid, '.j')
tkey = self.get_key_for_group(gid, '.t')
skey = self.get_key_for_group(gid, '.s')
result = self.encode_result(result, state)
encoded = self.encode([1, tid, state, result])
with client.pipeline() as pipe:
pipeline = (pipe.zadd(jkey, {
encoded: group_index
}).zcount(jkey, "-inf",
"+inf") if self._chord_zset else pipe.rpush(
jkey, encoded).llen(jkey)).get(tkey).get(skey)
if self.expires:
pipeline = pipeline \
.expire(jkey, self.expires) \
.expire(tkey, self.expires) \
.expire(skey, self.expires)
_, readycount, totaldiff, chord_size_bytes = pipeline.execute()[:4]
totaldiff = int(totaldiff or 0)
if chord_size_bytes:
try:
callback = maybe_signature(request.chord, app=app)
total = int(chord_size_bytes) + totaldiff
if readycount == total:
header_result = GroupResult.restore(gid)
if header_result is not None:
# If we manage to restore a `GroupResult`, then it must
# have been complex and saved by `apply_chord()` earlier.
#
# Before we can join the `GroupResult`, it needs to be
# manually marked as ready to avoid blocking
header_result.on_ready()
# We'll `join()` it to get the results and ensure they are
# structured as intended rather than the flattened version
# we'd construct without any other information.
join_func = (header_result.join_native
if header_result.supports_native_join else
header_result.join)
with allow_join_result():
resl = join_func(
timeout=app.conf.result_chord_join_timeout,
propagate=True)
else:
# Otherwise simply extract and decode the results we
# stashed along the way, which should be faster for large
# numbers of simple results in the chord header.
decode, unpack = self.decode, self._unpack_chord_result
with client.pipeline() as pipe:
if self._chord_zset:
pipeline = pipe.zrange(jkey, 0, -1)
else:
pipeline = pipe.lrange(jkey, 0, total)
resl, = pipeline.execute()
resl = [unpack(tup, decode) for tup in resl]
try:
callback.delay(resl)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord callback for %r raised: %r',
request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Callback error: {exc!r}'),
)
finally:
with client.pipeline() as pipe:
pipe \
.delete(jkey) \
.delete(tkey) \
.delete(skey) \
.execute()
except ChordError as exc:
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(callback, exc)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Join error: {exc!r}'),
)
def on_chord_part_return(self, request, state, result,
propagate=None, **kwargs):
app = self.app
tid, gid, group_index = request.id, request.group, request.group_index
if not gid or not tid:
return
if group_index is None:
group_index = '+inf'
client = self.client
jkey = self.get_key_for_group(gid, '.j')
tkey = self.get_key_for_group(gid, '.t')
result = self.encode_result(result, state)
encoded = self.encode([1, tid, state, result])
with client.pipeline() as pipe:
pipeline = (
pipe.zadd(jkey, {encoded: group_index}).zcount(jkey, "-inf", "+inf")
if self._chord_zset
else pipe.rpush(jkey, encoded).llen(jkey)
).get(tkey)
if self.expires is not None:
pipeline = pipeline \
.expire(jkey, self.expires) \
.expire(tkey, self.expires)
_, readycount, totaldiff = pipeline.execute()[:3]
totaldiff = int(totaldiff or 0)
try:
callback = maybe_signature(request.chord, app=app)
total = callback['chord_size'] + totaldiff
if readycount == total:
decode, unpack = self.decode, self._unpack_chord_result
with client.pipeline() as pipe:
if self._chord_zset:
pipeline = pipe.zrange(jkey, 0, -1)
else:
pipeline = pipe.lrange(jkey, 0, total)
resl, = pipeline.execute()
try:
callback.delay([unpack(tup, decode) for tup in resl])
with client.pipeline() as pipe:
_, _ = pipe \
.delete(jkey) \
.delete(tkey) \
.execute()
except Exception as exc: # pylint: disable=broad-except
logger.exception(
'Chord callback for %r raised: %r', request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Callback error: {exc!r}'),
)
except ChordError as exc:
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(callback, exc)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Join error: {exc!r}'),
)
