def apply_async(self, target, args=[], kwargs={}, **options):
"""Equivalent of the :func:`apply` built-in function.
Callbacks should optimally return as soon as possible since
otherwise the thread which handles the result will get blocked.
"""
if self._does_debug:
logger.debug('TaskPool: Apply %s (args:%s kwargs:%s)',
target, truncate(safe_repr(args), 1024),
truncate(safe_repr(kwargs), 1024))
return self.on_apply(target, args, kwargs,
waitforslot=self.putlocks,
callbacks_propagate=self.callbacks_propagate,
**options)
def dump_body(m, body):
# v2 protocol does not deserialize body
body = m.body if body is None else body
if isinstance(body, buffer_t):
body = bytes_t(body)
return '{0} ({1}b)'.format(truncate(safe_repr(body), 1024),
len(m.body))
def join(self, r, propagate=False, max_retries=10, **kwargs):
if self.no_join:
return
received = []
def on_result(task_id, value):
received.append(task_id)
for i in range(max_retries) if max_retries else count(0):
received[:] = []
try:
return r.get(callback=on_result, propagate=propagate, **kwargs)
except (socket.timeout, TimeoutError) as exc:
waiting_for = self.missing_results(r)
self.speaker.beep()
marker(
'Still waiting for {0}/{1}: [{2}]: {3!r}'.format(
len(r) - len(received), len(r),
truncate(', '.join(waiting_for)), exc), '!',
)
self.fbi.diag(waiting_for)
except self.connerrors as exc:
self.speaker.beep()
marker('join: connection lost: {0!r}'.format(exc), '!')
raise StopSuite('Test failed: Missing task results')
def repr_result(self, result, maxlen=46):
# 46 is the length needed to fit
# 'the quick brown fox jumps over the lazy dog' :)
return truncate(safe_repr(result), maxlen)
def dump_body(m, body):
if isinstance(body, buffer_t):
body = bytes_t(buffer)
return '{0} ({1}b)'.format(truncate(safe_repr(body), 1024),
len(m.body))
def trace_task(uuid, args, kwargs, request=None):
# R - is the possibly prepared return value.
# I - is the Info object.
# T - runtime
# Rstr - textual representation of return value
# retval - is the always unmodified return value.
# state - is the resulting task state.
# This function is very long because we have unrolled all the calls
# for performance reasons, and because the function is so long
# we want the main variables (I, and R) to stand out visually from the
# the rest of the variables, so breaking PEP8 is worth it ;)
R = I = T = Rstr = retval = state = None
task_request = None
time_start = monotonic()
try:
try:
kwargs.items
except AttributeError:
raise InvalidTaskError(
'Task keyword arguments is not a mapping')
push_task(task)
task_request = Context(request or {}, args=args,
called_directly=False, kwargs=kwargs)
push_request(task_request)
try:
# -*- PRE -*-
if prerun_receivers:
send_prerun(sender=task, task_id=uuid, task=task,
args=args, kwargs=kwargs)
loader_task_init(uuid, task)
if track_started:
store_result(
uuid, {'pid': pid, 'hostname': hostname}, STARTED,
request=task_request,
)
# -*- TRACE -*-
try:
R = retval = fun(*args, **kwargs)
state = SUCCESS
except Reject as exc:
I, R = Info(REJECTED, exc), ExceptionInfo(internal=True)
state, retval = I.state, I.retval
I.handle_reject(task, task_request)
except Ignore as exc:
I, R = Info(IGNORED, exc), ExceptionInfo(internal=True)
state, retval = I.state, I.retval
I.handle_ignore(task, task_request)
except Retry as exc:
I, R, state, retval = on_error(
task_request, exc, uuid, RETRY, call_errbacks=False,
)
except Exception as exc:
I, R, state, retval = on_error(task_request, exc, uuid)
if task_request.chord:
on_chord_part_return(task, state, exc)
except BaseException as exc:
raise
else:
try:
# callback tasks must be applied before the result is
# stored, so that result.children is populated.
# groups are called inline and will store trail
# separately, so need to call them separately
# so that the trail's not added multiple times :(
# (Issue #1936)
callbacks = task.request.callbacks
if callbacks:
if len(task.request.callbacks) > 1:
sigs, groups = [], []
for sig in callbacks:
sig = signature(sig, app=app)
if isinstance(sig, group):
groups.append(sig)
else:
sigs.append(sig)
for group_ in groups:
group.apply_async((retval, ))
if sigs:
group(sigs).apply_async((retval, ))
else:
signature(callbacks[0], app=app).delay(retval)
if publish_result:
store_result(
uuid, retval, SUCCESS, request=task_request,
)
except EncodeError as exc:
I, R, state, retval = on_error(task_request, exc, uuid)
else:
if task_request.chord:
on_chord_part_return(task, state, retval)
if task_on_success:
task_on_success(retval, uuid, args, kwargs)
if success_receivers:
send_success(sender=task, result=retval)
if _does_info:
T = monotonic() - time_start
Rstr = truncate(safe_repr(R), 256)
info(LOG_SUCCESS, {
'id': uuid, 'name': name,
'return_value': Rstr, 'runtime': T,
})
# -* POST *-
if state not in IGNORE_STATES:
if task_after_return:
task_after_return(
state, retval, uuid, args, kwargs, None,
)
finally:
try:
if postrun_receivers:
send_postrun(sender=task, task_id=uuid, task=task,
args=args, kwargs=kwargs,
retval=retval, state=state)
finally:
pop_task()
pop_request()
if not eager:
try:
backend_cleanup()
loader_cleanup()
except (KeyboardInterrupt, SystemExit, MemoryError):
raise
except Exception as exc:
logger.error('Process cleanup failed: %r', exc,
exc_info=True)
except MemoryError:
raise
except Exception as exc:
if eager:
raise
R = report_internal_error(task, exc)
if task_request is not None:
I, _, _, _ = on_error(task_request, exc, uuid)
return trace_ok_t(R, I, T, Rstr)
def test_truncate_text(s, maxsize, expected):
assert truncate(s, maxsize) == expected
def test_truncate_text(self):
self.assertEqual(truncate('ABCDEFGHI', 3), 'ABC...')
self.assertEqual(truncate('ABCDEFGHI', 10), 'ABCDEFGHI')
def dump_body(m, body):
# v2 protocol does not deserialize body
body = m.body if body is None else body
if isinstance(body, buffer_t):
body = bytes_t(body)
return '{0} ({1}b)'.format(truncate(safe_repr(body), 1024), len(m.body))
def repr_result(self, result, maxlen=RESULT_MAXLEN):
# 46 is the length needed to fit
# 'the quick brown fox jumps over the lazy dog' :)
if not isinstance(result, string_t):
result = safe_repr(result)
return truncate(result) if len(result) > maxlen else result
def test_truncate_text(self):
self.assertEqual(text.truncate('ABCDEFGHI', 3), 'ABC...')
self.assertEqual(text.truncate('ABCDEFGHI', 10), 'ABCDEFGHI')
def prepare_result(self, result, maxlen=RESULT_MAXLEN, truncate=truncate):
if not isinstance(result, ExceptionInfo):
return truncate(_repr_result(result), maxlen)
return result
def test_truncate_text(self):
self.assertEqual(text.truncate("ABCDEFGHI", 3), "ABC...")
self.assertEqual(text.truncate("ABCDEFGHI", 10), "ABCDEFGHI")
def test_truncate_text(self):
self.assertEqual(text.truncate("ABCDEFGHI", 3), "ABC...")
self.assertEqual(text.truncate("ABCDEFGHI", 10), "ABCDEFGHI")
def prepare_result(self, result, RESULT_MAXLEN=RESULT_MAXLEN):
if not isinstance(result, ExceptionInfo):
return truncate(repr(result), RESULT_MAXLEN)
return result
def __repr__(self):
task, it = self._unpack_args(self.kwargs)
return '[{0}(*x) for x in {1}]'.format(
task.task, truncate(repr(it), 100))
def __str__(self):
return truncate(self.body, maxlen=30)
def __repr__(self):
task, it = self._unpack_args(self.kwargs)
return '[%s(*x) for x in %s]' % (task.task, truncate(repr(it), 100))
def dump_body(m, body):
return "{0} ({1}b)".format(text.truncate(safe_repr(body), 1024), len(m.body))
def dump_body(m, body):
return '{0} ({1}b)'.format(truncate(safe_repr(body), 1024),
len(m.body))
def prepare_result(self, result, maxlen=RESULT_MAXLEN, truncate=truncate):
if not isinstance(result, ExceptionInfo):
return truncate(_repr_result(result), maxlen)
return result
def test_truncate_text(s, maxsize, expected):
assert truncate(s, maxsize) == expected
def prepare_result(self, result):
if not isinstance(result, ExceptionInfo):
return truncate(repr(result), 46)
def dump_body(m, body):
"""Format message body for debugging purposes."""
# v2 protocol does not deserialize body
body = m.body if body is None else body
return '{} ({}b)'.format(truncate(safe_repr(body), 1024),
len(m.body))
def prepare_result(self, result, RESULT_MAXLEN=RESULT_MAXLEN):
if not isinstance(result, ExceptionInfo):
return truncate(repr(result), RESULT_MAXLEN)
return result
def repr_result(self, result, maxlen=46):
# 46 is the length needed to fit
# 'the quick brown fox jumps over the lazy dog' :)
return truncate(safe_repr(result), maxlen)
def repr_result(self, result, maxlen=46):
# 46 is the length needed to fit
# 'the quick brown fox jumps over the lazy dog' :)
if not isinstance(result, string_t):
result = safe_repr(result)
return truncate(result) if len(result) > maxlen else result
def dump_body(m, body):
if isinstance(body, buffer_t):
body = bytes_t(body)
return '{0} ({1}b)'.format(truncate(safe_repr(body), 1024), len(m.body))
def __repr__(self):
task, it = self._unpack_args(self.kwargs)
return '[%s(*x) for x in %s]' % (task.task, truncate(repr(it), 100))
def __repr__(self):
task, it = self._unpack_args(self.kwargs)
return '[{0}(*x) for x in {1}]'.format(task.task,
truncate(repr(it), 100))
def prepare_result(self, result):
if not isinstance(result, ExceptionInfo):
return truncate(repr(result), 46)
def trace_task(uuid, args, kwargs, request=None):
# R - is the possibly prepared return value.
# I - is the Info object.
# T - runtime
# Rstr - textual representation of return value
# retval - is the always unmodified return value.
# state - is the resulting task state.
# This function is very long because we have unrolled all the calls
# for performance reasons, and because the function is so long
# we want the main variables (I, and R) to stand out visually from the
# the rest of the variables, so breaking PEP8 is worth it ;)
R = I = T = Rstr = retval = state = None
task_request = None
time_start = monotonic()
try:
try:
kwargs.items
except AttributeError:
raise InvalidTaskError(
'Task keyword arguments is not a mapping')
push_task(task)
task_request = Context(request or {}, args=args,
called_directly=False, kwargs=kwargs)
push_request(task_request)
try:
# -*- PRE -*-
if prerun_receivers:
send_prerun(sender=task, task_id=uuid, task=task,
args=args, kwargs=kwargs)
loader_task_init(uuid, task)
if track_started:
store_result(
uuid, {'pid': pid, 'hostname': hostname}, STARTED,
request=task_request,
)
# -*- TRACE -*-
try:
R = retval = fun(*args, **kwargs)
state = SUCCESS
except Reject as exc:
I, R = Info(REJECTED, exc), ExceptionInfo(internal=True)
state, retval = I.state, I.retval
I.handle_reject(task, task_request)
except Ignore as exc:
I, R = Info(IGNORED, exc), ExceptionInfo(internal=True)
state, retval = I.state, I.retval
I.handle_ignore(task, task_request)
except Retry as exc:
I, R, state, retval = on_error(
task_request, exc, uuid, RETRY, call_errbacks=False,
)
except Exception as exc:
I, R, state, retval = on_error(task_request, exc, uuid)
except BaseException as exc:
raise
else:
try:
# callback tasks must be applied before the result is
# stored, so that result.children is populated.
# groups are called inline and will store trail
# separately, so need to call them separately
# so that the trail's not added multiple times :(
# (Issue #1936)
callbacks = task.request.callbacks
if callbacks:
if len(task.request.callbacks) > 1:
sigs, groups = [], []
for sig in callbacks:
sig = signature(sig, app=app)
if isinstance(sig, group):
groups.append(sig)
else:
sigs.append(sig)
for group_ in groups:
group.apply_async((retval,))
if sigs:
group(sigs).apply_async((retval,))
else:
signature(callbacks[0], app=app).delay(retval)
# execute first task in chain
chain = task.request.chain
if chain:
signature(chain.pop(), app=app).apply_async(
(retval,), chain=chain)
mark_as_done(
uuid, retval, task_request, publish_result,
)
except EncodeError as exc:
I, R, state, retval = on_error(task_request, exc, uuid)
else:
if task_on_success:
task_on_success(retval, uuid, args, kwargs)
if success_receivers:
send_success(sender=task, result=retval)
if _does_info:
T = monotonic() - time_start
Rstr = truncate(safe_repr(R), 256)
info(LOG_SUCCESS, {
'id': uuid, 'name': name,
'return_value': Rstr, 'runtime': T,
})
# -* POST *-
if state not in IGNORE_STATES:
if task_after_return:
task_after_return(
state, retval, uuid, args, kwargs, None,
)
finally:
try:
if postrun_receivers:
send_postrun(sender=task, task_id=uuid, task=task,
args=args, kwargs=kwargs,
retval=retval, state=state)
finally:
pop_task()
pop_request()
if not eager:
try:
backend_cleanup()
loader_cleanup()
except (KeyboardInterrupt, SystemExit, MemoryError):
raise
except Exception as exc:
logger.error('Process cleanup failed: %r', exc,
exc_info=True)
except MemoryError:
raise
except Exception as exc:
if eager:
raise
R = report_internal_error(task, exc)
if task_request is not None:
I, _, _, _ = on_error(task_request, exc, uuid)
return trace_ok_t(R, I, T, Rstr)
def dump_body(m, body):
return '{0} ({1}b)'.format(truncate(safe_repr(body), 1024),
len(m.body))
def __str__(self):
return truncate(self.title, maxlen=30)
