def proto1_to_proto2(message, body):
"""Convert Task message protocol 1 arguments to protocol 2.
Returns:
Tuple: of ``(body, headers, already_decoded_status, utc)``
"""
try:
args, kwargs = body.get("args", ()), body.get("kwargs", {})
kwargs.items # pylint: disable=pointless-statement
except KeyError:
raise InvalidTaskError("Message does not have args/kwargs")
except AttributeError:
raise InvalidTaskError("Task keyword arguments must be a mapping", )
body.update(
argsrepr=saferepr(args),
kwargsrepr=saferepr(kwargs),
headers=message.headers,
)
try:
body["group"] = body["taskset"]
except KeyError:
pass
embed = {
"callbacks": body.get("callbacks"),
"errbacks": body.get("errbacks"),
"chord": body.get("chord"),
"chain": None,
}
return (args, kwargs, embed), body, True, body.get("utc", True)
def proto1_to_proto2(message, body):
"""Converts Task message protocol 1 arguments to protocol 2.
Returns tuple of ``(body, headers, already_decoded_status, utc)``
"""
try:
args, kwargs = body["args"], body["kwargs"]
kwargs.items
except KeyError:
raise InvalidTaskError("Message does not have args/kwargs")
except AttributeError:
raise InvalidTaskError("Task keyword arguments must be a mapping")
body.update(argsrepr=saferepr(args), kwargsrepr=saferepr(kwargs), headers=message.headers)
try:
body["group"] = body["taskset"]
except KeyError:
pass
embed = {
"callbacks": body.get("callbacks"),
"errbacks": body.get("errbacks"),
"chord": body.get("chord"),
"chain": None,
}
return (args, kwargs, embed), body, True, body.get("utc", True)
def proto1_to_proto2(message, body):
"""Converts Task message protocol 1 arguments to protocol 2.
Returns tuple of ``(body, headers, already_decoded_status, utc)``
"""
try:
args, kwargs = body['args'], body['kwargs']
kwargs.items
except KeyError:
raise InvalidTaskError('Message does not have args/kwargs')
except AttributeError:
raise InvalidTaskError(
'Task keyword arguments must be a mapping',
)
body.update(
argsrepr=saferepr(args),
kwargsrepr=saferepr(kwargs),
headers=message.headers,
)
try:
body['group'] = body['taskset']
except KeyError:
pass
return (args, kwargs), body, True, body.get('utc', True)
def proto1_to_proto2(message, body):
"""Convert Task message protocol 1 arguments to protocol 2.
Returns:
Tuple: of ``(body, headers, already_decoded_status, utc)``
"""
try:
args, kwargs = body.get('args', ()), body.get('kwargs', {})
kwargs.items # pylint: disable=pointless-statement
except KeyError:
raise InvalidTaskError('Message does not have args/kwargs')
except AttributeError:
raise InvalidTaskError(
'Task keyword arguments must be a mapping',
)
body.update(
argsrepr=saferepr(args),
kwargsrepr=saferepr(kwargs),
headers=message.headers,
)
try:
body['group'] = body['taskset']
except KeyError:
pass
embed = {
'callbacks': body.get('callbacks'),
'errbacks': body.get('errbacks'),
'chord': body.get('chord'),
'chain': None,
}
return (args, kwargs, embed), body, True, body.get('utc', True)
def as_task_v1(self, task_id, name, args=None, kwargs=None,
countdown=None, eta=None, group_id=None, group_index=None,
expires=None, retries=0,
chord=None, callbacks=None, errbacks=None, reply_to=None,
time_limit=None, soft_time_limit=None,
create_sent_event=False, root_id=None, parent_id=None,
shadow=None, now=None, timezone=None,
**compat_kwargs):
args = args or ()
kwargs = kwargs or {}
utc = self.utc
if not isinstance(args, (list, tuple)):
raise TypeError('task args must be a list or tuple')
if not isinstance(kwargs, Mapping):
raise TypeError('task keyword arguments must be a mapping')
if countdown: # convert countdown to ETA
self._verify_seconds(countdown, 'countdown')
now = now or self.app.now()
eta = now + timedelta(seconds=countdown)
if isinstance(expires, numbers.Real):
self._verify_seconds(expires, 'expires')
now = now or self.app.now()
expires = now + timedelta(seconds=expires)
eta = eta and eta.isoformat()
expires = expires and expires.isoformat()
return task_message(
headers={},
properties={
'correlation_id': task_id,
'reply_to': reply_to or '',
},
body={
'task': name,
'id': task_id,
'args': args,
'kwargs': kwargs,
'group': group_id,
'group_index': group_index,
'retries': retries,
'eta': eta,
'expires': expires,
'utc': utc,
'callbacks': callbacks,
'errbacks': errbacks,
'timelimit': (time_limit, soft_time_limit),
'taskset': group_id,
'chord': chord,
},
sent_event={
'uuid': task_id,
'name': name,
'args': saferepr(args),
'kwargs': saferepr(kwargs),
'retries': retries,
'eta': eta,
'expires': expires,
} if create_sent_event else None,
)
def test_same_as_repr(self):
# Simple objects, small containers and classes that overwrite __repr__
# For those the result should be the same as repr().
# Ahem. The docs don't say anything about that -- this appears to
# be testing an implementation quirk. Starting in Python 2.5, it's
# not true for dicts: pprint always sorts dicts by key now; before,
# it sorted a dict display if and only if the display required
# multiple lines. For that reason, dicts with more than one element
# aren't tested here.
types = (0, 0, 0 + 0j, 0.0, '', b'',
(), tuple2(), tuple3(), [], list2(), list3(), set(), set2(),
set3(), frozenset(), frozenset2(), frozenset3(), {}, dict2(),
dict3(), self.assertTrue, pprint, -6, -6, -6 - 6j, -1.5, 'x',
b'x', (3, ), [3], {
3: 6
}, (1, 2), [3, 4], {
5: 6
}, tuple2((1, 2)), tuple3((1, 2)), tuple3(range(100)), [3, 4],
list2([3, 4]), list3([3,
4]), list3(range(100)), {7}, set2({7}),
set3({7}), frozenset({8}), frozenset2({8}), frozenset3({8}),
dict2({5: 6}), dict3({5: 6}), range(10, -11, -1))
for simple in types:
native = old_repr(simple)
self.assertEqual(saferepr(simple), native)
def test_same_as_repr(self):
# Simple objects, small containers and classes that overwrite __repr__
# For those the result should be the same as repr().
# Ahem. The docs don't say anything about that -- this appears to
# be testing an implementation quirk. Starting in Python 2.5, it's
# not true for dicts: pprint always sorts dicts by key now; before,
# it sorted a dict display if and only if the display required
# multiple lines. For that reason, dicts with more than one element
# aren't tested here.
types = (
0, 0, 0 + 0j, 0.0, '', b'',
(), tuple2(), tuple3(),
[], list2(), list3(),
set(), set2(), set3(),
frozenset(), frozenset2(), frozenset3(),
{}, dict2(), dict3(),
self.assertTrue, pprint,
-6, -6, -6 - 6j, -1.5, 'x', b'x', (3,), [3], {3: 6},
(1, 2), [3, 4], {5: 6},
tuple2((1, 2)), tuple3((1, 2)), tuple3(range(100)),
[3, 4], list2([3, 4]), list3([3, 4]), list3(range(100)),
{7}, set2({7}), set3({7}),
frozenset({8}), frozenset2({8}), frozenset3({8}),
dict2({5: 6}), dict3({5: 6}),
range(10, -11, -1)
)
for simple in types:
native = old_repr(simple)
self.assertEqual(saferepr(simple), native)
def test_binary_bytes__long(self):
val = struct.pack('>QQQ', 12223, 1234, 3123) * 1024
result = saferepr(val, maxlen=128)
if hasattr(bytes, 'hex'): # Python 3.5+
assert '2fbf' in result
assert result.endswith("...'")
else: # Python 3.4
assert result
def test_binary_bytes__long(self):
val = struct.pack('>QQQ', 12223, 1234, 3123) * 1024
result = saferepr(val, maxlen=128)
if hasattr(bytes, 'hex'): # Python 3.5+
assert '2fbf' in result
assert result.endswith("...'")
else: # Python 3.4
assert result
def test_bytes_with_unicode(self):
class X(object):
def __repr__(self):
return 'æ e i a æ å'.encode('utf-8',
errors='backslash replace')
val = X()
assert repr(val)
assert saferepr(val)
def test_bytes_with_unicode(self):
class X(object):
def __repr__(self):
return 'æ e i a æ å'.encode(
'utf-8', errors='backslash replace')
val = X()
assert repr(val)
assert saferepr(val)
def test_same_as_repr(self, value):
# Simple objects, small containers, and classes that overwrite __repr__
# For those the result should be the same as repr().
# Ahem. The docs don't say anything about that -- this appears to
# be testing an implementation quirk. Starting in Python 2.5, it's
# not true for dicts: pprint always sorts dicts by key now; before,
# it sorted a dict display if and only if the display required
# multiple lines. For that reason, dicts with more than one element
# aren't tested here.
native = old_repr(value)
assert saferepr(value) == native
def test_same_as_repr(self, value):
# Simple objects, small containers, and classes that overwrite __repr__
# For those the result should be the same as repr().
# Ahem. The docs don't say anything about that -- this appears to
# be testing an implementation quirk. Starting in Python 2.5, it's
# not true for dicts: pprint always sorts dicts by key now; before,
# it sorted a dict display if and only if the display required
# multiple lines. For that reason, dicts with more than one element
# aren't tested here.
native = old_repr(value)
assert saferepr(value) == native
def send_notify(self, channel, data, **kw):
try:
with self.connection_or_acquire() as conn:
data = json.dumps((channel, data))
channel = conn.default_channel
msg = channel.prepare_message(data,
content_type='application/json')
return channel.basic_publish(msg,
exchange='xnotify',
routing_key='xnotify',
**kw)
except:
logger.exception('send notify error %s / %s ', channel,
saferepr(data, 128))
def send_message(self, data, type_='message', **kw):
info = self.brief()
exchange = kw.get('exchange', 'xmessage')
routing_key = kw.get('routing_key', 'xmessage')
self.logger.info(u'Notify %s <%s %s>' % (saferepr(data, 128), exchange, routing_key))
if not isinstance(data, dict):
data = {
'data': data
}
data.update(info)
current_app.send_message(type_, data, exchange, routing_key)
save_kw = {}
save_dt = ('task', self.name, {self.request.id: {'message': [data]}})
self.send_task('xspider.save_log', save_dt, save_kw)
def test_text_maxlen(self):
assert saferepr(D_D_TEXT, 100).endswith("...', ...}}")
def test_recursion(self):
d = {1: 2, 3: {4: 5}}
d[3][6] = d
res = saferepr(d)
assert 'Recursion on' in res
def test_numbers_dict(self):
assert saferepr(D_NUMBERS) == old_repr(D_NUMBERS)
def test_numbers_keys(self):
assert saferepr(D_INT_KEYS) == old_repr(D_INT_KEYS)
def test_text(self):
self.assertEqual(saferepr(D_TEXT), old_repr(D_TEXT).replace("u'", "'"))
def test_recursion(self):
d = {1: 2, 3: {4: 5}}
d[3][6] = d
res = saferepr(d)
self.assertIn('Recursion on', res)
def test_recursion(self):
d = {1: 2, 3: {4: 5}}
d[3][6] = d
res = saferepr(d)
assert 'Recursion on' in res
def test_numbers_list(self):
self.assertEqual(saferepr(L_NUMBERS), old_repr(L_NUMBERS))
def test_text_maxlen(self):
assert saferepr(D_D_TEXT, 100).endswith("...', ...}}")
def test_maxlevels(self):
saferepr(D_ALL, maxlevels=1)
def test_text(self):
assert saferepr(D_TEXT) == old_repr(D_TEXT).replace("u'", "'")
def test_numbers_keys(self):
assert saferepr(D_INT_KEYS) == old_repr(D_INT_KEYS)
def test_numbers_list(self):
assert saferepr(L_NUMBERS) == old_repr(L_NUMBERS)
def test_numbers_keys(self):
self.assertEqual(saferepr(D_INT_KEYS), old_repr(D_INT_KEYS))
def trace_task(uuid, args, kwargs, request=None):
# R - is the possibly prepared return value.
# I - is the Info object.
# T - runtime
# Rstr - textual representation of return value
# retval - is the always unmodified return value.
# state - is the resulting task state.
# This function is very long because we've unrolled all the calls
# for performance reasons, and because the function is so long
# we want the main variables (I, and R) to stand out visually from the
# the rest of the variables, so breaking PEP8 is worth it ;)
R = I = T = Rstr = retval = state = None
task_request = None
time_start = monotonic()
try:
try:
kwargs.items
except AttributeError:
raise InvalidTaskError(
'Task keyword arguments is not a mapping')
push_task(task)
task_request = Context(request or {}, args=args,
called_directly=False, kwargs=kwargs)
root_id = task_request.root_id or uuid
push_request(task_request)
try:
# -*- PRE -*-
if prerun_receivers:
send_prerun(sender=task, task_id=uuid, task=task,
args=args, kwargs=kwargs)
loader_task_init(uuid, task)
if track_started:
store_result(
uuid, {'pid': pid, 'hostname': hostname}, STARTED,
request=task_request,
)
# -*- TRACE -*-
try:
R = retval = fun(*args, **kwargs)
state = SUCCESS
except Reject as exc:
I, R = Info(REJECTED, exc), ExceptionInfo(internal=True)
state, retval = I.state, I.retval
I.handle_reject(task, task_request)
except Ignore as exc:
I, R = Info(IGNORED, exc), ExceptionInfo(internal=True)
state, retval = I.state, I.retval
I.handle_ignore(task, task_request)
except Retry as exc:
I, R, state, retval = on_error(
task_request, exc, uuid, RETRY, call_errbacks=False)
except Exception as exc:
I, R, state, retval = on_error(task_request, exc, uuid)
except BaseException as exc:
raise
else:
try:
# callback tasks must be applied before the result is
# stored, so that result.children is populated.
# groups are called inline and will store trail
# separately, so need to call them separately
# so that the trail's not added multiple times :(
# (Issue #1936)
callbacks = task.request.callbacks
if callbacks:
if len(task.request.callbacks) > 1:
sigs, groups = [], []
for sig in callbacks:
sig = signature(sig, app=app)
if isinstance(sig, group):
groups.append(sig)
else:
sigs.append(sig)
for group_ in groups:
group_.apply_async(
(retval,),
parent_id=uuid, root_id=root_id,
)
if sigs:
group(sigs, app=app).apply_async(
(retval,),
parent_id=uuid, root_id=root_id,
)
else:
signature(callbacks[0], app=app).apply_async(
(retval,), parent_id=uuid, root_id=root_id,
)
# execute first task in chain
chain = task_request.chain
if chain:
signature(chain.pop(), app=app).apply_async(
(retval,), chain=chain,
parent_id=uuid, root_id=root_id,
)
mark_as_done(
uuid, retval, task_request, publish_result,
)
except EncodeError as exc:
I, R, state, retval = on_error(task_request, exc, uuid)
else:
Rstr = saferepr(R, resultrepr_maxsize)
T = monotonic() - time_start
if task_on_success:
task_on_success(retval, uuid, args, kwargs)
if success_receivers:
send_success(sender=task, result=retval)
if _does_info:
info(LOG_SUCCESS, {
'id': uuid, 'name': name,
'return_value': Rstr, 'runtime': T,
})
# -* POST *-
if state not in IGNORE_STATES:
if task_after_return:
task_after_return(
state, retval, uuid, args, kwargs, None,
)
finally:
try:
if postrun_receivers:
send_postrun(sender=task, task_id=uuid, task=task,
args=args, kwargs=kwargs,
retval=retval, state=state)
finally:
pop_task()
pop_request()
if not eager:
try:
backend_cleanup()
loader_cleanup()
except (KeyboardInterrupt, SystemExit, MemoryError):
raise
except Exception as exc:
logger.error('Process cleanup failed: %r', exc,
exc_info=True)
except MemoryError:
raise
except Exception as exc:
if eager:
raise
R = report_internal_error(task, exc)
if task_request is not None:
I, _, _, _ = on_error(task_request, exc, uuid)
return trace_ok_t(R, I, T, Rstr)
def test_unicode_bytes__long(self):
val = 'øystein'.encode('utf-8') * 1024
assert saferepr(val, maxlen=128).endswith("...'")
def test_single_quote(self):
val = {"foo's": "bar's"}
assert ast.literal_eval(saferepr(val)) == val
def test_numbers_dict(self):
assert saferepr(D_NUMBERS) == old_repr(D_NUMBERS)
def test_unicode_bytes(self):
val = 'øystein'.encode('utf-8')
assert saferepr(val) == "b'øystein'"
def test_recursion(self):
d = {1: 2, 3: {4: 5}}
d[3][6] = d
res = saferepr(d)
self.assertIn('Recursion on', res)
def test_unicode_bytes__long(self):
val = 'øystein'.encode('utf-8') * 1024
assert saferepr(val, maxlen=128).endswith("...'")
def test_safe_types(self, value):
assert saferepr(value) == old_repr(value)
def test_binary_bytes(self):
val = struct.pack('>QQQ', 12223, 1234, 3123)
if hasattr(bytes, 'hex'): # Python 3.5+
assert '2fbf' in saferepr(val, maxlen=128)
else: # Python 3.4
assert saferepr(val, maxlen=128)
def test_numbers_list(self):
assert saferepr(L_NUMBERS) == old_repr(L_NUMBERS)
def test_bytes_with_unicode_py2_and_3(self):
assert saferepr([b'foo', 'a®rgs'.encode('utf-8')])
def test_text(self):
assert saferepr(D_TEXT) == old_repr(D_TEXT).replace("u'", "'")
def as_task_v2(self,
task_id,
name,
args=None,
kwargs=None,
countdown=None,
eta=None,
group_id=None,
expires=None,
retries=0,
chord=None,
callbacks=None,
errbacks=None,
reply_to=None,
time_limit=None,
soft_time_limit=None,
create_sent_event=False,
root_id=None,
parent_id=None,
shadow=None,
chain=None,
now=None,
timezone=None,
origin=None,
argsrepr=None,
kwargsrepr=None):
args = args or ()
kwargs = kwargs or {}
if not isinstance(args, (list, tuple)):
raise TypeError('task args must be a list or tuple')
if not isinstance(kwargs, Mapping):
raise TypeError('task keyword arguments must be a mapping')
if countdown: # convert countdown to ETA
self._verify_seconds(countdown, 'countdown')
now = now or self.app.now()
timezone = timezone or self.app.timezone
eta = maybe_make_aware(
now + timedelta(seconds=countdown),
tz=timezone,
)
if isinstance(expires, numbers.Real):
self._verify_seconds(expires, 'expires')
now = now or self.app.now()
timezone = timezone or self.app.timezone
expires = maybe_make_aware(
now + timedelta(seconds=expires),
tz=timezone,
)
if not isinstance(eta, string_t):
eta = eta and eta.isoformat()
# If we retry a task `expires` will already be ISO8601-formatted.
if not isinstance(expires, string_t):
expires = expires and expires.isoformat()
if argsrepr is None:
argsrepr = saferepr(args, self.argsrepr_maxsize)
if kwargsrepr is None:
kwargsrepr = saferepr(kwargs, self.kwargsrepr_maxsize)
if JSON_NEEDS_UNICODE_KEYS: # pragma: no cover
if callbacks:
callbacks = [utf8dict(callback) for callback in callbacks]
if errbacks:
errbacks = [utf8dict(errback) for errback in errbacks]
if chord:
chord = utf8dict(chord)
if not root_id: # empty root_id defaults to task_id
root_id = task_id
return task_message(
headers={
'lang': 'py',
'task': name,
'id': task_id,
'shadow': shadow,
'eta': eta,
'expires': expires,
'group': group_id,
'retries': retries,
'timelimit': [time_limit, soft_time_limit],
'root_id': root_id,
'parent_id': parent_id,
'argsrepr': argsrepr,
'kwargsrepr': kwargsrepr,
'origin': origin or anon_nodename()
},
properties={
'correlation_id': task_id,
'reply_to': reply_to or '',
},
body=(
args,
kwargs,
{
'callbacks': callbacks,
'errbacks': errbacks,
'chain': chain,
'chord': chord,
},
),
sent_event={
'uuid': task_id,
'root_id': root_id,
'parent_id': parent_id,
'name': name,
'args': argsrepr,
'kwargs': kwargsrepr,
'retries': retries,
'eta': eta,
'expires': expires,
} if create_sent_event else None,
)
def test_maxlevels(self):
saferepr(D_ALL, maxlevels=1)
def as_task_v1(self, task_id, name, args=None, kwargs=None,
countdown=None, eta=None, group_id=None,
expires=None, retries=0,
chord=None, callbacks=None, errbacks=None, reply_to=None,
time_limit=None, soft_time_limit=None,
create_sent_event=False, root_id=None, parent_id=None,
shadow=None, now=None, timezone=None):
args = args or ()
kwargs = kwargs or {}
utc = self.utc
if not isinstance(args, (list, tuple)):
raise ValueError('task args must be a list or tuple')
if not isinstance(kwargs, Mapping):
raise ValueError('task keyword arguments must be a mapping')
if countdown: # convert countdown to ETA
self._verify_seconds(countdown, 'countdown')
now = now or self.app.now()
timezone = timezone or self.app.timezone
eta = now + timedelta(seconds=countdown)
if utc:
eta = to_utc(eta).astimezone(timezone)
if isinstance(expires, numbers.Real):
self._verify_seconds(expires, 'expires')
now = now or self.app.now()
timezone = timezone or self.app.timezone
expires = now + timedelta(seconds=expires)
if utc:
expires = to_utc(expires).astimezone(timezone)
eta = eta and eta.isoformat()
expires = expires and expires.isoformat()
if JSON_NEEDS_UNICODE_KEYS: # pragma: no cover
if callbacks:
callbacks = [utf8dict(callback) for callback in callbacks]
if errbacks:
errbacks = [utf8dict(errback) for errback in errbacks]
if chord:
chord = utf8dict(chord)
return task_message(
headers={},
properties={
'correlation_id': task_id,
'reply_to': reply_to or '',
},
body={
'task': name,
'id': task_id,
'args': args,
'kwargs': kwargs,
'group': group_id,
'retries': retries,
'eta': eta,
'expires': expires,
'utc': utc,
'callbacks': callbacks,
'errbacks': errbacks,
'timelimit': (time_limit, soft_time_limit),
'taskset': group_id,
'chord': chord,
},
sent_event={
'uuid': task_id,
'name': name,
'args': saferepr(args),
'kwargs': saferepr(kwargs),
'retries': retries,
'eta': eta,
'expires': expires,
} if create_sent_event else None,
)
def test_text(self):
self.assertEqual(saferepr(D_TEXT), old_repr(D_TEXT).replace("u'", "'"))
def test_numbers_dict(self):
self.assertEqual(saferepr(D_NUMBERS), old_repr(D_NUMBERS))
def test_unicode_bytes(self):
val = 'øystein'.encode('utf-8')
assert saferepr(val) == "b'øystein'"
def test_numbers_list(self):
self.assertEqual(saferepr(L_NUMBERS), old_repr(L_NUMBERS))
def test_binary_bytes(self):
val = struct.pack('>QQQ', 12223, 1234, 3123)
if hasattr(bytes, 'hex'): # Python 3.5+
assert '2fbf' in saferepr(val, maxlen=128)
else: # Python 3.4
assert saferepr(val, maxlen=128)
def test_numbers_keys(self):
self.assertEqual(saferepr(D_INT_KEYS), old_repr(D_INT_KEYS))
def test_repr_raises(self):
class O(object):
def __repr__(self):
raise KeyError('foo')
assert 'Unrepresentable' in saferepr(O())
def test_safe_types(self, value):
assert saferepr(value) == old_repr(value)
def trace_task(uuid, args, kwargs, request=None):
# R - is the possibly prepared return value.
# I - is the Info object.
# T - runtime
# Rstr - textual representation of return value
# retval - is the always unmodified return value.
# state - is the resulting task state.
# This function is very long because we've unrolled all the calls
# for performance reasons, and because the function is so long
# we want the main variables (I, and R) to stand out visually from the
# the rest of the variables, so breaking PEP8 is worth it ;)
R = I = T = Rstr = retval = state = None
task_request = None
time_start = monotonic()
try:
try:
kwargs.items
except AttributeError:
raise InvalidTaskError(
'Task keyword arguments is not a mapping')
push_task(task)
task_request = Context(request or {}, args=args,
called_directly=False, kwargs=kwargs)
root_id = task_request.root_id or uuid
task_priority = task_request.delivery_info.get('priority') if \
inherit_parent_priority else None
push_request(task_request)
try:
# -*- PRE -*-
if prerun_receivers:
send_prerun(sender=task, task_id=uuid, task=task,
args=args, kwargs=kwargs)
loader_task_init(uuid, task)
if track_started:
store_result(
uuid, {'pid': pid, 'hostname': hostname}, STARTED,
request=task_request,
)
# -*- TRACE -*-
try:
R = retval = fun(*args, **kwargs)
state = SUCCESS
except Reject as exc:
I, R = Info(REJECTED, exc), ExceptionInfo(internal=True)
state, retval = I.state, I.retval
I.handle_reject(task, task_request)
traceback_clear(exc)
except Ignore as exc:
I, R = Info(IGNORED, exc), ExceptionInfo(internal=True)
state, retval = I.state, I.retval
I.handle_ignore(task, task_request)
traceback_clear(exc)
except Retry as exc:
I, R, state, retval = on_error(
task_request, exc, uuid, RETRY, call_errbacks=False)
traceback_clear(exc)
except Exception as exc:
I, R, state, retval = on_error(task_request, exc, uuid)
traceback_clear(exc)
except BaseException:
raise
else:
try:
# callback tasks must be applied before the result is
# stored, so that result.children is populated.
# groups are called inline and will store trail
# separately, so need to call them separately
# so that the trail's not added multiple times :(
# (Issue #1936)
callbacks = task.request.callbacks
if callbacks:
if len(task.request.callbacks) > 1:
sigs, groups = [], []
for sig in callbacks:
sig = signature(sig, app=app)
if isinstance(sig, group):
groups.append(sig)
else:
sigs.append(sig)
for group_ in groups:
group_.apply_async(
(retval,),
parent_id=uuid, root_id=root_id,
priority=task_priority
)
if sigs:
group(sigs, app=app).apply_async(
(retval,),
parent_id=uuid, root_id=root_id,
priority=task_priority
)
else:
signature(callbacks[0], app=app).apply_async(
(retval,), parent_id=uuid, root_id=root_id,
priority=task_priority
)
# execute first task in chain
chain = task_request.chain
if chain:
_chsig = signature(chain.pop(), app=app)
_chsig.apply_async(
(retval,), chain=chain,
parent_id=uuid, root_id=root_id,
priority=task_priority
)
mark_as_done(
uuid, retval, task_request, publish_result,
)
except EncodeError as exc:
I, R, state, retval = on_error(task_request, exc, uuid)
else:
Rstr = saferepr(R, resultrepr_maxsize)
T = monotonic() - time_start
if task_on_success:
task_on_success(retval, uuid, args, kwargs)
if success_receivers:
send_success(sender=task, result=retval)
if _does_info:
info(LOG_SUCCESS, {
'id': uuid,
'name': get_task_name(task_request, name),
'return_value': Rstr,
'runtime': T,
})
# -* POST *-
if state not in IGNORE_STATES:
if task_after_return:
task_after_return(
state, retval, uuid, args, kwargs, None,
)
finally:
try:
if postrun_receivers:
send_postrun(sender=task, task_id=uuid, task=task,
args=args, kwargs=kwargs,
retval=retval, state=state)
finally:
pop_task()
pop_request()
if not eager:
try:
backend_cleanup()
loader_cleanup()
except (KeyboardInterrupt, SystemExit, MemoryError):
raise
except Exception as exc:
logger.error('Process cleanup failed: %r', exc,
exc_info=True)
except MemoryError:
raise
except Exception as exc:
_signal_internal_error(task, uuid, args, kwargs, request, exc)
if eager:
raise
R = report_internal_error(task, exc)
if task_request is not None:
I, _, _, _ = on_error(task_request, exc, uuid)
return trace_ok_t(R, I, T, Rstr)
def test_bytes_with_unicode_py2_and_3(self):
assert saferepr([b'foo', 'a®rgs'.encode('utf-8')])
def as_task_v2(self, task_id, name, args=None, kwargs=None,
countdown=None, eta=None, group_id=None,
expires=None, retries=0, chord=None,
callbacks=None, errbacks=None, reply_to=None,
time_limit=None, soft_time_limit=None,
create_sent_event=False, root_id=None, parent_id=None,
shadow=None, chain=None, now=None, timezone=None,
origin=None, argsrepr=None, kwargsrepr=None):
args = args or ()
kwargs = kwargs or {}
if not isinstance(args, (list, tuple)):
raise TypeError('task args must be a list or tuple')
if not isinstance(kwargs, Mapping):
raise TypeError('task keyword arguments must be a mapping')
if countdown: # convert countdown to ETA
self._verify_seconds(countdown, 'countdown')
now = now or self.app.now()
timezone = timezone or self.app.timezone
eta = maybe_make_aware(
now + timedelta(seconds=countdown), tz=timezone,
)
if isinstance(expires, numbers.Real):
self._verify_seconds(expires, 'expires')
now = now or self.app.now()
timezone = timezone or self.app.timezone
expires = maybe_make_aware(
now + timedelta(seconds=expires), tz=timezone,
)
eta = eta and eta.isoformat()
expires = expires and expires.isoformat()
if argsrepr is None:
argsrepr = saferepr(args, self.argsrepr_maxsize)
if kwargsrepr is None:
kwargsrepr = saferepr(kwargs, self.kwargsrepr_maxsize)
if JSON_NEEDS_UNICODE_KEYS: # pragma: no cover
if callbacks:
callbacks = [utf8dict(callback) for callback in callbacks]
if errbacks:
errbacks = [utf8dict(errback) for errback in errbacks]
if chord:
chord = utf8dict(chord)
return task_message(
headers={
'lang': 'py',
'task': name,
'id': task_id,
'eta': eta,
'expires': expires,
'group': group_id,
'retries': retries,
'timelimit': [time_limit, soft_time_limit],
'root_id': root_id,
'parent_id': parent_id,
'argsrepr': argsrepr,
'kwargsrepr': kwargsrepr,
'origin': origin or anon_nodename()
},
properties={
'correlation_id': task_id,
'reply_to': reply_to or '',
},
body=(
args, kwargs, {
'callbacks': callbacks,
'errbacks': errbacks,
'chain': chain,
'chord': chord,
},
),
sent_event={
'uuid': task_id,
'root_id': root_id,
'parent_id': parent_id,
'name': name,
'args': argsrepr,
'kwargs': kwargsrepr,
'retries': retries,
'eta': eta,
'expires': expires,
} if create_sent_event else None,
)
def test_numbers_dict(self):
self.assertEqual(saferepr(D_NUMBERS), old_repr(D_NUMBERS))
def test_repr_raises(self):
class O(object):
def __repr__(self):
raise KeyError('foo')
assert 'Unrepresentable' in saferepr(O())
def test_safe_types(self):
for value in values(D_NUMBERS):
self.assertEqual(saferepr(value), old_repr(value))
