def MockMessage(self, id=None, receipt_handle=None, body=None):
m = Mock(name='message')
m.id = id or uuid()
m.receipt_handle = receipt_handle or uuid()
m._body = body
def _get_body():
return m._body
m.get_body.side_effect = _get_body
def _set_body(value):
m._body = value
m.set_body.side_effect = _set_body
return m
def test_can_consume(self, stdouts):
stderr = io.StringIO()
_restored = []
class RestoreChannel(virtual.Channel):
do_restore = True
def _restore(self, message):
_restored.append(message)
assert self.q.can_consume()
for i in range(self.q.prefetch_count - 1):
self.q.append(i, uuid())
assert self.q.can_consume()
self.q.append(i + 1, uuid())
assert not self.q.can_consume()
tag1 = next(iter(self.q._delivered))
self.q.ack(tag1)
assert self.q.can_consume()
tag2 = uuid()
self.q.append(i + 2, tag2)
assert not self.q.can_consume()
self.q.reject(tag2)
assert self.q.can_consume()
self.q.channel = RestoreChannel(self.q.channel.connection)
tag3 = uuid()
self.q.append(i + 3, tag3)
self.q.reject(tag3, requeue=True)
self.q._flush()
assert self.q._delivered
assert not self.q._delivered.restored
self.q.restore_unacked_once(stderr=stderr)
assert _restored == [11, 9, 8, 7, 6, 5, 4, 3, 2, 1]
assert self.q._delivered.restored
assert not self.q._delivered
self.q.restore_unacked_once(stderr=stderr)
self.q._delivered.restored = False
self.q.restore_unacked_once(stderr=stderr)
assert stderr.getvalue()
assert not stdouts.stdout.getvalue()
self.q.restore_at_shutdown = False
self.q.restore_unacked_once()
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
assert trace.trace_task_ret is trace._fast_trace_task
tid = uuid()
message = self.TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding)
assert not failed
assert res == repr(4 ** 4)
assert runtime is not None
assert isinstance(runtime, numbers.Real)
finally:
reset_worker_optimizations()
assert trace.trace_task_ret is trace._trace_task_ret
delattr(self.mytask, '__trace__')
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding, app=self.app,
)
assert not failed
assert res == repr(4 ** 4)
assert runtime is not None
assert isinstance(runtime, numbers.Real)
def freeze(self, _id=None, group_id=None, chord=None,
root_id=None, parent_id=None):
"""Finalize the signature by adding a concrete task id.
The task won't be called and you shouldn't call the signature
twice after freezing it as that'll result in two task messages
using the same task id.
Returns:
~@AsyncResult: promise of future evaluation.
"""
# pylint: disable=redefined-outer-name
# XXX chord is also a class in outer scope.
opts = self.options
try:
tid = opts['task_id']
except KeyError:
tid = opts['task_id'] = _id or uuid()
if root_id:
opts['root_id'] = root_id
if parent_id:
opts['parent_id'] = parent_id
if 'reply_to' not in opts:
opts['reply_to'] = self.app.oid
if group_id:
opts['group_id'] = group_id
if chord:
opts['chord'] = chord
# pylint: disable=too-many-function-args
# Borks on this, as it's a property.
return self.AsyncResult(tid)
def freeze(self, _id=None, group_id=None, chord=None,
root_id=None, parent_id=None):
"""Finalize the signature by adding a concrete task id.
The task won't be called and you shouldn't call the signature
twice after freezing it as that'll result in two task messages
using the same task id.
Returns:
~@AsyncResult: promise of future evaluation.
"""
opts = self.options
try:
tid = opts['task_id']
except KeyError:
tid = opts['task_id'] = _id or uuid()
if root_id:
opts['root_id'] = root_id
if parent_id:
opts['parent_id'] = parent_id
if 'reply_to' not in opts:
opts['reply_to'] = self.app.oid
if group_id:
opts['group_id'] = group_id
if chord:
opts['chord'] = chord
return self.AsyncResult(tid)
def test_on_success__with_events(self):
job = self.zRequest(id=uuid())
job.send_event = Mock(name='send_event')
job.on_success((False, 'foo', 1.0))
job.send_event.assert_called_with(
'task-succeeded', result='foo', runtime=1.0,
)
def _freeze_gid(self, options):
# remove task_id and use that as the group_id,
# if we don't remove it then every task will have the same id...
options = dict(self.options, **options)
options['group_id'] = group_id = (
options.pop('task_id', uuid()))
return options, group_id, options.get('root_id')
def freeze(self, _id=None, group_id=None, chord=None,
root_id=None, parent_id=None):
# pylint: disable=redefined-outer-name
# XXX chord is also a class in outer scope.
opts = self.options
try:
gid = opts['task_id']
except KeyError:
gid = opts['task_id'] = uuid()
if group_id:
opts['group_id'] = group_id
if chord:
opts['chord'] = chord
root_id = opts.setdefault('root_id', root_id)
parent_id = opts.setdefault('parent_id', parent_id)
new_tasks = []
# Need to unroll subgroups early so that chord gets the
# right result instance for chord_unlock etc.
results = list(self._freeze_unroll(
new_tasks, group_id, chord, root_id, parent_id,
))
if isinstance(self.tasks, MutableSequence):
self.tasks[:] = new_tasks
else:
self.tasks = new_tasks
return self.app.GroupResult(gid, results)
def test_execute_using_pool__revoked(self):
tid = uuid()
job = self.zRequest(id=tid, revoked_tasks={tid})
job.revoked = Mock()
job.revoked.return_value = True
with pytest.raises(TaskRevokedError):
job.execute_using_pool(self.pool)
def test_revoke_terminate(self):
request = Mock()
request.id = tid = uuid()
state = self.create_state()
state.consumer = Mock()
worker_state.task_reserved(request)
try:
r = control.revoke(state, tid, terminate=True)
self.assertIn(tid, revoked)
self.assertTrue(request.terminate.call_count)
self.assertIn('terminate:', r['ok'])
# unknown task id only revokes
r = control.revoke(state, uuid(), terminate=True)
self.assertIn('tasks unknown', r['ok'])
finally:
worker_state.task_ready(request)
def run(self, header, body, partial_args, app=None, interval=None,
countdown=1, max_retries=None, eager=False,
task_id=None, **options):
app = app or self._get_app(body)
group_id = header.options.get('task_id') or uuid()
root_id = body.options.get('root_id')
body.chord_size = self.__length_hint__()
options = dict(self.options, **options) if options else self.options
if options:
options.pop('task_id', None)
body.options.update(options)
results = header.freeze(
group_id=group_id, chord=body, root_id=root_id).results
bodyres = body.freeze(task_id, root_id=root_id)
# Chains should not be passed to the header tasks. See #3771
options.pop('chain', None)
parent = app.backend.apply_chord(
header, partial_args, group_id, body,
interval=interval, countdown=countdown,
options=options, max_retries=max_retries,
result=results)
bodyres.parent = parent
return bodyres
def test_revoke_terminate(self):
request = Mock()
request.id = tid = uuid()
state = self.create_state()
state.consumer = Mock()
worker_state.task_reserved(request)
try:
r = control.revoke(state, tid, terminate=True)
assert tid in revoked
assert request.terminate.call_count
assert 'terminate:' in r['ok']
# unknown task id only revokes
r = control.revoke(state, uuid(), terminate=True)
assert 'tasks unknown' in r['ok']
finally:
worker_state.task_ready(request)
def send_task(self, name, args=None, kwargs=None, countdown=None,
eta=None, task_id=None, producer=None, connection=None,
router=None, result_cls=None, expires=None,
publisher=None, link=None, link_error=None,
add_to_parent=True, group_id=None, retries=0, chord=None,
reply_to=None, time_limit=None, soft_time_limit=None,
root_id=None, parent_id=None, route_name=None,
shadow=None, chain=None, task_type=None, **options):
"""Send task by name.
Supports the same arguments as :meth:`@-Task.apply_async`.
Arguments:
name (str): Name of task to call (e.g. `"tasks.add"`).
result_cls (~@AsyncResult): Specify custom result class.
"""
parent = have_parent = None
amqp = self.amqp
task_id = task_id or uuid()
producer = producer or publisher # XXX compat
router = router or amqp.router
conf = self.conf
if conf.task_always_eager: # pragma: no cover
warnings.warn(AlwaysEagerIgnored(
'task_always_eager has no effect on send_task',
), stacklevel=2)
options = router.route(
options, route_name or name, args, kwargs, task_type)
if root_id is None:
parent, have_parent = self.current_worker_task, True
if parent:
root_id = parent.request.root_id or parent.request.id
if parent_id is None:
if not have_parent:
parent, have_parent = self.current_worker_task, True
if parent:
parent_id = parent.request.id
message = amqp.create_task_message(
task_id, name, args, kwargs, countdown, eta, group_id,
expires, retries, chord,
maybe_list(link), maybe_list(link_error),
reply_to or self.oid, time_limit, soft_time_limit,
self.conf.task_send_sent_event,
root_id, parent_id, shadow, chain,
)
if connection:
producer = amqp.Producer(connection)
with self.producer_or_acquire(producer) as P:
self.backend.on_task_call(P, task_id)
amqp.send_task_message(P, name, message, **options)
result = (result_cls or self.AsyncResult)(task_id)
if add_to_parent:
if not have_parent:
parent, have_parent = self.current_worker_task, True
if parent:
parent.add_trail(result)
return result
def test_execute(self):
tid = uuid()
job = self.xRequest(id=tid, args=[4], kwargs={})
assert job.execute() == 256
meta = self.mytask.backend.get_task_meta(tid)
assert meta['status'] == states.SUCCESS
assert meta['result'] == 256
def test_execute(self):
tid = uuid()
job = self.xRequest(id=tid, args=[4], kwargs={})
self.assertEqual(job.execute(), 256)
meta = self.mytask.backend.get_task_meta(tid)
self.assertEqual(meta['status'], states.SUCCESS)
self.assertEqual(meta['result'], 256)
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
tid = uuid()
message = TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding)
self.assertFalse(failed)
self.assertEqual(res, repr(4 ** 4))
self.assertIsNotNone(runtime)
self.assertIsInstance(runtime, numbers.Real)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(self.mytask, '__trace__')
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding, app=self.app,
)
self.assertFalse(failed)
self.assertEqual(res, repr(4 ** 4))
self.assertIsNotNone(runtime)
self.assertIsInstance(runtime, numbers.Real)
def test_revoke_with_name_not_in_registry(self):
tid = uuid()
m = {'method': 'revoke',
'destination': hostname,
'arguments': {'task_id': tid,
'task_name': 'xxxxxxxxx33333333388888'}}
self.panel.handle_message(m, None)
self.assertIn(tid, revoked)
def test_revoke_with_name(self):
tid = uuid()
m = {'method': 'revoke',
'destination': hostname,
'arguments': {'task_id': tid,
'task_name': self.mytask.name}}
self.panel.handle_message(m, None)
self.assertIn(tid, revoked)
def create_message(channel, **data):
data.setdefault('id', uuid())
m = Message(channel, body=pickle.dumps(dict(**data)),
content_type='application/x-python-serialize',
content_encoding='binary',
delivery_info={'consumer_tag': 'mock'})
m.accept = ['application/x-python-serialize']
return m
def test_process_cleanup_fails(self, _logger):
self.mytask.backend = Mock()
self.mytask.backend.process_cleanup = Mock(side_effect=KeyError())
tid = uuid()
ret = jail(self.app, tid, self.mytask.name, [2], {})
self.assertEqual(ret, 4)
self.mytask.backend.mark_as_done.assert_called()
self.assertIn('Process cleanup failed', _logger.error.call_args[0][0])
def test_execute_using_pool__expired(self):
tid = uuid()
job = self.zRequest(id=tid, revoked_tasks=set())
job.expires = 1232133
job.revoked = Mock()
job.revoked.return_value = True
with self.assertRaises(TaskRevokedError):
job.execute_using_pool(self.pool)
def test_can_consume(self, stdout, stderr):
_restored = []
class RestoreChannel(virtual.Channel):
do_restore = True
def _restore(self, message):
_restored.append(message)
self.assertTrue(self.q.can_consume())
for i in range(self.q.prefetch_count - 1):
self.q.append(i, uuid())
self.assertTrue(self.q.can_consume())
self.q.append(i + 1, uuid())
self.assertFalse(self.q.can_consume())
tag1 = next(iter(self.q._delivered))
self.q.ack(tag1)
self.assertTrue(self.q.can_consume())
tag2 = uuid()
self.q.append(i + 2, tag2)
self.assertFalse(self.q.can_consume())
self.q.reject(tag2)
self.assertTrue(self.q.can_consume())
self.q.channel = RestoreChannel(self.q.channel.connection)
tag3 = uuid()
self.q.append(i + 3, tag3)
self.q.reject(tag3, requeue=True)
self.q._flush()
self.q.restore_unacked_once()
self.assertListEqual(_restored, [11, 9, 8, 7, 6, 5, 4, 3, 2, 1])
self.assertTrue(self.q._delivered.restored)
self.assertFalse(self.q._delivered)
self.q.restore_unacked_once()
self.q._delivered.restored = False
self.q.restore_unacked_once()
self.assertTrue(stderr.getvalue())
self.assertFalse(stdout.getvalue())
self.q.restore_at_shutdown = False
self.q.restore_unacked_once()
def test_trace_catches_exception(self):
@self.app.task(request=None, shared=False)
def raising():
raise KeyError('baz')
with pytest.warns(RuntimeWarning):
res = trace_task(raising, uuid(), [], {}, app=self.app)[0]
assert isinstance(res, ExceptionInfo)
def test_revoke(self):
tid = uuid()
m = {"method": "revoke", "destination": hostname, "arguments": {"task_id": tid}}
self.panel.handle_message(m, None)
assert tid in revoked
m = {"method": "revoke", "destination": "does.not.exist", "arguments": {"task_id": tid + "xxx"}}
self.panel.handle_message(m, None)
assert tid + "xxx" not in revoked
def test_execute_using_pool__defaults_of_hybrid_to_proto2(self):
weakref_ref = Mock(name='weakref.ref')
headers = strategy.hybrid_to_proto2('', {'id': uuid(),
'task': self.mytask.name})[1]
job = self.zRequest(revoked_tasks=set(), ref=weakref_ref, **headers)
job.execute_using_pool(self.pool)
assert job._apply_result
weakref_ref.assert_called_with(self.pool.apply_async())
assert job._apply_result is weakref_ref()
def test_from_message(self):
us = 'æØåveéðƒeæ'
tid = uuid()
m = TaskMessage(self.mytask.name, tid, args=[2], kwargs={us: 'bar'})
job = Request(m, app=self.app)
self.assertIsInstance(job, Request)
self.assertEqual(job.name, self.mytask.name)
self.assertEqual(job.id, tid)
self.assertIs(job.message, m)
def test_revoke_with_name_not_in_registry(self):
tid = uuid()
m = {
"method": "revoke",
"destination": hostname,
"arguments": {"task_id": tid, "task_name": "xxxxxxxxx33333333388888"},
}
self.panel.handle_message(m, None)
assert tid in revoked
def test_process_cleanup_fails(self, patching):
_logger = patching('celery.app.trace.logger')
self.mytask.backend = Mock()
self.mytask.backend.process_cleanup = Mock(side_effect=KeyError())
tid = uuid()
ret = jail(self.app, tid, self.mytask.name, [2], {})
assert ret == 4
self.mytask.backend.mark_as_done.assert_called()
assert 'Process cleanup failed' in _logger.error.call_args[0][0]
def test_marked_as_started(self):
_started = []
def store_result(tid, meta, state, **kwargs):
if state == states.STARTED:
_started.append(tid)
self.mytask.backend.store_result = Mock(name='store_result')
self.mytask.backend.store_result.side_effect = store_result
self.mytask.track_started = True
tid = uuid()
jail(self.app, tid, self.mytask.name, [2], {})
assert tid in _started
self.mytask.ignore_result = True
tid = uuid()
jail(self.app, tid, self.mytask.name, [2], {})
assert tid not in _started
def election(self):
type = self.type
app = type.app
tid = self.options.get('task_id') or uuid()
with app.producer_or_acquire(None) as P:
props = type.backend.on_task_call(P, tid)
app.control.election(tid, 'task', self.clone(task_id=tid, **props),
connection=P.connection)
return type.AsyncResult(tid)
def test_trace_task_ret(self):
self.mytask.__trace__ = build_tracer(
self.mytask.name,
self.mytask,
self.app.loader,
'test',
app=self.app,
)
tid = uuid()
message = self.TaskMessage(self.mytask.name, tid, args=[4])
_, R, _ = _trace_task_ret(
self.mytask.name,
tid,
message.headers,
message.body,
message.content_type,
message.content_encoding,
app=self.app,
)
assert R == repr(4**4)
def test_worker_task_trace_handle_retry(self):
tid = uuid()
self.mytask.push_request(id=tid)
try:
raise ValueError('foo')
except Exception as exc:
try:
raise Retry(str(exc), exc=exc)
except Retry as exc:
w = TraceInfo(states.RETRY, exc)
w.handle_retry(
self.mytask, self.mytask.request, store_errors=False,
)
assert self.mytask.backend.get_status(tid) == states.PENDING
w.handle_retry(
self.mytask, self.mytask.request, store_errors=True,
)
assert self.mytask.backend.get_status(tid) == states.RETRY
finally:
self.mytask.pop_request()
def test_execute_using_pool_with_use_fast_trace_task(self):
self.app.use_fast_trace_task = True
weakref_ref = Mock(name='weakref.ref')
job = self.zRequest(id=uuid(), revoked_tasks=set(), ref=weakref_ref)
job.execute_using_pool(self.pool)
self.pool.apply_async.assert_called_with(
fast_trace_task,
args=(job.type, job.id, job.request_dict, job.body,
job.content_type, job.content_encoding),
accept_callback=job.on_accepted,
timeout_callback=job.on_timeout,
callback=job.on_success,
error_callback=job.on_failure,
soft_timeout=self.task.soft_time_limit,
timeout=self.task.time_limit,
correlation_id=job.id,
)
assert job._apply_result
weakref_ref.assert_called_with(self.pool.apply_async())
assert job._apply_result is weakref_ref()
def test_reply__collect_uses_default_channel(self):
class ConsumerCalled(Exception):
pass
def fake_Consumer(channel, *args, **kwargs):
raise ConsumerCalled(channel)
ticket = uuid()
with patch('kombu.pidbox.Consumer') as Consumer:
mailbox = pidbox.Mailbox('test_reply__collect')(self.connection)
assert mailbox.connection.default_channel is not None
Consumer.side_effect = fake_Consumer
try:
mailbox._collect(ticket, limit=1)
except ConsumerCalled as c:
assert c.args[0] is not None
except Exception:
raise
else:
assert False, "Consumer not called"
def test_fast_trace_task(self):
assert self.app.use_fast_trace_task is False
setup_worker_optimizations(self.app)
assert self.app.use_fast_trace_task is True
tid = uuid()
message = self.TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name,
self.mytask,
self.app.loader,
'test',
app=self.app,
)
failed, res, runtime = fast_trace_task(self.mytask.name, tid,
message.headers,
message.body,
message.content_type,
message.content_encoding)
assert not failed
assert res == repr(4**4)
assert runtime is not None
assert isinstance(runtime, numbers.Real)
finally:
reset_worker_optimizations(self.app)
assert self.app.use_fast_trace_task is False
delattr(self.mytask, '__trace__')
failed, res, runtime = trace_task_ret(
self.mytask.name,
tid,
message.headers,
message.body,
message.content_type,
message.content_encoding,
app=self.app,
)
assert not failed
assert res == repr(4**4)
assert runtime is not None
assert isinstance(runtime, numbers.Real)
def test_revoke(self):
tid = uuid()
m = {
'method': 'revoke',
'destination': hostname,
'arguments': {
'task_id': tid
}
}
self.panel.handle_message(m, None)
self.assertIn(tid, revoked)
m = {
'method': 'revoke',
'destination': 'does.not.exist',
'arguments': {
'task_id': tid + 'xxx'
}
}
self.panel.handle_message(m, None)
self.assertNotIn(tid + 'xxx', revoked)
def test_revoke(self):
tid = uuid()
m = {
'method': 'revoke',
'destination': hostname,
'arguments': {
'task_id': tid,
},
}
self.panel.handle_message(m, None)
assert tid in revoked
m = {
'method': 'revoke',
'destination': 'does.not.exist',
'arguments': {
'task_id': tid + 'xxx',
},
}
self.panel.handle_message(m, None)
assert tid + 'xxx' not in revoked
def apply_async(self,
args=None,
kwargs=None,
task_id=None,
producer=None,
link=None,
link_error=None,
shadow=None,
**options):
if self.singleton_config.app.conf.task_always_eager:
return super(Singleton,
self).apply_async(args, kwargs, task_id, producer,
link, link_error, shadow, **options)
args = args or []
kwargs = kwargs or {}
task_id = task_id or uuid()
lock = self.generate_lock(self.name, args, kwargs)
run_args = dict(lock=lock,
args=args,
kwargs=kwargs,
task_id=task_id,
producer=producer,
link=link,
link_error=link_error,
shadow=shadow,
**options)
task = self.lock_and_run(**run_args)
if task:
return task
existing_task_id = self.get_existing_task_id(lock)
while not existing_task_id:
task = self.lock_and_run(**run_args)
if task:
return task
existing_task_id = self.get_existing_task_id(lock)
return self.on_duplicate(existing_task_id)
def run(self, header, body, partial_args, app=None, interval=None,
countdown=1, max_retries=None, eager=False,
task_id=None, **options):
app = app or self._get_app(body)
group_id = header.options.get('task_id') or uuid()
root_id = body.options.get('root_id')
body.chord_size = self.__length_hint__()
options = dict(self.options, **options) if options else self.options
if options:
options.pop('task_id', None)
body.options.update(options)
bodyres = body.freeze(task_id, root_id=root_id)
# Chains should not be passed to the header tasks. See #3771
options.pop('chain', None)
# Neither should chords, for deeply nested chords to work
options.pop('chord', None)
options.pop('task_id', None)
header.freeze(group_id=group_id, chord=body, root_id=root_id)
header_result = header(*partial_args, task_id=group_id, **options)
if len(header_result) > 0:
app.backend.apply_chord(
header_result,
body,
interval=interval,
countdown=countdown,
max_retries=max_retries,
)
# The execution of a chord body is normally triggered by its header's
# tasks completing. If the header is empty this will never happen, so
# we execute the body manually here.
else:
body.delay([])
bodyres.parent = header_result
return bodyres
def apply_async(
self,
args=None,
kwargs=None,
task_id=None,
producer=None,
link=None,
link_error=None,
shadow=None,
**options
):
args = args or []
kwargs = kwargs or {}
task_id = task_id or uuid()
lock = self.generate_lock(self.name, args, kwargs)
run_args = dict(
lock=lock,
args=args,
kwargs=kwargs,
task_id=task_id,
producer=producer,
link=link,
link_error=link_error,
shadow=shadow,
**options
)
task = self.lock_and_run(**run_args)
if task:
return task
existing_task_id = self.get_existing_task_id(lock)
while not existing_task_id:
task = self.lock_and_run(**run_args)
if task:
return task
existing_task_id = self.get_existing_task_id(lock)
return self.AsyncResult(existing_task_id)
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
tid = uuid()
message = TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name,
self.mytask,
self.app.loader,
'test',
app=self.app,
)
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding)
self.assertFalse(failed)
self.assertEqual(res, repr(4**4))
self.assertIsNotNone(runtime)
self.assertIsInstance(runtime, numbers.Real)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(self.mytask, '__trace__')
failed, res, runtime = trace.trace_task_ret(
self.mytask.name,
tid,
message.headers,
message.body,
message.content_type,
message.content_encoding,
app=self.app,
)
self.assertFalse(failed)
self.assertEqual(res, repr(4**4))
self.assertIsNotNone(runtime)
self.assertIsInstance(runtime, numbers.Real)
def test_worker_task_trace_handle_failure(self):
tid = uuid()
self.mytask.push_request()
try:
self.mytask.request.id = tid
try:
raise ValueError('foo')
except Exception as exc:
w = TraceInfo(states.FAILURE, exc)
w.handle_failure(
self.mytask,
self.mytask.request,
store_errors=False,
)
assert self.mytask.backend.get_status(tid) == states.PENDING
w.handle_failure(
self.mytask,
self.mytask.request,
store_errors=True,
)
assert self.mytask.backend.get_status(tid) == states.FAILURE
finally:
self.mytask.pop_request()
def test_execute_using_pool_with_none_timelimit_header(self):
from celery.app.trace import trace_task_ret as trace
weakref_ref = Mock(name='weakref.ref')
job = self.zRequest(id=uuid(),
revoked_tasks=set(),
ref=weakref_ref,
headers={'timelimit': None})
job.execute_using_pool(self.pool)
self.pool.apply_async.assert_called_with(
trace,
args=(job.type, job.id, job.request_dict, job.body,
job.content_type, job.content_encoding),
accept_callback=job.on_accepted,
timeout_callback=job.on_timeout,
callback=job.on_success,
error_callback=job.on_failure,
soft_timeout=self.task.soft_time_limit,
timeout=self.task.time_limit,
correlation_id=job.id,
)
assert job._apply_result
weakref_ref.assert_called_with(self.pool.apply_async())
assert job._apply_result is weakref_ref()
def Mutex(client, name, expire):
lock_id = uuid()
i_won = client.setnx(name, lock_id)
try:
if i_won:
client.expire(name, expire)
yield
else:
if not client.ttl(name):
client.expire(name, expire)
raise MutexHeld()
finally:
if i_won:
try:
with client.pipeline(True) as pipe:
pipe.watch(name)
if pipe.get(name) == lock_id:
pipe.multi()
pipe.delete(name)
pipe.execute()
pipe.unwatch()
except redis.WatchError:
pass
def run(self, header, body, partial_args, app=None, interval=None,
countdown=1, max_retries=None, eager=False,
task_id=None, **options):
app = app or self._get_app(body)
group_id = header.options.get('task_id') or uuid()
root_id = body.options.get('root_id')
body.chord_size = self.__length_hint__()
options = dict(self.options, **options) if options else self.options
if options:
options.pop('task_id', None)
body.options.update(options)
results = header.freeze(
group_id=group_id, chord=body, root_id=root_id).results
bodyres = body.freeze(task_id, root_id=root_id)
parent = app.backend.apply_chord(
header, partial_args, group_id, body,
interval=interval, countdown=countdown,
options=options, max_retries=max_retries,
result=results)
bodyres.parent = parent
return bodyres
def test_basic_publish__get__consume__restore(self,
n='test_basic_publish'):
c = memory_client().channel()
c.exchange_declare(n)
c.queue_declare(n)
c.queue_bind(n, n, n)
c.queue_declare(n + '2')
c.queue_bind(n + '2', n, n)
m = c.prepare_message('nthex quick brown fox...')
c.basic_publish(m, n, n)
r1 = c.message_to_python(c.basic_get(n))
assert r1
assert r1.body == 'nthex quick brown fox...'.encode('utf-8')
assert c.basic_get(n) is None
consumer_tag = uuid()
c.basic_consume(n + '2', False,
consumer_tag=consumer_tag, callback=lambda *a: None)
assert n + '2' in c._active_queues
r2, _ = c.drain_events()
r2 = c.message_to_python(r2)
assert r2.body == 'nthex quick brown fox...'.encode('utf-8')
assert r2.delivery_info['exchange'] == n
assert r2.delivery_info['routing_key'] == n
with pytest.raises(virtual.Empty):
c.drain_events()
c.basic_cancel(consumer_tag)
c._restore(r2)
r3 = c.message_to_python(c.basic_get(n))
assert r3
assert r3.body == 'nthex quick brown fox...'.encode('utf-8')
assert c.basic_get(n) is None
def freeze(self, _id=None, group_id=None, chord=None,
root_id=None, parent_id=None):
opts = self.options
try:
gid = opts['task_id']
except KeyError:
gid = opts['task_id'] = uuid()
if group_id:
opts['group_id'] = group_id
if chord:
opts['chord'] = chord
root_id = opts.setdefault('root_id', root_id)
parent_id = opts.setdefault('parent_id', parent_id)
new_tasks = []
# Need to unroll subgroups early so that chord gets the
# right result instance for chord_unlock etc.
results = list(self._freeze_unroll(
new_tasks, group_id, chord, root_id, parent_id,
))
if isinstance(self.tasks, MutableSequence):
self.tasks[:] = new_tasks
else:
self.tasks = new_tasks
return self.app.GroupResult(gid, results)
def Mutex(client, name, expire):
"""The Redis lock implementation (probably shaky)."""
lock_id = uuid()
i_won = client.setnx(name, lock_id)
try:
if i_won:
client.expire(name, expire)
yield
else:
if not client.ttl(name):
client.expire(name, expire)
raise MutexHeld()
finally:
if i_won:
try:
with client.pipeline(True) as pipe:
pipe.watch(name)
if bytes_to_str(pipe.get(name)) == lock_id:
pipe.multi()
pipe.delete(name)
pipe.execute()
pipe.unwatch()
except redis.WatchError:
pass
def test_on_success__calls_failure(self):
job = self.zRequest(id=uuid())
einfo = Mock(name='einfo')
job.on_failure = Mock(name='on_failure')
job.on_success((True, einfo, 1.0))
job.on_failure.assert_called_with(einfo, return_ok=True)
def test_on_success__no_events(self):
self.eventer = None
job = self.zRequest(id=uuid())
job.send_event = Mock(name='send_event')
job.on_success((False, 'foo', 1.0))
job.send_event.assert_not_called()
def apply(self,
args=None,
kwargs=None,
link=None,
link_error=None,
task_id=None,
retries=None,
throw=None,
logfile=None,
loglevel=None,
headers=None,
**options):
"""Execute this task locally, by blocking until the task returns.
Arguments:
args (Tuple): positional arguments passed on to the task.
kwargs (Dict): keyword arguments passed on to the task.
throw (bool): Re-raise task exceptions.
Defaults to the :setting:`task_eager_propagates` setting.
Returns:
celery.result.EagerResult: pre-evaluated result.
"""
# trace imports Task, so need to import inline.
from celery.app.trace import build_tracer
app = self._get_app()
args = args or ()
kwargs = kwargs or {}
task_id = task_id or uuid()
retries = retries or 0
if throw is None:
throw = app.conf.task_eager_propagates
# Make sure we get the task instance, not class.
task = app._tasks[self.name]
request = {
'id': task_id,
'retries': retries,
'is_eager': True,
'logfile': logfile,
'loglevel': loglevel or 0,
'hostname': gethostname(),
'callbacks': maybe_list(link),
'errbacks': maybe_list(link_error),
'headers': headers,
'delivery_info': {
'is_eager': True
},
}
tb = None
tracer = build_tracer(
task.name,
task,
eager=True,
propagate=throw,
app=self._get_app(),
)
ret = tracer(task_id, args, kwargs, request)
retval = ret.retval
if isinstance(retval, ExceptionInfo):
retval, tb = retval.exception, retval.traceback
state = states.SUCCESS if ret.info is None else ret.info.state
return EagerResult(task_id, retval, state, traceback=tb)
def apply_async(self,
args=None,
kwargs=None,
task_id=None,
producer=None,
link=None,
link_error=None,
shadow=None,
**options):
"""Apply tasks asynchronously by sending a message.
Arguments:
args (Tuple): The positional arguments to pass on to the task.
kwargs (Dict): The keyword arguments to pass on to the task.
countdown (float): Number of seconds into the future that the
task should execute. Defaults to immediate execution.
eta (~datetime.datetime): Absolute time and date of when the task
should be executed. May not be specified if `countdown`
is also supplied.
expires (float, ~datetime.datetime): Datetime or
seconds in the future for the task should expire.
The task won't be executed after the expiration time.
shadow (str): Override task name used in logs/monitoring.
Default is retrieved from :meth:`shadow_name`.
connection (kombu.Connection): Re-use existing broker connection
instead of acquiring one from the connection pool.
retry (bool): If enabled sending of the task message will be
retried in the event of connection loss or failure.
Default is taken from the :setting:`task_publish_retry`
setting. Note that you need to handle the
producer/connection manually for this to work.
retry_policy (Mapping): Override the retry policy used.
See the :setting:`task_publish_retry_policy` setting.
queue (str, kombu.Queue): The queue to route the task to.
This must be a key present in :setting:`task_queues`, or
:setting:`task_create_missing_queues` must be
enabled. See :ref:`guide-routing` for more
information.
exchange (str, kombu.Exchange): Named custom exchange to send the
task to. Usually not used in combination with the ``queue``
argument.
routing_key (str): Custom routing key used to route the task to a
worker server. If in combination with a ``queue`` argument
only used to specify custom routing keys to topic exchanges.
priority (int): The task priority, a number between 0 and 9.
Defaults to the :attr:`priority` attribute.
serializer (str): Serialization method to use.
Can be `pickle`, `json`, `yaml`, `msgpack` or any custom
serialization method that's been registered
with :mod:`kombu.serialization.registry`.
Defaults to the :attr:`serializer` attribute.
compression (str): Optional compression method
to use. Can be one of ``zlib``, ``bzip2``,
or any custom compression methods registered with
:func:`kombu.compression.register`.
Defaults to the :setting:`task_compression` setting.
link (Signature): A single, or a list of tasks signatures
to apply if the task returns successfully.
link_error (Signature): A single, or a list of task signatures
to apply if an error occurs while executing the task.
producer (kombu.Producer): custom producer to use when publishing
the task.
add_to_parent (bool): If set to True (default) and the task
is applied while executing another task, then the result
will be appended to the parent tasks ``request.children``
attribute. Trailing can also be disabled by default using the
:attr:`trail` attribute
publisher (kombu.Producer): Deprecated alias to ``producer``.
headers (Dict): Message headers to be included in the message.
Returns:
celery.result.AsyncResult: Promise of future evaluation.
Raises:
TypeError: If not enough arguments are passed, or too many
arguments are passed. Note that signature checks may
be disabled by specifying ``@task(typing=False)``.
kombu.exceptions.OperationalError: If a connection to the
transport cannot be made, or if the connection is lost.
Note:
Also supports all keyword arguments supported by
:meth:`kombu.Producer.publish`.
"""
if self.typing:
try:
check_arguments = self.__header__
except AttributeError: # pragma: no cover
pass
else:
check_arguments(*(args or ()), **(kwargs or {}))
app = self._get_app()
if app.conf.task_always_eager:
with app.producer_or_acquire(producer) as eager_producer:
serializer = options.get(
'serializer',
(eager_producer.serializer if eager_producer.serializer
else app.conf.task_serializer))
body = args, kwargs
content_type, content_encoding, data = serialization.dumps(
body,
serializer,
)
args, kwargs = serialization.loads(data,
content_type,
content_encoding,
accept=[content_type])
with denied_join_result():
return self.apply(args,
kwargs,
task_id=task_id or uuid(),
link=link,
link_error=link_error,
**options)
if self.__v2_compat__:
shadow = shadow or self.shadow_name(self(), args, kwargs, options)
else:
shadow = shadow or self.shadow_name(args, kwargs, options)
preopts = self._get_exec_options()
options = dict(preopts, **options) if options else preopts
options.setdefault('ignore_result', self.ignore_result)
if self.priority:
options.setdefault('priority', self.priority)
return app.send_task(self.name,
args,
kwargs,
task_id=task_id,
producer=producer,
link=link,
link_error=link_error,
result_cls=self.AsyncResult,
shadow=shadow,
task_type=self,
**options)
def send_task(self,
name,
args=None,
kwargs=None,
countdown=None,
eta=None,
task_id=None,
producer=None,
connection=None,
router=None,
result_cls=None,
expires=None,
publisher=None,
link=None,
link_error=None,
add_to_parent=True,
group_id=None,
retries=0,
chord=None,
reply_to=None,
time_limit=None,
soft_time_limit=None,
root_id=None,
parent_id=None,
route_name=None,
shadow=None,
chain=None,
task_type=None,
**options):
"""Send task by name.
Supports the same arguments as :meth:`@-Task.apply_async`.
Arguments:
name (str): Name of task to call (e.g., `"tasks.add"`).
result_cls (AsyncResult): Specify custom result class.
"""
parent = have_parent = None
amqp = self.amqp
task_id = task_id or uuid()
producer = producer or publisher # XXX compat
router = router or amqp.router
conf = self.conf
if conf.task_always_eager: # pragma: no cover
warnings.warn(AlwaysEagerIgnored(
'task_always_eager has no effect on send_task', ),
stacklevel=2)
ignored_result = options.pop('ignore_result', False)
options = router.route(options, route_name or name, args, kwargs,
task_type)
if not root_id or not parent_id:
parent = self.current_worker_task
if parent:
if not root_id:
root_id = parent.request.root_id or parent.request.id
if not parent_id:
parent_id = parent.request.id
message = amqp.create_task_message(
task_id,
name,
args,
kwargs,
countdown,
eta,
group_id,
expires,
retries,
chord,
maybe_list(link),
maybe_list(link_error),
reply_to or self.oid,
time_limit,
soft_time_limit,
self.conf.task_send_sent_event,
root_id,
parent_id,
shadow,
chain,
argsrepr=options.get('argsrepr'),
kwargsrepr=options.get('kwargsrepr'),
)
if connection:
producer = amqp.Producer(connection, auto_declare=False)
with self.producer_or_acquire(producer) as P:
with P.connection._reraise_as_library_errors():
if not ignored_result:
self.backend.on_task_call(P, task_id)
amqp.send_task_message(P, name, message, **options)
result = (result_cls or self.AsyncResult)(task_id)
# We avoid using the constructor since a custom result class
# can be used, in which case the constructor may still use
# the old signature.
result.ignored = ignored_result
if add_to_parent:
if not have_parent:
parent, have_parent = self.current_worker_task, True
if parent:
parent.add_trail(result)
return result
def test_process_cleanup_BaseException(self):
self.mytask.backend = Mock()
self.mytask.backend.process_cleanup = Mock(side_effect=SystemExit())
with pytest.raises(SystemExit):
jail(self.app, uuid(), self.mytask.name, [2], {})
def test_execute_jail_success(self):
ret = jail(self.app, uuid(), self.mytask.name, [2], {})
assert ret == 4
def test_from_message_empty_args(self):
tid = uuid()
m = self.TaskMessage(self.mytask.name, tid, args=[], kwargs={})
job = Request(m, app=self.app)
assert isinstance(job, Request)
def test_uuid(self) -> None:
i1 = uuid()
i2 = uuid()
assert isinstance(i1, str)
assert i1 != i2
def test_on_success__acks_late_disabled(self):
self.task.acks_late = False
job = self.zRequest(id=uuid())
job.acknowledge = Mock(name='ack')
job.on_success((False, 'foo', 1.0))
job.acknowledge.assert_not_called()
def _next_delivery_tag(self):
return uuid()
def test_uuid4(self) -> None:
assert uuid() != uuid()
