def test_no_conn_driver_info(self):
app = Celery(set_as_current=False)
app.connection = Mock()
conn = app.connection.return_value = Mock()
conn.transport = None
bugreport(app)
def _chord_context(self, ResultCls, setup=None, **kwargs):
with patch('celery.result.GroupResult'):
@task()
def callback(*args, **kwargs):
pass
pts, result.GroupResult = result.GroupResult, ResultCls
callback.apply_async = Mock()
callback_s = callback.s()
callback_s.id = 'callback_id'
fail_current = self.app.backend.fail_from_current_stack = Mock()
try:
with patch_unlock_retry() as (unlock, retry):
subtask, canvas.maybe_subtask = (
canvas.maybe_subtask,
passthru,
)
if setup:
setup(callback)
try:
unlock('group_id',
callback_s,
result=[AsyncResult(r) for r in ['1', 2, 3]],
GroupResult=ResultCls,
**kwargs)
finally:
canvas.maybe_subtask = subtask
yield callback_s, retry, fail_current
finally:
result.GroupResult = pts
def test_when_not_ready(self, retry, TaskSetResult):
callback = Mock()
result = Mock(attrs=dict(ready=lambda: False))
TaskSetResult.restore = lambda setid: result
chords._unlock_chord("setid", callback, interval=10, max_retries=30)
self.assertFalse(callback.delay.call_count)
# did retry
chords._unlock_chord.retry.assert_called_with(countdown=10,
max_retries=30)
def test_outside_body_error(self, report_internal_error, build_tracer):
tracer = Mock()
tracer.side_effect = KeyError("foo")
build_tracer.return_value = tracer
@current_app.task
def xtask():
pass
trace_task(xtask, "uuid", (), {})
self.assertTrue(report_internal_error.call_count)
self.assertIs(xtask.__tracer__, tracer)
def test_outside_body_error(self, report_internal_error, build_tracer):
tracer = Mock()
tracer.side_effect = KeyError('foo')
build_tracer.return_value = tracer
@current_app.task
def xtask():
pass
trace_task(xtask, 'uuid', (), {})
self.assertTrue(report_internal_error.call_count)
self.assertIs(xtask.__trace__, tracer)
def test_run(self):
prev, current_app.backend = current_app.backend, Mock()
current_app.backend.cleanup = Mock()
current_app.backend.cleanup.__name__ = 'cleanup'
try:
Chord = current_app.tasks['celery.chord']
body = dict()
Chord(TaskSet(add.subtask((i, i)) for i in xrange(5)), body)
Chord([add.subtask((i, i)) for i in xrange(5)], body)
self.assertEqual(current_app.backend.on_chord_apply.call_count, 2)
finally:
current_app.backend = prev
def test_unlock_ready(self, retry, TaskSetResult):
callback = Mock()
result = Mock(attrs=dict(ready=lambda: True,
join=lambda **kw: [2, 4, 8, 6]))
TaskSetResult.restore = lambda setid: result
subtask, chords.subtask = chords.subtask, passthru
try:
chords._unlock_chord("setid", callback)
finally:
chords.subtask = subtask
callback.delay.assert_called_with([2, 4, 8, 6])
result.delete.assert_called_with()
# did not retry
self.assertFalse(retry.call_count)
def test_migrate(self, name="testcelery"):
x = BrokerConnection("memory://foo")
y = BrokerConnection("memory://foo")
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, "direct")
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish("foo", exchange=name, routing_key=name)
Producer(x).publish("bar", exchange=name, routing_key=name)
Producer(x).publish("baz", exchange=name, routing_key=name)
self.assertTrue(x.default_channel.queues)
self.assertFalse(y.default_channel.queues)
migrate_tasks(x, y)
yq = q(y.default_channel)
self.assertEqual(yq.get().body, "foo")
self.assertEqual(yq.get().body, "bar")
self.assertEqual(yq.get().body, "baz")
Producer(x).publish("foo", exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y, callback=callback)
self.assertTrue(callback.called)
migrate = Mock()
Producer(x).publish("baz", exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback, migrate=migrate)
self.assertTrue(migrate.called)
with patch("kombu.transport.virtual.Channel.queue_declare") as qd:
def effect(*args, **kwargs):
if kwargs.get("passive"):
raise StdChannelError()
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y)
x = BrokerConnection("memory://")
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y, callback=callback)
self.assertFalse(callback.called)
def test_migrate(self, name='testcelery'):
x = Connection('memory://foo')
y = Connection('memory://foo')
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, 'direct')
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish('foo', exchange=name, routing_key=name)
Producer(x).publish('bar', exchange=name, routing_key=name)
Producer(x).publish('baz', exchange=name, routing_key=name)
self.assertTrue(x.default_channel.queues)
self.assertFalse(y.default_channel.queues)
migrate_tasks(x, y, accept=['text/plain'])
yq = q(y.default_channel)
self.assertEqual(yq.get().body, ensure_bytes('foo'))
self.assertEqual(yq.get().body, ensure_bytes('bar'))
self.assertEqual(yq.get().body, ensure_bytes('baz'))
Producer(x).publish('foo', exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y, callback=callback, accept=['text/plain'])
self.assertTrue(callback.called)
migrate = Mock()
Producer(x).publish('baz', exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback,
migrate=migrate, accept=['text/plain'])
self.assertTrue(migrate.called)
with patch('kombu.transport.virtual.Channel.queue_declare') as qd:
def effect(*args, **kwargs):
if kwargs.get('passive'):
raise StdChannelError()
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y)
x = Connection('memory://')
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y, callback=callback, accept=['text/plain'])
self.assertFalse(callback.called)
def test_apply(self):
self.app.conf.CELERY_ALWAYS_EAGER = False
from celery import chord
m = Mock()
m.app.conf.CELERY_ALWAYS_EAGER = False
m.AsyncResult = AsyncResult
prev, chord.Chord = chord.Chord, m
try:
x = chord(add.s(i, i) for i in xrange(10))
body = add.s(2)
result = x(body)
self.assertEqual(result.id, body.options["task_id"])
self.assertTrue(chord.Chord.called)
finally:
chord.Chord = prev
def test_apply(self):
self.app.conf.CELERY_ALWAYS_EAGER = False
from celery import chord
m = Mock()
m.app.conf.CELERY_ALWAYS_EAGER = False
m.AsyncResult = AsyncResult
prev, chord.Chord = chord.Chord, m
try:
x = chord(add.s(i, i) for i in xrange(10))
body = add.s(2)
result = x(body)
self.assertEqual(result.id, body.options['task_id'])
self.assertTrue(chord.Chord.called)
finally:
chord.Chord = prev
def test_unlock_ready(self, GroupResult):
class AlwaysReady(TSR):
is_ready = True
value = [2, 4, 8, 6]
@task()
def callback(*args, **kwargs):
pass
pts, result.GroupResult = result.GroupResult, AlwaysReady
callback.apply_async = Mock()
callback_s = callback.s()
try:
with patch_unlock_retry() as (unlock, retry):
subtask, canvas.maybe_subtask = canvas.maybe_subtask, passthru
try:
unlock('group_id',
callback_s,
result=map(AsyncResult, [1, 2, 3]))
finally:
canvas.maybe_subtask = subtask
callback.apply_async.assert_called_with(([2, 4, 8, 6], ), {})
# did not retry
self.assertFalse(retry.call_count)
finally:
result.GroupResult = pts
def test_handle_error_state(self):
x = self.TI(states.FAILURE)
x.handle_failure = Mock()
x.handle_error_state(add_cast)
x.handle_failure.assert_called_with(
add_cast,
store_errors=add_cast.store_errors_even_if_ignored,
)
def test_current_routing_key(self):
task = Mock()
_task_stack.push(task)
try:
task.request.reply_to = 'reply_to'
self.assertEqual(self.b._routing_key('task_id'), 'reply_to')
finally:
_task_stack.pop()
def test_colorize(self):
from celery import Celery
app = Celery(set_as_current=False)
app.log.setup = Mock()
b = beatapp.Beat(app=app, no_color=True)
b.setup_logging()
self.assertTrue(app.log.setup.called)
self.assertEqual(app.log.setup.call_args[1]['colorize'], False)
def Message(body, exchange='exchange', routing_key='rkey',
compression=None, content_type='application/json',
content_encoding='utf-8'):
return Mock(attrs=dict(body=body,
delivery_info=dict(exchange=exchange, routing_key=routing_key),
headers=dict(compression=compression),
content_type=content_type, content_encoding=content_encoding,
properties={}))
def patch_unlock_retry():
unlock = current_app.tasks['celery.chord_unlock']
retry = Mock()
prev, unlock.retry = unlock.retry, retry
try:
yield unlock, retry
finally:
unlock.retry = prev
def test_apply(self):
self.app.conf.CELERY_ALWAYS_EAGER = False
from celery import chord
m = Mock()
m.app.conf.CELERY_ALWAYS_EAGER = False
m.AsyncResult = AsyncResult
prev, chord.Chord = chord.Chord, m
try:
x = chord(add.s(i, i) for i in xrange(10))
body = add.s(2)
result = x(body)
# does not modify original subtask
with self.assertRaises(KeyError):
body.options['task_id']
self.assertTrue(chord.Chord.called)
finally:
chord.Chord = prev
def test_apply(self):
self.app.conf.CELERY_ALWAYS_EAGER = False
from celery import chord
m = Mock()
m.app.conf.CELERY_ALWAYS_EAGER = False
m.AsyncResult = AsyncResult
prev, chord.Chord = chord.Chord, m
try:
x = chord(add.s(i, i) for i in xrange(10))
body = add.s(2)
result = x(body)
# does not modify original subtask
with self.assertRaises(KeyError):
body.options['task_id']
self.assertTrue(chord.Chord.called)
finally:
chord.Chord = prev
def test_removes_compression_header(self):
x = Message('foo', compression='zlib')
producer = Mock()
migrate_task(producer, x.body, x)
self.assertTrue(producer.publish.called)
args, kwargs = producer.publish.call_args
self.assertIsInstance(args[0], bytes_t)
self.assertNotIn('compression', kwargs['headers'])
self.assertEqual(kwargs['compression'], 'zlib')
self.assertEqual(kwargs['content_type'], 'application/json')
self.assertEqual(kwargs['content_encoding'], 'utf-8')
self.assertEqual(kwargs['exchange'], 'exchange')
self.assertEqual(kwargs['routing_key'], 'rkey')
def test_removes_compression_header(self):
x = Message("foo", compression="zlib")
producer = Mock()
migrate_task(producer, x.body, x)
self.assertTrue(producer.publish.called)
args, kwargs = producer.publish.call_args
self.assertIsInstance(args[0], bytes_t)
self.assertNotIn("compression", kwargs["headers"])
self.assertEqual(kwargs["compression"], "zlib")
self.assertEqual(kwargs["content_type"], "application/json")
self.assertEqual(kwargs["content_encoding"], "utf-8")
self.assertEqual(kwargs["exchange"], "exchange")
self.assertEqual(kwargs["routing_key"], "rkey")
def Message(body,
exchange="exchange",
routing_key="rkey",
compression=None,
content_type="application/json",
content_encoding="utf-8"):
return Mock(attrs=dict(body=body,
delivery_info=dict(exchange=exchange,
routing_key=routing_key),
headers=dict(compression=compression),
content_type=content_type,
content_encoding=content_encoding,
properties={}))
def test_unlock_ready(self, retry):
callback.apply_async = Mock()
pts, chords.TaskSetResult = chords.TaskSetResult, TSR
subtask, chords.subtask = chords.subtask, passthru
try:
chords._unlock_chord("setid",
callback.subtask(),
result=map(AsyncResult, [1, 2, 3]))
finally:
chords.subtask = subtask
chords.TaskSetResult = pts
callback.apply_async.assert_called_with(([2, 4, 8, 6], ), {})
# did not retry
self.assertFalse(retry.call_count)
def _result_context(self):
results = Queue()
class Message(object):
acked = 0
requeued = 0
def __init__(self, **merge):
self.payload = dict({
'status': states.STARTED,
'result': None
}, **merge)
self.body = pickle.dumps(self.payload)
self.content_type = 'application/x-python-serialize'
self.content_encoding = 'binary'
def ack(self, *args, **kwargs):
self.acked += 1
def requeue(self, *args, **kwargs):
self.requeued += 1
class MockBinding(object):
def __init__(self, *args, **kwargs):
self.channel = Mock()
def __call__(self, *args, **kwargs):
return self
def declare(self):
pass
def get(self, no_ack=False):
try:
return results.get(block=False)
except Empty:
pass
def is_bound(self):
return True
class MockBackend(AMQPBackend):
Queue = MockBinding
backend = MockBackend()
backend._republish = Mock()
yield results, backend, Message
def test_when_not_ready(self, GroupResult):
with patch_unlock_retry() as (unlock, retry):
class NeverReady(TSR):
is_ready = False
pts, result.GroupResult = result.GroupResult, NeverReady
try:
callback = Mock()
unlock('group_id', callback, interval=10, max_retries=30,
result=map(AsyncResult, [1, 2, 3]))
self.assertFalse(callback.delay.call_count)
# did retry
unlock.retry.assert_called_with(countdown=10, max_retries=30)
finally:
result.GroupResult = pts
def Message(body, exchange='exchange', routing_key='rkey',
compression=None, content_type='application/json',
content_encoding='utf-8'):
return Mock(
attrs={
'body': body,
'delivery_info': {
'exchange': exchange,
'routing_key': routing_key,
},
'headers': {
'compression': compression,
},
'content_type': content_type,
'content_encoding': content_encoding,
'properties': {}
},
)
def test_when_not_ready(self, retry, TaskSetResult):
callback.apply_async = Mock()
class NeverReady(TSR):
is_ready = False
pts, chords.TaskSetResult = chords.TaskSetResult, NeverReady
try:
chords._unlock_chord("setid",
callback.subtask,
interval=10,
max_retries=30,
result=map(AsyncResult, [1, 2, 3]))
self.assertFalse(callback.apply_async.call_count)
# did retry
chords._unlock_chord.retry.assert_called_with(countdown=10,
max_retries=30)
finally:
chords.TaskSetResult = pts
def __init__(self, *args, **kwargs):
self.channel = Mock()
class chord(chords.chord):
Chord = Mock()
class Chord(chords.Chord):
backend = Mock()
def test_poll_result(self):
results = Queue()
class Message(object):
acked = 0
requeued = 0
def __init__(self, **merge):
self.payload = dict({'status': states.STARTED,
'result': None}, **merge)
self.body = pickle.dumps(self.payload)
self.content_type = 'application/x-python-serialize'
self.content_encoding = 'binary'
def ack(self, *args, **kwargs):
self.acked += 1
def requeue(self, *args, **kwargs):
self.requeued += 1
class MockBinding(object):
def __init__(self, *args, **kwargs):
self.channel = Mock()
def __call__(self, *args, **kwargs):
return self
def declare(self):
pass
def get(self, no_ack=False):
try:
return results.get(block=False)
except Empty:
pass
def is_bound(self):
return True
class MockBackend(AMQPBackend):
Queue = MockBinding
backend = MockBackend()
backend._republish = Mock()
# FFWD's to the latest state.
state_messages = [
Message(status=states.RECEIVED, seq=1),
Message(status=states.STARTED, seq=2),
Message(status=states.FAILURE, seq=3),
]
for state_message in state_messages:
results.put(state_message)
r1 = backend.get_task_meta(uuid())
self.assertDictContainsSubset({'status': states.FAILURE,
'seq': 3}, r1,
'FFWDs to the last state')
# Caches last known state.
results.put(Message())
tid = uuid()
backend.get_task_meta(tid)
self.assertIn(tid, backend._cache, 'Caches last known state')
self.assertTrue(state_messages[-1].requeued)
# Returns cache if no new states.
results.queue.clear()
assert not results.qsize()
backend._cache[tid] = 'hello'
self.assertEqual(backend.get_task_meta(tid), 'hello',
'Returns cache if no new states')
def patch_unlock_retry():
unlock = current_app.tasks["celery.chord_unlock"]
retry = Mock()
prev, unlock.retry = unlock.retry, retry
yield unlock, retry
unlock.retry = prev
