def test_no_conn_driver_info(self): app = Celery(set_as_current=False) app.connection = Mock() conn = app.connection.return_value = Mock() conn.transport = None bugreport(app)
def _chord_context(self, ResultCls, setup=None, **kwargs): with patch('celery.result.GroupResult'): @task() def callback(*args, **kwargs): pass pts, result.GroupResult = result.GroupResult, ResultCls callback.apply_async = Mock() callback_s = callback.s() callback_s.id = 'callback_id' fail_current = self.app.backend.fail_from_current_stack = Mock() try: with patch_unlock_retry() as (unlock, retry): subtask, canvas.maybe_subtask = ( canvas.maybe_subtask, passthru, ) if setup: setup(callback) try: unlock('group_id', callback_s, result=[AsyncResult(r) for r in ['1', 2, 3]], GroupResult=ResultCls, **kwargs) finally: canvas.maybe_subtask = subtask yield callback_s, retry, fail_current finally: result.GroupResult = pts
def test_when_not_ready(self, retry, TaskSetResult): callback = Mock() result = Mock(attrs=dict(ready=lambda: False)) TaskSetResult.restore = lambda setid: result chords._unlock_chord("setid", callback, interval=10, max_retries=30) self.assertFalse(callback.delay.call_count) # did retry chords._unlock_chord.retry.assert_called_with(countdown=10, max_retries=30)
def test_outside_body_error(self, report_internal_error, build_tracer): tracer = Mock() tracer.side_effect = KeyError("foo") build_tracer.return_value = tracer @current_app.task def xtask(): pass trace_task(xtask, "uuid", (), {}) self.assertTrue(report_internal_error.call_count) self.assertIs(xtask.__tracer__, tracer)
def test_outside_body_error(self, report_internal_error, build_tracer): tracer = Mock() tracer.side_effect = KeyError('foo') build_tracer.return_value = tracer @current_app.task def xtask(): pass trace_task(xtask, 'uuid', (), {}) self.assertTrue(report_internal_error.call_count) self.assertIs(xtask.__trace__, tracer)
def test_run(self): prev, current_app.backend = current_app.backend, Mock() current_app.backend.cleanup = Mock() current_app.backend.cleanup.__name__ = 'cleanup' try: Chord = current_app.tasks['celery.chord'] body = dict() Chord(TaskSet(add.subtask((i, i)) for i in xrange(5)), body) Chord([add.subtask((i, i)) for i in xrange(5)], body) self.assertEqual(current_app.backend.on_chord_apply.call_count, 2) finally: current_app.backend = prev
def test_unlock_ready(self, retry, TaskSetResult): callback = Mock() result = Mock(attrs=dict(ready=lambda: True, join=lambda **kw: [2, 4, 8, 6])) TaskSetResult.restore = lambda setid: result subtask, chords.subtask = chords.subtask, passthru try: chords._unlock_chord("setid", callback) finally: chords.subtask = subtask callback.delay.assert_called_with([2, 4, 8, 6]) result.delete.assert_called_with() # did not retry self.assertFalse(retry.call_count)
def test_migrate(self, name="testcelery"): x = BrokerConnection("memory://foo") y = BrokerConnection("memory://foo") # use separate state x.default_channel.queues = {} y.default_channel.queues = {} ex = Exchange(name, "direct") q = Queue(name, exchange=ex, routing_key=name) q(x.default_channel).declare() Producer(x).publish("foo", exchange=name, routing_key=name) Producer(x).publish("bar", exchange=name, routing_key=name) Producer(x).publish("baz", exchange=name, routing_key=name) self.assertTrue(x.default_channel.queues) self.assertFalse(y.default_channel.queues) migrate_tasks(x, y) yq = q(y.default_channel) self.assertEqual(yq.get().body, "foo") self.assertEqual(yq.get().body, "bar") self.assertEqual(yq.get().body, "baz") Producer(x).publish("foo", exchange=name, routing_key=name) callback = Mock() migrate_tasks(x, y, callback=callback) self.assertTrue(callback.called) migrate = Mock() Producer(x).publish("baz", exchange=name, routing_key=name) migrate_tasks(x, y, callback=callback, migrate=migrate) self.assertTrue(migrate.called) with patch("kombu.transport.virtual.Channel.queue_declare") as qd: def effect(*args, **kwargs): if kwargs.get("passive"): raise StdChannelError() return 0, 3, 0 qd.side_effect = effect migrate_tasks(x, y) x = BrokerConnection("memory://") x.default_channel.queues = {} y.default_channel.queues = {} callback = Mock() migrate_tasks(x, y, callback=callback) self.assertFalse(callback.called)
def test_migrate(self, name='testcelery'): x = Connection('memory://foo') y = Connection('memory://foo') # use separate state x.default_channel.queues = {} y.default_channel.queues = {} ex = Exchange(name, 'direct') q = Queue(name, exchange=ex, routing_key=name) q(x.default_channel).declare() Producer(x).publish('foo', exchange=name, routing_key=name) Producer(x).publish('bar', exchange=name, routing_key=name) Producer(x).publish('baz', exchange=name, routing_key=name) self.assertTrue(x.default_channel.queues) self.assertFalse(y.default_channel.queues) migrate_tasks(x, y, accept=['text/plain']) yq = q(y.default_channel) self.assertEqual(yq.get().body, ensure_bytes('foo')) self.assertEqual(yq.get().body, ensure_bytes('bar')) self.assertEqual(yq.get().body, ensure_bytes('baz')) Producer(x).publish('foo', exchange=name, routing_key=name) callback = Mock() migrate_tasks(x, y, callback=callback, accept=['text/plain']) self.assertTrue(callback.called) migrate = Mock() Producer(x).publish('baz', exchange=name, routing_key=name) migrate_tasks(x, y, callback=callback, migrate=migrate, accept=['text/plain']) self.assertTrue(migrate.called) with patch('kombu.transport.virtual.Channel.queue_declare') as qd: def effect(*args, **kwargs): if kwargs.get('passive'): raise StdChannelError() return 0, 3, 0 qd.side_effect = effect migrate_tasks(x, y) x = Connection('memory://') x.default_channel.queues = {} y.default_channel.queues = {} callback = Mock() migrate_tasks(x, y, callback=callback, accept=['text/plain']) self.assertFalse(callback.called)
def test_apply(self): self.app.conf.CELERY_ALWAYS_EAGER = False from celery import chord m = Mock() m.app.conf.CELERY_ALWAYS_EAGER = False m.AsyncResult = AsyncResult prev, chord.Chord = chord.Chord, m try: x = chord(add.s(i, i) for i in xrange(10)) body = add.s(2) result = x(body) self.assertEqual(result.id, body.options["task_id"]) self.assertTrue(chord.Chord.called) finally: chord.Chord = prev
def test_apply(self): self.app.conf.CELERY_ALWAYS_EAGER = False from celery import chord m = Mock() m.app.conf.CELERY_ALWAYS_EAGER = False m.AsyncResult = AsyncResult prev, chord.Chord = chord.Chord, m try: x = chord(add.s(i, i) for i in xrange(10)) body = add.s(2) result = x(body) self.assertEqual(result.id, body.options['task_id']) self.assertTrue(chord.Chord.called) finally: chord.Chord = prev
def test_unlock_ready(self, GroupResult): class AlwaysReady(TSR): is_ready = True value = [2, 4, 8, 6] @task() def callback(*args, **kwargs): pass pts, result.GroupResult = result.GroupResult, AlwaysReady callback.apply_async = Mock() callback_s = callback.s() try: with patch_unlock_retry() as (unlock, retry): subtask, canvas.maybe_subtask = canvas.maybe_subtask, passthru try: unlock('group_id', callback_s, result=map(AsyncResult, [1, 2, 3])) finally: canvas.maybe_subtask = subtask callback.apply_async.assert_called_with(([2, 4, 8, 6], ), {}) # did not retry self.assertFalse(retry.call_count) finally: result.GroupResult = pts
def test_handle_error_state(self): x = self.TI(states.FAILURE) x.handle_failure = Mock() x.handle_error_state(add_cast) x.handle_failure.assert_called_with( add_cast, store_errors=add_cast.store_errors_even_if_ignored, )
def test_current_routing_key(self): task = Mock() _task_stack.push(task) try: task.request.reply_to = 'reply_to' self.assertEqual(self.b._routing_key('task_id'), 'reply_to') finally: _task_stack.pop()
def test_colorize(self): from celery import Celery app = Celery(set_as_current=False) app.log.setup = Mock() b = beatapp.Beat(app=app, no_color=True) b.setup_logging() self.assertTrue(app.log.setup.called) self.assertEqual(app.log.setup.call_args[1]['colorize'], False)
def Message(body, exchange='exchange', routing_key='rkey', compression=None, content_type='application/json', content_encoding='utf-8'): return Mock(attrs=dict(body=body, delivery_info=dict(exchange=exchange, routing_key=routing_key), headers=dict(compression=compression), content_type=content_type, content_encoding=content_encoding, properties={}))
def patch_unlock_retry(): unlock = current_app.tasks['celery.chord_unlock'] retry = Mock() prev, unlock.retry = unlock.retry, retry try: yield unlock, retry finally: unlock.retry = prev
def test_apply(self): self.app.conf.CELERY_ALWAYS_EAGER = False from celery import chord m = Mock() m.app.conf.CELERY_ALWAYS_EAGER = False m.AsyncResult = AsyncResult prev, chord.Chord = chord.Chord, m try: x = chord(add.s(i, i) for i in xrange(10)) body = add.s(2) result = x(body) # does not modify original subtask with self.assertRaises(KeyError): body.options['task_id'] self.assertTrue(chord.Chord.called) finally: chord.Chord = prev
def test_removes_compression_header(self): x = Message('foo', compression='zlib') producer = Mock() migrate_task(producer, x.body, x) self.assertTrue(producer.publish.called) args, kwargs = producer.publish.call_args self.assertIsInstance(args[0], bytes_t) self.assertNotIn('compression', kwargs['headers']) self.assertEqual(kwargs['compression'], 'zlib') self.assertEqual(kwargs['content_type'], 'application/json') self.assertEqual(kwargs['content_encoding'], 'utf-8') self.assertEqual(kwargs['exchange'], 'exchange') self.assertEqual(kwargs['routing_key'], 'rkey')
def test_removes_compression_header(self): x = Message("foo", compression="zlib") producer = Mock() migrate_task(producer, x.body, x) self.assertTrue(producer.publish.called) args, kwargs = producer.publish.call_args self.assertIsInstance(args[0], bytes_t) self.assertNotIn("compression", kwargs["headers"]) self.assertEqual(kwargs["compression"], "zlib") self.assertEqual(kwargs["content_type"], "application/json") self.assertEqual(kwargs["content_encoding"], "utf-8") self.assertEqual(kwargs["exchange"], "exchange") self.assertEqual(kwargs["routing_key"], "rkey")
def Message(body, exchange="exchange", routing_key="rkey", compression=None, content_type="application/json", content_encoding="utf-8"): return Mock(attrs=dict(body=body, delivery_info=dict(exchange=exchange, routing_key=routing_key), headers=dict(compression=compression), content_type=content_type, content_encoding=content_encoding, properties={}))
def test_unlock_ready(self, retry): callback.apply_async = Mock() pts, chords.TaskSetResult = chords.TaskSetResult, TSR subtask, chords.subtask = chords.subtask, passthru try: chords._unlock_chord("setid", callback.subtask(), result=map(AsyncResult, [1, 2, 3])) finally: chords.subtask = subtask chords.TaskSetResult = pts callback.apply_async.assert_called_with(([2, 4, 8, 6], ), {}) # did not retry self.assertFalse(retry.call_count)
def _result_context(self): results = Queue() class Message(object): acked = 0 requeued = 0 def __init__(self, **merge): self.payload = dict({ 'status': states.STARTED, 'result': None }, **merge) self.body = pickle.dumps(self.payload) self.content_type = 'application/x-python-serialize' self.content_encoding = 'binary' def ack(self, *args, **kwargs): self.acked += 1 def requeue(self, *args, **kwargs): self.requeued += 1 class MockBinding(object): def __init__(self, *args, **kwargs): self.channel = Mock() def __call__(self, *args, **kwargs): return self def declare(self): pass def get(self, no_ack=False): try: return results.get(block=False) except Empty: pass def is_bound(self): return True class MockBackend(AMQPBackend): Queue = MockBinding backend = MockBackend() backend._republish = Mock() yield results, backend, Message
def test_when_not_ready(self, GroupResult): with patch_unlock_retry() as (unlock, retry): class NeverReady(TSR): is_ready = False pts, result.GroupResult = result.GroupResult, NeverReady try: callback = Mock() unlock('group_id', callback, interval=10, max_retries=30, result=map(AsyncResult, [1, 2, 3])) self.assertFalse(callback.delay.call_count) # did retry unlock.retry.assert_called_with(countdown=10, max_retries=30) finally: result.GroupResult = pts
def Message(body, exchange='exchange', routing_key='rkey', compression=None, content_type='application/json', content_encoding='utf-8'): return Mock( attrs={ 'body': body, 'delivery_info': { 'exchange': exchange, 'routing_key': routing_key, }, 'headers': { 'compression': compression, }, 'content_type': content_type, 'content_encoding': content_encoding, 'properties': {} }, )
def test_when_not_ready(self, retry, TaskSetResult): callback.apply_async = Mock() class NeverReady(TSR): is_ready = False pts, chords.TaskSetResult = chords.TaskSetResult, NeverReady try: chords._unlock_chord("setid", callback.subtask, interval=10, max_retries=30, result=map(AsyncResult, [1, 2, 3])) self.assertFalse(callback.apply_async.call_count) # did retry chords._unlock_chord.retry.assert_called_with(countdown=10, max_retries=30) finally: chords.TaskSetResult = pts
def __init__(self, *args, **kwargs): self.channel = Mock()
class chord(chords.chord): Chord = Mock()
class Chord(chords.Chord): backend = Mock()
def test_poll_result(self): results = Queue() class Message(object): acked = 0 requeued = 0 def __init__(self, **merge): self.payload = dict({'status': states.STARTED, 'result': None}, **merge) self.body = pickle.dumps(self.payload) self.content_type = 'application/x-python-serialize' self.content_encoding = 'binary' def ack(self, *args, **kwargs): self.acked += 1 def requeue(self, *args, **kwargs): self.requeued += 1 class MockBinding(object): def __init__(self, *args, **kwargs): self.channel = Mock() def __call__(self, *args, **kwargs): return self def declare(self): pass def get(self, no_ack=False): try: return results.get(block=False) except Empty: pass def is_bound(self): return True class MockBackend(AMQPBackend): Queue = MockBinding backend = MockBackend() backend._republish = Mock() # FFWD's to the latest state. state_messages = [ Message(status=states.RECEIVED, seq=1), Message(status=states.STARTED, seq=2), Message(status=states.FAILURE, seq=3), ] for state_message in state_messages: results.put(state_message) r1 = backend.get_task_meta(uuid()) self.assertDictContainsSubset({'status': states.FAILURE, 'seq': 3}, r1, 'FFWDs to the last state') # Caches last known state. results.put(Message()) tid = uuid() backend.get_task_meta(tid) self.assertIn(tid, backend._cache, 'Caches last known state') self.assertTrue(state_messages[-1].requeued) # Returns cache if no new states. results.queue.clear() assert not results.qsize() backend._cache[tid] = 'hello' self.assertEqual(backend.get_task_meta(tid), 'hello', 'Returns cache if no new states')
def patch_unlock_retry(): unlock = current_app.tasks["celery.chord_unlock"] retry = Mock() prev, unlock.retry = unlock.retry, retry yield unlock, retry unlock.retry = prev