def test_run(self): ready_queue = Queue() condition = [None] def mycallback(value): condition[0].set() m = Mediator(ready_queue, mycallback) condition[0] = m._is_shutdown ready_queue.put(MockTask('Elaine M. Benes')) m.run() self.assertTrue(m._is_shutdown.isSet()) self.assertTrue(m._is_stopped.isSet())
def test_mediator_body_exception(self): ready_queue = Queue() def mycallback(value): raise KeyError('foo') m = Mediator(ready_queue, mycallback) ready_queue.put(MockTask('Elaine M. Benes')) m.body()
def test_poll_result(self): results = Queue() class Message(object): def __init__(self, **merge): self.payload = dict({'status': states.STARTED, 'result': None}, **merge) class MockBinding(object): def __init__(self, *args, **kwargs): pass def __call__(self, *args, **kwargs): return self def declare(self): pass def get(self, no_ack=False): try: return results.get(block=False) except Empty: pass class MockBackend(AMQPBackend): Queue = MockBinding backend = MockBackend() # FFWD's to the latest state. results.put(Message(status=states.RECEIVED, seq=1)) results.put(Message(status=states.STARTED, seq=2)) results.put(Message(status=states.FAILURE, seq=3)) r1 = backend.get_task_meta(uuid()) self.assertDictContainsSubset({'status': states.FAILURE, 'seq': 3}, r1, 'FFWDs to the last state') # Caches last known state. results.put(Message()) tid = uuid() backend.get_task_meta(tid) self.assertIn(tid, backend._cache, 'Caches last known state') # Returns cache if no new states. results.queue.clear() assert not results.qsize() backend._cache[tid] = 'hello' self.assertEqual(backend.get_task_meta(tid), 'hello', 'Returns cache if no new states')
def setup(self): self.buffer = FastQueue() self.timer = Timer() @self.app.task(shared=False) def foo_task(x, y, z): return x * y * z self.foo_task = foo_task
def test_mediator_body(self): ready_queue = Queue() got = {} def mycallback(value): got['value'] = value.value m = Mediator(ready_queue, mycallback) ready_queue.put(MockTask('George Costanza')) m.body() self.assertEqual(got['value'], 'George Costanza') ready_queue.put(MockTask('Jerry Seinfeld')) m._does_debug = False m.body() self.assertEqual(got['value'], 'Jerry Seinfeld')
def _result_context(self, serializer='pickle'): results = Queue() class Message(object): acked = 0 requeued = 0 def __init__(self, **merge): self.payload = dict({ 'status': states.STARTED, 'result': None }, **merge) if serializer == 'json': self.body = json.dumps(self.payload) self.content_type = 'application/json' else: self.body = pickle.dumps(self.payload) self.content_type = 'application/x-python-serialize' self.content_encoding = 'binary' def ack(self, *args, **kwargs): self.acked += 1 def requeue(self, *args, **kwargs): self.requeued += 1 class MockBinding(object): def __init__(self, *args, **kwargs): self.channel = Mock() def __call__(self, *args, **kwargs): return self def declare(self): pass def get(self, no_ack=False, accept=None): try: m = results.get(block=False) if m: m.accept = accept return m except Empty: pass def is_bound(self): return True class MockBackend(AMQPBackend): Queue = MockBinding backend = MockBackend(self.app, max_cached_results=100) backend.serializer = serializer backend._republish = Mock() yield results, backend, Message
def test_mediator_start__stop(self): ready_queue = Queue() m = Mediator(ready_queue, lambda t: t) m.start() self.assertFalse(m._is_shutdown.isSet()) self.assertFalse(m._is_stopped.isSet()) m.stop() m.join() self.assertTrue(m._is_shutdown.isSet()) self.assertTrue(m._is_stopped.isSet())
def test_mediator_crash(self, _exit): ms = [None] class _Mediator(Mediator): def body(self): try: raise KeyError('foo') finally: ms[0]._is_shutdown.set() ready_queue = Queue() ms[0] = m = _Mediator(ready_queue, None) ready_queue.put(MockTask('George Constanza')) stderr = Mock() p, sys.stderr = sys.stderr, stderr try: m.run() finally: sys.stderr = p self.assertTrue(_exit.call_count) self.assertTrue(stderr.write.call_count)
class TokenBucketQueue(object): """Queue with rate limited get operations. This uses the token bucket algorithm to rate limit the queue on get operations. :param fill_rate: The rate in tokens/second that the bucket will be refilled. :keyword capacity: Maximum number of tokens in the bucket. Default is 1. """ RateLimitExceeded = RateLimitExceeded def __init__(self, fill_rate, queue=None, capacity=1): self._bucket = TokenBucket(fill_rate, capacity) self.queue = queue if not self.queue: self.queue = Queue() def put(self, item, block=True): """Put an item onto the queue.""" self.queue.put(item, block=block) def put_nowait(self, item): """Put an item into the queue without blocking. :raises Queue.Full: If a free slot is not immediately available. """ return self.put(item, block=False) def get(self, block=True): """Remove and return an item from the queue. :raises RateLimitExceeded: If a token could not be consumed from the token bucket (consuming from the queue too fast). :raises Queue.Empty: If an item is not immediately available. """ get = block and self.queue.get or self.queue.get_nowait if not block and not self.items: raise Empty() if not self._bucket.can_consume(1): raise RateLimitExceeded() return get() def get_nowait(self): """Remove and return an item from the queue without blocking. :raises RateLimitExceeded: If a token could not be consumed from the token bucket (consuming from the queue too fast). :raises Queue.Empty: If an item is not immediately available. """ return self.get(block=False) def qsize(self): """Returns the size of the queue.""" return self.queue.qsize() def empty(self): """Returns :const:`True` if the queue is empty.""" return self.queue.empty() def clear(self): """Delete all data in the queue.""" return self.items.clear() def wait(self, block=False): """Wait until a token can be retrieved from the bucket and return the next item.""" get = self.get expected_time = self.expected_time while 1: remaining = expected_time() if not remaining: return get(block=block) sleep(remaining) def expected_time(self, tokens=1): """Returns the expected time in seconds of when a new token should be available.""" if not self.items: return 0 return self._bucket.expected_time(tokens) @property def items(self): """Underlying data. Do not modify.""" return self.queue.queue
def setUp(self): self.buffer = FastQueue() self.timer = Timer()
def __init__(self): self._buffer = Queue() self._count = count(1) self._tref = None self._pool = None
class Batches(Task): abstract = True #: Maximum number of message in buffer. flush_every = 10 #: Timeout in seconds before buffer is flushed anyway. flush_interval = 30 def __init__(self): self._buffer = Queue() self._count = count(1) self._tref = None self._pool = None def run(self, requests): raise NotImplementedError('must implement run(requests)') def Strategy(self, task, app, consumer): self._pool = consumer.pool hostname = consumer.hostname eventer = consumer.event_dispatcher Req = Request connection_errors = consumer.connection_errors timer = consumer.timer put_buffer = self._buffer.put flush_buffer = self._do_flush def task_message_handler(message, body, ack, reject, callbacks, **kw): request = Req(body, on_ack=ack, app=app, hostname=hostname, events=eventer, task=task, connection_errors=connection_errors, delivery_info=message.delivery_info) put_buffer(request) if self._tref is None: # first request starts flush timer. self._tref = timer.call_repeatedly( self.flush_interval, flush_buffer, ) if not next(self._count) % self.flush_every: flush_buffer() return task_message_handler def flush(self, requests): return self.apply_buffer(requests, ([SimpleRequest.from_request(r) for r in requests], )) def _do_flush(self): logger.debug('Batches: Wake-up to flush buffer...') requests = None if self._buffer.qsize(): requests = list(consume_queue(self._buffer)) if requests: logger.debug('Batches: Buffer complete: %s', len(requests)) self.flush(requests) if not requests: logger.debug('Batches: Canceling timer: Nothing in buffer.') if self._tref: self._tref.cancel() # cancel timer. self._tref = None def apply_buffer(self, requests, args=(), kwargs={}): acks_late = [], [] [acks_late[r.task.acks_late].append(r) for r in requests] assert requests and (acks_late[True] or acks_late[False]) def on_accepted(pid, time_accepted): [req.acknowledge() for req in acks_late[False]] def on_return(result): [req.acknowledge() for req in acks_late[True]] return self._pool.apply_async( apply_batches_task, (self, args, 0, None), accept_callback=on_accepted, callback=acks_late[True] and on_return or noop, )
def setup(self): self.buffer = FastQueue() self.timer = Timer()
def test_poll_result(self): results = Queue() class Message(object): def __init__(self, **merge): self.payload = dict({ 'status': states.STARTED, 'result': None }, **merge) self.body = pickle.dumps(self.payload) self.content_type = 'application/x-python-serialize' self.content_encoding = 'binary' class MockBinding(object): def __init__(self, *args, **kwargs): self.channel = Mock() def __call__(self, *args, **kwargs): return self def declare(self): pass def get(self, no_ack=False): try: return results.get(block=False) except Empty: pass def is_bound(self): return True class MockBackend(AMQPBackend): Queue = MockBinding backend = MockBackend() backend._republish = Mock() # FFWD's to the latest state. results.put(Message(status=states.RECEIVED, seq=1)) results.put(Message(status=states.STARTED, seq=2)) results.put(Message(status=states.FAILURE, seq=3)) r1 = backend.get_task_meta(uuid()) self.assertDictContainsSubset({ 'status': states.FAILURE, 'seq': 3 }, r1, 'FFWDs to the last state') # Caches last known state. results.put(Message()) tid = uuid() backend.get_task_meta(tid) self.assertIn(tid, backend._cache, 'Caches last known state') self.assertTrue(backend._republish.called) # Returns cache if no new states. results.queue.clear() assert not results.qsize() backend._cache[tid] = 'hello' self.assertEqual(backend.get_task_meta(tid), 'hello', 'Returns cache if no new states')
def __init__(self, fill_rate, queue=None, capacity=1): self._bucket = TokenBucket(fill_rate, capacity) self.queue = queue if not self.queue: self.queue = Queue()
class Batches(Task): abstract = True #: Maximum number of message in buffer. flush_every = 10 #: Timeout in seconds before buffer is flushed anyway. flush_interval = 30 def __init__(self): self._buffer = Queue() self._count = count(1) self._tref = None self._pool = None def run(self, requests): raise NotImplementedError('must implement run(requests)') def Strategy(self, task, app, consumer): self._pool = consumer.pool hostname = consumer.hostname eventer = consumer.event_dispatcher Req = Request connection_errors = consumer.connection_errors timer = consumer.timer put_buffer = self._buffer.put flush_buffer = self._do_flush body_can_be_buffer = consumer.pool.body_can_be_buffer def task_message_handler(message, body, ack, reject, callbacks, **kw): if body is None: body, headers, decoded, utc = ( message.body, message.headers, False, True, ) if not body_can_be_buffer: body = bytes(body) if isinstance(body, buffer_t) else body else: body, headers, decoded, utc = proto1_to_proto2(message, body) request = Req( message, on_ack=ack, on_reject=reject, app=app, hostname=hostname, eventer=eventer, task=task, body=body, headers=headers, decoded=decoded, utc=utc, connection_errors=connection_errors, ) put_buffer(request) if self._tref is None: # first request starts flush timer. self._tref = timer.call_repeatedly( self.flush_interval, flush_buffer, ) if not next(self._count) % self.flush_every: flush_buffer() return task_message_handler def flush(self, requests): return self.apply_buffer(requests, ([SimpleRequest.from_request(r) for r in requests],)) def _do_flush(self): logger.debug('Batches: Wake-up to flush buffer...') requests = None if self._buffer.qsize(): requests = list(consume_queue(self._buffer)) if requests: logger.debug('Batches: Buffer complete: %s', len(requests)) self.flush(requests) if not requests: logger.debug('Batches: Canceling timer: Nothing in buffer.') if self._tref: self._tref.cancel() # cancel timer. self._tref = None def apply_buffer(self, requests, args=(), kwargs={}): acks_late = [], [] [acks_late[r.task.acks_late].append(r) for r in requests] assert requests and (acks_late[True] or acks_late[False]) def on_accepted(pid, time_accepted): [req.acknowledge() for req in acks_late[False]] def on_return(result): [req.acknowledge() for req in acks_late[True]] return self._pool.apply_async( apply_batches_task, (self, args, 0, None), accept_callback=on_accepted, callback=acks_late[True] and on_return or noop, )
class Batches(Task): abstract = True #: Maximum number of message in buffer. flush_every = 10 #: Timeout in seconds before buffer is flushed anyway. flush_interval = 30 def __init__(self): self._buffer = Queue() self._count = count(1) self._tref = None self._pool = None def run(self, requests): raise NotImplementedError("must implement run(requests)") def Strategy(self, task, app, consumer): self._pool = consumer.pool hostname = consumer.hostname eventer = consumer.event_dispatcher Req = Request connection_errors = consumer.connection_errors timer = consumer.timer put_buffer = self._buffer.put flush_buffer = self._do_flush def task_message_handler(message, body, ack): request = Req( body, on_ack=ack, app=app, hostname=hostname, events=eventer, task=task, connection_errors=connection_errors, delivery_info=message.delivery_info, ) put_buffer(request) if self._tref is None: # first request starts flush timer. self._tref = timer.apply_interval(self.flush_interval * 1000.0, flush_buffer) if not next(self._count) % self.flush_every: flush_buffer() return task_message_handler def flush(self, requests): return self.apply_buffer(requests, ([SimpleRequest.from_request(r) for r in requests],)) def _do_flush(self): logger.debug("Batches: Wake-up to flush buffer...") requests = None if self._buffer.qsize(): requests = list(consume_queue(self._buffer)) if requests: logger.debug("Batches: Buffer complete: %s", len(requests)) self.flush(requests) if not requests: logger.debug("Batches: Cancelling timer: Nothing in buffer.") self._tref.cancel() # cancel timer. self._tref = None def apply_buffer(self, requests, args=(), kwargs={}): acks_late = [], [] [acks_late[r.task.acks_late].append(r) for r in requests] assert requests and (acks_late[True] or acks_late[False]) def on_accepted(pid, time_accepted): [req.acknowledge() for req in acks_late[False]] def on_return(result): [req.acknowledge() for req in acks_late[True]] return self._pool.apply_async( apply_batches_task, (self, args, 0, None), accept_callback=on_accepted, callback=acks_late[True] and on_return or None, )
class Batches(Task): abstract = True #: Maximum number of message in buffer. flush_every = 10 #: Timeout in seconds before buffer is flushed anyway. flush_interval = 30 def __init__(self): self._buffer = Queue() self._count = count(1) self._tref = None self._pool = None def run(self, requests): raise NotImplementedError('must implement run(requests)') def Strategy(self, task, app, consumer): self._pool = consumer.pool hostname = consumer.hostname eventer = consumer.event_dispatcher Req = Request connection_errors = consumer.connection_errors timer = consumer.timer put_buffer = self._buffer.put flush_buffer = self._do_flush body_can_be_buffer = consumer.pool.body_can_be_buffer def task_message_handler(message, body, ack, reject, callbacks, **kw): if body is None: body, headers, decoded, utc = ( message.body, message.headers, False, True, ) if not body_can_be_buffer: body = bytes(body) if isinstance(body, buffer_t) else body else: body, headers, decoded, utc = proto1_to_proto2(message, body) request = Req( message, on_ack=ack, on_reject=reject, app=app, hostname=hostname, eventer=eventer, task=task, body=body, headers=headers, decoded=decoded, utc=utc, connection_errors=connection_errors, ) put_buffer(request) if self._tref is None: # first request starts flush timer. self._tref = timer.call_repeatedly( self.flush_interval, flush_buffer, ) if not next(self._count) % self.flush_every: flush_buffer() return task_message_handler def apply(self, args=None, kwargs=None, *_args, **_kwargs): """ Execute this task locally as a batch of size 1, by blocking until the task returns. Arguments: args (Tuple): positional arguments passed on to the task. Returns: celery.result.EagerResult: pre-evaluated result. """ request = SimpleRequest( id=_kwargs.get("task_id", uuid()), name="batch request", args=args or (), kwargs=kwargs or {}, delivery_info=None, hostname="localhost", ) return super(Batches, self).apply(([request], ), {}, *_args, **_kwargs) def flush(self, requests): return self.apply_buffer( requests, ([SimpleRequest.from_request(r) for r in requests], )) def _do_flush(self): logger.debug('Batches: Wake-up to flush buffer...') requests = None if self._buffer.qsize(): requests = list(consume_queue(self._buffer)) if requests: logger.debug('Batches: Buffer complete: %s', len(requests)) self.flush(requests) if not requests: logger.debug('Batches: Canceling timer: Nothing in buffer.') if self._tref: self._tref.cancel() # cancel timer. self._tref = None def apply_buffer(self, requests, args=(), kwargs={}): acks_late = [], [] [acks_late[r.task.acks_late].append(r) for r in requests] assert requests and (acks_late[True] or acks_late[False]) def on_accepted(pid, time_accepted): [req.acknowledge() for req in acks_late[False]] def on_return(result): [req.acknowledge() for req in acks_late[True]] return self._pool.apply_async( apply_batches_task, (self, args, 0, None), accept_callback=on_accepted, callback=acks_late[True] and on_return or noop, )
class test_Consumer(AppCase): def setup(self): self.buffer = FastQueue() self.timer = Timer() @self.app.task(shared=False) def foo_task(x, y, z): return x * y * z self.foo_task = foo_task def teardown(self): self.timer.stop() def test_info(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.task_consumer = Mock() l.qos = QoS(l.task_consumer.qos, 10) l.connection = Mock() l.connection.info.return_value = {'foo': 'bar'} l.controller = l.app.WorkController() l.controller.pool = Mock() l.controller.pool.info.return_value = [Mock(), Mock()] l.controller.consumer = l info = l.controller.stats() self.assertEqual(info['prefetch_count'], 10) self.assertTrue(info['broker']) def test_start_when_closed(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = CLOSE l.start() def test_connection(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.start(l) self.assertIsInstance(l.connection, Connection) l.blueprint.state = RUN l.event_dispatcher = None l.blueprint.restart(l) self.assertTrue(l.connection) l.blueprint.state = RUN l.shutdown() self.assertIsNone(l.connection) self.assertIsNone(l.task_consumer) l.blueprint.start(l) self.assertIsInstance(l.connection, Connection) l.blueprint.restart(l) l.stop() l.shutdown() self.assertIsNone(l.connection) self.assertIsNone(l.task_consumer) def test_close_connection(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN step = find_step(l, consumer.Connection) conn = l.connection = Mock() step.shutdown(l) self.assertTrue(conn.close.called) self.assertIsNone(l.connection) l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) eventer = l.event_dispatcher = Mock() eventer.enabled = True heart = l.heart = MockHeart() l.blueprint.state = RUN Events = find_step(l, consumer.Events) Events.shutdown(l) Heart = find_step(l, consumer.Heart) Heart.shutdown(l) self.assertTrue(eventer.close.call_count) self.assertTrue(heart.closed) @patch('celery.worker.consumer.warn') def test_receive_message_unknown(self, warn): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() backend = Mock() m = create_message(backend, unknown={'baz': '!!!'}) l.event_dispatcher = Mock() l.node = MockNode() callback = self._get_on_message(l) callback(m.decode(), m) self.assertTrue(warn.call_count) @patch('celery.worker.strategy.to_timestamp') def test_receive_message_eta_OverflowError(self, to_timestamp): to_timestamp.side_effect = OverflowError() l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() m = create_message(Mock(), task=self.foo_task.name, args=('2, 2'), kwargs={}, eta=datetime.now().isoformat()) l.event_dispatcher = Mock() l.node = MockNode() l.update_strategies() l.qos = Mock() callback = self._get_on_message(l) callback(m.decode(), m) self.assertTrue(m.acknowledged) @patch('celery.worker.consumer.error') def test_receive_message_InvalidTaskError(self, error): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.event_dispatcher = Mock() l.steps.pop() m = create_message(Mock(), task=self.foo_task.name, args=(1, 2), kwargs='foobarbaz', id=1) l.update_strategies() l.event_dispatcher = Mock() callback = self._get_on_message(l) callback(m.decode(), m) self.assertIn('Received invalid task message', error.call_args[0][0]) @patch('celery.worker.consumer.crit') def test_on_decode_error(self, crit): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) class MockMessage(Mock): content_type = 'application/x-msgpack' content_encoding = 'binary' body = 'foobarbaz' message = MockMessage() l.on_decode_error(message, KeyError('foo')) self.assertTrue(message.ack.call_count) self.assertIn("Can't decode message body", crit.call_args[0][0]) def _get_on_message(self, l): if l.qos is None: l.qos = Mock() l.event_dispatcher = Mock() l.task_consumer = Mock() l.connection = Mock() l.connection.drain_events.side_effect = SystemExit() with self.assertRaises(SystemExit): l.loop(*l.loop_args()) self.assertTrue(l.task_consumer.register_callback.called) return l.task_consumer.register_callback.call_args[0][0] def test_receieve_message(self): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.event_dispatcher = Mock() m = create_message(Mock(), task=self.foo_task.name, args=[2, 4, 8], kwargs={}) l.update_strategies() callback = self._get_on_message(l) callback(m.decode(), m) in_bucket = self.buffer.get_nowait() self.assertIsInstance(in_bucket, Request) self.assertEqual(in_bucket.name, self.foo_task.name) self.assertEqual(in_bucket.execute(), 2 * 4 * 8) self.assertTrue(self.timer.empty()) def test_start_channel_error(self): class MockConsumer(Consumer): iterations = 0 def loop(self, *args, **kwargs): if not self.iterations: self.iterations = 1 raise KeyError('foo') raise SyntaxError('bar') l = MockConsumer(self.buffer.put, timer=self.timer, send_events=False, pool=BasePool(), app=self.app) l.channel_errors = (KeyError, ) with self.assertRaises(KeyError): l.start() l.timer.stop() def test_start_connection_error(self): class MockConsumer(Consumer): iterations = 0 def loop(self, *args, **kwargs): if not self.iterations: self.iterations = 1 raise KeyError('foo') raise SyntaxError('bar') l = MockConsumer(self.buffer.put, timer=self.timer, send_events=False, pool=BasePool(), app=self.app) l.connection_errors = (KeyError, ) self.assertRaises(SyntaxError, l.start) l.timer.stop() def test_loop_ignores_socket_timeout(self): class Connection(self.app.connection().__class__): obj = None def drain_events(self, **kwargs): self.obj.connection = None raise socket.timeout(10) l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.connection = Connection() l.task_consumer = Mock() l.connection.obj = l l.qos = QoS(l.task_consumer.qos, 10) l.loop(*l.loop_args()) def test_loop_when_socket_error(self): class Connection(self.app.connection().__class__): obj = None def drain_events(self, **kwargs): self.obj.connection = None raise socket.error('foo') l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN c = l.connection = Connection() l.connection.obj = l l.task_consumer = Mock() l.qos = QoS(l.task_consumer.qos, 10) with self.assertRaises(socket.error): l.loop(*l.loop_args()) l.blueprint.state = CLOSE l.connection = c l.loop(*l.loop_args()) def test_loop(self): class Connection(self.app.connection().__class__): obj = None def drain_events(self, **kwargs): self.obj.connection = None l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.connection = Connection() l.connection.obj = l l.task_consumer = Mock() l.qos = QoS(l.task_consumer.qos, 10) l.loop(*l.loop_args()) l.loop(*l.loop_args()) self.assertTrue(l.task_consumer.consume.call_count) l.task_consumer.qos.assert_called_with(prefetch_count=10) self.assertEqual(l.qos.value, 10) l.qos.decrement_eventually() self.assertEqual(l.qos.value, 9) l.qos.update() self.assertEqual(l.qos.value, 9) l.task_consumer.qos.assert_called_with(prefetch_count=9) def test_ignore_errors(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.connection_errors = (AttributeError, KeyError, ) l.channel_errors = (SyntaxError, ) ignore_errors(l, Mock(side_effect=AttributeError('foo'))) ignore_errors(l, Mock(side_effect=KeyError('foo'))) ignore_errors(l, Mock(side_effect=SyntaxError('foo'))) with self.assertRaises(IndexError): ignore_errors(l, Mock(side_effect=IndexError('foo'))) def test_apply_eta_task(self): from celery.worker import state l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.qos = QoS(None, 10) task = object() qos = l.qos.value l.apply_eta_task(task) self.assertIn(task, state.reserved_requests) self.assertEqual(l.qos.value, qos - 1) self.assertIs(self.buffer.get_nowait(), task) def test_receieve_message_eta_isoformat(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() m = create_message( Mock(), task=self.foo_task.name, eta=(datetime.now() + timedelta(days=1)).isoformat(), args=[2, 4, 8], kwargs={}, ) l.task_consumer = Mock() l.qos = QoS(l.task_consumer.qos, 1) current_pcount = l.qos.value l.event_dispatcher = Mock() l.enabled = False l.update_strategies() callback = self._get_on_message(l) callback(m.decode(), m) l.timer.stop() l.timer.join(1) items = [entry[2] for entry in self.timer.queue] found = 0 for item in items: if item.args[0].name == self.foo_task.name: found = True self.assertTrue(found) self.assertGreater(l.qos.value, current_pcount) l.timer.stop() def test_pidbox_callback(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) con = find_step(l, consumer.Control).box con.node = Mock() con.reset = Mock() con.on_message('foo', 'bar') con.node.handle_message.assert_called_with('foo', 'bar') con.node = Mock() con.node.handle_message.side_effect = KeyError('foo') con.on_message('foo', 'bar') con.node.handle_message.assert_called_with('foo', 'bar') con.node = Mock() con.node.handle_message.side_effect = ValueError('foo') con.on_message('foo', 'bar') con.node.handle_message.assert_called_with('foo', 'bar') self.assertTrue(con.reset.called) def test_revoke(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() backend = Mock() id = uuid() t = create_message(backend, task=self.foo_task.name, args=[2, 4, 8], kwargs={}, id=id) from celery.worker.state import revoked revoked.add(id) callback = self._get_on_message(l) callback(t.decode(), t) self.assertTrue(self.buffer.empty()) def test_receieve_message_not_registered(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() backend = Mock() m = create_message(backend, task='x.X.31x', args=[2, 4, 8], kwargs={}) l.event_dispatcher = Mock() callback = self._get_on_message(l) self.assertFalse(callback(m.decode(), m)) with self.assertRaises(Empty): self.buffer.get_nowait() self.assertTrue(self.timer.empty()) @patch('celery.worker.consumer.warn') @patch('celery.worker.consumer.logger') def test_receieve_message_ack_raises(self, logger, warn): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN backend = Mock() m = create_message(backend, args=[2, 4, 8], kwargs={}) l.event_dispatcher = Mock() l.connection_errors = (socket.error, ) m.reject = Mock() m.reject.side_effect = socket.error('foo') callback = self._get_on_message(l) self.assertFalse(callback(m.decode(), m)) self.assertTrue(warn.call_count) with self.assertRaises(Empty): self.buffer.get_nowait() self.assertTrue(self.timer.empty()) m.reject.assert_called_with() self.assertTrue(logger.critical.call_count) def test_receive_message_eta(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.steps.pop() l.event_dispatcher = Mock() l.event_dispatcher._outbound_buffer = deque() backend = Mock() m = create_message( backend, task=self.foo_task.name, args=[2, 4, 8], kwargs={}, eta=(datetime.now() + timedelta(days=1)).isoformat(), ) try: l.blueprint.start(l) p = l.app.conf.BROKER_CONNECTION_RETRY l.app.conf.BROKER_CONNECTION_RETRY = False l.blueprint.start(l) l.app.conf.BROKER_CONNECTION_RETRY = p l.blueprint.restart(l) l.event_dispatcher = Mock() callback = self._get_on_message(l) callback(m.decode(), m) finally: l.timer.stop() l.timer.join() in_hold = l.timer.queue[0] self.assertEqual(len(in_hold), 3) eta, priority, entry = in_hold task = entry.args[0] self.assertIsInstance(task, Request) self.assertEqual(task.name, self.foo_task.name) self.assertEqual(task.execute(), 2 * 4 * 8) with self.assertRaises(Empty): self.buffer.get_nowait() def test_reset_pidbox_node(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) con = find_step(l, consumer.Control).box con.node = Mock() chan = con.node.channel = Mock() l.connection = Mock() chan.close.side_effect = socket.error('foo') l.connection_errors = (socket.error, ) con.reset() chan.close.assert_called_with() def test_reset_pidbox_node_green(self): from celery.worker.pidbox import gPidbox pool = Mock() pool.is_green = True l = MyKombuConsumer(self.buffer.put, timer=self.timer, pool=pool, app=self.app) con = find_step(l, consumer.Control) self.assertIsInstance(con.box, gPidbox) con.start(l) l.pool.spawn_n.assert_called_with( con.box.loop, l, ) def test__green_pidbox_node(self): pool = Mock() pool.is_green = True l = MyKombuConsumer(self.buffer.put, timer=self.timer, pool=pool, app=self.app) l.node = Mock() controller = find_step(l, consumer.Control) class BConsumer(Mock): def __enter__(self): self.consume() return self def __exit__(self, *exc_info): self.cancel() controller.box.node.listen = BConsumer() connections = [] class Connection(object): calls = 0 def __init__(self, obj): connections.append(self) self.obj = obj self.default_channel = self.channel() self.closed = False def __enter__(self): return self def __exit__(self, *exc_info): self.close() def channel(self): return Mock() def as_uri(self): return 'dummy://' def drain_events(self, **kwargs): if not self.calls: self.calls += 1 raise socket.timeout() self.obj.connection = None controller.box._node_shutdown.set() def close(self): self.closed = True l.connection = Mock() l.connect = lambda: Connection(obj=l) controller = find_step(l, consumer.Control) controller.box.loop(l) self.assertTrue(controller.box.node.listen.called) self.assertTrue(controller.box.consumer) controller.box.consumer.consume.assert_called_with() self.assertIsNone(l.connection) self.assertTrue(connections[0].closed) @patch('kombu.connection.Connection._establish_connection') @patch('kombu.utils.sleep') def test_connect_errback(self, sleep, connect): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) from kombu.transport.memory import Transport Transport.connection_errors = (ChannelError, ) def effect(): if connect.call_count > 1: return raise ChannelError() connect.side_effect = effect l.connect() connect.assert_called_with() def test_stop_pidbox_node(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) cont = find_step(l, consumer.Control) cont._node_stopped = Event() cont._node_shutdown = Event() cont._node_stopped.set() cont.stop(l) def test_start__loop(self): class _QoS(object): prev = 3 value = 4 def update(self): self.prev = self.value class _Consumer(MyKombuConsumer): iterations = 0 def reset_connection(self): if self.iterations >= 1: raise KeyError('foo') init_callback = Mock() l = _Consumer(self.buffer.put, timer=self.timer, init_callback=init_callback, app=self.app) l.task_consumer = Mock() l.broadcast_consumer = Mock() l.qos = _QoS() l.connection = Connection() l.iterations = 0 def raises_KeyError(*args, **kwargs): l.iterations += 1 if l.qos.prev != l.qos.value: l.qos.update() if l.iterations >= 2: raise KeyError('foo') l.loop = raises_KeyError with self.assertRaises(KeyError): l.start() self.assertEqual(l.iterations, 2) self.assertEqual(l.qos.prev, l.qos.value) init_callback.reset_mock() l = _Consumer(self.buffer.put, timer=self.timer, app=self.app, send_events=False, init_callback=init_callback) l.qos = _QoS() l.task_consumer = Mock() l.broadcast_consumer = Mock() l.connection = Connection() l.loop = Mock(side_effect=socket.error('foo')) with self.assertRaises(socket.error): l.start() self.assertTrue(l.loop.call_count) def test_reset_connection_with_no_node(self): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.steps.pop() self.assertEqual(None, l.pool) l.blueprint.start(l)
class test_Consumer(ConsumerCase): def setup(self): self.buffer = FastQueue() self.timer = Timer() @self.app.task(shared=False) def foo_task(x, y, z): return x * y * z self.foo_task = foo_task def teardown(self): self.timer.stop() def LoopConsumer(self, buffer=None, controller=None, timer=None, app=None, without_mingle=True, without_gossip=True, without_heartbeat=True, **kwargs): if controller is None: controller = Mock(name='.controller') buffer = buffer if buffer is not None else self.buffer.put timer = timer if timer is not None else self.timer app = app if app is not None else self.app c = Consumer( buffer, timer=timer, app=app, controller=controller, without_mingle=without_mingle, without_gossip=without_gossip, without_heartbeat=without_heartbeat, **kwargs ) c.task_consumer = Mock(name='.task_consumer') c.qos = QoS(c.task_consumer.qos, 10) c.connection = Mock(name='.connection') c.controller = c.app.WorkController() c.heart = Mock(name='.heart') c.controller.consumer = c c.pool = c.controller.pool = Mock(name='.controller.pool') c.node = Mock(name='.node') c.event_dispatcher = mock_event_dispatcher() return c def NoopConsumer(self, *args, **kwargs): c = self.LoopConsumer(*args, **kwargs) c.loop = Mock(name='.loop') return c def test_info(self): c = self.NoopConsumer() c.connection.info.return_value = {'foo': 'bar'} c.controller.pool.info.return_value = [Mock(), Mock()] info = c.controller.stats() assert info['prefetch_count'] == 10 assert info['broker'] def test_start_when_closed(self): c = self.NoopConsumer() c.blueprint.state = CLOSE c.start() def test_connection(self): c = self.NoopConsumer() c.blueprint.start(c) assert isinstance(c.connection, Connection) c.blueprint.state = RUN c.event_dispatcher = None c.blueprint.restart(c) assert c.connection c.blueprint.state = RUN c.shutdown() assert c.connection is None assert c.task_consumer is None c.blueprint.start(c) assert isinstance(c.connection, Connection) c.blueprint.restart(c) c.stop() c.shutdown() assert c.connection is None assert c.task_consumer is None def test_close_connection(self): c = self.NoopConsumer() c.blueprint.state = RUN step = find_step(c, consumer.Connection) connection = c.connection step.shutdown(c) connection.close.assert_called() assert c.connection is None def test_close_connection__heart_shutdown(self): c = self.NoopConsumer() event_dispatcher = c.event_dispatcher heart = c.heart c.event_dispatcher.enabled = True c.blueprint.state = RUN Events = find_step(c, consumer.Events) Events.shutdown(c) Heart = find_step(c, consumer.Heart) Heart.shutdown(c) event_dispatcher.close.assert_called() heart.stop.assert_called_with() @patch('celery.worker.consumer.consumer.warn') def test_receive_message_unknown(self, warn): c = self.LoopConsumer() c.blueprint.state = RUN c.steps.pop() channel = Mock(name='.channeol') m = create_message(channel, unknown={'baz': '!!!'}) callback = self._get_on_message(c) callback(m) warn.assert_called() @patch('celery.worker.strategy.to_timestamp') def test_receive_message_eta_OverflowError(self, to_timestamp): to_timestamp.side_effect = OverflowError() c = self.LoopConsumer() c.blueprint.state = RUN c.steps.pop() m = self.create_task_message( Mock(), self.foo_task.name, args=('2, 2'), kwargs={}, eta=datetime.now().isoformat(), ) c.update_strategies() callback = self._get_on_message(c) callback(m) assert m.acknowledged @patch('celery.worker.consumer.consumer.error') def test_receive_message_InvalidTaskError(self, error): c = self.LoopConsumer() c.blueprint.state = RUN c.steps.pop() m = self.create_task_message( Mock(), self.foo_task.name, args=(1, 2), kwargs='foobarbaz', id=1) c.update_strategies() strat = c.strategies[self.foo_task.name] = Mock(name='strategy') strat.side_effect = InvalidTaskError() callback = self._get_on_message(c) callback(m) error.assert_called() assert 'Received invalid task message' in error.call_args[0][0] @patch('celery.worker.consumer.consumer.crit') def test_on_decode_error(self, crit): c = self.LoopConsumer() class MockMessage(Mock): content_type = 'application/x-msgpack' content_encoding = 'binary' body = 'foobarbaz' message = MockMessage() c.on_decode_error(message, KeyError('foo')) assert message.ack.call_count assert "Can't decode message body" in crit.call_args[0][0] def _get_on_message(self, c): if c.qos is None: c.qos = Mock() c.task_consumer = Mock() c.event_dispatcher = mock_event_dispatcher() c.connection = Mock(name='.connection') c.connection.get_heartbeat_interval.return_value = 0 c.connection.drain_events.side_effect = WorkerShutdown() with pytest.raises(WorkerShutdown): c.loop(*c.loop_args()) assert c.task_consumer.on_message return c.task_consumer.on_message def test_receieve_message(self): c = self.LoopConsumer() c.blueprint.state = RUN m = self.create_task_message( Mock(), self.foo_task.name, args=[2, 4, 8], kwargs={}, ) c.update_strategies() callback = self._get_on_message(c) callback(m) in_bucket = self.buffer.get_nowait() assert isinstance(in_bucket, Request) assert in_bucket.name == self.foo_task.name assert in_bucket.execute() == 2 * 4 * 8 assert self.timer.empty() def test_start_channel_error(self): c = self.NoopConsumer(task_events=False, pool=BasePool()) c.loop.on_nth_call_do_raise(KeyError('foo'), SyntaxError('bar')) c.channel_errors = (KeyError,) try: with pytest.raises(KeyError): c.start() finally: c.timer and c.timer.stop() def test_start_connection_error(self): c = self.NoopConsumer(task_events=False, pool=BasePool()) c.loop.on_nth_call_do_raise(KeyError('foo'), SyntaxError('bar')) c.connection_errors = (KeyError,) try: with pytest.raises(SyntaxError): c.start() finally: c.timer and c.timer.stop() def test_loop_ignores_socket_timeout(self): class Connection(self.app.connection_for_read().__class__): obj = None def drain_events(self, **kwargs): self.obj.connection = None raise socket.timeout(10) c = self.NoopConsumer() c.connection = Connection(self.app.conf.broker_url) c.connection.obj = c c.qos = QoS(c.task_consumer.qos, 10) c.loop(*c.loop_args()) def test_loop_when_socket_error(self): class Connection(self.app.connection_for_read().__class__): obj = None def drain_events(self, **kwargs): self.obj.connection = None raise socket.error('foo') c = self.LoopConsumer() c.blueprint.state = RUN conn = c.connection = Connection(self.app.conf.broker_url) c.connection.obj = c c.qos = QoS(c.task_consumer.qos, 10) with pytest.raises(socket.error): c.loop(*c.loop_args()) c.blueprint.state = CLOSE c.connection = conn c.loop(*c.loop_args()) def test_loop(self): class Connection(self.app.connection_for_read().__class__): obj = None def drain_events(self, **kwargs): self.obj.connection = None @property def supports_heartbeats(self): return False c = self.LoopConsumer() c.blueprint.state = RUN c.connection = Connection(self.app.conf.broker_url) c.connection.obj = c c.connection.get_heartbeat_interval = Mock(return_value=None) c.qos = QoS(c.task_consumer.qos, 10) c.loop(*c.loop_args()) c.loop(*c.loop_args()) assert c.task_consumer.consume.call_count c.task_consumer.qos.assert_called_with(prefetch_count=10) assert c.qos.value == 10 c.qos.decrement_eventually() assert c.qos.value == 9 c.qos.update() assert c.qos.value == 9 c.task_consumer.qos.assert_called_with(prefetch_count=9) def test_ignore_errors(self): c = self.NoopConsumer() c.connection_errors = (AttributeError, KeyError,) c.channel_errors = (SyntaxError,) ignore_errors(c, Mock(side_effect=AttributeError('foo'))) ignore_errors(c, Mock(side_effect=KeyError('foo'))) ignore_errors(c, Mock(side_effect=SyntaxError('foo'))) with pytest.raises(IndexError): ignore_errors(c, Mock(side_effect=IndexError('foo'))) def test_apply_eta_task(self): c = self.NoopConsumer() c.qos = QoS(None, 10) task = Mock(name='task', id='1234213') qos = c.qos.value c.apply_eta_task(task) assert task in state.reserved_requests assert c.qos.value == qos - 1 assert self.buffer.get_nowait() is task def test_receieve_message_eta_isoformat(self): c = self.LoopConsumer() c.blueprint.state = RUN c.steps.pop() m = self.create_task_message( Mock(), self.foo_task.name, eta=(datetime.now() + timedelta(days=1)).isoformat(), args=[2, 4, 8], kwargs={}, ) c.qos = QoS(c.task_consumer.qos, 1) current_pcount = c.qos.value c.event_dispatcher.enabled = False c.update_strategies() callback = self._get_on_message(c) callback(m) c.timer.stop() c.timer.join(1) items = [entry[2] for entry in self.timer.queue] found = 0 for item in items: if item.args[0].name == self.foo_task.name: found = True assert found assert c.qos.value > current_pcount c.timer.stop() def test_pidbox_callback(self): c = self.NoopConsumer() con = find_step(c, consumer.Control).box con.node = Mock() con.reset = Mock() con.on_message('foo', 'bar') con.node.handle_message.assert_called_with('foo', 'bar') con.node = Mock() con.node.handle_message.side_effect = KeyError('foo') con.on_message('foo', 'bar') con.node.handle_message.assert_called_with('foo', 'bar') con.node = Mock() con.node.handle_message.side_effect = ValueError('foo') con.on_message('foo', 'bar') con.node.handle_message.assert_called_with('foo', 'bar') con.reset.assert_called() def test_revoke(self): c = self.LoopConsumer() c.blueprint.state = RUN c.steps.pop() channel = Mock(name='channel') id = uuid() t = self.create_task_message( channel, self.foo_task.name, args=[2, 4, 8], kwargs={}, id=id, ) state.revoked.add(id) callback = self._get_on_message(c) callback(t) assert self.buffer.empty() def test_receieve_message_not_registered(self): c = self.LoopConsumer() c.blueprint.state = RUN c.steps.pop() channel = Mock(name='channel') m = self.create_task_message( channel, 'x.X.31x', args=[2, 4, 8], kwargs={}, ) callback = self._get_on_message(c) assert not callback(m) with pytest.raises(Empty): self.buffer.get_nowait() assert self.timer.empty() @patch('celery.worker.consumer.consumer.warn') @patch('celery.worker.consumer.consumer.logger') def test_receieve_message_ack_raises(self, logger, warn): c = self.LoopConsumer() c.blueprint.state = RUN channel = Mock(name='channel') m = self.create_task_message( channel, self.foo_task.name, args=[2, 4, 8], kwargs={}, ) m.headers = None c.update_strategies() c.connection_errors = (socket.error,) m.reject = Mock() m.reject.side_effect = socket.error('foo') callback = self._get_on_message(c) assert not callback(m) warn.assert_called() with pytest.raises(Empty): self.buffer.get_nowait() assert self.timer.empty() m.reject_log_error.assert_called_with(logger, c.connection_errors) def test_receive_message_eta(self): if os.environ.get('C_DEBUG_TEST'): pp = partial(print, file=sys.__stderr__) else: def pp(*args, **kwargs): pass pp('TEST RECEIVE MESSAGE ETA') pp('+CREATE MYKOMBUCONSUMER') c = self.LoopConsumer() pp('-CREATE MYKOMBUCONSUMER') c.steps.pop() channel = Mock(name='channel') pp('+ CREATE MESSAGE') m = self.create_task_message( channel, self.foo_task.name, args=[2, 4, 8], kwargs={}, eta=(datetime.now() + timedelta(days=1)).isoformat(), ) pp('- CREATE MESSAGE') try: pp('+ BLUEPRINT START 1') c.blueprint.start(c) pp('- BLUEPRINT START 1') p = c.app.conf.broker_connection_retry c.app.conf.broker_connection_retry = False pp('+ BLUEPRINT START 2') c.blueprint.start(c) pp('- BLUEPRINT START 2') c.app.conf.broker_connection_retry = p pp('+ BLUEPRINT RESTART') c.blueprint.restart(c) pp('- BLUEPRINT RESTART') pp('+ GET ON MESSAGE') callback = self._get_on_message(c) pp('- GET ON MESSAGE') pp('+ CALLBACK') callback(m) pp('- CALLBACK') finally: pp('+ STOP TIMER') c.timer.stop() pp('- STOP TIMER') try: pp('+ JOIN TIMER') c.timer.join() pp('- JOIN TIMER') except RuntimeError: pass in_hold = c.timer.queue[0] assert len(in_hold) == 3 eta, priority, entry = in_hold task = entry.args[0] assert isinstance(task, Request) assert task.name == self.foo_task.name assert task.execute() == 2 * 4 * 8 with pytest.raises(Empty): self.buffer.get_nowait() def test_reset_pidbox_node(self): c = self.NoopConsumer() con = find_step(c, consumer.Control).box con.node = Mock() chan = con.node.channel = Mock() chan.close.side_effect = socket.error('foo') c.connection_errors = (socket.error,) con.reset() chan.close.assert_called_with() def test_reset_pidbox_node_green(self): c = self.NoopConsumer(pool=Mock(is_green=True)) con = find_step(c, consumer.Control) assert isinstance(con.box, gPidbox) con.start(c) c.pool.spawn_n.assert_called_with(con.box.loop, c) def test_green_pidbox_node(self): pool = Mock() pool.is_green = True c = self.NoopConsumer(pool=Mock(is_green=True)) controller = find_step(c, consumer.Control) class BConsumer(Mock): def __enter__(self): self.consume() return self def __exit__(self, *exc_info): self.cancel() controller.box.node.listen = BConsumer() connections = [] class Connection(object): calls = 0 def __init__(self, obj): connections.append(self) self.obj = obj self.default_channel = self.channel() self.closed = False def __enter__(self): return self def __exit__(self, *exc_info): self.close() def channel(self): return Mock() def as_uri(self): return 'dummy://' def drain_events(self, **kwargs): if not self.calls: self.calls += 1 raise socket.timeout() self.obj.connection = None controller.box._node_shutdown.set() def close(self): self.closed = True c.connection_for_read = lambda: Connection(obj=c) controller = find_step(c, consumer.Control) controller.box.loop(c) controller.box.node.listen.assert_called() assert controller.box.consumer controller.box.consumer.consume.assert_called_with() assert c.connection is None assert connections[0].closed @patch('kombu.connection.Connection._establish_connection') @patch('kombu.utils.functional.sleep') def test_connect_errback(self, sleep, connect): c = self.NoopConsumer() Transport.connection_errors = (ChannelError,) connect.on_nth_call_do(ChannelError('error'), n=1) c.connect() connect.assert_called_with() def test_stop_pidbox_node(self): c = self.NoopConsumer() cont = find_step(c, consumer.Control) cont._node_stopped = Event() cont._node_shutdown = Event() cont._node_stopped.set() cont.stop(c) def test_start__loop(self): class _QoS(object): prev = 3 value = 4 def update(self): self.prev = self.value init_callback = Mock(name='init_callback') c = self.NoopConsumer(init_callback=init_callback) c.qos = _QoS() c.connection = Connection(self.app.conf.broker_url) c.connection.get_heartbeat_interval = Mock(return_value=None) c.iterations = 0 def raises_KeyError(*args, **kwargs): c.iterations += 1 if c.qos.prev != c.qos.value: c.qos.update() if c.iterations >= 2: raise KeyError('foo') c.loop = raises_KeyError with pytest.raises(KeyError): c.start() assert c.iterations == 2 assert c.qos.prev == c.qos.value init_callback.reset_mock() c = self.NoopConsumer(task_events=False, init_callback=init_callback) c.qos = _QoS() c.connection = Connection(self.app.conf.broker_url) c.connection.get_heartbeat_interval = Mock(return_value=None) c.loop = Mock(side_effect=socket.error('foo')) with pytest.raises(socket.error): c.start() c.loop.assert_called() def test_reset_connection_with_no_node(self): c = self.NoopConsumer() c.steps.pop() c.blueprint.start(c)
def test_poll_result(self): results = Queue() class Message(object): def __init__(self, **merge): self.payload = dict({'status': states.STARTED, 'result': None}, **merge) self.body = pickle.dumps(self.payload) self.content_type = 'application/x-python-serialize' self.content_encoding = 'binary' class MockBinding(object): def __init__(self, *args, **kwargs): self.channel = Mock() def __call__(self, *args, **kwargs): return self def declare(self): pass def get(self, no_ack=False): try: return results.get(block=False) except Empty: pass def is_bound(self): return True class MockBackend(AMQPBackend): Queue = MockBinding backend = MockBackend() backend._republish = Mock() # FFWD's to the latest state. results.put(Message(status=states.RECEIVED, seq=1)) results.put(Message(status=states.STARTED, seq=2)) results.put(Message(status=states.FAILURE, seq=3)) r1 = backend.get_task_meta(uuid()) self.assertDictContainsSubset({'status': states.FAILURE, 'seq': 3}, r1, 'FFWDs to the last state') # Caches last known state. results.put(Message()) tid = uuid() backend.get_task_meta(tid) self.assertIn(tid, backend._cache, 'Caches last known state') self.assertTrue(backend._republish.called) # Returns cache if no new states. results.queue.clear() assert not results.qsize() backend._cache[tid] = 'hello' self.assertEqual(backend.get_task_meta(tid), 'hello', 'Returns cache if no new states')
class test_Consumer(AppCase): def setup(self): self.buffer = FastQueue() self.timer = Timer() @self.app.task(shared=False) def foo_task(x, y, z): return x * y * z self.foo_task = foo_task def teardown(self): self.timer.stop() def test_info(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.task_consumer = Mock() l.qos = QoS(l.task_consumer.qos, 10) l.connection = Mock() l.connection.info.return_value = {'foo': 'bar'} l.controller = l.app.WorkController() l.controller.pool = Mock() l.controller.pool.info.return_value = [Mock(), Mock()] l.controller.consumer = l info = l.controller.stats() self.assertEqual(info['prefetch_count'], 10) self.assertTrue(info['broker']) def test_start_when_closed(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = CLOSE l.start() def test_connection(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.start(l) self.assertIsInstance(l.connection, Connection) l.blueprint.state = RUN l.event_dispatcher = None l.blueprint.restart(l) self.assertTrue(l.connection) l.blueprint.state = RUN l.shutdown() self.assertIsNone(l.connection) self.assertIsNone(l.task_consumer) l.blueprint.start(l) self.assertIsInstance(l.connection, Connection) l.blueprint.restart(l) l.stop() l.shutdown() self.assertIsNone(l.connection) self.assertIsNone(l.task_consumer) def test_close_connection(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN step = find_step(l, consumer.Connection) conn = l.connection = Mock() step.shutdown(l) self.assertTrue(conn.close.called) self.assertIsNone(l.connection) l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) eventer = l.event_dispatcher = Mock() eventer.enabled = True heart = l.heart = MockHeart() l.blueprint.state = RUN Events = find_step(l, consumer.Events) Events.shutdown(l) Heart = find_step(l, consumer.Heart) Heart.shutdown(l) self.assertTrue(eventer.close.call_count) self.assertTrue(heart.closed) @patch('celery.worker.consumer.warn') def test_receive_message_unknown(self, warn): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() backend = Mock() m = create_message(backend, unknown={'baz': '!!!'}) l.event_dispatcher = Mock() l.node = MockNode() callback = self._get_on_message(l) callback(m.decode(), m) self.assertTrue(warn.call_count) @patch('celery.worker.strategy.to_timestamp') def test_receive_message_eta_OverflowError(self, to_timestamp): to_timestamp.side_effect = OverflowError() l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() m = create_message(Mock(), task=self.foo_task.name, args=('2, 2'), kwargs={}, eta=datetime.now().isoformat()) l.event_dispatcher = Mock() l.node = MockNode() l.update_strategies() l.qos = Mock() callback = self._get_on_message(l) callback(m.decode(), m) self.assertTrue(m.acknowledged) @patch('celery.worker.consumer.error') def test_receive_message_InvalidTaskError(self, error): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.event_dispatcher = Mock() l.steps.pop() m = create_message(Mock(), task=self.foo_task.name, args=(1, 2), kwargs='foobarbaz', id=1) l.update_strategies() l.event_dispatcher = Mock() callback = self._get_on_message(l) callback(m.decode(), m) self.assertIn('Received invalid task message', error.call_args[0][0]) @patch('celery.worker.consumer.crit') def test_on_decode_error(self, crit): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) class MockMessage(Mock): content_type = 'application/x-msgpack' content_encoding = 'binary' body = 'foobarbaz' message = MockMessage() l.on_decode_error(message, KeyError('foo')) self.assertTrue(message.ack.call_count) self.assertIn("Can't decode message body", crit.call_args[0][0]) def _get_on_message(self, l): if l.qos is None: l.qos = Mock() l.event_dispatcher = Mock() l.task_consumer = Mock() l.connection = Mock() l.connection.drain_events.side_effect = SystemExit() with self.assertRaises(SystemExit): l.loop(*l.loop_args()) self.assertTrue(l.task_consumer.register_callback.called) return l.task_consumer.register_callback.call_args[0][0] def test_receieve_message(self): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.event_dispatcher = Mock() m = create_message(Mock(), task=self.foo_task.name, args=[2, 4, 8], kwargs={}) l.update_strategies() callback = self._get_on_message(l) callback(m.decode(), m) in_bucket = self.buffer.get_nowait() self.assertIsInstance(in_bucket, Request) self.assertEqual(in_bucket.name, self.foo_task.name) self.assertEqual(in_bucket.execute(), 2 * 4 * 8) self.assertTrue(self.timer.empty()) def test_start_channel_error(self): class MockConsumer(Consumer): iterations = 0 def loop(self, *args, **kwargs): if not self.iterations: self.iterations = 1 raise KeyError('foo') raise SyntaxError('bar') l = MockConsumer(self.buffer.put, timer=self.timer, send_events=False, pool=BasePool(), app=self.app) l.channel_errors = (KeyError, ) with self.assertRaises(KeyError): l.start() l.timer.stop() def test_start_connection_error(self): class MockConsumer(Consumer): iterations = 0 def loop(self, *args, **kwargs): if not self.iterations: self.iterations = 1 raise KeyError('foo') raise SyntaxError('bar') l = MockConsumer(self.buffer.put, timer=self.timer, send_events=False, pool=BasePool(), app=self.app) l.connection_errors = (KeyError, ) self.assertRaises(SyntaxError, l.start) l.timer.stop() def test_loop_ignores_socket_timeout(self): class Connection(self.app.connection().__class__): obj = None def drain_events(self, **kwargs): self.obj.connection = None raise socket.timeout(10) l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.connection = Connection() l.task_consumer = Mock() l.connection.obj = l l.qos = QoS(l.task_consumer.qos, 10) l.loop(*l.loop_args()) def test_loop_when_socket_error(self): class Connection(self.app.connection().__class__): obj = None def drain_events(self, **kwargs): self.obj.connection = None raise socket.error('foo') l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN c = l.connection = Connection() l.connection.obj = l l.task_consumer = Mock() l.qos = QoS(l.task_consumer.qos, 10) with self.assertRaises(socket.error): l.loop(*l.loop_args()) l.blueprint.state = CLOSE l.connection = c l.loop(*l.loop_args()) def test_loop(self): class Connection(self.app.connection().__class__): obj = None def drain_events(self, **kwargs): self.obj.connection = None l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.connection = Connection() l.connection.obj = l l.task_consumer = Mock() l.qos = QoS(l.task_consumer.qos, 10) l.loop(*l.loop_args()) l.loop(*l.loop_args()) self.assertTrue(l.task_consumer.consume.call_count) l.task_consumer.qos.assert_called_with(prefetch_count=10) self.assertEqual(l.qos.value, 10) l.qos.decrement_eventually() self.assertEqual(l.qos.value, 9) l.qos.update() self.assertEqual(l.qos.value, 9) l.task_consumer.qos.assert_called_with(prefetch_count=9) def test_ignore_errors(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.connection_errors = ( AttributeError, KeyError, ) l.channel_errors = (SyntaxError, ) ignore_errors(l, Mock(side_effect=AttributeError('foo'))) ignore_errors(l, Mock(side_effect=KeyError('foo'))) ignore_errors(l, Mock(side_effect=SyntaxError('foo'))) with self.assertRaises(IndexError): ignore_errors(l, Mock(side_effect=IndexError('foo'))) def test_apply_eta_task(self): from celery.worker import state l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.qos = QoS(None, 10) task = object() qos = l.qos.value l.apply_eta_task(task) self.assertIn(task, state.reserved_requests) self.assertEqual(l.qos.value, qos - 1) self.assertIs(self.buffer.get_nowait(), task) def test_receieve_message_eta_isoformat(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() m = create_message( Mock(), task=self.foo_task.name, eta=(datetime.now() + timedelta(days=1)).isoformat(), args=[2, 4, 8], kwargs={}, ) l.task_consumer = Mock() l.qos = QoS(l.task_consumer.qos, 1) current_pcount = l.qos.value l.event_dispatcher = Mock() l.enabled = False l.update_strategies() callback = self._get_on_message(l) callback(m.decode(), m) l.timer.stop() l.timer.join(1) items = [entry[2] for entry in self.timer.queue] found = 0 for item in items: if item.args[0].name == self.foo_task.name: found = True self.assertTrue(found) self.assertGreater(l.qos.value, current_pcount) l.timer.stop() def test_pidbox_callback(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) con = find_step(l, consumer.Control).box con.node = Mock() con.reset = Mock() con.on_message('foo', 'bar') con.node.handle_message.assert_called_with('foo', 'bar') con.node = Mock() con.node.handle_message.side_effect = KeyError('foo') con.on_message('foo', 'bar') con.node.handle_message.assert_called_with('foo', 'bar') con.node = Mock() con.node.handle_message.side_effect = ValueError('foo') con.on_message('foo', 'bar') con.node.handle_message.assert_called_with('foo', 'bar') self.assertTrue(con.reset.called) def test_revoke(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() backend = Mock() id = uuid() t = create_message(backend, task=self.foo_task.name, args=[2, 4, 8], kwargs={}, id=id) from celery.worker.state import revoked revoked.add(id) callback = self._get_on_message(l) callback(t.decode(), t) self.assertTrue(self.buffer.empty()) def test_receieve_message_not_registered(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() backend = Mock() m = create_message(backend, task='x.X.31x', args=[2, 4, 8], kwargs={}) l.event_dispatcher = Mock() callback = self._get_on_message(l) self.assertFalse(callback(m.decode(), m)) with self.assertRaises(Empty): self.buffer.get_nowait() self.assertTrue(self.timer.empty()) @patch('celery.worker.consumer.warn') @patch('celery.worker.consumer.logger') def test_receieve_message_ack_raises(self, logger, warn): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN backend = Mock() m = create_message(backend, args=[2, 4, 8], kwargs={}) l.event_dispatcher = Mock() l.connection_errors = (socket.error, ) m.reject = Mock() m.reject.side_effect = socket.error('foo') callback = self._get_on_message(l) self.assertFalse(callback(m.decode(), m)) self.assertTrue(warn.call_count) with self.assertRaises(Empty): self.buffer.get_nowait() self.assertTrue(self.timer.empty()) m.reject.assert_called_with(requeue=False) self.assertTrue(logger.critical.call_count) def test_receive_message_eta(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.steps.pop() l.event_dispatcher = Mock() l.event_dispatcher._outbound_buffer = deque() backend = Mock() m = create_message( backend, task=self.foo_task.name, args=[2, 4, 8], kwargs={}, eta=(datetime.now() + timedelta(days=1)).isoformat(), ) try: l.blueprint.start(l) p = l.app.conf.BROKER_CONNECTION_RETRY l.app.conf.BROKER_CONNECTION_RETRY = False l.blueprint.start(l) l.app.conf.BROKER_CONNECTION_RETRY = p l.blueprint.restart(l) l.event_dispatcher = Mock() callback = self._get_on_message(l) callback(m.decode(), m) finally: l.timer.stop() l.timer.join() in_hold = l.timer.queue[0] self.assertEqual(len(in_hold), 3) eta, priority, entry = in_hold task = entry.args[0] self.assertIsInstance(task, Request) self.assertEqual(task.name, self.foo_task.name) self.assertEqual(task.execute(), 2 * 4 * 8) with self.assertRaises(Empty): self.buffer.get_nowait() def test_reset_pidbox_node(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) con = find_step(l, consumer.Control).box con.node = Mock() chan = con.node.channel = Mock() l.connection = Mock() chan.close.side_effect = socket.error('foo') l.connection_errors = (socket.error, ) con.reset() chan.close.assert_called_with() def test_reset_pidbox_node_green(self): from celery.worker.pidbox import gPidbox pool = Mock() pool.is_green = True l = MyKombuConsumer(self.buffer.put, timer=self.timer, pool=pool, app=self.app) con = find_step(l, consumer.Control) self.assertIsInstance(con.box, gPidbox) con.start(l) l.pool.spawn_n.assert_called_with( con.box.loop, l, ) def test__green_pidbox_node(self): pool = Mock() pool.is_green = True l = MyKombuConsumer(self.buffer.put, timer=self.timer, pool=pool, app=self.app) l.node = Mock() controller = find_step(l, consumer.Control) class BConsumer(Mock): def __enter__(self): self.consume() return self def __exit__(self, *exc_info): self.cancel() controller.box.node.listen = BConsumer() connections = [] class Connection(object): calls = 0 def __init__(self, obj): connections.append(self) self.obj = obj self.default_channel = self.channel() self.closed = False def __enter__(self): return self def __exit__(self, *exc_info): self.close() def channel(self): return Mock() def as_uri(self): return 'dummy://' def drain_events(self, **kwargs): if not self.calls: self.calls += 1 raise socket.timeout() self.obj.connection = None controller.box._node_shutdown.set() def close(self): self.closed = True l.connection = Mock() l.connect = lambda: Connection(obj=l) controller = find_step(l, consumer.Control) controller.box.loop(l) self.assertTrue(controller.box.node.listen.called) self.assertTrue(controller.box.consumer) controller.box.consumer.consume.assert_called_with() self.assertIsNone(l.connection) self.assertTrue(connections[0].closed) @patch('kombu.connection.Connection._establish_connection') @patch('kombu.utils.sleep') def test_connect_errback(self, sleep, connect): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) from kombu.transport.memory import Transport Transport.connection_errors = (ChannelError, ) def effect(): if connect.call_count > 1: return raise ChannelError('error') connect.side_effect = effect l.connect() connect.assert_called_with() def test_stop_pidbox_node(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) cont = find_step(l, consumer.Control) cont._node_stopped = Event() cont._node_shutdown = Event() cont._node_stopped.set() cont.stop(l) def test_start__loop(self): class _QoS(object): prev = 3 value = 4 def update(self): self.prev = self.value class _Consumer(MyKombuConsumer): iterations = 0 def reset_connection(self): if self.iterations >= 1: raise KeyError('foo') init_callback = Mock() l = _Consumer(self.buffer.put, timer=self.timer, init_callback=init_callback, app=self.app) l.task_consumer = Mock() l.broadcast_consumer = Mock() l.qos = _QoS() l.connection = Connection() l.iterations = 0 def raises_KeyError(*args, **kwargs): l.iterations += 1 if l.qos.prev != l.qos.value: l.qos.update() if l.iterations >= 2: raise KeyError('foo') l.loop = raises_KeyError with self.assertRaises(KeyError): l.start() self.assertEqual(l.iterations, 2) self.assertEqual(l.qos.prev, l.qos.value) init_callback.reset_mock() l = _Consumer(self.buffer.put, timer=self.timer, app=self.app, send_events=False, init_callback=init_callback) l.qos = _QoS() l.task_consumer = Mock() l.broadcast_consumer = Mock() l.connection = Connection() l.loop = Mock(side_effect=socket.error('foo')) with self.assertRaises(socket.error): l.start() self.assertTrue(l.loop.call_count) def test_reset_connection_with_no_node(self): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.steps.pop() self.assertEqual(None, l.pool) l.blueprint.start(l)
class test_Consumer(AppCase): def setup(self): self.buffer = FastQueue() self.timer = Timer() @self.app.task(shared=False) def foo_task(x, y, z): return x * y * z self.foo_task = foo_task def teardown(self): self.timer.stop() def test_info(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.task_consumer = Mock() l.qos = QoS(l.task_consumer.qos, 10) l.connection = Mock() l.connection.info.return_value = {"foo": "bar"} l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() l.controller.pool.info.return_value = [Mock(), Mock()] l.controller.consumer = l info = l.controller.stats() self.assertEqual(info["prefetch_count"], 10) self.assertTrue(info["broker"]) def test_start_when_closed(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = CLOSE l.start() def test_connection(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() l.blueprint.start(l) self.assertIsInstance(l.connection, Connection) l.blueprint.state = RUN l.event_dispatcher = None l.blueprint.restart(l) self.assertTrue(l.connection) l.blueprint.state = RUN l.shutdown() self.assertIsNone(l.connection) self.assertIsNone(l.task_consumer) l.blueprint.start(l) self.assertIsInstance(l.connection, Connection) l.blueprint.restart(l) l.stop() l.shutdown() self.assertIsNone(l.connection) self.assertIsNone(l.task_consumer) def test_close_connection(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN step = find_step(l, consumer.Connection) conn = l.connection = Mock() step.shutdown(l) self.assertTrue(conn.close.called) self.assertIsNone(l.connection) l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) eventer = l.event_dispatcher = mock_event_dispatcher() eventer.enabled = True heart = l.heart = MockHeart() l.blueprint.state = RUN Events = find_step(l, consumer.Events) Events.shutdown(l) Heart = find_step(l, consumer.Heart) Heart.shutdown(l) self.assertTrue(eventer.close.call_count) self.assertTrue(heart.closed) @patch("celery.worker.consumer.warn") def test_receive_message_unknown(self, warn): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() channel = Mock() m = create_message(channel, unknown={"baz": "!!!"}) l.event_dispatcher = mock_event_dispatcher() l.node = MockNode() callback = self._get_on_message(l) callback(m) self.assertTrue(warn.call_count) @patch("celery.worker.strategy.to_timestamp") def test_receive_message_eta_OverflowError(self, to_timestamp): to_timestamp.side_effect = OverflowError() l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() l.blueprint.state = RUN l.steps.pop() m = create_task_message(Mock(), self.foo_task.name, args=("2, 2"), kwargs={}, eta=datetime.now().isoformat()) l.event_dispatcher = mock_event_dispatcher() l.node = MockNode() l.update_strategies() l.qos = Mock() callback = self._get_on_message(l) callback(m) self.assertTrue(m.acknowledged) @patch("celery.worker.consumer.error") def test_receive_message_InvalidTaskError(self, error): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.event_dispatcher = mock_event_dispatcher() l.steps.pop() l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() m = create_task_message(Mock(), self.foo_task.name, args=(1, 2), kwargs="foobarbaz", id=1) l.update_strategies() l.event_dispatcher = mock_event_dispatcher() strat = l.strategies[self.foo_task.name] = Mock(name="strategy") strat.side_effect = InvalidTaskError() callback = self._get_on_message(l) callback(m) self.assertTrue(error.called) self.assertIn("Received invalid task message", error.call_args[0][0]) @patch("celery.worker.consumer.crit") def test_on_decode_error(self, crit): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) class MockMessage(Mock): content_type = "application/x-msgpack" content_encoding = "binary" body = "foobarbaz" message = MockMessage() l.on_decode_error(message, KeyError("foo")) self.assertTrue(message.ack.call_count) self.assertIn("Can't decode message body", crit.call_args[0][0]) def _get_on_message(self, l): if l.qos is None: l.qos = Mock() l.event_dispatcher = mock_event_dispatcher() l.task_consumer = Mock() l.connection = Mock() l.connection.drain_events.side_effect = WorkerShutdown() with self.assertRaises(WorkerShutdown): l.loop(*l.loop_args()) self.assertTrue(l.task_consumer.on_message) return l.task_consumer.on_message def test_receieve_message(self): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() l.blueprint.state = RUN l.event_dispatcher = mock_event_dispatcher() m = create_task_message(Mock(), self.foo_task.name, args=[2, 4, 8], kwargs={}) l.update_strategies() callback = self._get_on_message(l) callback(m) in_bucket = self.buffer.get_nowait() self.assertIsInstance(in_bucket, Request) self.assertEqual(in_bucket.name, self.foo_task.name) self.assertEqual(in_bucket.execute(), 2 * 4 * 8) self.assertTrue(self.timer.empty()) def test_start_channel_error(self): class MockConsumer(Consumer): iterations = 0 def loop(self, *args, **kwargs): if not self.iterations: self.iterations = 1 raise KeyError("foo") raise SyntaxError("bar") l = MockConsumer(self.buffer.put, timer=self.timer, send_events=False, pool=BasePool(), app=self.app) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() l.channel_errors = (KeyError,) with self.assertRaises(KeyError): l.start() l.timer.stop() def test_start_connection_error(self): class MockConsumer(Consumer): iterations = 0 def loop(self, *args, **kwargs): if not self.iterations: self.iterations = 1 raise KeyError("foo") raise SyntaxError("bar") l = MockConsumer(self.buffer.put, timer=self.timer, send_events=False, pool=BasePool(), app=self.app) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() l.connection_errors = (KeyError,) self.assertRaises(SyntaxError, l.start) l.timer.stop() def test_loop_ignores_socket_timeout(self): class Connection(self.app.connection().__class__): obj = None def drain_events(self, **kwargs): self.obj.connection = None raise socket.timeout(10) l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.connection = Connection() l.task_consumer = Mock() l.connection.obj = l l.qos = QoS(l.task_consumer.qos, 10) l.loop(*l.loop_args()) def test_loop_when_socket_error(self): class Connection(self.app.connection().__class__): obj = None def drain_events(self, **kwargs): self.obj.connection = None raise socket.error("foo") l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN c = l.connection = Connection() l.connection.obj = l l.task_consumer = Mock() l.qos = QoS(l.task_consumer.qos, 10) with self.assertRaises(socket.error): l.loop(*l.loop_args()) l.blueprint.state = CLOSE l.connection = c l.loop(*l.loop_args()) def test_loop(self): class Connection(self.app.connection().__class__): obj = None def drain_events(self, **kwargs): self.obj.connection = None l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.connection = Connection() l.connection.obj = l l.task_consumer = Mock() l.qos = QoS(l.task_consumer.qos, 10) l.loop(*l.loop_args()) l.loop(*l.loop_args()) self.assertTrue(l.task_consumer.consume.call_count) l.task_consumer.qos.assert_called_with(prefetch_count=10) self.assertEqual(l.qos.value, 10) l.qos.decrement_eventually() self.assertEqual(l.qos.value, 9) l.qos.update() self.assertEqual(l.qos.value, 9) l.task_consumer.qos.assert_called_with(prefetch_count=9) def test_ignore_errors(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.connection_errors = (AttributeError, KeyError) l.channel_errors = (SyntaxError,) ignore_errors(l, Mock(side_effect=AttributeError("foo"))) ignore_errors(l, Mock(side_effect=KeyError("foo"))) ignore_errors(l, Mock(side_effect=SyntaxError("foo"))) with self.assertRaises(IndexError): ignore_errors(l, Mock(side_effect=IndexError("foo"))) def test_apply_eta_task(self): from celery.worker import state l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() l.qos = QoS(None, 10) task = object() qos = l.qos.value l.apply_eta_task(task) self.assertIn(task, state.reserved_requests) self.assertEqual(l.qos.value, qos - 1) self.assertIs(self.buffer.get_nowait(), task) def test_receieve_message_eta_isoformat(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() l.blueprint.state = RUN l.steps.pop() m = create_task_message( Mock(), self.foo_task.name, eta=(datetime.now() + timedelta(days=1)).isoformat(), args=[2, 4, 8], kwargs={} ) l.task_consumer = Mock() l.qos = QoS(l.task_consumer.qos, 1) current_pcount = l.qos.value l.event_dispatcher = mock_event_dispatcher() l.enabled = False l.update_strategies() callback = self._get_on_message(l) callback(m) l.timer.stop() l.timer.join(1) items = [entry[2] for entry in self.timer.queue] found = 0 for item in items: if item.args[0].name == self.foo_task.name: found = True self.assertTrue(found) self.assertGreater(l.qos.value, current_pcount) l.timer.stop() def test_pidbox_callback(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) con = find_step(l, consumer.Control).box con.node = Mock() con.reset = Mock() con.on_message("foo", "bar") con.node.handle_message.assert_called_with("foo", "bar") con.node = Mock() con.node.handle_message.side_effect = KeyError("foo") con.on_message("foo", "bar") con.node.handle_message.assert_called_with("foo", "bar") con.node = Mock() con.node.handle_message.side_effect = ValueError("foo") con.on_message("foo", "bar") con.node.handle_message.assert_called_with("foo", "bar") self.assertTrue(con.reset.called) def test_revoke(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() channel = Mock() id = uuid() t = create_task_message(channel, self.foo_task.name, args=[2, 4, 8], kwargs={}, id=id) from celery.worker.state import revoked revoked.add(id) callback = self._get_on_message(l) callback(t) self.assertTrue(self.buffer.empty()) def test_receieve_message_not_registered(self): l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.blueprint.state = RUN l.steps.pop() channel = Mock(name="channel") m = create_task_message(channel, "x.X.31x", args=[2, 4, 8], kwargs={}) l.event_dispatcher = mock_event_dispatcher() callback = self._get_on_message(l) self.assertFalse(callback(m)) with self.assertRaises(Empty): self.buffer.get_nowait() self.assertTrue(self.timer.empty()) @patch("celery.worker.consumer.warn") @patch("celery.worker.consumer.logger") def test_receieve_message_ack_raises(self, logger, warn): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() l.blueprint.state = RUN channel = Mock() m = create_task_message(channel, self.foo_task.name, args=[2, 4, 8], kwargs={}) m.headers = None l.event_dispatcher = mock_event_dispatcher() l.update_strategies() l.connection_errors = (socket.error,) m.reject = Mock() m.reject.side_effect = socket.error("foo") callback = self._get_on_message(l) self.assertFalse(callback(m)) self.assertTrue(warn.call_count) with self.assertRaises(Empty): self.buffer.get_nowait() self.assertTrue(self.timer.empty()) m.reject_log_error.assert_called_with(logger, l.connection_errors) def test_receive_message_eta(self): import sys from functools import partial if os.environ.get("C_DEBUG_TEST"): pp = partial(print, file=sys.__stderr__) else: def pp(*args, **kwargs): pass pp("TEST RECEIVE MESSAGE ETA") pp("+CREATE MYKOMBUCONSUMER") l = _MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() pp("-CREATE MYKOMBUCONSUMER") l.steps.pop() l.event_dispatcher = mock_event_dispatcher() channel = Mock(name="channel") pp("+ CREATE MESSAGE") m = create_task_message( channel, self.foo_task.name, args=[2, 4, 8], kwargs={}, eta=(datetime.now() + timedelta(days=1)).isoformat() ) pp("- CREATE MESSAGE") try: pp("+ BLUEPRINT START 1") l.blueprint.start(l) pp("- BLUEPRINT START 1") p = l.app.conf.BROKER_CONNECTION_RETRY l.app.conf.BROKER_CONNECTION_RETRY = False pp("+ BLUEPRINT START 2") l.blueprint.start(l) pp("- BLUEPRINT START 2") l.app.conf.BROKER_CONNECTION_RETRY = p pp("+ BLUEPRINT RESTART") l.blueprint.restart(l) pp("- BLUEPRINT RESTART") l.event_dispatcher = mock_event_dispatcher() pp("+ GET ON MESSAGE") callback = self._get_on_message(l) pp("- GET ON MESSAGE") pp("+ CALLBACK") callback(m) pp("- CALLBACK") finally: pp("+ STOP TIMER") l.timer.stop() pp("- STOP TIMER") try: pp("+ JOIN TIMER") l.timer.join() pp("- JOIN TIMER") except RuntimeError: pass in_hold = l.timer.queue[0] self.assertEqual(len(in_hold), 3) eta, priority, entry = in_hold task = entry.args[0] self.assertIsInstance(task, Request) self.assertEqual(task.name, self.foo_task.name) self.assertEqual(task.execute(), 2 * 4 * 8) with self.assertRaises(Empty): self.buffer.get_nowait() def test_reset_pidbox_node(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) con = find_step(l, consumer.Control).box con.node = Mock() chan = con.node.channel = Mock() l.connection = Mock() chan.close.side_effect = socket.error("foo") l.connection_errors = (socket.error,) con.reset() chan.close.assert_called_with() def test_reset_pidbox_node_green(self): from celery.worker.pidbox import gPidbox pool = Mock() pool.is_green = True l = MyKombuConsumer(self.buffer.put, timer=self.timer, pool=pool, app=self.app) con = find_step(l, consumer.Control) self.assertIsInstance(con.box, gPidbox) con.start(l) l.pool.spawn_n.assert_called_with(con.box.loop, l) def test__green_pidbox_node(self): pool = Mock() pool.is_green = True l = MyKombuConsumer(self.buffer.put, timer=self.timer, pool=pool, app=self.app) l.node = Mock() controller = find_step(l, consumer.Control) class BConsumer(Mock): def __enter__(self): self.consume() return self def __exit__(self, *exc_info): self.cancel() controller.box.node.listen = BConsumer() connections = [] class Connection(object): calls = 0 def __init__(self, obj): connections.append(self) self.obj = obj self.default_channel = self.channel() self.closed = False def __enter__(self): return self def __exit__(self, *exc_info): self.close() def channel(self): return Mock() def as_uri(self): return "dummy://" def drain_events(self, **kwargs): if not self.calls: self.calls += 1 raise socket.timeout() self.obj.connection = None controller.box._node_shutdown.set() def close(self): self.closed = True l.connection = Mock() l.connect = lambda: Connection(obj=l) controller = find_step(l, consumer.Control) controller.box.loop(l) self.assertTrue(controller.box.node.listen.called) self.assertTrue(controller.box.consumer) controller.box.consumer.consume.assert_called_with() self.assertIsNone(l.connection) self.assertTrue(connections[0].closed) @patch("kombu.connection.Connection._establish_connection") @patch("kombu.utils.sleep") def test_connect_errback(self, sleep, connect): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) from kombu.transport.memory import Transport Transport.connection_errors = (ChannelError,) def effect(): if connect.call_count > 1: return raise ChannelError("error") connect.side_effect = effect l.connect() connect.assert_called_with() def test_stop_pidbox_node(self): l = MyKombuConsumer(self.buffer.put, timer=self.timer, app=self.app) cont = find_step(l, consumer.Control) cont._node_stopped = Event() cont._node_shutdown = Event() cont._node_stopped.set() cont.stop(l) def test_start__loop(self): class _QoS(object): prev = 3 value = 4 def update(self): self.prev = self.value class _Consumer(MyKombuConsumer): iterations = 0 def reset_connection(self): if self.iterations >= 1: raise KeyError("foo") init_callback = Mock() l = _Consumer(self.buffer.put, timer=self.timer, init_callback=init_callback, app=self.app) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() l.task_consumer = Mock() l.broadcast_consumer = Mock() l.qos = _QoS() l.connection = Connection() l.iterations = 0 def raises_KeyError(*args, **kwargs): l.iterations += 1 if l.qos.prev != l.qos.value: l.qos.update() if l.iterations >= 2: raise KeyError("foo") l.loop = raises_KeyError with self.assertRaises(KeyError): l.start() self.assertEqual(l.iterations, 2) self.assertEqual(l.qos.prev, l.qos.value) init_callback.reset_mock() l = _Consumer(self.buffer.put, timer=self.timer, app=self.app, send_events=False, init_callback=init_callback) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() l.qos = _QoS() l.task_consumer = Mock() l.broadcast_consumer = Mock() l.connection = Connection() l.loop = Mock(side_effect=socket.error("foo")) with self.assertRaises(socket.error): l.start() self.assertTrue(l.loop.call_count) def test_reset_connection_with_no_node(self): l = Consumer(self.buffer.put, timer=self.timer, app=self.app) l.controller = l.app.WorkController() l.pool = l.controller.pool = Mock() l.steps.pop() l.blueprint.start(l)