def test_start_reconnects_if_connectaion_failed(self): self.one_route_fixture['route'] = [ "asgard/counts", "asgard/counts/errors" ] consumer = Consumer(self.one_route_fixture, *self.connection_parameters) with unittest.mock.patch.object(consumer, 'keep_runnig', side_effect=[True, True, False]), \ asynctest.patch.object(asyncio, 'sleep') as sleep_mock: is_connected_mock = mock.PropertyMock( side_effect=[False, False, True]) queue_mock = CoroutineMock( consume=CoroutineMock(), connect=CoroutineMock(side_effect=[AioamqpException, True])) type(queue_mock).is_connected = is_connected_mock loop = asyncio.get_event_loop() consumer.queue = queue_mock loop.run_until_complete(consumer.start()) self.assertEqual(1, queue_mock.connect.await_count) self.assertEqual(2, queue_mock.consume.await_count) self.assertEqual([ mock.call(queue_name="asgard/counts"), mock.call(queue_name="asgard/counts/errors") ], queue_mock.consume.await_args_list) self.assertEqual(2, sleep_mock.await_count)
async def test_on_queue_error_logs_exception_and_acks_message(self): """ Logamos qualquer erro de parsing/validação de mensagem """ delivery_tag = 42 body = "not a JSON" queue_mock = CoroutineMock(ack=CoroutineMock()) consumer = Consumer(self.one_route_fixture, *self.connection_parameters) with mock.patch.object(conf, "logger") as logger_mock: await consumer.on_queue_error(body, delivery_tag, "Error: not a JSON", queue_mock) logger_mock.error.assert_called_with({"exception": "Error: not a JSON", "original_msg": body, "parse-error": True}) queue_mock.ack.assert_awaited_once_with(delivery_tag=delivery_tag)
async def test_on_connection_error_logs_exception(self): """ Logamos qualquer erro de conexão com o Rabbit, inclusive acesso negado """ consumer = Consumer(self.one_route_fixture, *self.connection_parameters) with mock.patch.object(conf, "logger") as logger_mock: try: 1 / 0 except Exception as e: await consumer.on_connection_error(e) logger_mock.error.assert_called_with({ "exc_message": "division by zero", "exc_traceback": mock.ANY, })
async def test_consume_all_queues(self): """ """ self.one_route_fixture['route'] = [ "asgard/counts", "asgard/counts/errors" ] consumer = Consumer(self.one_route_fixture, *self.connection_parameters) queue_mock = CoroutineMock(consume=CoroutineMock()) await consumer.consume_all_queues(queue_mock) self.assertEqual(2, queue_mock.consume.await_count) self.assertEqual([ mock.call(queue_name="asgard/counts"), mock.call(queue_name="asgard/counts/errors") ], queue_mock.consume.await_args_list)
async def test_on_exception_reject_message(self): @self.app.route( ["queue"], type=RouteTypes.AMQP_RABBITMQ, options={Events.ON_EXCEPTION: Actions.REJECT}, ) async def _handler(messages): raise Exception("BOOM!") route = self.app.routes_registry.amqp_routes[0] consumer = Consumer(route, *self.connection_parameters) with self.assertRaises(Exception): await consumer.on_queue_message(msg=self.mock_message) self.mock_message.reject.assert_awaited_once_with(requeue=False)
def test_consumer_adjusts_bulk_size(self): """ Se escolhermos um prefetch menor do que o bulk_size, significa que nosso "bucket" nunca vai encher e isso significa que nosso consumer ficará congelado, em um deadlock: Ele estará esperando o bucket encher E ao mesmo tempo estará o esperando o bucket esvaziar para que possa receber mais mensagens do RabbitMQ Vamos setar o bulk_size com sendo min(bulk_size, prefetch_count) """ self.one_route_fixture["options"]["bulk_size"] = 2048 consumer = Consumer(self.one_route_fixture, *self.connection_parameters) self.assertEqual(1024, consumer.bucket.size) self.one_route_fixture["options"]["bulk_size"] = 4096 consumer = Consumer( self.one_route_fixture, host="127.0.0.1", username="******", password="******", prefetch_count=8192, ) self.assertEqual(4096, consumer.bucket.size)
async def test_do_not_flush_if_bucket_is_already_empty_when_timeout_expires( self ): class MyBucket(Bucket): def pop_all(self): global items items = self._items self._items = [] return items handler_mock = CoroutineMock() self.one_route_fixture["handler"] = handler_mock self.one_route_fixture["options"]["bulk_size"] = 3 consumer = Consumer( self.one_route_fixture, *self.connection_parameters, bucket_class=MyBucket, ) self.loop.create_task(consumer._flush_clocked()) # Realizando sleep para devolver o loop para o clock await asyncio.sleep(0.1) self.assertEqual(0, handler_mock.await_count)
async def test_on_message_handle_error_logs_exception(self): """ Logamos a exception lançada pelo handler. Aqui o try/except serve apenas para termos uma exception real, com traceback. """ consumer = Consumer(self.one_route_fixture, *self.connection_parameters) with mock.patch.object(conf, "logger") as logger_mock: try: 1 / 0 except Exception as e: await consumer.on_message_handle_error(e) logger_mock.error.assert_called_with({ "exc_message": "division by zero", "exc_traceback": mock.ANY, })
async def test_on_queue_message_auto_ack_on_success(self): """ Se o handler registrado no @app.route() rodar com sucesso, devemos fazer o ack da mensagem """ consumer = Consumer(self.one_route_fixture, *self.connection_parameters) expected_body = {"key": "value"} message_mock = RabbitMQMessage(body=expected_body, delivery_tag=10) result = await consumer.on_queue_message(expected_body, delivery_tag=10, queue=self.queue_mock) self.assertEqual((42, expected_body), result) self.queue_mock.ack.assert_awaited_once_with(delivery_tag=10) self.queue_mock.reject.assert_not_awaited()
async def test_on_queue_message_rejects_on_exception(self): """ Se o handler der raise em qualquer exception, devemos dar reject() na mensagem """ async def exception_handler(message): return message.do_not_exist self.one_route_fixture['handler'] = exception_handler consumer = Consumer(self.one_route_fixture, *self.connection_parameters) queue_mock = CoroutineMock(ack=CoroutineMock(), reject=CoroutineMock()) with self.assertRaises(AttributeError): await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=queue_mock) queue_mock.reject.assert_awaited_once_with(delivery_tag=10, requeue=True) queue_mock.ack.assert_not_awaited
async def test_on_exception_requeue_message(self): """ Confirma que em caso de exceção, será feito `message.reject(requeue=True)` """ @self.app.route(["queue"], options={Events.ON_EXCEPTION: Actions.REQUEUE}) async def _handler(messages): raise Exception("BOOM!") consumer = Consumer(self.app.routes_registry[_handler], *self.connection_parameters) with self.assertRaises(Exception): await consumer.on_queue_message({"key": 42}, delivery_tag=10, queue=self.queue_mock) self.queue_mock.reject.assert_awaited_with(delivery_tag=10, requeue=True)
async def test_on_exception_requeue_message(self): """ Confirma que em caso de exceção, será feito `message.reject(requeue=True)` """ @self.app.route( ["queue"], type=RouteTypes.AMQP_RABBITMQ, options={Events.ON_EXCEPTION: Actions.REQUEUE}, ) async def _handler(messages): raise Exception("BOOM!") route = self.app.routes_registry.amqp_routes[0] consumer = Consumer(route, *self.connection_parameters) with self.assertRaises(Exception): await consumer.on_queue_message(msg=self.mock_message) self.mock_message.reject.assert_awaited_once_with(requeue=True)
async def test_consume_all_queues(self): """ """ self.one_route_fixture["routes"] = [ "asgard/counts", "asgard/counts/errors", ] consumer = Consumer(self.one_route_fixture, *self.connection_parameters) queue_mock = CoroutineMock(consume=CoroutineMock()) await consumer.consume_all_queues(queue_mock) queue_mock.consume.assert_has_awaits( [ mock.call(queue_name="asgard/counts", delegate=consumer), mock.call(queue_name="asgard/counts/errors", delegate=consumer), ], any_order=True, )
async def test_on_queue_message_bulk_size_one(self): class MyBucket(Bucket): def pop_all(self): return self._items; handler_mock = CoroutineMock() self.one_route_fixture['handler'] = handler_mock self.one_route_fixture['options']['bulk_size'] = 1 consumer = Consumer(self.one_route_fixture, *self.connection_parameters, bucket_class=MyBucket) queue_mock = CoroutineMock(ack=CoroutineMock()) await consumer.on_queue_message({"key": "value"}, delivery_tag=20, queue=queue_mock) handler_mock.assert_awaited_once_with(consumer.bucket._items) self.assertEqual([mock.call(delivery_tag=20)], queue_mock.ack.await_args_list) self.assertEqual(0, queue_mock.reject.call_count)
async def test_on_queue_message_bulk_mixed_ack_and_reject(self): async def handler(messages): messages[1].reject() messages[2].reject() self.one_route_fixture['handler'] = handler self.one_route_fixture['options']['bulk_size'] = 5 consumer = Consumer(self.one_route_fixture, *self.connection_parameters) queue_mock = CoroutineMock(ack=CoroutineMock(), reject=CoroutineMock()) await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=queue_mock) await consumer.on_queue_message({"key": "value"}, delivery_tag=11, queue=queue_mock) await consumer.on_queue_message({"key": "value"}, delivery_tag=12, queue=queue_mock) await consumer.on_queue_message({"key": "value"}, delivery_tag=13, queue=queue_mock) await consumer.on_queue_message({"key": "value"}, delivery_tag=14, queue=queue_mock) self.assertEqual([mock.call(delivery_tag=10), mock.call(delivery_tag=13), mock.call(delivery_tag=14)], queue_mock.ack.await_args_list) self.assertEqual([mock.call(delivery_tag=11, requeue=True), mock.call(delivery_tag=12, requeue=True)], queue_mock.reject.await_args_list)
async def test_on_exception_default_action_bulk_messages(self): """ Se o handler der raise em qualquer exception, devemos dar reject() na mensagem """ async def exception_handler(message): return message.do_not_exist self.one_route_fixture['handler'] = exception_handler self.one_route_fixture['options']['bulk_size'] = 2 consumer = Consumer(self.one_route_fixture, *self.connection_parameters) queue_mock = CoroutineMock(ack=CoroutineMock(), reject=CoroutineMock()) with self.assertRaises(AttributeError): await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=queue_mock) await consumer.on_queue_message({"key": "value"}, delivery_tag=11, queue=queue_mock) self.assertCountEqual([mock.call(delivery_tag=10, requeue=True), mock.call(delivery_tag=11, requeue=True)], queue_mock.reject.await_args_list) queue_mock.ack.assert_not_awaited
async def test_on_queue_message_bulk_size_one(self): class MyBucket(Bucket): def pop_all(self): return self._items handler_mock = CoroutineMock(__name__="handler") self.one_route_fixture["handler"] = handler_mock self.one_route_fixture["options"]["bulk_size"] = 1 consumer = Consumer( self.one_route_fixture, *self.connection_parameters, bucket_class=MyBucket, ) msg = self._make_msg(delivery_tag=20) await consumer.on_queue_message(msg=msg) handler_mock.assert_awaited_once_with(consumer.bucket._items) msg.ack.assert_awaited_once() msg.reject.assert_not_awaited()
async def test_restart_all_consumers_if_channel_is_closed(self): """ Se detectamos que o channel está fechado, devemos reiniciar todos os consumers. Isso vale pois atualmente todos eles compartilham o mesmo channel. """ self.one_route_fixture["routes"] = [ "asgard/counts", "asgard/counts/errors", ] consumer = Consumer(self.one_route_fixture, *self.connection_parameters) queue_mock = CoroutineMock(consume=CoroutineMock()) with asynctest.patch.object( consumer, "queue", queue_mock), unittest.mock.patch.object( consumer, "keep_runnig", side_effect=[ True, True, True, False ]), asynctest.patch.object( asyncio, "sleep") as sleep_mock, asynctest.patch.object( consumer, "clock_task", side_effect=[True, True]), asynctest.patch.object( consumer.queue.connection, "has_channel_ready", Mock(side_effect=[False, True, False]), ): await consumer.start() queue_mock.consume.assert_has_awaits( [ mock.call(queue_name="asgard/counts", delegate=consumer), mock.call(queue_name="asgard/counts/errors", delegate=consumer), mock.call(queue_name="asgard/counts", delegate=consumer), mock.call(queue_name="asgard/counts/errors", delegate=consumer), ], any_order=True, )
async def test_on_queue_message_bulk_mixed_ack_and_reject_on_success_reject( self): self.maxDiff = None async def handler(messages): messages[1].reject(requeue=True) messages[2].reject(requeue=True) self.one_route_fixture['handler'] = handler self.one_route_fixture['options']['bulk_size'] = 5 self.one_route_fixture['options'][Events.ON_SUCCESS] = Actions.REJECT consumer = Consumer(self.one_route_fixture, *self.connection_parameters) queue_mock = CoroutineMock(ack=CoroutineMock(), reject=CoroutineMock()) await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=queue_mock) await consumer.on_queue_message({"key": "value"}, delivery_tag=11, queue=queue_mock) await consumer.on_queue_message({"key": "value"}, delivery_tag=12, queue=queue_mock) await consumer.on_queue_message({"key": "value"}, delivery_tag=13, queue=queue_mock) await consumer.on_queue_message({"key": "value"}, delivery_tag=14, queue=queue_mock) self.assertCountEqual([ mock.call(delivery_tag=10, requeue=False), mock.call(delivery_tag=11, requeue=True), mock.call(delivery_tag=12, requeue=True), mock.call(delivery_tag=13, requeue=False), mock.call(delivery_tag=14, requeue=False) ], queue_mock.reject.await_args_list)
async def test_bulk_flushes_on_timeout_even_with_bucket_not_full(self): """ Se nosso bucket não chegar ao total de mensagens do nosso bulk_size, temos que fazer flush de tempos em tempos, senão poderemos ficar eternamente com mensagens presas. """ handler_mock = CoroutineMock() self.one_route_fixture['handler'] = handler_mock self.one_route_fixture['options']['bulk_size'] = 5 self.one_route_fixture['options']['bulk_flush_interval'] = 3 consumer = Consumer(self.one_route_fixture, *self.connection_parameters) queue_mock = CoroutineMock(ack=CoroutineMock(), reject=CoroutineMock()) await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=queue_mock) await consumer.on_queue_message({"key": "value"}, delivery_tag=11, queue=queue_mock) #await consumer.on_queue_message({"key": "value"}, delivery_tag=12, queue=queue_mock) #await consumer.on_queue_message({"key": "value"}, delivery_tag=13, queue=queue_mock) #await consumer.on_queue_message({"key": "value"}, delivery_tag=14, queue=queue_mock) await asyncio.sleep(4) #handler_mock.assert_awaited_once self.assertEqual(1, handler_mock.await_count) self.assertEqual([mock.call(delivery_tag=10), mock.call(delivery_tag=11)], queue_mock.ack.await_args_list)
async def test_on_queue_message_bulk_mixed_ack_and_reject_on_success_reject( self): self.maxDiff = None async def handler(messages): messages[1].reject(requeue=True) messages[2].reject(requeue=True) self.one_route_fixture["handler"] = handler self.one_route_fixture["options"]["bulk_size"] = 5 self.one_route_fixture["options"][Events.ON_SUCCESS] = Actions.REJECT consumer = Consumer(self.one_route_fixture, *self.connection_parameters) msgs = [self._make_msg(delivery_tag=i) for i in range(5)] for msg in msgs: await consumer.on_queue_message(msg) msgs[0].reject.assert_awaited_once_with(requeue=False) msgs[1].reject.assert_awaited_once_with(requeue=True) msgs[2].reject.assert_awaited_once_with(requeue=True) msgs[3].reject.assert_awaited_once_with(requeue=False) msgs[4].reject.assert_awaited_once_with(requeue=False)
async def test_on_exception_default_action_bulk_messages(self): """ Se o handler der raise em qualquer exception, devemos dar reject() na mensagem """ async def exception_handler(message): return message.do_not_exist self.one_route_fixture["handler"] = exception_handler self.one_route_fixture["options"]["bulk_size"] = 2 consumer = Consumer(self.one_route_fixture, *self.connection_parameters) msgs = [self._make_msg(delivery_tag=1), self._make_msg(delivery_tag=2)] with self.assertRaises(AttributeError): await consumer.on_queue_message(msg=msgs[0]) await consumer.on_queue_message(msg=msgs[1]) msgs[0].reject.assert_awaited_once_with(requeue=True) msgs[1].reject.assert_awaited_once_with(requeue=True) msgs[0].ack.assert_not_awaited() msgs[1].ack.assert_not_awaited()
async def test_on_queue_message_calls_inner_handler(self): consumer = Consumer(self.one_route_fixture, *self.connection_parameters) result = await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=self.queue_mock) self.assertEqual((42, {"key": "value"}), result)
async def test_on_queue_message_calls_inner_handler(self): consumer = Consumer(self.one_route_fixture, *self.connection_parameters) result = await consumer.on_queue_message(msg=self.mock_message) self.assertEqual((42, self.mock_message.deserialized_data), result)
def test_consumer_returns_correct_queue_name(self): consumer = Consumer(self.one_route_fixture, *self.connection_parameters) self.assertEqual(["/asgard/counts/ok"], consumer.queue_name)
def test_consumer_instantiate_correct_size_bucket(self): consumer = Consumer(self.one_route_fixture, *self.connection_parameters) self.assertEqual(self.one_route_fixture["options"]["bulk_size"], consumer.bucket.size)
def test_consumer_instantiate_using_bucket_class(self): class MyBucket(Bucket): pass consumer = Consumer(self.one_route_fixture, *self.connection_parameters, bucket_class=MyBucket) self.assertTrue(isinstance(consumer.bucket, MyBucket))
def test_return_correct_queue_name(self): """ consumer.quene_name deve retornar o nome da fila que está sendo consumida. """ consumer = Consumer(self.one_route_fixture, *self.connection_parameters) self.assertEquals(self.one_route_fixture['route'], consumer.queue_name)
async def test_on_queue_message_precondition_failed_on_ack(self): consumer = Consumer(self.one_route_fixture, *self.connection_parameters) self.mock_message.ack.side_effect = AioamqpException with self.assertRaises(AioamqpException): await consumer.on_queue_message(msg=self.mock_message)
async def test_on_queue_message_precondition_failed_on_ack(self): consumer = Consumer(self.one_route_fixture, *self.connection_parameters) queue_mock = CoroutineMock(ack=CoroutineMock(side_effect=AioamqpException)) with self.assertRaises(AioamqpException): await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=queue_mock)