def test_start_reconnects_if_connectaion_failed(self):
self.one_route_fixture['route'] = [
"asgard/counts", "asgard/counts/errors"
]
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
with unittest.mock.patch.object(consumer, 'keep_runnig', side_effect=[True, True, False]), \
asynctest.patch.object(asyncio, 'sleep') as sleep_mock:
is_connected_mock = mock.PropertyMock(
side_effect=[False, False, True])
queue_mock = CoroutineMock(
consume=CoroutineMock(),
connect=CoroutineMock(side_effect=[AioamqpException, True]))
type(queue_mock).is_connected = is_connected_mock
loop = asyncio.get_event_loop()
consumer.queue = queue_mock
loop.run_until_complete(consumer.start())
self.assertEqual(1, queue_mock.connect.await_count)
self.assertEqual(2, queue_mock.consume.await_count)
self.assertEqual([
mock.call(queue_name="asgard/counts"),
mock.call(queue_name="asgard/counts/errors")
], queue_mock.consume.await_args_list)
self.assertEqual(2, sleep_mock.await_count)
async def test_on_queue_error_logs_exception_and_acks_message(self):
"""
Logamos qualquer erro de parsing/validação de mensagem
"""
delivery_tag = 42
body = "not a JSON"
queue_mock = CoroutineMock(ack=CoroutineMock())
consumer = Consumer(self.one_route_fixture, *self.connection_parameters)
with mock.patch.object(conf, "logger") as logger_mock:
await consumer.on_queue_error(body, delivery_tag, "Error: not a JSON", queue_mock)
logger_mock.error.assert_called_with({"exception": "Error: not a JSON",
"original_msg": body,
"parse-error": True})
queue_mock.ack.assert_awaited_once_with(delivery_tag=delivery_tag)
async def test_on_connection_error_logs_exception(self):
"""
Logamos qualquer erro de conexão com o Rabbit, inclusive acesso negado
"""
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
with mock.patch.object(conf, "logger") as logger_mock:
try:
1 / 0
except Exception as e:
await consumer.on_connection_error(e)
logger_mock.error.assert_called_with({
"exc_message": "division by zero",
"exc_traceback": mock.ANY,
})
async def test_consume_all_queues(self):
"""
"""
self.one_route_fixture['route'] = [
"asgard/counts", "asgard/counts/errors"
]
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
queue_mock = CoroutineMock(consume=CoroutineMock())
await consumer.consume_all_queues(queue_mock)
self.assertEqual(2, queue_mock.consume.await_count)
self.assertEqual([
mock.call(queue_name="asgard/counts"),
mock.call(queue_name="asgard/counts/errors")
], queue_mock.consume.await_args_list)
async def test_on_exception_reject_message(self):
@self.app.route(
["queue"],
type=RouteTypes.AMQP_RABBITMQ,
options={Events.ON_EXCEPTION: Actions.REJECT},
)
async def _handler(messages):
raise Exception("BOOM!")
route = self.app.routes_registry.amqp_routes[0]
consumer = Consumer(route, *self.connection_parameters)
with self.assertRaises(Exception):
await consumer.on_queue_message(msg=self.mock_message)
self.mock_message.reject.assert_awaited_once_with(requeue=False)
def test_consumer_adjusts_bulk_size(self):
"""
Se escolhermos um prefetch menor do que o bulk_size, significa que nosso "bucket"
nunca vai encher e isso significa que nosso consumer ficará congelado, em um deadlock:
Ele estará esperando o bucket encher
E ao mesmo tempo estará o esperando o bucket esvaziar para que possa receber mais mensagens do RabbitMQ
Vamos setar o bulk_size com sendo min(bulk_size, prefetch_count)
"""
self.one_route_fixture["options"]["bulk_size"] = 2048
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
self.assertEqual(1024, consumer.bucket.size)
self.one_route_fixture["options"]["bulk_size"] = 4096
consumer = Consumer(
self.one_route_fixture,
host="127.0.0.1",
username="******",
password="******",
prefetch_count=8192,
)
self.assertEqual(4096, consumer.bucket.size)
async def test_do_not_flush_if_bucket_is_already_empty_when_timeout_expires(
self
):
class MyBucket(Bucket):
def pop_all(self):
global items
items = self._items
self._items = []
return items
handler_mock = CoroutineMock()
self.one_route_fixture["handler"] = handler_mock
self.one_route_fixture["options"]["bulk_size"] = 3
consumer = Consumer(
self.one_route_fixture,
*self.connection_parameters,
bucket_class=MyBucket,
)
self.loop.create_task(consumer._flush_clocked())
# Realizando sleep para devolver o loop para o clock
await asyncio.sleep(0.1)
self.assertEqual(0, handler_mock.await_count)
async def test_on_message_handle_error_logs_exception(self):
"""
Logamos a exception lançada pelo handler.
Aqui o try/except serve apenas para termos uma exception real, com traceback.
"""
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
with mock.patch.object(conf, "logger") as logger_mock:
try:
1 / 0
except Exception as e:
await consumer.on_message_handle_error(e)
logger_mock.error.assert_called_with({
"exc_message": "division by zero",
"exc_traceback": mock.ANY,
})
async def test_on_queue_message_auto_ack_on_success(self):
"""
Se o handler registrado no @app.route() rodar com sucesso,
devemos fazer o ack da mensagem
"""
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
expected_body = {"key": "value"}
message_mock = RabbitMQMessage(body=expected_body, delivery_tag=10)
result = await consumer.on_queue_message(expected_body,
delivery_tag=10,
queue=self.queue_mock)
self.assertEqual((42, expected_body), result)
self.queue_mock.ack.assert_awaited_once_with(delivery_tag=10)
self.queue_mock.reject.assert_not_awaited()
async def test_on_queue_message_rejects_on_exception(self):
"""
Se o handler der raise em qualquer exception, devemos
dar reject() na mensagem
"""
async def exception_handler(message):
return message.do_not_exist
self.one_route_fixture['handler'] = exception_handler
consumer = Consumer(self.one_route_fixture, *self.connection_parameters)
queue_mock = CoroutineMock(ack=CoroutineMock(), reject=CoroutineMock())
with self.assertRaises(AttributeError):
await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=queue_mock)
queue_mock.reject.assert_awaited_once_with(delivery_tag=10, requeue=True)
queue_mock.ack.assert_not_awaited
async def test_on_exception_requeue_message(self):
"""
Confirma que em caso de exceção, será feito `message.reject(requeue=True)`
"""
@self.app.route(["queue"],
options={Events.ON_EXCEPTION: Actions.REQUEUE})
async def _handler(messages):
raise Exception("BOOM!")
consumer = Consumer(self.app.routes_registry[_handler],
*self.connection_parameters)
with self.assertRaises(Exception):
await consumer.on_queue_message({"key": 42},
delivery_tag=10,
queue=self.queue_mock)
self.queue_mock.reject.assert_awaited_with(delivery_tag=10,
requeue=True)
async def test_on_exception_requeue_message(self):
"""
Confirma que em caso de exceção, será feito `message.reject(requeue=True)`
"""
@self.app.route(
["queue"],
type=RouteTypes.AMQP_RABBITMQ,
options={Events.ON_EXCEPTION: Actions.REQUEUE},
)
async def _handler(messages):
raise Exception("BOOM!")
route = self.app.routes_registry.amqp_routes[0]
consumer = Consumer(route, *self.connection_parameters)
with self.assertRaises(Exception):
await consumer.on_queue_message(msg=self.mock_message)
self.mock_message.reject.assert_awaited_once_with(requeue=True)
async def test_consume_all_queues(self):
"""
"""
self.one_route_fixture["routes"] = [
"asgard/counts",
"asgard/counts/errors",
]
consumer = Consumer(self.one_route_fixture, *self.connection_parameters)
queue_mock = CoroutineMock(consume=CoroutineMock())
await consumer.consume_all_queues(queue_mock)
queue_mock.consume.assert_has_awaits(
[
mock.call(queue_name="asgard/counts", delegate=consumer),
mock.call(queue_name="asgard/counts/errors", delegate=consumer),
],
any_order=True,
)
async def test_on_queue_message_bulk_size_one(self):
class MyBucket(Bucket):
def pop_all(self):
return self._items;
handler_mock = CoroutineMock()
self.one_route_fixture['handler'] = handler_mock
self.one_route_fixture['options']['bulk_size'] = 1
consumer = Consumer(self.one_route_fixture, *self.connection_parameters, bucket_class=MyBucket)
queue_mock = CoroutineMock(ack=CoroutineMock())
await consumer.on_queue_message({"key": "value"}, delivery_tag=20, queue=queue_mock)
handler_mock.assert_awaited_once_with(consumer.bucket._items)
self.assertEqual([mock.call(delivery_tag=20)], queue_mock.ack.await_args_list)
self.assertEqual(0, queue_mock.reject.call_count)
async def test_on_queue_message_bulk_mixed_ack_and_reject(self):
async def handler(messages):
messages[1].reject()
messages[2].reject()
self.one_route_fixture['handler'] = handler
self.one_route_fixture['options']['bulk_size'] = 5
consumer = Consumer(self.one_route_fixture, *self.connection_parameters)
queue_mock = CoroutineMock(ack=CoroutineMock(), reject=CoroutineMock())
await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=queue_mock)
await consumer.on_queue_message({"key": "value"}, delivery_tag=11, queue=queue_mock)
await consumer.on_queue_message({"key": "value"}, delivery_tag=12, queue=queue_mock)
await consumer.on_queue_message({"key": "value"}, delivery_tag=13, queue=queue_mock)
await consumer.on_queue_message({"key": "value"}, delivery_tag=14, queue=queue_mock)
self.assertEqual([mock.call(delivery_tag=10), mock.call(delivery_tag=13), mock.call(delivery_tag=14)], queue_mock.ack.await_args_list)
self.assertEqual([mock.call(delivery_tag=11, requeue=True), mock.call(delivery_tag=12, requeue=True)], queue_mock.reject.await_args_list)
async def test_on_exception_default_action_bulk_messages(self):
"""
Se o handler der raise em qualquer exception, devemos
dar reject() na mensagem
"""
async def exception_handler(message):
return message.do_not_exist
self.one_route_fixture['handler'] = exception_handler
self.one_route_fixture['options']['bulk_size'] = 2
consumer = Consumer(self.one_route_fixture, *self.connection_parameters)
queue_mock = CoroutineMock(ack=CoroutineMock(), reject=CoroutineMock())
with self.assertRaises(AttributeError):
await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=queue_mock)
await consumer.on_queue_message({"key": "value"}, delivery_tag=11, queue=queue_mock)
self.assertCountEqual([mock.call(delivery_tag=10, requeue=True), mock.call(delivery_tag=11, requeue=True)], queue_mock.reject.await_args_list)
queue_mock.ack.assert_not_awaited
async def test_on_queue_message_bulk_size_one(self):
class MyBucket(Bucket):
def pop_all(self):
return self._items
handler_mock = CoroutineMock(__name__="handler")
self.one_route_fixture["handler"] = handler_mock
self.one_route_fixture["options"]["bulk_size"] = 1
consumer = Consumer(
self.one_route_fixture,
*self.connection_parameters,
bucket_class=MyBucket,
)
msg = self._make_msg(delivery_tag=20)
await consumer.on_queue_message(msg=msg)
handler_mock.assert_awaited_once_with(consumer.bucket._items)
msg.ack.assert_awaited_once()
msg.reject.assert_not_awaited()
async def test_restart_all_consumers_if_channel_is_closed(self):
"""
Se detectamos que o channel está fechado, devemos reiniciar todos os
consumers. Isso vale pois atualmente todos eles compartilham o mesmo channel.
"""
self.one_route_fixture["routes"] = [
"asgard/counts",
"asgard/counts/errors",
]
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
queue_mock = CoroutineMock(consume=CoroutineMock())
with asynctest.patch.object(
consumer, "queue", queue_mock), unittest.mock.patch.object(
consumer,
"keep_runnig",
side_effect=[
True, True, True, False
]), asynctest.patch.object(
asyncio,
"sleep") as sleep_mock, asynctest.patch.object(
consumer, "clock_task",
side_effect=[True, True]), asynctest.patch.object(
consumer.queue.connection,
"has_channel_ready",
Mock(side_effect=[False, True, False]),
):
await consumer.start()
queue_mock.consume.assert_has_awaits(
[
mock.call(queue_name="asgard/counts", delegate=consumer),
mock.call(queue_name="asgard/counts/errors",
delegate=consumer),
mock.call(queue_name="asgard/counts", delegate=consumer),
mock.call(queue_name="asgard/counts/errors",
delegate=consumer),
],
any_order=True,
)
async def test_on_queue_message_bulk_mixed_ack_and_reject_on_success_reject(
self):
self.maxDiff = None
async def handler(messages):
messages[1].reject(requeue=True)
messages[2].reject(requeue=True)
self.one_route_fixture['handler'] = handler
self.one_route_fixture['options']['bulk_size'] = 5
self.one_route_fixture['options'][Events.ON_SUCCESS] = Actions.REJECT
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
queue_mock = CoroutineMock(ack=CoroutineMock(), reject=CoroutineMock())
await consumer.on_queue_message({"key": "value"},
delivery_tag=10,
queue=queue_mock)
await consumer.on_queue_message({"key": "value"},
delivery_tag=11,
queue=queue_mock)
await consumer.on_queue_message({"key": "value"},
delivery_tag=12,
queue=queue_mock)
await consumer.on_queue_message({"key": "value"},
delivery_tag=13,
queue=queue_mock)
await consumer.on_queue_message({"key": "value"},
delivery_tag=14,
queue=queue_mock)
self.assertCountEqual([
mock.call(delivery_tag=10, requeue=False),
mock.call(delivery_tag=11, requeue=True),
mock.call(delivery_tag=12, requeue=True),
mock.call(delivery_tag=13, requeue=False),
mock.call(delivery_tag=14, requeue=False)
], queue_mock.reject.await_args_list)
async def test_bulk_flushes_on_timeout_even_with_bucket_not_full(self):
"""
Se nosso bucket não chegar ao total de mensagens do nosso bulk_size, temos
que fazer flush de tempos em tempos, senão poderemos ficar eternamente com mensagens presas.
"""
handler_mock = CoroutineMock()
self.one_route_fixture['handler'] = handler_mock
self.one_route_fixture['options']['bulk_size'] = 5
self.one_route_fixture['options']['bulk_flush_interval'] = 3
consumer = Consumer(self.one_route_fixture, *self.connection_parameters)
queue_mock = CoroutineMock(ack=CoroutineMock(), reject=CoroutineMock())
await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=queue_mock)
await consumer.on_queue_message({"key": "value"}, delivery_tag=11, queue=queue_mock)
#await consumer.on_queue_message({"key": "value"}, delivery_tag=12, queue=queue_mock)
#await consumer.on_queue_message({"key": "value"}, delivery_tag=13, queue=queue_mock)
#await consumer.on_queue_message({"key": "value"}, delivery_tag=14, queue=queue_mock)
await asyncio.sleep(4)
#handler_mock.assert_awaited_once
self.assertEqual(1, handler_mock.await_count)
self.assertEqual([mock.call(delivery_tag=10), mock.call(delivery_tag=11)], queue_mock.ack.await_args_list)
async def test_on_queue_message_bulk_mixed_ack_and_reject_on_success_reject(
self):
self.maxDiff = None
async def handler(messages):
messages[1].reject(requeue=True)
messages[2].reject(requeue=True)
self.one_route_fixture["handler"] = handler
self.one_route_fixture["options"]["bulk_size"] = 5
self.one_route_fixture["options"][Events.ON_SUCCESS] = Actions.REJECT
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
msgs = [self._make_msg(delivery_tag=i) for i in range(5)]
for msg in msgs:
await consumer.on_queue_message(msg)
msgs[0].reject.assert_awaited_once_with(requeue=False)
msgs[1].reject.assert_awaited_once_with(requeue=True)
msgs[2].reject.assert_awaited_once_with(requeue=True)
msgs[3].reject.assert_awaited_once_with(requeue=False)
msgs[4].reject.assert_awaited_once_with(requeue=False)
async def test_on_exception_default_action_bulk_messages(self):
"""
Se o handler der raise em qualquer exception, devemos
dar reject() na mensagem
"""
async def exception_handler(message):
return message.do_not_exist
self.one_route_fixture["handler"] = exception_handler
self.one_route_fixture["options"]["bulk_size"] = 2
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
msgs = [self._make_msg(delivery_tag=1), self._make_msg(delivery_tag=2)]
with self.assertRaises(AttributeError):
await consumer.on_queue_message(msg=msgs[0])
await consumer.on_queue_message(msg=msgs[1])
msgs[0].reject.assert_awaited_once_with(requeue=True)
msgs[1].reject.assert_awaited_once_with(requeue=True)
msgs[0].ack.assert_not_awaited()
msgs[1].ack.assert_not_awaited()
async def test_on_queue_message_calls_inner_handler(self):
consumer = Consumer(self.one_route_fixture, *self.connection_parameters)
result = await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=self.queue_mock)
self.assertEqual((42, {"key": "value"}), result)
async def test_on_queue_message_calls_inner_handler(self):
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
result = await consumer.on_queue_message(msg=self.mock_message)
self.assertEqual((42, self.mock_message.deserialized_data), result)
def test_consumer_returns_correct_queue_name(self):
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
self.assertEqual(["/asgard/counts/ok"], consumer.queue_name)
def test_consumer_instantiate_correct_size_bucket(self):
consumer = Consumer(self.one_route_fixture,
*self.connection_parameters)
self.assertEqual(self.one_route_fixture["options"]["bulk_size"],
consumer.bucket.size)
def test_consumer_instantiate_using_bucket_class(self):
class MyBucket(Bucket):
pass
consumer = Consumer(self.one_route_fixture, *self.connection_parameters, bucket_class=MyBucket)
self.assertTrue(isinstance(consumer.bucket, MyBucket))
def test_return_correct_queue_name(self):
"""
consumer.quene_name deve retornar o nome da fila que está sendo consumida.
"""
consumer = Consumer(self.one_route_fixture, *self.connection_parameters)
self.assertEquals(self.one_route_fixture['route'], consumer.queue_name)
async def test_on_queue_message_precondition_failed_on_ack(self):
consumer = Consumer(self.one_route_fixture, *self.connection_parameters)
self.mock_message.ack.side_effect = AioamqpException
with self.assertRaises(AioamqpException):
await consumer.on_queue_message(msg=self.mock_message)
async def test_on_queue_message_precondition_failed_on_ack(self):
consumer = Consumer(self.one_route_fixture, *self.connection_parameters)
queue_mock = CoroutineMock(ack=CoroutineMock(side_effect=AioamqpException))
with self.assertRaises(AioamqpException):
await consumer.on_queue_message({"key": "value"}, delivery_tag=10, queue=queue_mock)
