def test_consume_same_consume_key(self):
"""
Verify that calling consume once more with the same queue name,
exchange name and routing key will result in an exception.
"""
self.consumer.consume(
ConsumeParams(lambda _: ...), QueueParams("queue")
)
# Assertions
with self.assertRaises(ValueError):
self.consumer.consume(
ConsumeParams(lambda _: ...), QueueParams("queue")
)
def test_on_msg(self):
"""
Verify on_msg calls result in a call to the consumer's function as
well as long as a matching consumer tag entry is found.
"""
# Prep
on_message_callback_called = False
def on_message_callback(msg):
if not msg == b'body' or isinstance(msg, ConsumeOK):
return
nonlocal on_message_callback_called
on_message_callback_called = True
queue_params = QueueParams("queue")
self.consumer.consume(ConsumeParams(on_message_callback),
queue_params=queue_params)
frame_mock = Mock()
frame_mock.method.consumer_tag = "123"
self.consumer.on_consume_ok(frame_mock, queue_params=queue_params)
deliver_mock = Mock()
deliver_mock.consumer_tag = "123"
# Run test
self.consumer.on_msg(Mock(), deliver_mock, Mock(), b"body")
# Assertions
self.assertTrue(on_message_callback_called)
def test_on_consume_ok(self):
"""
Verify that the correct actions follows a consume OK.
"""
# Prep
on_message_callback_called = False
def on_message_callback(msg):
if not isinstance(msg, ConsumeOK):
return
nonlocal on_message_callback_called
on_message_callback_called = True
consume_params = ConsumeParams(on_message_callback)
queue_params = QueueParams("queue")
frame = Mock()
self.consumer.consume(consume_params, queue_params=queue_params)
# Run test
self.consumer.on_consume_ok(frame,
queue_params=queue_params)
# Assertions
consume_instance = self.consumer._consumes[
_gen_consume_key(queue=queue_params.queue)
]
self.assertEqual(consume_instance.consumer_tag,
frame.method.consumer_tag)
self.assertTrue(on_message_callback_called) # Called with ConsumeOK
def test_consume_neither_queue_nor_exchange_provided(self):
"""
Verify that consume raises an exception if neither queue nor exchange
is provided.
"""
with self.assertRaises(ValueError):
self.consumer.consume(ConsumeParams(lambda _: ...))
def test_on_exchange_declared(self):
"""
Verify that the correct action follows a successful exchange
declaration.
"""
# Prep
self.consumer.bind_queue = Mock()
exchange_params = ExchangeParams("exchange")
consume_params = ConsumeParams(lambda _: ...)
queue_params = QueueParams("queue")
# Run test
self.consumer.on_exchange_declared(exchange_params,
None,
consume_params=consume_params,
queue_params=queue_params)
# Assertions
call_args = self.consumer.bind_queue.call_args
queue_bind_params = call_args.args[0]
self.consumer.bind_queue.assert_called_with(
queue_bind_params, callback=ANY
)
self.assertEqual(queue_bind_params.queue, consume_params.queue)
self.assertEqual(queue_bind_params.exchange, exchange_params.exchange)
self.assertEqual(queue_bind_params.routing_key, None)
def test_consume_from_exchange_and_queue(self):
"""
Verify RMQConsumer can consume from an exchange, binding it to a
specific queue.
"""
msg_received = None
event = threading.Event()
def on_msg(msg):
event.set()
nonlocal msg_received
msg_received = msg
# Consume
consume_key = self.consumer.consume(
ConsumeParams(on_msg),
exchange_params=ExchangeParams("exchange_fanout",
exchange_type=ExchangeType.fanout),
queue_params=QueueParams("queue_fanout_receiver"))
self.assertEqual(consume_key, "queue_fanout_receiver|exchange_fanout")
# Wait for ConsumeOK
event.wait(timeout=1.0)
self.assertTrue(isinstance(msg_received, ConsumeOK))
# Send a message and verify it is delivered OK
event.clear()
self.producer.publish(b"body",
exchange_params=ExchangeParams(
"exchange_fanout",
exchange_type=ExchangeType.fanout))
event.wait(timeout=1.0)
self.assertEqual(msg_received, b"body")
def test_consume_queue_exchange_and_routing_key(self):
"""
Verify possibility to provide both an exchange, routing_key and a queue
to the consume method.
"""
# Prep
def on_msg(): pass
consume = ConsumeParams(on_msg)
queue = QueueParams("queue")
exchange = ExchangeParams("exchange")
# Run test
self.consumer.consume(consume,
queue_params=queue,
exchange_params=exchange,
routing_key="routing_key")
# Assertions
consume_key = _gen_consume_key(queue=queue.queue,
exchange=exchange.exchange,
routing_key="routing_key")
consume_instance = self.consumer._consumes[consume_key]
self.assertEqual(consume_instance.consume_params, consume)
self.assertEqual(consume_instance.queue_params, queue)
self.assertEqual(consume_instance.exchange_params, exchange)
self.assertEqual(consume_instance.routing_key, "routing_key")
self.consumer.declare_queue.assert_called_with(queue, callback=ANY)
def test_consume_from_queue(self):
"""Verify RMQConsumer can consume from a queue."""
msg_received = None
event = threading.Event()
def on_msg(msg):
event.set()
nonlocal msg_received
msg_received = msg
# Consume
consume_key = self.consumer.consume(ConsumeParams(on_msg),
queue_params=QueueParams("queue"))
self.assertEqual(consume_key, "queue")
# Wait for ConsumeOK
event.wait(timeout=1.0)
self.assertTrue(isinstance(msg_received, ConsumeOK))
# Send a message and verify it is delivered OK
event.clear() # clear to re-use event
self.producer.publish(b"body", queue_params=QueueParams("queue"))
event.wait(timeout=1.0)
self.assertEqual(msg_received, b"body")
def test_consume_from_queue_wo_override(self):
"""
Verify consuming from a queue without the callback override.
"""
# Prep
def consume_on_msg():
...
consume_params = ConsumeParams(consume_on_msg)
def on_consume_ok():
...
# Test
self.conn_imp.basic_consume(consume_params, callback=on_consume_ok)
# Assert
self.conn_imp._channel.basic_consume.assert_called_with(
consume_params.queue,
consume_params.on_message_callback,
auto_ack=consume_params.auto_ack,
exclusive=consume_params.exclusive,
consumer_tag=consume_params.consumer_tag,
arguments=consume_params.arguments,
callback=on_consume_ok)
def test_consume_from_direct_exchange(self):
"""
Verify RMQConsumer can consume from a direct exchange and routing key.
"""
msg_received = None
event = threading.Event()
def on_msg(msg):
event.set()
nonlocal msg_received
msg_received = msg
# Consume
consume_key = self.consumer.consume(
ConsumeParams(on_msg),
exchange_params=ExchangeParams("exchange"),
routing_key="bla.bla")
self.assertEqual(consume_key, "exchange|bla.bla")
# Wait for ConsumeOK
event.wait(timeout=1.0)
self.assertTrue(isinstance(msg_received, ConsumeOK))
# Send a message and verify it is delivered OK
event.clear()
self.producer.publish(b"body",
exchange_params=ExchangeParams("exchange"),
routing_key="bla.bla")
event.wait(timeout=1.0)
self.assertEqual(msg_received, b"body")
def set_up_confirmed_consume(self) -> str:
"""Helper that sets up queue-only confirmed consume."""
queue_params = QueueParams("queue")
self.consumer.consume(ConsumeParams(lambda _: ...),
queue_params=queue_params)
frame_mock = Mock()
frame_mock.method.consumer_tag = "123"
self.consumer.on_consume_ok(frame_mock, queue_params=queue_params)
return "123"
def test_on_close_marks_consumes_stopped(self):
"""Verify a call to on_close marks all consumes as stopped."""
queue_consume = ConsumeParams(lambda _: ...)
queue = QueueParams("queue")
self.consumer.consume(queue_consume, queue_params=queue)
exchange_consume = ConsumeParams(lambda _: ...)
exchange = ExchangeParams("exchange")
self.consumer.consume(exchange_consume, exchange_params=exchange)
# Fake having all consumes started
for _, consume in self.consumer._consumes.items():
consume.consumer_tag = "some tag"
# Run test
self.consumer.on_close()
# Checks
for _, consume in self.consumer._consumes.items():
if consume.consumer_tag == "some tag":
self.fail(
"a consume did not have its consumer tag cleared when "
"on_close was called"
)
def test_on_ready_starts_consumes(self):
"""
Verify that a call to on_ready results in re-issuing of all active
consumes.
"""
# Prep
self.consumer._handle_consume = Mock()
queue_params = QueueParams("queue")
exchange_params = ExchangeParams("exchange")
self.consumer.consume(ConsumeParams(lambda _: ...),
queue_params=queue_params)
self.consumer.consume(ConsumeParams(lambda _: ...),
exchange_params=exchange_params)
# Run test
self.consumer.on_ready()
# Assertions
self.consumer._handle_consume.assert_has_calls(
(call(ANY, queue_params, None, None),
call(ANY, None, exchange_params, None),)
)
def test_consume_queue_only(self):
"""Verify possibility to consume from a queue."""
# Prep
def on_msg(): pass
consume = ConsumeParams(on_msg)
queue = QueueParams("queue")
# Run test
self.consumer.consume(consume, queue_params=queue)
#
# Assertions
consume_key = _gen_consume_key(queue=queue.queue)
consume_instance = self.consumer._consumes[consume_key]
self.assertEqual(consume_instance.consume_params, consume)
self.assertEqual(consume_instance.queue_params, queue)
self.assertEqual(consume_instance.exchange_params, None)
self.consumer.declare_queue.assert_called_with(queue, callback=ANY)
def test_on_consume_ok_callback_raises_exception(self):
"""
Verify that an exception raised from the consume OK notification does
not crash the consumer.
"""
# Prep
def crappy_callback(msg):
# Random exception chosen, ANY exception shall be possible to
# handle.
raise ValueError
queue_params = QueueParams("queue")
self.consumer.consume(ConsumeParams(crappy_callback),
queue_params=queue_params)
# Run test
# Assertion is that no unhandled exception happens :-)
self.consumer.on_consume_ok(Mock(), queue_params=queue_params)
def test_on_queue_declared_no_exchange(self):
"""
Verify that the correct action follows a successful queue declare.
"""
# Prep
self.consumer.basic_consume = Mock()
frame = Mock()
consume_params = ConsumeParams(lambda _: ...)
queue_params = QueueParams("queue")
# Run test
self.consumer.on_queue_declared(frame,
consume_params=consume_params,
queue_params=queue_params)
# Assertions
self.consumer.basic_consume.assert_called_with(
consume_params,
on_message_callback_override=self.consumer.on_msg,
callback=ANY)
def test_on_queue_declared_exchange(self):
"""
Verify that the correct action follows a successful queue declare when
exchange is specified.
"""
# Prep
self.consumer.declare_exchange = Mock()
frame = Mock()
consume_params = ConsumeParams(lambda _: ...)
queue_params = QueueParams("queue")
exchange_params = ExchangeParams("exchange")
# Run test
self.consumer.on_queue_declared(frame,
consume_params=consume_params,
queue_params=queue_params,
exchange_params=exchange_params)
# Assertions
self.assertTrue(isinstance(consume_params.queue, Mock))
self.consumer.declare_exchange.assert_called_with(
exchange_params, callback=ANY)
def test_consume_exchange_only(self):
"""Verify possibility to consume from an exchange."""
# Prep
def on_msg(): pass
consume = ConsumeParams(on_msg)
exchange = ExchangeParams("exchange")
# Run test
self.consumer.consume(consume, exchange_params=exchange)
#
# Assertions
# Verify consume instance exists in consumer instance
consume_key = _gen_consume_key(exchange=exchange.exchange)
consume_instance = self.consumer._consumes[consume_key]
self.assertEqual(consume_instance.consume_params, consume)
self.assertEqual(consume_instance.exchange_params, exchange)
# Always starts with queue declare, can't verify call params since
# queue is autogenerated if not provided, and the other arg is a
# callback constructed from functools.partial.
self.consumer.declare_queue.assert_called()
def test_on_msg_callback_raises_exception(self):
"""
Verify that exceptions raised from the consumer callback as a result of
an on_msg invocation does not lead to an exception in RMQConsumer.
"""
# Prep
def on_message_callback(msg):
if isinstance(msg, ConsumeOK):
return
raise ValueError
self.consumer.consume(ConsumeParams(on_message_callback),
queue_params=QueueParams("queue"))
frame_mock = Mock()
frame_mock.method.consumer_tag = "123"
self.consumer.on_consume_ok(frame_mock,
queue_params=QueueParams("queue"))
deliver_mock = Mock()
deliver_mock.consumer_tag = "123"
# Run test, assertion is that no exception is uncaught
self.consumer.on_msg(Mock(), deliver_mock, Mock(), b"body")
def test_on_queue_bound(self):
"""
Verify that the correct action follows a successful queue bind
operation.
"""
# Prep
self.consumer.basic_consume = Mock()
frame = Mock()
consume_params = ConsumeParams(lambda _: ...)
queue_params = QueueParams("queue")
# Run test
self.consumer.on_queue_bound(frame,
consume_params=consume_params,
queue_params=queue_params,
exchange="exchange",
routing_key=None)
# Assertions
self.consumer.basic_consume.assert_called_with(
consume_params,
on_message_callback_override=self.consumer.on_msg,
callback=ANY
)
logger.addHandler(handler)
consumer = RMQConsumer()
consumer.start()
def stop(*args):
consumer.stop()
sys.exit(0)
signal.signal(signal.SIGINT, stop)
def on_msg(message):
if isinstance(message, ConsumeOK):
print("GOT CONSUME OK")
else:
print(f"GOT MESSAGE: {message}")
consumer.consume(
ConsumeParams(on_msg),
exchange_params=ExchangeParams("direct",
exchange_type=ExchangeType.fanout),
routing_key="rkey"
)
threading.Event().wait()
def ready(self):
"""
Called once the application is ready.
https://docs.djangoproject.com/en/3.1/ref/applications/
"""
# Development specific, due to Django code reload ...
# DEBUG indicates development, RUN_MAIN indicates it's not the
# reloader reaching this block.
if settings.DEBUG and not os.environ.get("RUN_MAIN"):
return
if BrokerConfig.has_started:
return
BrokerConfig.has_started = True
# Set up and start the RabbitMQ client
rmq_client_log_level = logging.INFO
logger = logging.getLogger("rabbitmq_client")
logger.setLevel(rmq_client_log_level)
# Create handler to actually print something
stream_handler = logging.StreamHandler()
logger.addHandler(stream_handler)
formatter = logging.Formatter(fmt="{asctime} {levelname:^8} "
"{name} - {message}",
style="{",
datefmt="%d/%m/%Y %H:%M:%S")
stream_handler.setFormatter(formatter)
stream_handler.setLevel(rmq_client_log_level)
credentials = pika.PlainCredentials(settings.HUME_BROKER_USERNAME,
settings.HUME_BROKER_PASSWORD)
connection_params = pika.ConnectionParameters(
host=settings.HUME_BROKER_IP,
port=settings.HUME_BROKER_PORT,
virtual_host='/',
credentials=credentials)
hint_master_queue_parameters = QueueParams(
settings.MASTER_COMMAND_QUEUE_NAME, durable=True)
consumer = RMQConsumer(connection_parameters=connection_params)
consumer.consume(ConsumeParams(incoming_message),
queue_params=hint_master_queue_parameters)
producer = RMQProducer(connection_parameters=connection_params)
producer_module.init(producer)
consumer.start()
producer.start()
def stop_func(_s, _f, consumer=None, producer=None):
"""
:param _s: signal leading to stop
:param _f: frame when stop was called
:param consumer: rabbitmq_client.RMQConsumer
:param producer: rabbitmq_client.RMQProducer
"""
consumer.stop()
producer.stop()
stop_callback = functools.partial(stop_func,
consumer=consumer,
producer=producer)
signal.signal(signal.SIGINT, stop_callback)
signal.signal(signal.SIGTERM, stop_callback)