def test_message_without_retry_dead_letter(settings, mocker, caplog):
settings.CQRS['replica']['CQRS_MAX_RETRIES'] = 1
produce_message = mocker.patch(
'dj_cqrs.transport.rabbit_mq.RabbitMQTransport._produce_message', )
channel = mocker.MagicMock()
payload = TransportPayload(
SignalType.SAVE,
'basic',
{'id': 1},
1,
correlation_id='abc',
retries=2,
)
delay_queue = DelayQueue()
PublicRabbitMQTransport.fail_message(channel, 1, payload, None,
delay_queue)
assert delay_queue.qsize() == 0
assert channel.basic_nack.call_count == 1
assert produce_message.call_count == 1
produce_payload = produce_message.call_args[0][2]
assert produce_payload is payload
assert getattr(produce_message, 'is_dead_letter', False)
assert 'CQRS is failed: pk = 1 (basic), correlation_id = abc, retries = 2.' in caplog.text
assert (
'CQRS is added to dead letter queue: pk = 1 (basic), correlation_id = abc'
in caplog.text)
def test_fail_message_with_retry(mocker):
payload = TransportPayload(SignalType.SAVE, 'basic', {'id': 1}, 1)
delay_queue = DelayQueue()
PublicRabbitMQTransport.fail_message(mocker.MagicMock(), 100, payload,
None, delay_queue)
assert delay_queue.qsize() == 1
delay_message = delay_queue.get()
assert delay_message.delivery_tag == 100
assert delay_message.payload is payload
def consume(cls):
consumer_rabbit_settings = cls._get_consumer_settings()
common_rabbit_settings = cls._get_common_settings()
while True:
connection = None
try:
delay_queue = DelayQueue(max_size=get_delay_queue_max_size())
connection, channel, consumer_generator = cls._get_consumer_rmq_objects(
*(common_rabbit_settings + consumer_rabbit_settings), )
for method_frame, properties, body in consumer_generator:
if method_frame is not None:
cls._consume_message(
channel,
method_frame,
properties,
body,
delay_queue,
)
cls._process_delay_messages(channel, delay_queue)
except (exceptions.AMQPError, exceptions.ChannelError,
exceptions.ReentrancyError, gaierror):
logger.error('AMQP connection error. Reconnecting...',
exc_info=True)
time.sleep(cls.CONSUMER_RETRY_TIMEOUT)
finally:
if connection and not connection.is_closed:
connection.close()
def test_delay_queue_get_ready(mocker):
fake_put_now = datetime(2020, 1, 1, second=0, tzinfo=timezone.utc)
mocker.patch('django.utils.timezone.now', return_value=fake_put_now)
delay_queue = DelayQueue()
delay_messages = []
for delay in (1, 0, 3600, 2):
eta = fake_put_now + timedelta(seconds=delay)
delay_message = DelayMessage(None, None, eta)
delay_queue.put(delay_message)
delay_messages.append(delay_message)
mocker.stopall()
fake_get_ready_now = datetime(2020, 1, 1, second=3, tzinfo=timezone.utc)
mocker.patch('django.utils.timezone.now', return_value=fake_get_ready_now)
ready_messages = list(delay_queue.get_ready())
assert len(ready_messages) == 3
sorted_expected = sorted(delay_messages, key=lambda x: x.eta)
expected_not_ready = sorted_expected.pop()
for expected, result in zip(sorted_expected, ready_messages):
assert expected is result
assert delay_queue.qsize() == 1
result_message = delay_queue.get()
assert result_message is expected_not_ready
def test_fail_message_invalid_model(mocker, caplog):
nack = mocker.patch(
'dj_cqrs.transport.rabbit_mq.RabbitMQTransport._nack', )
payload = TransportPayload(SignalType.SAVE, 'not_existing', {'id': 1}, 1)
delay_queue = DelayQueue()
delivery_tag = 101
PublicRabbitMQTransport.fail_message(
mocker.MagicMock(),
delivery_tag,
payload,
None,
delay_queue,
)
assert delay_queue.qsize() == 0
assert nack.call_count == 1
assert nack.call_args[0][1] == delivery_tag
assert 'Model for cqrs_id not_existing is not found.' in caplog.text
def test_delay_queue_put_full():
eta = datetime(2020, 1, 1, second=0, tzinfo=timezone.utc)
delay_queue = DelayQueue(max_size=1)
delay_queue.put(
DelayMessage(1, None, eta),
)
with pytest.raises(Full):
delay_queue.put(
DelayMessage(2, None, eta),
)
assert delay_queue.qsize() == 1
assert delay_queue.get().delivery_tag == 1
def test_process_delay_messages(mocker, caplog):
channel = mocker.MagicMock()
produce = mocker.patch(
'dj_cqrs.transport.rabbit_mq.RabbitMQTransport.produce')
payload = TransportPayload(SignalType.SAVE, 'CQRS_ID', {'id': 1}, 1)
delay_queue = DelayQueue()
delay_queue.put(
DelayMessage(delivery_tag=1,
payload=payload,
eta=datetime.now(tz=timezone.utc)), )
PublicRabbitMQTransport.process_delay_messages(channel, delay_queue)
assert delay_queue.qsize() == 0
assert channel.basic_nack.call_count == 1
assert produce.call_count == 1
produce_payload = produce.call_args[0][0]
assert produce_payload is payload
assert produce_payload.retries == 1
assert getattr(produce_payload, 'is_requeue', False)
assert 'CQRS is requeued: pk = 1 (CQRS_ID)' in caplog.text
def _get_delay_queue(cls):
replica_settings = settings.CQRS.get('replica', {})
max_size = DEFAULT_DELAY_QUEUE_MAX_SIZE
if 'delay_queue_max_size' in replica_settings:
max_size = replica_settings['delay_queue_max_size']
if max_size is not None and max_size <= 0:
logger.warning(
"Settings delay_queue_max_size=%s is invalid, using default %s.",
max_size,
DEFAULT_DELAY_QUEUE_MAX_SIZE,
)
max_size = DEFAULT_DELAY_QUEUE_MAX_SIZE
return DelayQueue(max_size)
def test_delay_queue_put_same_eta():
eta = datetime(2020, 1, 1, second=0, tzinfo=timezone.utc)
delay_messages = [DelayMessage(delivery_tag, None, eta) for delivery_tag in range(10)]
delay_queue = DelayQueue()
for delay_message in delay_messages:
delay_queue.put(delay_message)
assert delay_queue.qsize() == 10
assert delay_queue.get()
def test_delay_queue_put():
fake_now = datetime(2020, 1, 1, second=0, tzinfo=timezone.utc)
delay_message = DelayMessage(1, {'test': 'data'}, fake_now)
delay_queue = DelayQueue()
delay_queue.put(delay_message)
assert delay_queue.qsize() == 1
result_message = delay_queue.get()
assert result_message is delay_message
def test_delay_message_with_requeue(mocker, caplog):
channel = mocker.MagicMock()
requeue_message = mocker.patch(
'dj_cqrs.transport.rabbit_mq.RabbitMQTransport._requeue_message', )
delay_messages = []
for delay in (2, 1, 3):
payload = TransportPayload(SignalType.SAVE, 'CQRS_ID', {'id': delay},
delay)
eta = datetime.now(tz=timezone.utc) + timedelta(hours=delay)
delay_message = DelayMessage(delivery_tag=delay,
payload=payload,
eta=eta)
delay_messages.append(delay_message)
delay_queue = DelayQueue(max_size=3)
for delay_message in delay_messages:
delay_queue.put(delay_message)
exceeding_delay = 0
exceeding_payload = TransportPayload(SignalType.SAVE, 'CQRS_ID', {'id': 4},
4)
PublicRabbitMQTransport.delay_message(
channel,
4,
exceeding_payload,
exceeding_delay,
delay_queue,
)
assert delay_queue.qsize() == 3
assert delay_queue.get().payload is exceeding_payload
assert (
'CQRS is delayed: pk = 4 (CQRS_ID), correlation_id = None, delay = 0 sec'
in caplog.text)
assert requeue_message.call_count == 1
requeue_payload = requeue_message.call_args[0][2]
min_eta_delay_message = sorted(delay_messages, key=lambda x: x.eta)[0]
assert requeue_payload is min_eta_delay_message.payload
def test_delay_queue_invalid_max_size():
with pytest.raises(AssertionError) as e:
DelayQueue(max_size=0)
assert e.value.args[0] == "Delay queue max_size should be positive integer."