def test_send_messages_through_a_fifo_queue_and_make_sure_they_are_always_ordered(
backend, request):
def delete_queue():
backend.delete_queue(queue)
request.addfinalizer(delete_queue)
queue_name = "test-queue-{}.fifo".format(uuid.uuid4())
queue, _ = backend.declare_queue(queue_name)
message_group_id = "my-app"
for i in range(100):
message_dedup_id = str(uuid.uuid4())
backend.publish_to_queue(
Message.create(f"my-message-{i}", "my-event-type", 40),
queue,
message_group_id=message_group_id,
message_deduplication_id=message_dedup_id,
)
received_messages = []
retrieved_messages = backend.retrieve_messages(queue)
while retrieved_messages:
for m in retrieved_messages:
received_messages.append(m.content)
backend.acknowledge(m)
retrieved_messages = backend.retrieve_messages(queue)
expected_array = [f"my-message-{i}" for i in range(100)]
assert_that(received_messages, contains_exactly(*expected_array))
def test_send_messages_through_a_non_fifo_queue_does_not_guarantee_order(
backend, request):
def delete_queue():
backend.delete_queue(queue)
request.addfinalizer(delete_queue)
queue_name = "test-queue-{}".format(uuid.uuid4())
queue, _ = backend.declare_queue(queue_name)
for i in range(100):
backend.publish_to_queue(
Message.create(f"my-message-{i}", "my-event-type", 40), queue)
received_messages = []
retrieved_messages = backend.retrieve_messages(queue)
while retrieved_messages:
for m in retrieved_messages:
received_messages.append(m.content)
backend.acknowledge(m)
retrieved_messages = backend.retrieve_messages(queue)
expected_array = [f"my-message-{i}" for i in range(100)]
assert_that(received_messages, contains_inanyorder(*expected_array))
def test_backend_declare_a_queue_for_a_topic_filtering_the_events_it_sends_to_certain_queues(
backend, topic):
try:
queue_1, _ = backend.declare_queue("dev-queue-{}".format(uuid.uuid4()),
topic,
filter_events=["MyBananaEvent"])
queue_2, _ = backend.declare_queue(
"dev-queue-{}".format(uuid.uuid4()),
topic,
filter_events=["MyNonBananaEvent"],
)
queue_3, _ = backend.declare_queue("dev-queue-{}".format(uuid.uuid4()),
topic)
backend.publish_to_topic(
Message.create(
json.dumps({
"event_type_name": "MyBananaEvent",
"value": 3
}),
"MyBananaEvent",
40,
),
topic,
)
backend.publish_to_topic(
Message.create(
json.dumps({
"event_type_name": "MyNonBananaEvent",
"value": 5
}),
"MyNonBananaEvent",
40,
),
topic,
)
time.sleep(2)
_assert_messages(backend, queue_3,
["MyNonBananaEvent", "MyBananaEvent"], 2)
_assert_messages(backend, queue_1, ["MyBananaEvent"], 1)
_assert_messages(backend, queue_2, ["MyNonBananaEvent"], 1)
finally:
backend.delete_queue(queue_1)
backend.delete_queue(queue_2)
backend.delete_queue(queue_3)
def retrieve_messages(self, queue):
method_frame, properties, body = self.channel.basic_get(queue=queue,
no_ack=False)
if not method_frame:
return []
message = Message(
message_id=str(method_frame.delivery_tag),
content=json.loads(body.decode("utf-8")),
metadata=properties,
)
return [message]
def _construct_message(self, message: Any) -> Message:
body = message.body
manifest: Optional[str] = None
serializer_id: Optional[int] = None
try:
message_content = json.loads(body)
if "Message" in message_content:
content = message_content["Message"]
# Does the content have more attributes? If so, it is very likely that the message came from a non-raw
# SNS redirection
if "MessageAttributes" in message_content:
manifest = (
message_content["MessageAttributes"]
.get("manifest", {})
.get("Value")
or ""
)
serializer_id = (
message_content["MessageAttributes"]
.get("serializer_id", {})
.get("Value")
)
else:
content = message_content
except JSONDecodeError:
content = body
try:
manifest = manifest or message.message_attributes.get("manifest", {}).get(
"StringValue"
)
serializer_id = int(
serializer_id
or message.message_attributes.get("serializer_id", {}).get(
"StringValue"
)
)
except Exception:
manifest = None
serializer_id = None
return Message(message.message_id, content, message, serializer_id, manifest)
def test_if_a_consumer_raises_an_exception_the_message_is_not_acknowledged(
self):
serializer = SerializerStub()
serialized_event = serializer.serialize(BananaHappened("apple"))
messages = [
Message.create(serialized_event, None, serializer.identifier())
for _ in range(2)
]
backend = a_backend_with_messages(messages)
consumer = ExceptionaleConsumer()
registry = SerializerRegistry(serializer_settings)
sut = SimpleMessageDispatcher(consumer,
serializer_registry=registry,
backend=backend)
sut.consume_event("queue")
assert_that(backend.acknowledge, never(called()))
def _dispatch_message(self, message: Message) -> None:
manifest = message.get_message_manifest()
# If the message cannot be deserialized, just ignore it.
# ACK it anyway to avoid hanging on the same message over an over again
try:
if message.serializer_id is not None:
message_data = self.serializer_registry.deserialize_with_serializerid(
message.content, message.serializer_id, manifest=manifest)
else:
# TODO: If no serializerid is supplied, at least we need to grab some
# clue in the message to know which serializer must be used.
# for now, rely on the default.
message_data = self.serializer_registry.deserialize_with_class(
message.content, object, manifest=manifest)
except SerializationError as e:
logger.error(e)
self._backend.acknowledge(message)
return
consumers = self._get_consumers(message_data)
successful = 0
for consumer in consumers:
try:
# Store into the cache
message_key = (
f"{get_fully_qualified_name(consumer)}.{message.message_id}"
)
if message_key in self.cache:
logger.info("detected a duplicated message, ignoring")
else:
consumer.process(message_data,
message_id=message.message_id)
successful += 1
self.cache.store(message_key, message_key)
except Exception as e:
logger.exception(e)
if self.always_ack or successful == len(consumers):
self._backend.acknowledge(message)
def test_consume_on_queue_but_no_consumer_interested_in_the_messages(self):
serializer = SerializerStub()
serialized_event = serializer.serialize(BananaHappened("apple"))
messages = [
Message.create(serialized_event, None, serializer.identifier())
for _ in range(2)
]
backend = a_backend_with_messages(messages)
consumer = ProxySpy(NoBananaConsumer())
registry = SerializerRegistry(serializer_settings)
sut = SimpleMessageDispatcher(consumer,
serializer_registry=registry,
backend=backend)
sut.consume_event("queue")
assert_that(consumer.process, never(called()))
assert_that(backend.acknowledge, called().times(2))
def test_not_acknowledging_any_of_the_messages_on_a_fifo_queue_will_delay_the_delivery_of_the_rest(
backend, request):
def delete_queue():
backend.delete_queue(queue)
request.addfinalizer(delete_queue)
queue_name = "test-queue-{}.fifo".format(uuid.uuid4())
queue, _ = backend.declare_queue(queue_name)
message_group_id = "my-app"
for i in range(20):
message_dedup_id = str(uuid.uuid4())
backend.publish_to_queue(
Message.create(f"my-message-{i}", "my-event-type", 40),
queue,
message_group_id=message_group_id,
message_deduplication_id=message_dedup_id,
)
received_messages = []
retrieved_messages = backend.retrieve_messages(queue)
num_acks = 0
while num_acks < 20:
for i, m in enumerate(retrieved_messages):
if i == 5:
break
else:
received_messages.append(m.content)
backend.acknowledge(m)
num_acks += 1
retrieved_messages = backend.retrieve_messages(queue)
expected_array = [f"my-message-{i}" for i in range(20)]
assert_that(received_messages, contains_exactly(*expected_array))