def test_consumer_with_connection_issues(self, local_file, s3_mocked,
broker_stage_messages,
event_loop):
total_messages = 4
topic = 'platform.upload.validation'
with FakeMQ(connection_failing_attempt_countdown=1,
disconnect_in_operation=2) as mq:
client = ReconnectingClient(mq, "consumer")
consumer = client.get_callback(app.handle_validation)
for _ in range(total_messages):
message = self._create_message_s3(local_file,
broker_stage_messages,
avoid_produce_queue=True,
topic=topic)
mq.send_and_wait(topic,
json.dumps(message).encode('utf-8'), True)
assert mq.produce_calls_count == total_messages
assert mq.count_topic_messages(topic) == total_messages
assert len(app.produce_queue) == 0
assert mq.consume_calls_count == 0
event_loop.run_until_complete(self.coroutine_test(consumer))
assert mq.consume_calls_count > 0
assert mq.consume_return_messages_count == 1
assert mq.count_topic_messages(topic) == 0
assert mq.disconnect_in_operation_called is True
assert mq.trying_to_connect_failures_calls == 1
assert len(app.produce_queue) == 4
def test_consumer_with_validation_failure(self, local_file, s3_mocked, broker_stage_messages, event_loop):
total_messages = 4
topic = 'platform.upload.validation'
s3_storage.s3.create_bucket(Bucket=s3_storage.REJECT)
produced_messages = []
with FakeMQ() as mq:
client = ReconnectingClient(mq, "consumer")
consumer = client.get_callback(app.handle_validation)
for _ in range(total_messages):
message = self._create_message_s3(
local_file, broker_stage_messages, avoid_produce_queue=True, topic=topic, validation='failure'
)
mq.send_and_wait(topic, json.dumps(message).encode('utf-8'), True)
produced_messages.append(message)
assert mq.produce_calls_count == total_messages
assert mq.count_topic_messages(topic) == total_messages
assert len(app.produce_queue) == 0
assert mq.consume_calls_count == 0
event_loop.run_until_complete(self.coroutine_test(consumer))
for m in produced_messages:
assert s3_storage.ls(s3_storage.QUARANTINE, m['payload_id'])['ResponseMetadata']['HTTPStatusCode'] == 404
assert s3_storage.ls(s3_storage.REJECT, m['payload_id'])['ResponseMetadata']['HTTPStatusCode'] == 200
assert mq.consume_calls_count > 0
assert mq.consume_return_messages_count == 1
assert mq.count_topic_messages(topic) == 0
assert mq.disconnect_in_operation_called is False
assert mq.trying_to_connect_failures_calls == 0
assert len(app.produce_queue) == 0
def test_consumer_with_validation_unknown(self, local_file, s3_mocked,
broker_stage_messages,
event_loop):
total_messages = 4
topic = 'platform.upload.validation'
s3_storage.s3.create_bucket(Bucket=s3_storage.REJECT)
produced_messages = []
with FakeMQ() as mq:
consumer = app.MQClient(mq, "consumer").run(app.handle_validation)
for _ in range(total_messages):
message = self._create_message_s3(local_file,
broker_stage_messages,
avoid_produce_queue=True,
topic=topic,
validation='unknown')
mq.send_and_wait(topic,
json.dumps(message).encode('utf-8'), True)
produced_messages.append(message)
assert mq.produce_calls_count == total_messages
assert mq.count_topic_messages(topic) == total_messages
assert len(app.produce_queue) == 0
assert mq.consume_calls_count == 0
event_loop.run_until_complete(self.coroutine_test(consumer))
assert mq.consume_calls_count > 0
assert mq.consume_return_messages_count == 1
assert mq.count_topic_messages(topic) == 0
assert mq.disconnect_in_operation_called is False
assert mq.trying_to_connect_failures_calls == 0
assert len(app.produce_queue) == 0
def test_producer_with_s3_bucket(self, local_file, s3_mocked, broker_stage_messages, event_loop):
total_messages = 4
[self._create_message_s3(local_file, broker_stage_messages) for _ in range(total_messages)]
with FakeMQ() as mq:
client = ReconnectingClient(mq, "producer")
producer = client.get_callback(app.make_preprocessor())
assert mq.produce_calls_count == 0
assert len(app.produce_queue) == total_messages
event_loop.run_until_complete(self.coroutine_test(producer))
assert mq.produce_calls_count == total_messages
assert len(app.produce_queue) == 0
assert mq.disconnect_in_operation_called is False
assert mq.trying_to_connect_failures_calls == 0
def test_producer_with_connection_issues(self, local_file, s3_mocked,
broker_stage_messages,
event_loop):
total_messages = 4
[
self._create_message_s3(local_file, broker_stage_messages)
for _ in range(total_messages)
]
with FakeMQ(connection_failing_attempt_countdown=1,
disconnect_in_operation=2) as mq:
producer = app.MQClient(mq,
"producer").run(app.make_preprocessor())
assert mq.produce_calls_count == 0
assert len(app.produce_queue) == total_messages
event_loop.run_until_complete(self.coroutine_test(producer))
assert mq.produce_calls_count == total_messages
assert len(app.produce_queue) == 0
assert mq.disconnect_in_operation_called is True
assert mq.trying_to_connect_failures_calls == 1
