def test_cli_integration_dequeues_batch(self):
"""Test that CLI dequeues batched message from CLI"""
tq = Queue(self.config)
self.assertEqual(get_kafka_end_offset(tq, self.queue_name), 0)
for i in range(3):
message = {f"cli-input-key-{i}": f"cli-input-value-{i}"}
tq.enqueue(message)
tq.engine.producer.close()
self.assertEqual(get_kafka_end_offset(tq, self.queue_name), 3)
runner = CliRunner()
result = runner.invoke(
cli.main,
[
"--broker-url",
self.broker_url,
"--queue-name",
self.queue_name,
"dequeue",
"--batch",
"3",
],
obj={},
)
self.assertTrue(result.exit_code == 0)
expected_messages = [
'{"cli-input-key-0": "cli-input-value-0"}\n',
'{"cli-input-key-1": "cli-input-value-1"}\n',
'{"cli-input-key-2": "cli-input-value-2"}\n',
]
self.assertEqual(result.output, "".join(expected_messages))
def test_message_queue_unit_enqueue(self, mock_connection):
"""Test that Queue mock enqueues message"""
message = {"key": "value"}
tq = Queue(self.config)
tq.enqueue(message)
tq.engine.queue.put.assert_called_once_with(message)
def test_message_queue_integration_enqueue_single_message(self):
"""Test that Queue enqueues single message properly"""
tq = Queue(self.config)
message = {"key": "value"}
tq.enqueue(message)
tq.engine.producer.close()
self.assertEqual(get_kafka_end_offset(tq, self.queue_name), 1)
def test_message_queue_unit_enqueue(self, mock_producer, mock_consumer):
"""Test that Queue mock enqueues message"""
message = {"key": "value"}
tq = Queue(self.config)
tq.enqueue(message)
tq.engine.producer.send.assert_called_once_with(self.queue_name,
value=message)
def test_message_queue_integration_dequeue_single_message(self):
"""Test that Queue dequeues single message properly"""
tq = Queue(self.config)
self.assertEqual(get_kafka_end_offset(tq, self.queue_name), 0)
message = {"key-kafka-dequeue": "value"}
tq.enqueue(message)
tq.engine.producer.close()
self.assertEqual(get_kafka_end_offset(tq, self.queue_name), 1)
result = tq.dequeue()
tq.engine.consumer.commit()
self.assertEqual(result, b'{"key-kafka-dequeue": "value"}')
self.assertEqual(get_kafka_committed_offset(tq, self.queue_name), 1)
def test_message_queue_integration_dequeue_multiple_messages(self):
"""Test that Queue dequeues multiple message properly"""
tq = Queue(self.config)
self.assertEqual(get_kafka_end_offset(tq, self.queue_name), 0)
messages = [
{
"key1": "value1"
},
{
"key2": "value2"
},
{
"key3": "value3"
},
{
"key4": "value4"
},
]
for message in messages:
tq.enqueue(message)
tq.engine.producer.close()
self.assertEqual(get_kafka_end_offset(tq, self.queue_name), 4)
result = tq.dequeue()
tq.engine.consumer.commit()
self.assertEqual(result, b'{"key1": "value1"}')
self.assertEqual(get_kafka_committed_offset(tq, self.queue_name), 1)
result = tq.dequeue()
tq.engine.consumer.commit()
self.assertEqual(result, b'{"key2": "value2"}')
self.assertEqual(get_kafka_committed_offset(tq, self.queue_name), 2)
result = tq.dequeue()
tq.engine.consumer.commit()
self.assertEqual(result, b'{"key3": "value3"}')
self.assertEqual(get_kafka_committed_offset(tq, self.queue_name), 3)
result = tq.dequeue()
tq.engine.consumer.commit()
self.assertEqual(result, b'{"key4": "value4"}')
self.assertEqual(get_kafka_committed_offset(tq, self.queue_name), 4)
def test_cli_integration_dequeues_message(self):
"""Test that CLI dequeues message from CLI"""
tq = Queue(self.config)
message = {"cli-input-key": "cli-input-value"}
self.assertEqual(get_kafka_end_offset(tq, self.queue_name), 0)
tq.enqueue(message)
tq.engine.producer.close()
self.assertEqual(get_kafka_end_offset(tq, self.queue_name), 1)
runner = CliRunner()
result = runner.invoke(
cli.main,
[
"--broker-url",
self.broker_url,
"--queue-name",
self.queue_name,
"dequeue",
],
obj={},
)
self.assertTrue(result.exit_code == 0)
self.assertEqual({"cli-input-key": "cli-input-value"}, message)
def test_message_queue_integration_enqueue_multiple_messages(self):
"""Test that Queue enqueues multiple messages properly"""
tq = Queue(self.config)
self.assertEqual(get_kafka_end_offset(tq, self.queue_name), 0)
messages = [
{
"key1": "value1"
},
{
"key2": "value2"
},
{
"key3": "value3"
},
{
"key4": "value4"
},
]
for message in messages:
tq.enqueue(message)
tq.engine.producer.close()
self.assertEqual(get_kafka_end_offset(tq, self.queue_name), 4)
class TestCLIIntegrationKombu(unittest.TestCase):
"""Integration tests for poppy CLI backed by kombu"""
def setUp(self):
self.broker_url = CONFIG_TEST_IN_MEMORY_URL
self.queue_name = "test-message-queue"
self.config = deepcopy(Queue.get_default_config())
self.config["BROKER_URL"] = self.broker_url
self.config["QUEUE_NAME"] = self.queue_name
self.tq = Queue(self.config)
def tearDown(self):
self.tq.engine.queue.queue.delete()
self.tq.close()
def test_cli_integration_enqueues_raw(self):
"""Test that CLI enqueues raw messages from CLI options"""
self.assertEqual(self.tq.engine.queue.qsize(), 0)
runner = CliRunner()
with runner.isolated_filesystem():
with open("raw_input.txt", "w") as f:
f.write(
'{"raw-key-1": "raw-value-1"}\n{"raw-key-2": "raw-value-2"}\n'
)
result = runner.invoke(
cli.main,
[
"--broker-url",
self.broker_url,
"--queue-name",
self.queue_name,
"enqueue-raw",
"--message-input",
"raw_input.txt",
],
obj={},
)
self.assertTrue(result.exit_code == 0)
self.assertEqual(self.tq.engine.queue.qsize(), 2)
result = self.tq.dequeue()
self.assertEqual(result, b'{"raw-key-1": "raw-value-1"}')
result = self.tq.dequeue()
self.assertEqual(result, b'{"raw-key-2": "raw-value-2"}')
def test_cli_integration_enqueues_message(self):
"""Test that CLI enqueues message from CLI options"""
self.assertEqual(self.tq.engine.queue.qsize(), 0)
runner = CliRunner()
result = runner.invoke(
cli.main,
[
"--broker-url",
self.broker_url,
"--queue-name",
self.queue_name,
"enqueue",
"--message-entry",
"cli-input-key",
"cli-input-value",
],
obj={},
)
self.assertTrue(result.exit_code == 0)
self.assertEqual(self.tq.engine.queue.qsize(), 1)
result = self.tq.dequeue()
self.assertEqual(result, b'{"cli-input-key": "cli-input-value"}')
def test_cli_integration_enqueues_message_multiple_keys(self):
"""Test that CLI enqueues message from CLI options"""
self.assertEqual(self.tq.engine.queue.qsize(), 0)
runner = CliRunner()
result = runner.invoke(
cli.main,
[
"--broker-url",
self.broker_url,
"--queue-name",
self.queue_name,
"enqueue",
"--message-entry",
"k1",
"v1",
"--message-entry",
"k2",
"v2",
"--message-entry",
"k3",
"v3",
],
obj={},
)
self.assertTrue(result.exit_code == 0)
self.assertEqual(self.tq.engine.queue.qsize(), 1)
result = self.tq.dequeue()
self.assertEqual(
result,
b'{"k1": "v1", "k2": "v2", "k3": "v3"}',
)
def test_cli_integration_dequeues_message(self):
"""Test that CLI dequeues message from CLI"""
message = {"cli-input-key": "cli-input-value"}
self.assertEqual(self.tq.engine.queue.qsize(), 0)
self.tq.enqueue(message)
self.assertEqual(self.tq.engine.queue.qsize(), 1)
runner = CliRunner()
result = runner.invoke(
cli.main,
[
"--broker-url",
self.broker_url,
"--queue-name",
self.queue_name,
"dequeue",
],
obj={},
)
self.assertTrue(result.exit_code == 0)
result = json.loads(result.output)
self.assertDictEqual(result, message)
def test_cli_integration_dequeues_batch(self):
"""Test that CLI dequeues batched message from CLI"""
self.assertEqual(self.tq.engine.queue.qsize(), 0)
for i in range(3):
message = {f"cli-input-key-{i}": f"cli-input-value-{i}"}
self.tq.enqueue(message)
self.assertEqual(self.tq.engine.queue.qsize(), 3)
runner = CliRunner()
result = runner.invoke(
cli.main,
[
"--broker-url",
self.broker_url,
"--queue-name",
self.queue_name,
"dequeue",
"--batch",
"3",
],
obj={},
)
self.assertTrue(result.exit_code == 0)
expected_messages = [
'{"cli-input-key-0": "cli-input-value-0"}\n',
'{"cli-input-key-1": "cli-input-value-1"}\n',
'{"cli-input-key-2": "cli-input-value-2"}\n',
]
self.assertEqual(result.output, "".join(expected_messages))
def test_cli_integration_dequeues_empty(self):
"""Test that CLI dequeues empty message from CLI"""
self.assertEqual(self.tq.engine.queue.qsize(), 0)
runner = CliRunner()
result = runner.invoke(
cli.main,
[
"--broker-url",
self.broker_url,
"--queue-name",
self.queue_name,
"dequeue",
"--blocking-dequeue-timeout",
"10",
],
obj={},
)
self.assertTrue(result.exit_code == 0)
class TestQueueKombuIntegration(unittest.TestCase):
"""Integration test for poppy Queue with kombu backend"""
def setUp(self):
self.broker_url = "memory://"
self.queue_name = "test-message-queue"
self.config = deepcopy(Queue.get_default_config())
self.config["BROKER_URL"] = self.broker_url
self.config["QUEUE_NAME"] = self.queue_name
self.tq = Queue(self.config)
def tearDown(self):
self.tq.engine.queue.queue.delete()
self.tq.close()
def test_message_queue_integration_connection(self):
"""Test that Queue connects properly"""
with closing(Queue(self.config)) as tq:
self.assertTrue(tq.engine.conn.connected)
self.assertFalse(tq.engine.conn.connected)
def test_message_queue_integration_enqueue_single_message(self):
"""Test that Queue enqueues single message properly"""
self.assertEqual(self.tq.engine.queue.qsize(), 0)
message = {"key": "value"}
self.tq.enqueue(message)
self.assertEqual(self.tq.engine.queue.qsize(), 1)
def test_message_queue_integration_enqueue_multiple_messages(self):
"""Test that Queue enqueues multiple messages properly"""
self.assertEqual(self.tq.engine.queue.qsize(), 0)
messages = [
{
"key1": "value1"
},
{
"key2": "value2"
},
{
"key3": "value3"
},
{
"key4": "value4"
},
]
for message in messages:
self.tq.enqueue(message)
self.assertEqual(self.tq.engine.queue.qsize(), 4)
def test_message_queue_integration_dequeue_single_message(self):
"""Test that Queue dequeues single message properly"""
self.assertEqual(self.tq.engine.queue.qsize(), 0)
message = {"key-kombu-dequeue-integration": "value"}
self.tq.enqueue(message)
self.assertEqual(self.tq.engine.queue.qsize(), 1)
result = self.tq.dequeue()
self.assertEqual(result, b'{"key-kombu-dequeue-integration": "value"}')
self.assertEqual(self.tq.engine.queue.qsize(), 0)
def test_message_queue_integration_dequeue_multiple_messages(self):
"""Test that Queue dequeues multiple message properly"""
self.assertEqual(self.tq.engine.queue.qsize(), 0)
messages = [
{
"key1": "value1"
},
{
"key2": "value2"
},
{
"key3": "value3"
},
{
"key4": "value4"
},
]
for message in messages:
self.tq.enqueue(message)
self.assertEqual(self.tq.engine.queue.qsize(), 4)
result = self.tq.dequeue()
self.assertEqual(result, b'{"key1": "value1"}')
self.assertEqual(self.tq.engine.queue.qsize(), 3)
result = self.tq.dequeue()
self.assertEqual(result, b'{"key2": "value2"}')
self.assertEqual(self.tq.engine.queue.qsize(), 2)