async def test_consumer_transactional_commit(self):
producer = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts,
transactional_id="sobaka_producer")
await producer.start()
self.add_cleanup(producer.stop)
producer2 = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts)
await producer2.start()
self.add_cleanup(producer2.stop)
consumer = AIOKafkaConsumer(
self.topic, loop=self.loop,
bootstrap_servers=self.hosts,
auto_offset_reset="earliest",
isolation_level="read_committed")
await consumer.start()
self.add_cleanup(consumer.stop)
# We will produce from a transactional producer and then from a
# non-transactional. This should block consumption on that partition
# until transaction is committed.
await producer.begin_transaction()
meta = await producer.send_and_wait(
self.topic, b'Hello from transaction', partition=0)
meta2 = await producer2.send_and_wait(
self.topic, b'Hello from non-transaction', partition=0)
# The transaction blocked consumption
task = self.loop.create_task(consumer.getone())
await asyncio.sleep(1, loop=self.loop)
self.assertFalse(task.done())
tp = TopicPartition(self.topic, 0)
self.assertEqual(consumer.last_stable_offset(tp), 0)
self.assertEqual(consumer.highwater(tp), 2)
await producer.commit_transaction()
# Order should be preserved. We first yield the first message, although
# it belongs to a committed afterwards transaction
msg = await task
self.assertEqual(msg.offset, meta.offset)
self.assertEqual(msg.timestamp, meta.timestamp)
self.assertEqual(msg.value, b"Hello from transaction")
self.assertEqual(msg.key, None)
msg = await consumer.getone()
self.assertEqual(msg.offset, meta2.offset)
self.assertEqual(msg.timestamp, meta2.timestamp)
self.assertEqual(msg.value, b"Hello from non-transaction")
self.assertEqual(msg.key, None)
# 3, because we have a commit marker also
tp = TopicPartition(self.topic, 0)
self.assertEqual(consumer.last_stable_offset(tp), 3)
self.assertEqual(consumer.highwater(tp), 3)
