def test_manual_subscribe_pattern(self):
msgs1 = yield from self.send_messages(0, range(0, 10))
msgs2 = yield from self.send_messages(1, range(10, 20))
available_msgs = msgs1 + msgs2
consumer = AIOKafkaConsumer(loop=self.loop,
group_id='test-group',
bootstrap_servers=self.hosts,
auto_offset_reset='earliest',
enable_auto_commit=False)
consumer.subscribe(pattern="topic-test_manual_subs*")
yield from consumer.start()
yield from consumer.seek_to_committed()
result = []
for i in range(20):
msg = yield from consumer.getone()
result.append(msg.value)
self.assertEqual(set(available_msgs), set(result))
yield from consumer.commit(
{TopicPartition(self.topic, 0): OffsetAndMetadata(9, '')})
yield from consumer.seek_to_committed(TopicPartition(self.topic, 0))
msg = yield from consumer.getone(TopicPartition(self.topic, 0))
self.assertEqual(msg.value, b'9')
yield from consumer.commit(
{TopicPartition(self.topic, 0): OffsetAndMetadata(10, '')})
yield from consumer.stop()
# subscribe by topic
consumer = AIOKafkaConsumer(loop=self.loop,
group_id='test-group',
bootstrap_servers=self.hosts,
auto_offset_reset='earliest',
enable_auto_commit=False)
consumer.subscribe(topics=(self.topic, ))
yield from consumer.start()
yield from consumer.seek_to_committed()
result = []
for i in range(10):
msg = yield from consumer.getone()
result.append(msg.value)
self.assertEqual(set(msgs2), set(result))
self.assertEqual(consumer.subscription(), set([self.topic]))
yield from consumer.stop()
def test_manual_subscribe_pattern(self):
msgs1 = yield from self.send_messages(0, range(0, 10))
msgs2 = yield from self.send_messages(1, range(10, 20))
available_msgs = msgs1 + msgs2
consumer = AIOKafkaConsumer(
loop=self.loop, group_id='test-group',
bootstrap_servers=self.hosts, auto_offset_reset='earliest',
enable_auto_commit=False)
consumer.subscribe(pattern="topic-test_manual_subs*")
yield from consumer.start()
yield from consumer.seek_to_committed()
result = []
for i in range(20):
msg = yield from consumer.getone()
result.append(msg.value)
self.assertEqual(set(available_msgs), set(result))
yield from consumer.commit(
{TopicPartition(self.topic, 0): OffsetAndMetadata(9, '')})
yield from consumer.seek_to_committed(TopicPartition(self.topic, 0))
msg = yield from consumer.getone()
self.assertEqual(msg.value, b'9')
yield from consumer.commit(
{TopicPartition(self.topic, 0): OffsetAndMetadata(10, '')})
yield from consumer.stop()
# subscribe by topic
consumer = AIOKafkaConsumer(
loop=self.loop, group_id='test-group',
bootstrap_servers=self.hosts, auto_offset_reset='earliest',
enable_auto_commit=False)
consumer.subscribe(topics=(self.topic,))
yield from consumer.start()
yield from consumer.seek_to_committed()
result = []
for i in range(10):
msg = yield from consumer.getone()
result.append(msg.value)
self.assertEqual(set(msgs2), set(result))
self.assertEqual(consumer.subscription(), set([self.topic]))
yield from consumer.stop()
def consumer_factory(self, **kwargs):
enable_auto_commit = kwargs.pop('enable_auto_commit', True)
auto_offset_reset = kwargs.pop('auto_offset_reset', 'earliest')
group = kwargs.pop('group', 'group-%s' % self.id())
consumer = AIOKafkaConsumer(
self.topic, loop=self.loop, group_id=group,
bootstrap_servers=self.hosts,
enable_auto_commit=enable_auto_commit,
auto_offset_reset=auto_offset_reset,
**kwargs)
yield from consumer.start()
if group is not None:
yield from consumer.seek_to_committed()
return consumer
def consumer_factory(self, **kwargs):
enable_auto_commit = kwargs.pop('enable_auto_commit', True)
auto_offset_reset = kwargs.pop('auto_offset_reset', 'earliest')
group = kwargs.pop('group', 'group-%s' % self.id())
consumer = AIOKafkaConsumer(
self.topic, loop=self.loop, group_id=group,
bootstrap_servers=self.hosts,
enable_auto_commit=enable_auto_commit,
auto_offset_reset=auto_offset_reset,
**kwargs)
yield from consumer.start()
if group is not None:
yield from consumer.seek_to_committed()
return consumer
def test_consumer_seek_on_unassigned(self):
tp0 = TopicPartition(self.topic, 0)
tp1 = TopicPartition(self.topic, 1)
consumer = AIOKafkaConsumer(loop=self.loop,
group_id=None,
bootstrap_servers=self.hosts)
yield from consumer.start()
self.add_cleanup(consumer.stop)
consumer.assign([tp0])
with self.assertRaises(IllegalStateError):
yield from consumer.seek_to_beginning(tp1)
with self.assertRaises(IllegalStateError):
yield from consumer.seek_to_committed(tp1)
with self.assertRaises(IllegalStateError):
yield from consumer.seek_to_end(tp1)
