def test_single_consumergroup_join(self):
record_stream = DeferredQueue(backlog=1)
def processor(consumer, records):
log.debug('processor(%r, %r)', consumer, records)
record_stream.put(records)
coord = ConsumerGroup(
self.client,
self.id(),
topics=[self.topic],
processor=processor,
retry_backoff_ms=100,
heartbeat_interval_ms=1000,
fatal_backoff_ms=3000,
)
join_de = self.when_called(coord, 'on_join_complete')
coord.start()
self.addCleanup(coord.stop)
yield join_de
self.assertIn(self.topic, coord.consumers)
self.assertEqual(len(coord.consumers[self.topic]), self.num_partitions)
self.assertEqual(coord.consumers[self.topic][0].topic, self.topic)
self.assertEqual(coord.consumers[self.topic][0].partition, 0)
for part in range(self.num_partitions):
values = yield self.send_messages(part, [part])
msgs = yield record_stream.get()
self.assertEqual(msgs[0].partition, part)
self.assertEqual(msgs[0].message.value, values[0])
def test_start_leave(self):
"""
start a consumergroup, join, start consumers, then get kicked out
"""
client = self.mock_client([])
processor = Mock()
group = ConsumerGroup(client, "group_id", "topic1", processor)
group.start()
group.on_join_prepare()
group.on_join_complete({"topic1": [1, 2, 3]})
self.assertEqual(len(group.consumers["topic1"]), 3)
group.on_group_leave()
self.assertEqual(len(group.consumers), 0)
def test_broker_restart(self):
"""
restart the kafka broker and verify that the group rejoins
"""
record_stream = DeferredQueue(backlog=1)
def processor(consumer, records):
log.debug('processor(%r, %r)', consumer, records)
record_stream.put(records)
coord = ConsumerGroup(
self.client,
self.id(),
topics=[self.topic],
processor=processor,
retry_backoff_ms=100,
heartbeat_interval_ms=1000,
fatal_backoff_ms=2000,
)
join_de = self.when_called(coord, 'on_join_complete')
coord.start()
self.addCleanup(coord.stop)
yield join_de
self.assertIn(self.topic, coord.consumers)
self.assertEqual(len(coord.consumers[self.topic]), self.num_partitions)
self.assertEqual(coord.consumers[self.topic][0].topic, self.topic)
self.assertEqual(coord.consumers[self.topic][0].partition, 0)
# restart the broker and see that we re-join and still work
leave_de = self.when_called(coord, 'on_group_leave')
prepare_de = self.when_called(coord, 'on_join_prepare')
join_de = self.when_called(coord, 'on_join_complete')
self.harness.brokers[0].stop()
yield leave_de
self.assertEqual(len(coord.consumers), 0)
self.harness.brokers[0].restart()
yield prepare_de
yield join_de
self.assertIn(self.topic, coord.consumers)
self.assertEqual(len(coord.consumers[self.topic]), self.num_partitions)
for part in range(self.num_partitions):
values = yield self.send_messages(part, [part])
msgs = yield record_stream.get()
self.assertEqual(msgs[0].partition, part)
self.assertEqual(msgs[0].message.value, values[0])
def test_consumer_error(self):
"""
get an unexpected stop error from a consumer
"""
client = self.mock_client([])
processor = Mock()
group = ConsumerGroup(client, "group_id", "topic1", processor)
start_d = group.start()
self.assertNoResult(start_d)
with patch('afkak._group.Consumer') as mock_consumer:
mock_consumer.return_value.start.return_value = d = defer.Deferred(
)
group.on_join_complete({"topic1": [1]})
self.assertEqual(mock_consumer.return_value.start.called, True)
d.errback(Failure(AssertionError()))
self.failureResultOf(start_d, AssertionError)
d.addErrback(lambda result: None)
def test_consumer_cancel_during_shutdown(self):
"""
get an unexpected CancelledError on the start() deferred
while shutting down a consumer becasue our heartbeat timed out
"""
client = self.mock_client([])
processor = Mock()
group = ConsumerGroup(client, "group_id", "topic1", processor)
start_d = group.start()
with patch('afkak._group.Consumer') as mock_consumer:
consumer_instance = mock_consumer.return_value
consumer_start_d = defer.Deferred()
consumer_instance.start.return_value = consumer_start_d
consumer_instance._start_d = consumer_start_d
group.on_join_complete({"topic1": [1]})
self.assertEqual(consumer_instance.start.called, True)
def stop():
consumer_start_d.errback(defer.CancelledError())
consumer_instance.stop.side_effect = stop
group.rejoin_after_error(Failure(RequestTimedOutError()))
self.assertEqual(consumer_instance.stop.called, True)
self.successResultOf(consumer_start_d)
self.assertNoResult(start_d)
def test_stop_error(self):
"""
get errors while stopping consumers
"""
client = self.mock_client([])
processor = Mock()
group = ConsumerGroup(client, "group_id", "topic1", processor)
group.start()
with patch('afkak._group.Consumer'):
group.on_join_complete({"topic1": [1]})
consumer = group.consumers["topic1"][0]
consumer.stop.side_effect = KeyError()
group.stop_consumers()
def test_repr(self):
"""
ConsumerGroup's repr displays the group name, instance ID (so that
instances can be differentiated), state, and any member ID assigned by
the broker.
"""
group = ConsumerGroup(self.mock_client([]), "group_id", ["topic1"],
lambda c, m: None)
self.assertRegex(
repr(group),
r"\A<afkak\.ConsumerGroup 0x[a-f0-9]+ for 'group_id' \[initialized\] member_id=''>\Z",
)
def test_rejoin_consumer(self):
client = self.mock_client([])
processor = Mock()
group = ConsumerGroup(client, "group_id", "topic1", processor)
start_d = group.start()
group.on_group_leave = Mock()
with patch('afkak._group.Consumer') as mock_consumer:
mock_consumer.return_value.start.return_value = d = defer.Deferred(
)
group.on_join_complete({"topic1": [1]})
self.assertEqual(mock_consumer.return_value.start.called, True)
d.errback(Failure(IllegalGeneration()))
self.assertEqual(group._rejoin_needed, True)
self.assertNoResult(start_d)
group.on_group_leave.assert_any_call()
def test_shutdown_error(self):
"""
get errors while shutting down consumers
"""
client = self.mock_client([])
processor = Mock()
group = ConsumerGroup(client, "group_id", "topic1", processor)
group.start()
with patch('afkak._group.Consumer', side_effect=[Mock(), Mock()]):
group.on_join_complete({"topic1": [1, 2]})
consumer = group.consumers["topic1"][0]
consumer._start_d = defer.Deferred()
consumer.shutdown.side_effect = KeyError()
consumer.stop.side_effect = KeyError()
consumer2 = group.consumers["topic1"][1]
consumer2.shutdown.return_value = defer.Deferred()
de = group.shutdown_consumers()
self.assertNoResult(de)
self.assertEqual(len(group.consumers), 0)
consumer2.shutdown.return_value.errback(KeyError())
consumer2.stop.assert_called_once_with()
self.successResultOf(de)
def test_consumer_rejoin(self):
"""
trigger a rejoin via consumer commit failure
"""
group = 'rejoin_group'
self.client2 = KafkaClient(self.harness.bootstrap_hosts,
clientId=self.topic + '2')
self.addCleanup(self.client2.close)
record_stream = DeferredQueue(backlog=1)
def processor(consumer, records):
log.debug('processor(%r, %r)', consumer, records)
record_stream.put(records)
coord = ConsumerGroup(
self.client,
group,
topics=[self.topic],
processor=processor,
session_timeout_ms=6000,
retry_backoff_ms=100,
heartbeat_interval_ms=1000,
fatal_backoff_ms=3000,
consumer_kwargs=dict(auto_commit_every_ms=1000),
)
coord_start_d = coord.start()
self.addCleanup(coord.stop)
# FIXME: This doesn't seem to get fired reliably.
coord_start_d
# self.addCleanup(lambda: coord_start_d)
yield wait_for_assignments(self.topic, self.num_partitions, [coord])
# kill the heartbeat timer and start joining the second consumer
while True:
if coord._heartbeat_looper.running:
coord._heartbeat_looper.stop()
break
else:
yield async_delay()
coord2 = ConsumerGroup(
self.client2,
group,
topics=[self.topic],
processor=processor,
session_timeout_ms=6000,
retry_backoff_ms=100,
heartbeat_interval_ms=1000,
fatal_backoff_ms=3000,
consumer_kwargs=dict(auto_commit_every_ms=1000),
)
coord2_start_d = coord2.start()
self.addCleanup(coord2.stop)
# FIXME: This doesn't seem to get fired reliably.
coord2_start_d
# self.addCleanup(lambda: coord2_start_d)
# send some messages and see that they're processed
# the commit will eventually fail because we're rebalancing
for part in range(15):
yield async_delay()
values = yield self.send_messages(part % self.num_partitions,
[part])
msgs = yield record_stream.get()
if msgs[0].partition != part:
# once the commit fails, we will see the msg twice
break
self.assertEqual(msgs[0].message.value, values[0])
yield wait_for_assignments(self.topic, self.num_partitions,
[coord, coord2])
# Once assignments have been received we need to ensure that the record
# stream is clear of any duplicate messages. We do this by producing
# a sentinel to each partition and consuming messages from the stream
# until all the sentinels have appeared at least once. At that point
# any churn should have cleared up and we can depend on lock-step
# delivery.
pending_sentinels = {}
for part in range(self.num_partitions):
[value] = yield self.send_messages(part, ['sentinel'])
pending_sentinels[part] = value
while pending_sentinels:
[message] = yield record_stream.get()
if pending_sentinels.get(
message.partition) == message.message.value:
del pending_sentinels[message.partition]
# after the cluster has re-formed, send some more messages
# and check that we get them too (and don't get the old messages again)
record_stream = DeferredQueue(backlog=1)
for part in range(self.num_partitions):
yield async_delay()
[value] = yield self.send_messages(part, [part])
log.debug('waiting for messages from partition %d', part)
[message] = yield record_stream.get()
self.assertEqual(message.partition, part)
self.assertEqual(message.message.value, value)
def test_two_consumergroup_join(self):
"""
When a second member joins the consumer group it triggers a rebalance.
After that completes some partitions are distributed to each member.
"""
group_id = 'group_for_two'
self.client2 = KafkaClient(self.harness.bootstrap_hosts,
clientId=self.topic + '2')
self.addCleanup(self.client2.close)
record_stream = DeferredQueue(backlog=1)
def processor(consumer, records):
log.debug("processor(%r, %r)", consumer, records)
record_stream.put(records)
coord = ConsumerGroup(
self.client,
group_id,
topics=[self.topic],
processor=processor,
retry_backoff_ms=100,
heartbeat_interval_ms=1000,
fatal_backoff_ms=3000,
)
de = self.when_called(coord, 'on_join_complete')
coord_start_d = coord.start()
self.addCleanup(coord.stop)
# FIXME: This doesn't seem to get fired reliably.
coord_start_d
# self.addCleanup(lambda: coord_start_d)
yield de
# send some messages and see that they're processed
for part in range(self.num_partitions):
values = yield self.send_messages(part, [part])
msgs = yield record_stream.get()
self.assertEqual(msgs[0].partition, part)
self.assertEqual(msgs[0].message.value, values[0])
coord2 = ConsumerGroup(
self.client2,
group_id,
topics=[self.topic],
processor=processor,
retry_backoff_ms=100,
heartbeat_interval_ms=1000,
fatal_backoff_ms=3000,
)
de = self.when_called(coord, 'on_join_complete')
de2 = self.when_called(coord2, 'on_join_complete')
coord2_start_d = coord2.start()
self.addCleanup(coord2.stop)
# FIXME: This doesn't seem to get fired reliably
coord2_start_d
# self.addCleanup(lambda: coord2_start_d)
yield de
yield de2
self.assertIn(self.topic, coord.consumers)
self.assertIn(self.topic, coord2.consumers)
self.assertEqual(len(coord.consumers[self.topic]), 3)
self.assertEqual(len(coord2.consumers[self.topic]), 3)
self.assertNotEqual(coord.consumers[self.topic][0].partition,
coord2.consumers[self.topic][0].partition)
# after the cluster has re-formed, send some more messages
# and check that we get them too (and don't get the old messages again)
for part in range(self.num_partitions):
values = yield self.send_messages(part, [part])
msgs = yield record_stream.get()
self.assertEqual(msgs[0].partition, part)
self.assertEqual(msgs[0].message.value, values[0])