def test_consumer_rejoin(self, static_members):
"""
Test validating that re-joining static member will not casuse rebalance
"""
self.setup_producer(20)
group = 'test-gr-1'
consumers = self.create_consumers(
2,
self.topic_spec.name,
group,
static_members=static_members,
consumer_properties={"session.timeout.ms": 40000})
# wait for some messages
wait_until(lambda: ConsumerGroupTest.consumed_at_least(consumers, 50),
30, 2)
rpk = RpkTool(self.redpanda)
# at this point we have 2 consumers in stable group
self.validate_group_state(group,
expected_state="Stable",
static_members=static_members)
# stop one of the consumers
consumers[0].stop()
consumers[0].wait()
rpk_group = rpk.group_describe(group)
if static_members:
# with static members group should still be in stable state
assert rpk_group.state == "Stable"
assert rpk_group.members == 2
else:
# consumer will request group leave when shutdown gracefully and it is dynamic
self.wait_for_members(group, 1)
# start the consumer again
consumers[0].start()
consumers[0].wait_for_started()
# wait for consumer to start
if static_members:
# with static members group should be stable immediately as the
# consumer is rejoining with the same instance id
self.validate_group_state(group,
expected_state="Stable",
static_members=static_members)
else:
# group should get back to its original 2 members state
self.wait_for_members(group, 2)
self.producer.wait()
self.producer.free()
for c in consumers:
c.stop()
c.wait()
c.free()
def validate_group_state(self, group, expected_state, static_members):
rpk = RpkTool(self.redpanda)
# validate group state
rpk_group = rpk.group_describe(group)
assert rpk_group.members == 2
assert rpk_group.state == expected_state
for p in rpk_group.partitions:
if static_members:
assert 'panda-consumer' in p.instance_id
else:
assert p.instance_id is None
def test_mixed_consumers_join(self):
"""
Test validating that dynamic and static consumers may exists in the same group
"""
self.setup_producer(20)
group = 'test-gr-1'
consumers = []
consumers.append(
self.create_consumer(self.topic_spec.name, group,
"panda-instance"))
consumers.append(
self.create_consumer(self.topic_spec.name, group, None))
for c in consumers:
c.start()
# wait for some messages
wait_until(lambda: ConsumerGroupTest.consumed_at_least(consumers, 50),
30, 2)
rpk = RpkTool(self.redpanda)
# validate group state
rpk_group = rpk.group_describe(group)
assert rpk_group.members == 2
assert rpk_group.state == "Stable"
static_members = set()
dynamic_members = set()
for p in rpk_group.partitions:
if p.instance_id:
static_members.add(p.client_id)
else:
dynamic_members.add(p.client_id)
assert len(static_members) == 1
assert len(dynamic_members) == 1
self.producer.wait()
self.producer.free()
for c in consumers:
c.stop()
c.wait()
c.free()
def test_consumer_group_mirroring(self, source_type):
# start redpanda
self.start_brokers(source_type=source_type)
consumer_group = "test-group-1"
# start mirror maker
self.mirror_maker = MirrorMaker2(self.test_context,
num_nodes=1,
source_cluster=self.source_broker,
target_cluster=self.redpanda,
consumer_group_pattern=consumer_group,
log_level="TRACE")
self.mirror_maker.start()
msg_size = 512
msg_cnt = 1000000 if self.redpanda.dedicated_nodes else 100
# produce some messages to source redpanda
producer = RpkProducer(self.test_context,
self.source_broker,
self.topic.name,
msg_size,
msg_cnt,
acks=-1)
producer.start()
producer.wait()
producer.free()
# consume some messages from source redpanda
consumer = RpkConsumer(self.test_context,
self.source_broker,
self.topic.name,
ignore_errors=False,
retries=3,
group=consumer_group,
save_msgs=False,
num_msgs=int(msg_cnt / 5))
consumer.start()
consumer.wait()
consumer.stop()
source_messages = consumer.messages
self.logger.info(f"source message count: {len(source_messages)}")
consumer.free()
src_rpk = RpkTool(self.source_broker)
source_group = src_rpk.group_describe(consumer_group)
target_rpk = RpkTool(self.redpanda)
def target_group_equal():
try:
target_group = target_rpk.group_describe(consumer_group)
except RpkException as e:
# e.g. COORDINATOR_NOT_AVAILABLE
self.logger.info(f"Error describing target cluster group: {e}")
return False
self.logger.info(
f"source {source_group}, target_group: {target_group}")
return target_group.partitions == source_group.partitions and target_group.name == source_group.name
# wait for consumer group sync
timeout = 600 if self.redpanda.dedicated_nodes else 60
wait_until(target_group_equal, timeout_sec=timeout, backoff_sec=5)
self.mirror_maker.stop()