def _consume_all(self, topic_names: list[str], msg_count_per_topic: int,
timeout_per_topic: int):
"""
Don't do anything with the messages, just consume them to demonstrate
that doing so does not exhaust redpanda resources.
"""
def consumer_saw_msgs(consumer):
self.logger.info(
f"Consumer message_count={consumer.message_count} / {msg_count_per_topic}"
)
# Tolerate greater-than, because if there were errors during production
# there can have been retries.
return consumer.message_count >= msg_count_per_topic
for tn in topic_names:
consumer = RpkConsumer(self._ctx,
self.redpanda,
tn,
save_msgs=False,
fetch_max_bytes=BIG_FETCH,
num_msgs=msg_count_per_topic)
consumer.start()
wait_until(lambda: consumer_saw_msgs(consumer),
timeout_sec=timeout_per_topic,
backoff_sec=5)
consumer.stop()
consumer.free()
def _consume_and_count_bytes(self):
consumer = RpkConsumer(self._ctx, self.redpanda, self.topic)
consumer.start()
self._bytes_received = 0
def count_bytes():
for msg in consumer.messages:
value = msg["value"]
# Ignore None values
if value is None:
return False
self._bytes_received += len(value)
# Returns true the first time any bytes
# are fetched.
return self._bytes_received > 0
wait_until(count_bytes,
timeout_sec=30,
backoff_sec=5,
err_msg="count_bytes() failed")
consumer.stop()
def test_kafka_streams(self):
example = self.create_example()
# This will raise TypeError if PRODUCER is undefined
producer = self.PRODUCER(self._ctx, self.redpanda, self.topics[0].name)
consumer = RpkConsumer(self._ctx, self.redpanda, self.topics[1].name)
# Start the example
example.start()
# Produce some data
producer.start()
producer.wait()
# Consume the data
consumer.start()
def try_cons():
i = 0
msgs = consumer.messages
while i < len(msgs) and not self.is_valid_msg(msgs[i]):
i += 1
return i < len(msgs)
wait_until(
try_cons,
timeout_sec=self._timeout,
backoff_sec=5,
err_msg=f"kafka-streams {self._ctx.cls_name} consumer failed")
consumer.stop()
producer.stop()
example.stop()
def test_consumer_group_mirroring(self, source_type):
# start redpanda
self.start_brokers(source_type=source_type)
consumer_group = "test-group-1"
# start mirror maker
self.mirror_maker = MirrorMaker2(self.test_context,
num_nodes=1,
source_cluster=self.source_broker,
target_cluster=self.redpanda,
consumer_group_pattern=consumer_group,
log_level="TRACE")
self.mirror_maker.start()
msg_size = 512
msg_cnt = 1000000 if self.redpanda.dedicated_nodes else 100
# produce some messages to source redpanda
producer = RpkProducer(self.test_context,
self.source_broker,
self.topic.name,
msg_size,
msg_cnt,
acks=-1)
producer.start()
producer.wait()
producer.free()
# consume some messages from source redpanda
consumer = RpkConsumer(self.test_context,
self.source_broker,
self.topic.name,
ignore_errors=False,
retries=3,
group=consumer_group,
save_msgs=False,
num_msgs=int(msg_cnt / 5))
consumer.start()
consumer.wait()
consumer.stop()
source_messages = consumer.messages
self.logger.info(f"source message count: {len(source_messages)}")
consumer.free()
src_rpk = RpkTool(self.source_broker)
source_group = src_rpk.group_describe(consumer_group)
target_rpk = RpkTool(self.redpanda)
def target_group_equal():
try:
target_group = target_rpk.group_describe(consumer_group)
except RpkException as e:
# e.g. COORDINATOR_NOT_AVAILABLE
self.logger.info(f"Error describing target cluster group: {e}")
return False
self.logger.info(
f"source {source_group}, target_group: {target_group}")
return target_group.partitions == source_group.partitions and target_group.name == source_group.name
# wait for consumer group sync
timeout = 600 if self.redpanda.dedicated_nodes else 60
wait_until(target_group_equal, timeout_sec=timeout, backoff_sec=5)
self.mirror_maker.stop()
def test_static(self):
"""
Move partitions with data, but no active producers or consumers.
"""
self.logger.info(f"Starting redpanda...")
self.start_redpanda(num_nodes=3)
topics = []
for partition_count in range(1, 5):
for replication_factor in (1, 3):
name = f"topic{len(topics)}"
spec = TopicSpec(name=name,
partition_count=partition_count,
replication_factor=replication_factor)
topics.append(spec)
self.logger.info(f"Creating topics...")
for spec in topics:
self.client().create_topic(spec)
num_records = 1000
produced = set(
((f"key-{i:08d}", f"record-{i:08d}") for i in range(num_records)))
for spec in topics:
self.logger.info(f"Producing to {spec}")
producer = KafProducer(self.test_context, self.redpanda, spec.name,
num_records)
producer.start()
self.logger.info(
f"Finished producing to {spec}, waiting for producer...")
producer.wait()
producer.free()
self.logger.info(f"Producer stop complete.")
for _ in range(25):
self._move_and_verify()
for spec in topics:
self.logger.info(f"Verifying records in {spec}")
consumer = RpkConsumer(self.test_context,
self.redpanda,
spec.name,
ignore_errors=False,
retries=0)
consumer.start()
timeout = 30
t1 = time.time()
consumed = set()
while consumed != produced:
if time.time() > t1 + timeout:
self.logger.error(
f"Validation failed for topic {spec.name}. Produced {len(produced)}, consumed {len(consumed)}"
)
self.logger.error(
f"Messages consumed but not produced: {sorted(consumed - produced)}"
)
self.logger.error(
f"Messages produced but not consumed: {sorted(produced - consumed)}"
)
assert set(consumed) == produced
else:
time.sleep(5)
for m in consumer.messages:
self.logger.info(f"message: {m}")
consumed = set([(m['key'], m['value'])
for m in consumer.messages])
self.logger.info(f"Stopping consumer...")
consumer.stop()
self.logger.info(f"Awaiting consumer...")
consumer.wait()
self.logger.info(f"Freeing consumer...")
consumer.free()
self.logger.info(f"Finished verifying records in {spec}")
