def test_consumer_group_mirroring(self, source_type): # start redpanda self.start_brokers(source_type=source_type) consumer_group = "test-group-1" # start mirror maker self.mirror_maker = MirrorMaker2(self.test_context, num_nodes=1, source_cluster=self.source_broker, target_cluster=self.redpanda, consumer_group_pattern=consumer_group, log_level="TRACE") self.mirror_maker.start() msg_size = 512 msg_cnt = 1000000 if self.redpanda.dedicated_nodes else 100 # produce some messages to source redpanda producer = RpkProducer(self.test_context, self.source_broker, self.topic.name, msg_size, msg_cnt, acks=-1) producer.start() producer.wait() producer.free() # consume some messages from source redpanda consumer = RpkConsumer(self.test_context, self.source_broker, self.topic.name, ignore_errors=False, retries=3, group=consumer_group, save_msgs=False, num_msgs=int(msg_cnt / 5)) consumer.start() consumer.wait() consumer.stop() source_messages = consumer.messages self.logger.info(f"source message count: {len(source_messages)}") consumer.free() src_rpk = RpkTool(self.source_broker) source_group = src_rpk.group_describe(consumer_group) target_rpk = RpkTool(self.redpanda) def target_group_equal(): try: target_group = target_rpk.group_describe(consumer_group) except RpkException as e: # e.g. COORDINATOR_NOT_AVAILABLE self.logger.info(f"Error describing target cluster group: {e}") return False self.logger.info( f"source {source_group}, target_group: {target_group}") return target_group.partitions == source_group.partitions and target_group.name == source_group.name # wait for consumer group sync timeout = 600 if self.redpanda.dedicated_nodes else 60 wait_until(target_group_equal, timeout_sec=timeout, backoff_sec=5) self.mirror_maker.stop()
def test_static(self): """ Move partitions with data, but no active producers or consumers. """ self.logger.info(f"Starting redpanda...") self.start_redpanda(num_nodes=3) topics = [] for partition_count in range(1, 5): for replication_factor in (1, 3): name = f"topic{len(topics)}" spec = TopicSpec(name=name, partition_count=partition_count, replication_factor=replication_factor) topics.append(spec) self.logger.info(f"Creating topics...") for spec in topics: self.client().create_topic(spec) num_records = 1000 produced = set( ((f"key-{i:08d}", f"record-{i:08d}") for i in range(num_records))) for spec in topics: self.logger.info(f"Producing to {spec}") producer = KafProducer(self.test_context, self.redpanda, spec.name, num_records) producer.start() self.logger.info( f"Finished producing to {spec}, waiting for producer...") producer.wait() producer.free() self.logger.info(f"Producer stop complete.") for _ in range(25): self._move_and_verify() for spec in topics: self.logger.info(f"Verifying records in {spec}") consumer = RpkConsumer(self.test_context, self.redpanda, spec.name, ignore_errors=False, retries=0) consumer.start() timeout = 30 t1 = time.time() consumed = set() while consumed != produced: if time.time() > t1 + timeout: self.logger.error( f"Validation failed for topic {spec.name}. Produced {len(produced)}, consumed {len(consumed)}" ) self.logger.error( f"Messages consumed but not produced: {sorted(consumed - produced)}" ) self.logger.error( f"Messages produced but not consumed: {sorted(produced - consumed)}" ) assert set(consumed) == produced else: time.sleep(5) for m in consumer.messages: self.logger.info(f"message: {m}") consumed = set([(m['key'], m['value']) for m in consumer.messages]) self.logger.info(f"Stopping consumer...") consumer.stop() self.logger.info(f"Awaiting consumer...") consumer.wait() self.logger.info(f"Freeing consumer...") consumer.free() self.logger.info(f"Finished verifying records in {spec}")