class ReassignPartitionsTest(ProduceConsumeValidateTest): """ These tests validate partition reassignment. Create a topic with few partitions, load some data, trigger partition re-assignment with and without broker failure, check that partition re-assignment can complete and there is no data loss. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ReassignPartitionsTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(test_context, num_nodes=4, zk=self.zk, topics={ self.topic: { "partitions": 20, "replication-factor": 3, 'configs': { "min.insync.replicas": 2 } } }) self.num_partitions = 20 self.timeout_sec = 60 self.producer_throughput = 1000 self.num_producers = 1 self.num_consumers = 1 def setUp(self): self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super( ReassignPartitionsTest, self).min_cluster_size() + self.num_producers + self.num_consumers def clean_bounce_some_brokers(self): """Bounce every other broker""" for node in self.kafka.nodes[::2]: self.kafka.restart_node(node, clean_shutdown=True) def reassign_partitions(self, bounce_brokers): partition_info = self.kafka.parse_describe_topic( self.kafka.describe_topic(self.topic)) self.logger.debug("Partitions before reassignment:" + str(partition_info)) # jumble partition assignment in dictionary seed = random.randint(0, 2**31 - 1) self.logger.debug("Jumble partition assignment with seed " + str(seed)) random.seed(seed) # The list may still be in order, but that's ok shuffled_list = range(0, self.num_partitions) random.shuffle(shuffled_list) for i in range(0, self.num_partitions): partition_info["partitions"][i]["partition"] = shuffled_list[i] self.logger.debug("Jumbled partitions: " + str(partition_info)) # send reassign partitions command self.kafka.execute_reassign_partitions(partition_info) if bounce_brokers: # bounce a few brokers at the same time self.clean_bounce_some_brokers() # Wait until finished or timeout wait_until( lambda: self.kafka.verify_reassign_partitions(partition_info), timeout_sec=self.timeout_sec, backoff_sec=.5) @cluster(num_nodes=7) @parametrize(security_protocol="PLAINTEXT", bounce_brokers=True) @parametrize(security_protocol="PLAINTEXT", bounce_brokers=False) def test_reassign_partitions(self, bounce_brokers, security_protocol): """Reassign partitions tests. Setup: 1 zk, 3 kafka nodes, 1 topic with partitions=3, replication-factor=3, and min.insync.replicas=2 - Produce messages in the background - Consume messages in the background - Reassign partitions - If bounce_brokers is True, also bounce a few brokers while partition re-assignment is in progress - When done reassigning partitions and bouncing brokers, stop producing, and finish consuming - Validate that every acked message was consumed """ self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = security_protocol new_consumer = False if self.kafka.security_protocol == "PLAINTEXT" else True self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, new_consumer=new_consumer, consumer_timeout_ms=60000, message_validator=is_int) self.kafka.start() self.run_produce_consume_validate( core_test_action=lambda: self.reassign_partitions(bounce_brokers))
class ThrottlingTest(ProduceConsumeValidateTest): """Tests throttled partition reassignment. This is essentially similar to the reassign_partitions_test, except that we throttle the reassignment and verify that it takes a sensible amount of time given the throttle and the amount of data being moved. Since the correctness is time dependent, this test also simplifies the cluster topology. In particular, we fix the number of brokers, the replication-factor, the number of partitions, the partition size, and the number of partitions being moved so that we can accurately predict the time throttled reassignment should take. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ThrottlingTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.zk = ZookeeperService(test_context, num_nodes=1) # Because we are starting the producer/consumer/validate cycle _after_ # seeding the cluster with big data (to test throttling), we need to # Start the consumer from the end of the stream. further, we need to # ensure that the consumer is fully started before the producer starts # so that we don't miss any messages. This timeout ensures the sufficient # condition. self.consumer_init_timeout_sec = 20 self.num_brokers = 6 self.num_partitions = 3 self.kafka = KafkaService(test_context, num_nodes=self.num_brokers, zk=self.zk, topics={ self.topic: { "partitions": self.num_partitions, "replication-factor": 2, "configs": { "segment.bytes": 64 * 1024 * 1024 } } }) self.producer_throughput = 1000 self.timeout_sec = 400 self.num_records = 2000 self.record_size = 4096 * 100 # 400 KB # 1 MB per partition on average. self.partition_size = (self.num_records * self.record_size) / self.num_partitions self.num_producers = 2 self.num_consumers = 1 self.throttle = 4 * 1024 * 1024 # 4 MB/s def setUp(self): self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super(ThrottlingTest, self).min_cluster_size() +\ self.num_producers + self.num_consumers def clean_bounce_some_brokers(self): """Bounce every other broker""" for node in self.kafka.nodes[::2]: self.kafka.restart_node(node, clean_shutdown=True) def reassign_partitions(self, bounce_brokers, throttle): """This method reassigns partitions using a throttle. It makes an assertion about the minimum amount of time the reassignment should take given the value of the throttle, the number of partitions being moved, and the size of each partition. """ partition_info = self.kafka.parse_describe_topic( self.kafka.describe_topic(self.topic)) self.logger.debug("Partitions before reassignment:" + str(partition_info)) max_num_moves = 0 for i in range(0, self.num_partitions): old_replicas = set(partition_info["partitions"][i]["replicas"]) new_part = (i + 1) % self.num_partitions new_replicas = set( partition_info["partitions"][new_part]["replicas"]) max_num_moves = max(len(new_replicas - old_replicas), max_num_moves) partition_info["partitions"][i]["partition"] = new_part self.logger.debug("Jumbled partitions: " + str(partition_info)) self.kafka.execute_reassign_partitions(partition_info, throttle=throttle) start = time.time() if bounce_brokers: # bounce a few brokers at the same time self.clean_bounce_some_brokers() # Wait until finished or timeout size_per_broker = max_num_moves * self.partition_size self.logger.debug("Max amount of data transfer per broker: %fb", size_per_broker) estimated_throttled_time = math.ceil( float(size_per_broker) / self.throttle) estimated_time_with_buffer = estimated_throttled_time * 2 self.logger.debug("Waiting %ds for the reassignment to complete", estimated_time_with_buffer) wait_until( lambda: self.kafka.verify_reassign_partitions(partition_info), timeout_sec=estimated_time_with_buffer, backoff_sec=.5) stop = time.time() time_taken = stop - start self.logger.debug("Transfer took %d second. Estimated time : %ds", time_taken, estimated_throttled_time) assert time_taken >= estimated_throttled_time, \ ("Expected rebalance to take at least %ds, but it took %ds" % ( estimated_throttled_time, time_taken)) @cluster(num_nodes=10) @parametrize(bounce_brokers=True) @parametrize(bounce_brokers=False) def test_throttled_reassignment(self, bounce_brokers): security_protocol = 'PLAINTEXT' self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = security_protocol producer_id = 'bulk_producer' bulk_producer = ProducerPerformanceService( context=self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, num_records=self.num_records, record_size=self.record_size, throughput=-1, client_id=producer_id, jmx_object_names=[ 'kafka.producer:type=producer-metrics,client-id=%s' % producer_id ], jmx_attributes=['outgoing-byte-rate']) self.producer = VerifiableProducer(context=self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, message_validator=is_int, throughput=self.producer_throughput) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=60000, message_validator=is_int, from_beginning=False) self.kafka.start() bulk_producer.run() self.run_produce_consume_validate( core_test_action=lambda: self.reassign_partitions( bounce_brokers, self.throttle))
class ReassignPartitionsTest(ProduceConsumeValidateTest): """ These tests validate partition reassignment. Create a topic with few partitions, load some data, trigger partition re-assignment with and without broker failure, check that partition re-assignment can complete and there is no data loss. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ReassignPartitionsTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(test_context, num_nodes=4, zk=self.zk, topics={self.topic: { "partitions": 20, "replication-factor": 3, 'configs': {"min.insync.replicas": 2}} }) self.num_partitions = 20 self.timeout_sec = 60 self.producer_throughput = 1000 self.num_producers = 1 self.num_consumers = 1 def setUp(self): self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super(ReassignPartitionsTest, self).min_cluster_size() + self.num_producers + self.num_consumers def clean_bounce_some_brokers(self): """Bounce every other broker""" for node in self.kafka.nodes[::2]: self.kafka.restart_node(node, clean_shutdown=True) def reassign_partitions(self, bounce_brokers): partition_info = self.kafka.parse_describe_topic(self.kafka.describe_topic(self.topic)) self.logger.debug("Partitions before reassignment:" + str(partition_info)) # jumble partition assignment in dictionary seed = random.randint(0, 2 ** 31 - 1) self.logger.debug("Jumble partition assignment with seed " + str(seed)) random.seed(seed) # The list may still be in order, but that's ok shuffled_list = range(0, self.num_partitions) random.shuffle(shuffled_list) for i in range(0, self.num_partitions): partition_info["partitions"][i]["partition"] = shuffled_list[i] self.logger.debug("Jumbled partitions: " + str(partition_info)) # send reassign partitions command self.kafka.execute_reassign_partitions(partition_info) if bounce_brokers: # bounce a few brokers at the same time self.clean_bounce_some_brokers() # Wait until finished or timeout wait_until(lambda: self.kafka.verify_reassign_partitions(partition_info), timeout_sec=self.timeout_sec, backoff_sec=.5) @parametrize(security_protocol="PLAINTEXT", bounce_brokers=True) @parametrize(security_protocol="PLAINTEXT", bounce_brokers=False) def test_reassign_partitions(self, bounce_brokers, security_protocol): """Reassign partitions tests. Setup: 1 zk, 3 kafka nodes, 1 topic with partitions=3, replication-factor=3, and min.insync.replicas=2 - Produce messages in the background - Consume messages in the background - Reassign partitions - If bounce_brokers is True, also bounce a few brokers while partition re-assignment is in progress - When done reassigning partitions and bouncing brokers, stop producing, and finish consuming - Validate that every acked message was consumed """ self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = security_protocol new_consumer = False if self.kafka.security_protocol == "PLAINTEXT" else True self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, new_consumer=new_consumer, consumer_timeout_ms=60000, message_validator=is_int) self.kafka.start() self.run_produce_consume_validate(core_test_action=lambda: self.reassign_partitions(bounce_brokers))
class ReassignPartitionsTest(ProduceConsumeValidateTest): """ These tests validate partition reassignment. Create a topic with few partitions, load some data, trigger partition re-assignment with and without broker failure, check that partition re-assignment can complete and there is no data loss. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ReassignPartitionsTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.num_partitions = 20 self.zk = ZookeeperService(test_context, num_nodes=1) # We set the min.insync.replicas to match the replication factor because # it makes the test more stringent. If min.isr = 2 and # replication.factor=3, then the test would tolerate the failure of # reassignment for upto one replica per partition, which is not # desirable for this test in particular. self.kafka = KafkaService(test_context, num_nodes=4, zk=self.zk, server_prop_overides=[ [config_property.LOG_ROLL_TIME_MS, "5000"], [config_property.LOG_RETENTION_CHECK_INTERVAL_MS, "5000"] ], topics={self.topic: { "partitions": self.num_partitions, "replication-factor": 3, 'configs': { "min.insync.replicas": 3, }} }) self.timeout_sec = 60 self.producer_throughput = 1000 self.num_producers = 1 self.num_consumers = 1 def setUp(self): self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super(ReassignPartitionsTest, self).min_cluster_size() + self.num_producers + self.num_consumers def clean_bounce_some_brokers(self): """Bounce every other broker""" for node in self.kafka.nodes[::2]: self.kafka.restart_node(node, clean_shutdown=True) def reassign_partitions(self, bounce_brokers): partition_info = self.kafka.parse_describe_topic(self.kafka.describe_topic(self.topic)) self.logger.debug("Partitions before reassignment:" + str(partition_info)) # jumble partition assignment in dictionary seed = random.randint(0, 2 ** 31 - 1) self.logger.debug("Jumble partition assignment with seed " + str(seed)) random.seed(seed) # The list may still be in order, but that's ok shuffled_list = range(0, self.num_partitions) random.shuffle(shuffled_list) for i in range(0, self.num_partitions): partition_info["partitions"][i]["partition"] = shuffled_list[i] self.logger.debug("Jumbled partitions: " + str(partition_info)) # send reassign partitions command self.kafka.execute_reassign_partitions(partition_info) if bounce_brokers: # bounce a few brokers at the same time self.clean_bounce_some_brokers() # Wait until finished or timeout wait_until(lambda: self.kafka.verify_reassign_partitions(partition_info), timeout_sec=self.timeout_sec, backoff_sec=.5) def move_start_offset(self): """We move the start offset of the topic by writing really old messages and waiting for them to be cleaned up. """ producer = VerifiableProducer(self.test_context, 1, self.kafka, self.topic, throughput=-1, enable_idempotence=True, create_time=1000) producer.start() wait_until(lambda: producer.num_acked > 0, timeout_sec=30, err_msg="Failed to get an acknowledgement for %ds" % 30) # Wait 8 seconds to let the topic be seeded with messages that will # be deleted. The 8 seconds is important, since we should get 2 deleted # segments in this period based on the configured log roll time and the # retention check interval. time.sleep(8) producer.stop() self.logger.info("Seeded topic with %d messages which will be deleted" %\ producer.num_acked) # Since the configured check interval is 5 seconds, we wait another # 6 seconds to ensure that at least one more cleaning so that the last # segment is deleted. An altenate to using timeouts is to poll each # partition until the log start offset matches the end offset. The # latter is more robust. time.sleep(6) @cluster(num_nodes=8) @matrix(bounce_brokers=[True, False], reassign_from_offset_zero=[True, False]) def test_reassign_partitions(self, bounce_brokers, reassign_from_offset_zero): """Reassign partitions tests. Setup: 1 zk, 4 kafka nodes, 1 topic with partitions=20, replication-factor=3, and min.insync.replicas=3 - Produce messages in the background - Consume messages in the background - Reassign partitions - If bounce_brokers is True, also bounce a few brokers while partition re-assignment is in progress - When done reassigning partitions and bouncing brokers, stop producing, and finish consuming - Validate that every acked message was consumed """ self.kafka.start() if not reassign_from_offset_zero: self.move_start_offset() self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, enable_idempotence=True) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=60000, message_validator=is_int) self.enable_idempotence=True self.run_produce_consume_validate(core_test_action=lambda: self.reassign_partitions(bounce_brokers))
class ReassignPartitionsTest(ProduceConsumeValidateTest): """ These tests validate partition reassignment. Create a topic with few partitions, load some data, trigger partition re-assignment with and without broker failure, check that partition re-assignment can complete and there is no data loss. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ReassignPartitionsTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.num_partitions = 20 self.zk = ZookeeperService(test_context, num_nodes=1) # We set the min.insync.replicas to match the replication factor because # it makes the test more stringent. If min.isr = 2 and # replication.factor=3, then the test would tolerate the failure of # reassignment for upto one replica per partition, which is not # desirable for this test in particular. self.kafka = KafkaService( test_context, num_nodes=4, zk=self.zk, server_prop_overides=[[ config_property.LOG_ROLL_TIME_MS, "5000" ], [config_property.LOG_RETENTION_CHECK_INTERVAL_MS, "5000"]], topics={ self.topic: { "partitions": self.num_partitions, "replication-factor": 3, 'configs': { "min.insync.replicas": 3, } } }) self.timeout_sec = 60 self.producer_throughput = 1000 self.num_producers = 1 self.num_consumers = 1 def setUp(self): self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super( ReassignPartitionsTest, self).min_cluster_size() + self.num_producers + self.num_consumers def clean_bounce_some_brokers(self): """Bounce every other broker""" for node in self.kafka.nodes[::2]: self.kafka.restart_node(node, clean_shutdown=True) def reassign_partitions(self, bounce_brokers): partition_info = self.kafka.parse_describe_topic( self.kafka.describe_topic(self.topic)) self.logger.debug("Partitions before reassignment:" + str(partition_info)) # jumble partition assignment in dictionary seed = random.randint(0, 2**31 - 1) self.logger.debug("Jumble partition assignment with seed " + str(seed)) random.seed(seed) # The list may still be in order, but that's ok shuffled_list = list(range(0, self.num_partitions)) random.shuffle(shuffled_list) for i in range(0, self.num_partitions): partition_info["partitions"][i]["partition"] = shuffled_list[i] self.logger.debug("Jumbled partitions: " + str(partition_info)) # send reassign partitions command self.kafka.execute_reassign_partitions(partition_info) if bounce_brokers: # bounce a few brokers at the same time self.clean_bounce_some_brokers() # Wait until finished or timeout wait_until( lambda: self.kafka.verify_reassign_partitions(partition_info), timeout_sec=self.timeout_sec, backoff_sec=.5) def move_start_offset(self): """We move the start offset of the topic by writing really old messages and waiting for them to be cleaned up. """ producer = VerifiableProducer(self.test_context, 1, self.kafka, self.topic, throughput=-1, enable_idempotence=True, create_time=1000) producer.start() wait_until(lambda: producer.num_acked > 0, timeout_sec=30, err_msg="Failed to get an acknowledgement for %ds" % 30) # Wait 8 seconds to let the topic be seeded with messages that will # be deleted. The 8 seconds is important, since we should get 2 deleted # segments in this period based on the configured log roll time and the # retention check interval. time.sleep(8) producer.stop() self.logger.info("Seeded topic with %d messages which will be deleted" %\ producer.num_acked) # Since the configured check interval is 5 seconds, we wait another # 6 seconds to ensure that at least one more cleaning so that the last # segment is deleted. An altenate to using timeouts is to poll each # partition until the log start offset matches the end offset. The # latter is more robust. time.sleep(6) @cluster(num_nodes=8) @matrix(bounce_brokers=[True, False], reassign_from_offset_zero=[True, False]) def test_reassign_partitions(self, bounce_brokers, reassign_from_offset_zero): """Reassign partitions tests. Setup: 1 zk, 4 kafka nodes, 1 topic with partitions=20, replication-factor=3, and min.insync.replicas=3 - Produce messages in the background - Consume messages in the background - Reassign partitions - If bounce_brokers is True, also bounce a few brokers while partition re-assignment is in progress - When done reassigning partitions and bouncing brokers, stop producing, and finish consuming - Validate that every acked message was consumed """ self.kafka.start() if not reassign_from_offset_zero: self.move_start_offset() self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, enable_idempotence=True) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=60000, message_validator=is_int) self.enable_idempotence = True self.run_produce_consume_validate( core_test_action=lambda: self.reassign_partitions(bounce_brokers))
class ThrottlingTest(ProduceConsumeValidateTest): """Tests throttled partition reassignment. This is essentially similar to the reassign_partitions_test, except that we throttle the reassignment and verify that it takes a sensible amount of time given the throttle and the amount of data being moved. Since the correctness is time dependent, this test also simplifies the cluster topology. In particular, we fix the number of brokers, the replication-factor, the number of partitions, the partition size, and the number of partitions being moved so that we can accurately predict the time throttled reassignment should take. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ThrottlingTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.zk = ZookeeperService(test_context, num_nodes=1) # Because we are starting the producer/consumer/validate cycle _after_ # seeding the cluster with big data (to test throttling), we need to # Start the consumer from the end of the stream. further, we need to # ensure that the consumer is fully started before the producer starts # so that we don't miss any messages. This timeout ensures the sufficient # condition. self.consumer_init_timeout_sec = 20 self.num_brokers = 6 self.num_partitions = 3 self.kafka = KafkaService(test_context, num_nodes=self.num_brokers, zk=self.zk, topics={ self.topic: { "partitions": self.num_partitions, "replication-factor": 2, "configs": { "segment.bytes": 64 * 1024 * 1024 } } }) self.producer_throughput = 1000 self.timeout_sec = 400 self.num_records = 2000 self.record_size = 4096 * 100 # 400 KB # 1 MB per partition on average. self.partition_size = (self.num_records * self.record_size) / self.num_partitions self.num_producers = 2 self.num_consumers = 1 self.throttle = 4 * 1024 * 1024 # 4 MB/s def setUp(self): self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super(ThrottlingTest, self).min_cluster_size() +\ self.num_producers + self.num_consumers def clean_bounce_some_brokers(self): """Bounce every other broker""" for node in self.kafka.nodes[::2]: self.kafka.restart_node(node, clean_shutdown=True) def reassign_partitions(self, bounce_brokers, throttle): """This method reassigns partitions using a throttle. It makes an assertion about the minimum amount of time the reassignment should take given the value of the throttle, the number of partitions being moved, and the size of each partition. """ partition_info = self.kafka.parse_describe_topic( self.kafka.describe_topic(self.topic)) self.logger.debug("Partitions before reassignment:" + str(partition_info)) max_num_moves = 0 for i in range(0, self.num_partitions): old_replicas = set(partition_info["partitions"][i]["replicas"]) new_part = (i+1) % self.num_partitions new_replicas = set(partition_info["partitions"][new_part]["replicas"]) max_num_moves = max(len(new_replicas - old_replicas), max_num_moves) partition_info["partitions"][i]["partition"] = new_part self.logger.debug("Jumbled partitions: " + str(partition_info)) self.kafka.execute_reassign_partitions(partition_info, throttle=throttle) start = time.time() if bounce_brokers: # bounce a few brokers at the same time self.clean_bounce_some_brokers() # Wait until finished or timeout size_per_broker = max_num_moves * self.partition_size self.logger.debug("Max amount of data transfer per broker: %fb", size_per_broker) estimated_throttled_time = math.ceil(float(size_per_broker) / self.throttle) estimated_time_with_buffer = estimated_throttled_time * 2 self.logger.debug("Waiting %ds for the reassignment to complete", estimated_time_with_buffer) wait_until(lambda: self.kafka.verify_reassign_partitions(partition_info), timeout_sec=estimated_time_with_buffer, backoff_sec=.5) stop = time.time() time_taken = stop - start self.logger.debug("Transfer took %d second. Estimated time : %ds", time_taken, estimated_throttled_time) assert time_taken >= estimated_throttled_time * 0.9, \ ("Expected rebalance to take at least %ds, but it took %ds" % ( estimated_throttled_time, time_taken)) @cluster(num_nodes=10) @parametrize(bounce_brokers=True) @parametrize(bounce_brokers=False) def test_throttled_reassignment(self, bounce_brokers): security_protocol = 'PLAINTEXT' self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = security_protocol producer_id = 'bulk_producer' bulk_producer = ProducerPerformanceService( context=self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, num_records=self.num_records, record_size=self.record_size, throughput=-1, client_id=producer_id) self.producer = VerifiableProducer(context=self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, message_validator=is_int, throughput=self.producer_throughput) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=60000, message_validator=is_int, from_beginning=False) self.kafka.start() bulk_producer.run() self.run_produce_consume_validate(core_test_action= lambda: self.reassign_partitions(bounce_brokers, self.throttle)) self.logger.debug("Bulk producer outgoing-byte-rates: %s", (metric.value for k, metrics in bulk_producer.metrics(group='producer-metrics', name='outgoing-byte-rate', client_id=producer_id) for metric in metrics) )