def test_multiple_consumers_random_group_partitions( self, metadata_quorum=quorum.zk): """ Runs multiple consumers in to read messages from specific partitions. Since a consumerGroup isn't specified, each consumer will get assigned a random group and consume from all partitions """ self.produce_messages(self.active_topics, max_messages=20000) consume_spec = ConsumeBenchWorkloadSpec( 0, TaskSpec.MAX_DURATION_MS, self.consumer_workload_service.consumer_node, self.consumer_workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=2000, consumer_conf={}, admin_client_conf={}, common_client_conf={}, threads_per_worker=4, active_topics=["consume_bench_topic1:[0-4]"]) consume_workload = self.trogdor.create_task("consume_workload", consume_spec) consume_workload.wait_for_done(timeout_sec=360) self.logger.debug("Consume workload finished") tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_multiple_consumers_specified_group_partitions_should_raise( self, metadata_quorum=quorum.zk): """ Runs multiple consumers in the same group to read messages from specific partitions. It is an invalid configuration to provide a consumer group and specific partitions. """ expected_error_msg = 'explicit partition assignment' self.produce_messages(self.active_topics, max_messages=20000) consume_spec = ConsumeBenchWorkloadSpec( 0, TaskSpec.MAX_DURATION_MS, self.consumer_workload_service.consumer_node, self.consumer_workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=2000, consumer_conf={}, admin_client_conf={}, common_client_conf={}, threads_per_worker=4, consumer_group="fail_group", active_topics=["consume_bench_topic1:[0-4]"]) consume_workload = self.trogdor.create_task("consume_workload", consume_spec) try: consume_workload.wait_for_done(timeout_sec=360) raise Exception( "Should have raised an exception due to an invalid configuration" ) except RuntimeError as e: if expected_error_msg not in str(e): raise RuntimeError("Unexpected Exception - " + str(e)) self.logger.info(e)
def test_single_partition(self, metadata_quorum=quorum.zk): """ Run a ConsumeBench against a single partition """ active_topics = { "consume_bench_topic": { "numPartitions": 2, "replicationFactor": 3 } } self.produce_messages(active_topics, 5000) consume_spec = ConsumeBenchWorkloadSpec( 0, TaskSpec.MAX_DURATION_MS, self.consumer_workload_service.consumer_node, self.consumer_workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=2500, consumer_conf={}, admin_client_conf={}, common_client_conf={}, active_topics=["consume_bench_topic:1"]) consume_workload = self.trogdor.create_task("consume_workload", consume_spec) consume_workload.wait_for_done(timeout_sec=180) self.logger.debug("Consume workload finished") tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_two_consumers_specified_group_topics(self, metadata_quorum=quorum.zk): """ Runs two consumers in the same consumer group to read messages from topics. Since a consumerGroup is specified, each consumer should dynamically get assigned a partition from group """ self.produce_messages(self.active_topics) consume_spec = ConsumeBenchWorkloadSpec( 0, TaskSpec.MAX_DURATION_MS, self.consumer_workload_service.consumer_node, self.consumer_workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=2000, # both should read at least 2k messages consumer_conf={}, admin_client_conf={}, common_client_conf={}, threads_per_worker=2, consumer_group="testGroup", active_topics=["consume_bench_topic[0-5]"]) consume_workload = self.trogdor.create_task("consume_workload", consume_spec) consume_workload.wait_for_done(timeout_sec=360) self.logger.debug("Consume workload finished") tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_multiple_consumers_random_group_topics(self): """ Runs multiple consumers group to read messages from topics. Since a consumerGroup isn't specified, each consumer should read from all topics independently """ self.produce_messages(self.active_topics, max_messages=5000) consume_spec = ConsumeBenchWorkloadSpec( 0, TaskSpec.MAX_DURATION_MS, self.consumer_workload_service.consumer_node, self.consumer_workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=5000, # all should read exactly 5k messages consumer_conf={}, admin_client_conf={}, common_client_conf={}, threads_per_worker=5, active_topics=["consume_bench_topic[0-5]"]) consume_workload = self.trogdor.create_task("consume_workload", consume_spec) consume_workload.wait_for_done(timeout_sec=360) self.logger.debug("Consume workload finished") tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_multiple_consumers_specified_group_partitions_should_raise(self): """ Runs multiple consumers in to read messages from specific partitions. Since a consumerGroup isn't specified, each consumer will get assigned a random group and consume from all partitions """ self.produce_messages(self.active_topics, max_messages=20000) consume_spec = ConsumeBenchWorkloadSpec( 0, TaskSpec.MAX_DURATION_MS, self.consumer_workload_service.consumer_node, self.consumer_workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=2000, consumer_conf={}, admin_client_conf={}, common_client_conf={}, threads_per_worker=4, consumer_group="fail_group", active_topics=["consume_bench_topic1:[0-4]"]) consume_workload = self.trogdor.create_task("consume_workload", consume_spec) try: consume_workload.wait_for_done(timeout_sec=360) raise Exception( "Should have raised an exception due to an invalid configuration" ) except RuntimeError as e: if 'Will not split partitions' not in str(e): raise RuntimeError("Unexpected Exception - " + str(e)) self.logger.info(e)
def test_produce_consume(self, topic_count, partition_count, replication_factor): topics_create_start_time = time.time() for i in range(topic_count): topic = "replicas_produce_consume_%d" % i print("Creating topic %s" % topic) # Force some stdout for Jenkins topic_cfg = { "topic": topic, "partitions": partition_count, "replication-factor": replication_factor, "configs": {"min.insync.replicas": 2} } self.kafka.create_topic(topic_cfg) topics_create_end_time = time.time() self.logger.info("Time to create topics: %d" % (topics_create_end_time - topics_create_start_time)) producer_workload_service = ProduceBenchWorkloadService(self.test_context, self.kafka) consumer_workload_service = ConsumeBenchWorkloadService(self.test_context, self.kafka) trogdor = TrogdorService(context=self.test_context, client_services=[self.kafka, producer_workload_service, consumer_workload_service]) trogdor.start() produce_spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS, producer_workload_service.producer_node, producer_workload_service.bootstrap_servers, target_messages_per_sec=10000, max_messages=3400000, producer_conf={}, admin_client_conf={}, common_client_conf={}, inactive_topics={}, active_topics={"replicas_produce_consume_[0-2]": { "numPartitions": partition_count, "replicationFactor": replication_factor }}) produce_workload = trogdor.create_task("replicas-produce-workload", produce_spec) produce_workload.wait_for_done(timeout_sec=600) self.logger.info("Completed produce bench") consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS, consumer_workload_service.consumer_node, consumer_workload_service.bootstrap_servers, target_messages_per_sec=10000, max_messages=3400000, consumer_conf={}, admin_client_conf={}, common_client_conf={}, active_topics=["replicas_produce_consume_[0-2]"]) consume_workload = trogdor.create_task("replicas-consume-workload", consume_spec) consume_workload.wait_for_done(timeout_sec=600) self.logger.info("Completed consume bench") trogdor.stop()
def test_consume_bench(self, topics): """ Runs a ConsumeBench workload to consume messages """ self.produce_messages(self.active_topics) consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS, self.consumer_workload_service.consumer_node, self.consumer_workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=10000, consumer_conf={}, admin_client_conf={}, common_client_conf={}, active_topics=topics) consume_workload = self.trogdor.create_task("consume_workload", consume_spec) consume_workload.wait_for_done(timeout_sec=360) self.logger.debug("Consume workload finished") tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_consume_group_bench(self): """ Runs two ConsumeBench workloads in the same consumer group to read messages from topics """ self.produce_messages(self.active_topics) consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS, self.consumer_workload_service.consumer_node, self.consumer_workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=2000, # both should read at least 2k messages consumer_conf={}, admin_client_conf={}, common_client_conf={}, consumer_group="testGroup", active_topics=["consume_bench_topic[0-5]"]) consume_workload_1 = self.trogdor.create_task("consume_workload_1", consume_spec) consume_workload_2 = self.trogdor.create_task("consume_workload_2", consume_spec) consume_workload_1.wait_for_done(timeout_sec=360) self.logger.debug("Consume workload 1 finished") consume_workload_2.wait_for_done(timeout_sec=360) self.logger.debug("Consume workload 2 finished") tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))