def test_produce_bench(self): active_topics = { "produce_bench_topic[0-1]": { "numPartitions": 1, "replicationFactor": 3 } } inactive_topics = { "produce_bench_topic[2-9]": { "numPartitions": 1, "replicationFactor": 3 } } spec = ProduceBenchWorkloadSpec( 0, TaskSpec.MAX_DURATION_MS, self.workload_service.producer_node, self.workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=100000, producer_conf={}, inactive_topics=inactive_topics, active_topics=active_topics) workload1 = self.trogdor.create_task("workload1", spec) workload1.wait_for_done(timeout_sec=360) tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_produce_bench(self): spec = ProduceBenchWorkloadSpec( 0, TaskSpec.MAX_DURATION_MS, self.workload_service.producer_node, self.workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=100000, producer_conf={}, admin_client_conf={}, common_client_conf={}, inactive_topics=self.inactive_topics, active_topics=self.active_topics) workload1 = self.trogdor.create_task("workload1", spec) # the trogdor service logs all requests() operations to INFO level, # which is too verbose. We explicitly change the level to WARNING and # set it back after the wait_for_done function returns self.trogdor.logger.setLevel('WARNING') workload1.wait_for_done(timeout_sec=360) # set it back to info self.trogdor.logger.setLevel('INFO') tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_produce_consume(self, topic_count, partition_count, replication_factor): topics_create_start_time = time.time() for i in range(topic_count): topic = "replicas_produce_consume_%d" % i print("Creating topic %s" % topic) # Force some stdout for Jenkins topic_cfg = { "topic": topic, "partitions": partition_count, "replication-factor": replication_factor, "configs": {"min.insync.replicas": 2} } self.kafka.create_topic(topic_cfg) topics_create_end_time = time.time() self.logger.info("Time to create topics: %d" % (topics_create_end_time - topics_create_start_time)) producer_workload_service = ProduceBenchWorkloadService(self.test_context, self.kafka) consumer_workload_service = ConsumeBenchWorkloadService(self.test_context, self.kafka) trogdor = TrogdorService(context=self.test_context, client_services=[self.kafka, producer_workload_service, consumer_workload_service]) trogdor.start() produce_spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS, producer_workload_service.producer_node, producer_workload_service.bootstrap_servers, target_messages_per_sec=10000, max_messages=3400000, producer_conf={}, admin_client_conf={}, common_client_conf={}, inactive_topics={}, active_topics={"replicas_produce_consume_[0-2]": { "numPartitions": partition_count, "replicationFactor": replication_factor }}) produce_workload = trogdor.create_task("replicas-produce-workload", produce_spec) produce_workload.wait_for_done(timeout_sec=600) self.logger.info("Completed produce bench") consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS, consumer_workload_service.consumer_node, consumer_workload_service.bootstrap_servers, target_messages_per_sec=10000, max_messages=3400000, consumer_conf={}, admin_client_conf={}, common_client_conf={}, active_topics=["replicas_produce_consume_[0-2]"]) consume_workload = trogdor.create_task("replicas-consume-workload", consume_spec) consume_workload.wait_for_done(timeout_sec=600) self.logger.info("Completed consume bench") trogdor.stop()
def produce_messages(self, topics, max_messages=10000): produce_spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS, self.producer_workload_service.producer_node, self.producer_workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=max_messages, producer_conf={}, admin_client_conf={}, common_client_conf={}, inactive_topics={}, active_topics=topics) produce_workload = self.trogdor.create_task("produce_workload", produce_spec) produce_workload.wait_for_done(timeout_sec=180) self.logger.debug("Produce workload finished")
def test_produce_bench(self): spec = ProduceBenchWorkloadSpec( 0, TaskSpec.MAX_DURATION_MS, self.workload_service.producer_node, self.workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=100000, producer_conf={}, total_topics=10, active_topics=2) workload1 = self.trogdor.create_task("workload1", spec) workload1.wait_for_done(timeout_sec=360) tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_produce_bench_transactions(self, metadata_quorum=quorum.zk): spec = ProduceBenchWorkloadSpec( 0, TaskSpec.MAX_DURATION_MS, self.workload_service.producer_node, self.workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=100000, producer_conf={}, admin_client_conf={}, common_client_conf={}, inactive_topics=self.inactive_topics, active_topics=self.active_topics, transaction_generator={ # 10 transactions with 10k messages "type": "uniform", "messagesPerTransaction": "10000" }) workload1 = self.trogdor.create_task("workload1", spec) workload1.wait_for_done(timeout_sec=360) tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))