def test_produce_bench(self):
active_topics = {
"produce_bench_topic[0-1]": {
"numPartitions": 1,
"replicationFactor": 3
}
}
inactive_topics = {
"produce_bench_topic[2-9]": {
"numPartitions": 1,
"replicationFactor": 3
}
}
spec = ProduceBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.workload_service.producer_node,
self.workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=100000,
producer_conf={},
inactive_topics=inactive_topics,
active_topics=active_topics)
workload1 = self.trogdor.create_task("workload1", spec)
workload1.wait_for_done(timeout_sec=360)
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
def test_produce_bench(self):
spec = ProduceBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.workload_service.producer_node,
self.workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=100000,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics=self.inactive_topics,
active_topics=self.active_topics)
workload1 = self.trogdor.create_task("workload1", spec)
# the trogdor service logs all requests() operations to INFO level,
# which is too verbose. We explicitly change the level to WARNING and
# set it back after the wait_for_done function returns
self.trogdor.logger.setLevel('WARNING')
workload1.wait_for_done(timeout_sec=360)
# set it back to info
self.trogdor.logger.setLevel('INFO')
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
def test_produce_consume(self, topic_count, partition_count, replication_factor):
topics_create_start_time = time.time()
for i in range(topic_count):
topic = "replicas_produce_consume_%d" % i
print("Creating topic %s" % topic) # Force some stdout for Jenkins
topic_cfg = {
"topic": topic,
"partitions": partition_count,
"replication-factor": replication_factor,
"configs": {"min.insync.replicas": 2}
}
self.kafka.create_topic(topic_cfg)
topics_create_end_time = time.time()
self.logger.info("Time to create topics: %d" % (topics_create_end_time - topics_create_start_time))
producer_workload_service = ProduceBenchWorkloadService(self.test_context, self.kafka)
consumer_workload_service = ConsumeBenchWorkloadService(self.test_context, self.kafka)
trogdor = TrogdorService(context=self.test_context,
client_services=[self.kafka, producer_workload_service, consumer_workload_service])
trogdor.start()
produce_spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
producer_workload_service.producer_node,
producer_workload_service.bootstrap_servers,
target_messages_per_sec=10000,
max_messages=3400000,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics={},
active_topics={"replicas_produce_consume_[0-2]": {
"numPartitions": partition_count, "replicationFactor": replication_factor
}})
produce_workload = trogdor.create_task("replicas-produce-workload", produce_spec)
produce_workload.wait_for_done(timeout_sec=600)
self.logger.info("Completed produce bench")
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
consumer_workload_service.consumer_node,
consumer_workload_service.bootstrap_servers,
target_messages_per_sec=10000,
max_messages=3400000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
active_topics=["replicas_produce_consume_[0-2]"])
consume_workload = trogdor.create_task("replicas-consume-workload", consume_spec)
consume_workload.wait_for_done(timeout_sec=600)
self.logger.info("Completed consume bench")
trogdor.stop()
def produce_messages(self, topics, max_messages=10000):
produce_spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.producer_workload_service.producer_node,
self.producer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=max_messages,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics={},
active_topics=topics)
produce_workload = self.trogdor.create_task("produce_workload", produce_spec)
produce_workload.wait_for_done(timeout_sec=180)
self.logger.debug("Produce workload finished")
def test_produce_bench(self):
spec = ProduceBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.workload_service.producer_node,
self.workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=100000,
producer_conf={},
total_topics=10,
active_topics=2)
workload1 = self.trogdor.create_task("workload1", spec)
workload1.wait_for_done(timeout_sec=360)
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
def test_produce_bench_transactions(self, metadata_quorum=quorum.zk):
spec = ProduceBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.workload_service.producer_node,
self.workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=100000,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics=self.inactive_topics,
active_topics=self.active_topics,
transaction_generator={
# 10 transactions with 10k messages
"type": "uniform",
"messagesPerTransaction": "10000"
})
workload1 = self.trogdor.create_task("workload1", spec)
workload1.wait_for_done(timeout_sec=360)
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))