def test_multiple_consumers_random_group_partitions(
self, metadata_quorum=quorum.zk):
"""
Runs multiple consumers in to read messages from specific partitions.
Since a consumerGroup isn't specified, each consumer will get assigned a random group
and consume from all partitions
"""
self.produce_messages(self.active_topics, max_messages=20000)
consume_spec = ConsumeBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=4,
active_topics=["consume_bench_topic1:[0-4]"])
consume_workload = self.trogdor.create_task("consume_workload",
consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
def test_multiple_consumers_specified_group_partitions_should_raise(
self, metadata_quorum=quorum.zk):
"""
Runs multiple consumers in the same group to read messages from specific partitions.
It is an invalid configuration to provide a consumer group and specific partitions.
"""
expected_error_msg = 'explicit partition assignment'
self.produce_messages(self.active_topics, max_messages=20000)
consume_spec = ConsumeBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=4,
consumer_group="fail_group",
active_topics=["consume_bench_topic1:[0-4]"])
consume_workload = self.trogdor.create_task("consume_workload",
consume_spec)
try:
consume_workload.wait_for_done(timeout_sec=360)
raise Exception(
"Should have raised an exception due to an invalid configuration"
)
except RuntimeError as e:
if expected_error_msg not in str(e):
raise RuntimeError("Unexpected Exception - " + str(e))
self.logger.info(e)
def test_single_partition(self, metadata_quorum=quorum.zk):
"""
Run a ConsumeBench against a single partition
"""
active_topics = {
"consume_bench_topic": {
"numPartitions": 2,
"replicationFactor": 3
}
}
self.produce_messages(active_topics, 5000)
consume_spec = ConsumeBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2500,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
active_topics=["consume_bench_topic:1"])
consume_workload = self.trogdor.create_task("consume_workload",
consume_spec)
consume_workload.wait_for_done(timeout_sec=180)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
def test_two_consumers_specified_group_topics(self,
metadata_quorum=quorum.zk):
"""
Runs two consumers in the same consumer group to read messages from topics.
Since a consumerGroup is specified, each consumer should dynamically get assigned a partition from group
"""
self.produce_messages(self.active_topics)
consume_spec = ConsumeBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2000, # both should read at least 2k messages
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=2,
consumer_group="testGroup",
active_topics=["consume_bench_topic[0-5]"])
consume_workload = self.trogdor.create_task("consume_workload",
consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
def test_multiple_consumers_random_group_topics(self):
"""
Runs multiple consumers group to read messages from topics.
Since a consumerGroup isn't specified, each consumer should read from all topics independently
"""
self.produce_messages(self.active_topics, max_messages=5000)
consume_spec = ConsumeBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=5000, # all should read exactly 5k messages
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=5,
active_topics=["consume_bench_topic[0-5]"])
consume_workload = self.trogdor.create_task("consume_workload",
consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
def test_multiple_consumers_specified_group_partitions_should_raise(self):
"""
Runs multiple consumers in to read messages from specific partitions.
Since a consumerGroup isn't specified, each consumer will get assigned a random group
and consume from all partitions
"""
self.produce_messages(self.active_topics, max_messages=20000)
consume_spec = ConsumeBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=4,
consumer_group="fail_group",
active_topics=["consume_bench_topic1:[0-4]"])
consume_workload = self.trogdor.create_task("consume_workload",
consume_spec)
try:
consume_workload.wait_for_done(timeout_sec=360)
raise Exception(
"Should have raised an exception due to an invalid configuration"
)
except RuntimeError as e:
if 'Will not split partitions' not in str(e):
raise RuntimeError("Unexpected Exception - " + str(e))
self.logger.info(e)
def test_produce_consume(self, topic_count, partition_count, replication_factor):
topics_create_start_time = time.time()
for i in range(topic_count):
topic = "replicas_produce_consume_%d" % i
print("Creating topic %s" % topic) # Force some stdout for Jenkins
topic_cfg = {
"topic": topic,
"partitions": partition_count,
"replication-factor": replication_factor,
"configs": {"min.insync.replicas": 2}
}
self.kafka.create_topic(topic_cfg)
topics_create_end_time = time.time()
self.logger.info("Time to create topics: %d" % (topics_create_end_time - topics_create_start_time))
producer_workload_service = ProduceBenchWorkloadService(self.test_context, self.kafka)
consumer_workload_service = ConsumeBenchWorkloadService(self.test_context, self.kafka)
trogdor = TrogdorService(context=self.test_context,
client_services=[self.kafka, producer_workload_service, consumer_workload_service])
trogdor.start()
produce_spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
producer_workload_service.producer_node,
producer_workload_service.bootstrap_servers,
target_messages_per_sec=10000,
max_messages=3400000,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics={},
active_topics={"replicas_produce_consume_[0-2]": {
"numPartitions": partition_count, "replicationFactor": replication_factor
}})
produce_workload = trogdor.create_task("replicas-produce-workload", produce_spec)
produce_workload.wait_for_done(timeout_sec=600)
self.logger.info("Completed produce bench")
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
consumer_workload_service.consumer_node,
consumer_workload_service.bootstrap_servers,
target_messages_per_sec=10000,
max_messages=3400000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
active_topics=["replicas_produce_consume_[0-2]"])
consume_workload = trogdor.create_task("replicas-consume-workload", consume_spec)
consume_workload.wait_for_done(timeout_sec=600)
self.logger.info("Completed consume bench")
trogdor.stop()
def test_consume_bench(self, topics):
"""
Runs a ConsumeBench workload to consume messages
"""
self.produce_messages(self.active_topics)
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=10000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
active_topics=topics)
consume_workload = self.trogdor.create_task("consume_workload", consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_consume_group_bench(self):
"""
Runs two ConsumeBench workloads in the same consumer group to read messages from topics
"""
self.produce_messages(self.active_topics)
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2000, # both should read at least 2k messages
consumer_conf={},
admin_client_conf={},
common_client_conf={},
consumer_group="testGroup",
active_topics=["consume_bench_topic[0-5]"])
consume_workload_1 = self.trogdor.create_task("consume_workload_1", consume_spec)
consume_workload_2 = self.trogdor.create_task("consume_workload_2", consume_spec)
consume_workload_1.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload 1 finished")
consume_workload_2.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload 2 finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))