class ProduceBenchTest(Test):
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ProduceBenchTest, self).__init__(test_context)
self.zk = ZookeeperService(test_context, num_nodes=3)
self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk)
self.workload_service = ProduceBenchWorkloadService(test_context, self.kafka)
self.trogdor = TrogdorService(context=self.test_context,
client_services=[self.kafka, self.workload_service])
self.active_topics = {"produce_bench_topic[0-1]": {"numPartitions": 1, "replicationFactor": 3}}
self.inactive_topics = {"produce_bench_topic[2-9]": {"numPartitions": 1, "replicationFactor": 3}}
def setUp(self):
self.trogdor.start()
self.zk.start()
self.kafka.start()
def teardown(self):
self.trogdor.stop()
self.kafka.stop()
self.zk.stop()
def test_produce_bench(self):
spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.workload_service.producer_node,
self.workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=100000,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics=self.inactive_topics,
active_topics=self.active_topics)
workload1 = self.trogdor.create_task("workload1", spec)
workload1.wait_for_done(timeout_sec=360)
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_produce_bench_transactions(self):
spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.workload_service.producer_node,
self.workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=100000,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics=self.inactive_topics,
active_topics=self.active_topics,
transaction_generator={
# 10 transactions with 10k messages
"type": "uniform",
"messagesPerTransaction": "10000"
})
workload1 = self.trogdor.create_task("workload1", spec)
workload1.wait_for_done(timeout_sec=360)
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
class ProduceBenchTest(Test):
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ProduceBenchTest, self).__init__(test_context)
self.redpanda = RedpandaService(test_context, num_nodes=3)
self.workload_service = ProduceBenchWorkloadService(
test_context, self.redpanda)
self.trogdor = TrogdorService(
context=self.test_context,
client_services=[self.redpanda, self.workload_service])
self.active_topics = {
"produce_bench_topic[0-1]": {
"numPartitions": 1,
"replicationFactor": 3
}
}
self.inactive_topics = {
"produce_bench_topic[2-9]": {
"numPartitions": 1,
"replicationFactor": 3
}
}
def setUp(self):
self.trogdor.start()
self.redpanda.start()
def teardown(self):
self.trogdor.stop()
self.redpanda.stop()
@cluster(num_nodes=3)
def test_produce_bench(self):
spec = ProduceBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.workload_service.producer_node,
self.workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=100000,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics=self.inactive_topics,
active_topics=self.active_topics)
workload1 = self.trogdor.create_task("workload1", spec)
# the trogdor service logs all requests() operations to INFO level,
# which is too verbose. We explicitly change the level to WARNING and
# set it back after the wait_for_done function returns
self.trogdor.logger.setLevel('WARNING')
workload1.wait_for_done(timeout_sec=360)
# set it back to info
self.trogdor.logger.setLevel('INFO')
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
class ProduceBenchTest(Test):
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ProduceBenchTest, self).__init__(test_context)
self.zk = ZookeeperService(test_context, num_nodes=3)
self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk)
self.workload_service = ProduceBenchWorkloadService(
test_context, self.kafka)
self.trogdor = TrogdorService(
context=self.test_context,
client_services=[self.kafka, self.workload_service])
def setUp(self):
self.trogdor.start()
self.zk.start()
self.kafka.start()
def teardown(self):
self.trogdor.stop()
self.kafka.stop()
self.zk.stop()
def test_produce_bench(self):
active_topics = {
"produce_bench_topic[0-1]": {
"numPartitions": 1,
"replicationFactor": 3
}
}
inactive_topics = {
"produce_bench_topic[2-9]": {
"numPartitions": 1,
"replicationFactor": 3
}
}
spec = ProduceBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.workload_service.producer_node,
self.workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=100000,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics=inactive_topics,
active_topics=active_topics)
workload1 = self.trogdor.create_task("workload1", spec)
workload1.wait_for_done(timeout_sec=360)
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
class TrogdorTest(Test):
"""
Tests the Trogdor fault injection daemon in isolation.
"""
def __init__(self, test_context):
super(TrogdorTest, self).__init__(test_context)
def set_up_trogdor(self, num_agent_nodes):
self.agent_nodes = self.test_context.cluster.alloc(ClusterSpec.simple_linux(num_agent_nodes))
self.trogdor = TrogdorService(context=self.test_context, agent_nodes=self.agent_nodes)
for agent_node in self.agent_nodes:
agent_node.account.logger = self.trogdor.logger
self.trogdor.start()
def setUp(self):
self.trogdor = None
self.agent_nodes = None
def tearDown(self):
if self.trogdor is not None:
self.trogdor.stop()
self.trogdor = None
if self.agent_nodes is not None:
self.test_context.cluster.free(self.agent_nodes)
self.agent_nodes = None
@cluster(num_nodes=4)
def test_trogdor_service(self):
"""
Test that we can bring up Trogdor and create a no-op fault.
"""
self.set_up_trogdor(3)
spec = NoOpTaskSpec(0, TaskSpec.MAX_DURATION_MS)
self.trogdor.create_task("myfault", spec)
def check_for_myfault():
faults = self.trogdor.tasks()["tasks"]
self.logger.info("tasks = %s" % faults)
return "myfault" in faults
wait_until(lambda: check_for_myfault,
timeout_sec=10, backoff_sec=.2, err_msg="Failed to read back myfault.")
self.trogdor.stop_task("myfault")
@cluster(num_nodes=4)
def test_network_partition_fault(self):
"""
Test that the network partition fault results in a true network partition between nodes.
"""
self.set_up_trogdor(3)
spec = NetworkPartitionFaultSpec(0, TaskSpec.MAX_DURATION_MS,
[[self.agent_nodes[0]], self.agent_nodes[1:]])
partitions = spec.message["partitions"]
assert 2 == len(partitions)
assert [self.agent_nodes[0].name] == partitions[0]
assert [self.agent_nodes[1].name, self.agent_nodes[2].name] == partitions[1]
self.trogdor.create_task("partition0", spec)
def verify_nodes_partitioned():
if node_is_reachable(self.agent_nodes[0], self.agent_nodes[1]):
return False
if node_is_reachable(self.agent_nodes[1], self.agent_nodes[0]):
return False
if node_is_reachable(self.agent_nodes[2], self.agent_nodes[0]):
return False
return True
wait_until(lambda: verify_nodes_partitioned,
timeout_sec=10, backoff_sec=.2, err_msg="Failed to verify that the nodes were partitioned.")
if not node_is_reachable(self.agent_nodes[0], self.agent_nodes[0]):
raise RuntimeError("Node 0 must be reachable from itself.")
if not node_is_reachable(self.agent_nodes[1], self.agent_nodes[2]):
raise RuntimeError("Node 2 must be reachable from node 1.")
if not node_is_reachable(self.agent_nodes[2], self.agent_nodes[1]):
raise RuntimeError("Node 1 must be reachable from node 2.")
class ProduceBenchTest(Test):
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ProduceBenchTest, self).__init__(test_context)
self.zk = ZookeeperService(test_context,
num_nodes=3) if quorum.for_test(
test_context) == quorum.zk else None
self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk)
self.workload_service = ProduceBenchWorkloadService(
test_context, self.kafka)
self.trogdor = TrogdorService(
context=self.test_context,
client_services=[self.kafka, self.workload_service])
self.active_topics = {
"produce_bench_topic[0-1]": {
"numPartitions": 1,
"replicationFactor": 3
}
}
self.inactive_topics = {
"produce_bench_topic[2-9]": {
"numPartitions": 1,
"replicationFactor": 3
}
}
def setUp(self):
self.trogdor.start()
if self.zk:
self.zk.start()
self.kafka.start()
def teardown(self):
self.trogdor.stop()
self.kafka.stop()
if self.zk:
self.zk.stop()
@cluster(num_nodes=8)
@matrix(metadata_quorum=quorum.all_non_upgrade)
def test_produce_bench(self, metadata_quorum=quorum.zk):
spec = ProduceBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.workload_service.producer_node,
self.workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=100000,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics=self.inactive_topics,
active_topics=self.active_topics)
workload1 = self.trogdor.create_task("workload1", spec)
workload1.wait_for_done(timeout_sec=360)
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
@cluster(num_nodes=8)
def test_produce_bench_transactions(self, metadata_quorum=quorum.zk):
spec = ProduceBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.workload_service.producer_node,
self.workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=100000,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics=self.inactive_topics,
active_topics=self.active_topics,
transaction_generator={
# 10 transactions with 10k messages
"type": "uniform",
"messagesPerTransaction": "10000"
})
workload1 = self.trogdor.create_task("workload1", spec)
workload1.wait_for_done(timeout_sec=360)
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
class ConsumeBenchTest(Test):
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ConsumeBenchTest, self).__init__(test_context)
self.zk = ZookeeperService(test_context,
num_nodes=3) if quorum.for_test(
test_context) == quorum.zk else None
self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk)
self.producer_workload_service = ProduceBenchWorkloadService(
test_context, self.kafka)
self.consumer_workload_service = ConsumeBenchWorkloadService(
test_context, self.kafka)
self.consumer_workload_service_2 = ConsumeBenchWorkloadService(
test_context, self.kafka)
self.active_topics = {
"consume_bench_topic[0-5]": {
"numPartitions": 5,
"replicationFactor": 3
}
}
self.trogdor = TrogdorService(context=self.test_context,
client_services=[
self.kafka,
self.producer_workload_service,
self.consumer_workload_service,
self.consumer_workload_service_2
])
def setUp(self):
self.trogdor.start()
if self.zk:
self.zk.start()
self.kafka.start()
def teardown(self):
self.trogdor.stop()
self.kafka.stop()
if self.zk:
self.zk.stop()
def produce_messages(self, topics, max_messages=10000):
produce_spec = ProduceBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.producer_workload_service.producer_node,
self.producer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=max_messages,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics={},
active_topics=topics)
produce_workload = self.trogdor.create_task("produce_workload",
produce_spec)
produce_workload.wait_for_done(timeout_sec=180)
self.logger.debug("Produce workload finished")
@cluster(num_nodes=10)
@matrix(topics=[["consume_bench_topic[0-5]"]],
metadata_quorum=quorum.all_non_upgrade) # topic subscription
@matrix(topics=[["consume_bench_topic[0-5]:[0-4]"]],
metadata_quorum=quorum.all_non_upgrade) # manual topic assignment
def test_consume_bench(self, topics, metadata_quorum=quorum.zk):
"""
Runs a ConsumeBench workload to consume messages
"""
self.produce_messages(self.active_topics)
consume_spec = ConsumeBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=10000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
active_topics=topics)
consume_workload = self.trogdor.create_task("consume_workload",
consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
@cluster(num_nodes=10)
@matrix(metadata_quorum=quorum.all_non_upgrade)
def test_single_partition(self, metadata_quorum=quorum.zk):
"""
Run a ConsumeBench against a single partition
"""
active_topics = {
"consume_bench_topic": {
"numPartitions": 2,
"replicationFactor": 3
}
}
self.produce_messages(active_topics, 5000)
consume_spec = ConsumeBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2500,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
active_topics=["consume_bench_topic:1"])
consume_workload = self.trogdor.create_task("consume_workload",
consume_spec)
consume_workload.wait_for_done(timeout_sec=180)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
@cluster(num_nodes=10)
@matrix(metadata_quorum=quorum.all_non_upgrade)
def test_multiple_consumers_random_group_topics(self,
metadata_quorum=quorum.zk):
"""
Runs multiple consumers group to read messages from topics.
Since a consumerGroup isn't specified, each consumer should read from all topics independently
"""
self.produce_messages(self.active_topics, max_messages=5000)
consume_spec = ConsumeBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=5000, # all should read exactly 5k messages
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=5,
active_topics=["consume_bench_topic[0-5]"])
consume_workload = self.trogdor.create_task("consume_workload",
consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
@cluster(num_nodes=10)
@matrix(metadata_quorum=quorum.all_non_upgrade)
def test_two_consumers_specified_group_topics(self,
metadata_quorum=quorum.zk):
"""
Runs two consumers in the same consumer group to read messages from topics.
Since a consumerGroup is specified, each consumer should dynamically get assigned a partition from group
"""
self.produce_messages(self.active_topics)
consume_spec = ConsumeBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2000, # both should read at least 2k messages
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=2,
consumer_group="testGroup",
active_topics=["consume_bench_topic[0-5]"])
consume_workload = self.trogdor.create_task("consume_workload",
consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
@cluster(num_nodes=10)
@matrix(metadata_quorum=quorum.all_non_upgrade)
def test_multiple_consumers_random_group_partitions(
self, metadata_quorum=quorum.zk):
"""
Runs multiple consumers in to read messages from specific partitions.
Since a consumerGroup isn't specified, each consumer will get assigned a random group
and consume from all partitions
"""
self.produce_messages(self.active_topics, max_messages=20000)
consume_spec = ConsumeBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=4,
active_topics=["consume_bench_topic1:[0-4]"])
consume_workload = self.trogdor.create_task("consume_workload",
consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" %
json.dumps(tasks, sort_keys=True, indent=2))
@cluster(num_nodes=10)
@matrix(metadata_quorum=quorum.all_non_upgrade)
def test_multiple_consumers_specified_group_partitions_should_raise(
self, metadata_quorum=quorum.zk):
"""
Runs multiple consumers in the same group to read messages from specific partitions.
It is an invalid configuration to provide a consumer group and specific partitions.
"""
expected_error_msg = 'explicit partition assignment'
self.produce_messages(self.active_topics, max_messages=20000)
consume_spec = ConsumeBenchWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=4,
consumer_group="fail_group",
active_topics=["consume_bench_topic1:[0-4]"])
consume_workload = self.trogdor.create_task("consume_workload",
consume_spec)
try:
consume_workload.wait_for_done(timeout_sec=360)
raise Exception(
"Should have raised an exception due to an invalid configuration"
)
except RuntimeError as e:
if expected_error_msg not in str(e):
raise RuntimeError("Unexpected Exception - " + str(e))
self.logger.info(e)
class ConsumeBenchTest(Test):
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ConsumeBenchTest, self).__init__(test_context)
self.zk = ZookeeperService(test_context, num_nodes=3)
self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk)
self.producer_workload_service = ProduceBenchWorkloadService(test_context, self.kafka)
self.consumer_workload_service = ConsumeBenchWorkloadService(test_context, self.kafka)
self.consumer_workload_service_2 = ConsumeBenchWorkloadService(test_context, self.kafka)
self.active_topics = {"consume_bench_topic[0-5]": {"numPartitions": 5, "replicationFactor": 3}}
self.trogdor = TrogdorService(context=self.test_context,
client_services=[self.kafka, self.producer_workload_service,
self.consumer_workload_service,
self.consumer_workload_service_2])
def setUp(self):
self.trogdor.start()
self.zk.start()
self.kafka.start()
def teardown(self):
self.trogdor.stop()
self.kafka.stop()
self.zk.stop()
def produce_messages(self, topics, max_messages=10000):
produce_spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.producer_workload_service.producer_node,
self.producer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=max_messages,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics={},
active_topics=topics)
produce_workload = self.trogdor.create_task("produce_workload", produce_spec)
produce_workload.wait_for_done(timeout_sec=180)
self.logger.debug("Produce workload finished")
@parametrize(topics=["consume_bench_topic[0-5]"]) # topic subscription
@parametrize(topics=["consume_bench_topic[0-5]:[0-4]"]) # manual topic assignment
def test_consume_bench(self, topics):
"""
Runs a ConsumeBench workload to consume messages
"""
self.produce_messages(self.active_topics)
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=10000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
active_topics=topics)
consume_workload = self.trogdor.create_task("consume_workload", consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_consume_bench_single_partition(self):
"""
Run a ConsumeBench against a single partition
"""
active_topics = {"consume_bench_topic": {"numPartitions": 2, "replicationFactor": 3}}
self.produce_messages(active_topics, 5000)
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2500,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
active_topics=["consume_bench_topic:1"])
consume_workload = self.trogdor.create_task("consume_workload", consume_spec)
consume_workload.wait_for_done(timeout_sec=180)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_consume_group_bench(self):
"""
Runs two ConsumeBench workloads in the same consumer group to read messages from topics
"""
self.produce_messages(self.active_topics)
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2000, # both should read at least 2k messages
consumer_conf={},
admin_client_conf={},
common_client_conf={},
consumer_group="testGroup",
active_topics=["consume_bench_topic[0-5]"])
consume_workload_1 = self.trogdor.create_task("consume_workload_1", consume_spec)
consume_workload_2 = self.trogdor.create_task("consume_workload_2", consume_spec)
consume_workload_1.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload 1 finished")
consume_workload_2.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload 2 finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
class ConsumeBenchTest(Test):
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ConsumeBenchTest, self).__init__(test_context)
self.zk = ZookeeperService(test_context, num_nodes=3)
self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk)
self.producer_workload_service = ProduceBenchWorkloadService(test_context, self.kafka)
self.consumer_workload_service = ConsumeBenchWorkloadService(test_context, self.kafka)
self.consumer_workload_service_2 = ConsumeBenchWorkloadService(test_context, self.kafka)
self.active_topics = {"consume_bench_topic[0-5]": {"numPartitions": 5, "replicationFactor": 3}}
self.trogdor = TrogdorService(context=self.test_context,
client_services=[self.kafka, self.producer_workload_service,
self.consumer_workload_service,
self.consumer_workload_service_2])
def setUp(self):
self.trogdor.start()
self.zk.start()
self.kafka.start()
def teardown(self):
self.trogdor.stop()
self.kafka.stop()
self.zk.stop()
def produce_messages(self, topics, max_messages=10000):
produce_spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.producer_workload_service.producer_node,
self.producer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=max_messages,
producer_conf={},
admin_client_conf={},
common_client_conf={},
inactive_topics={},
active_topics=topics)
produce_workload = self.trogdor.create_task("produce_workload", produce_spec)
produce_workload.wait_for_done(timeout_sec=180)
self.logger.debug("Produce workload finished")
@parametrize(topics=["consume_bench_topic[0-5]"]) # topic subscription
@parametrize(topics=["consume_bench_topic[0-5]:[0-4]"]) # manual topic assignment
def test_consume_bench(self, topics):
"""
Runs a ConsumeBench workload to consume messages
"""
self.produce_messages(self.active_topics)
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=10000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
active_topics=topics)
consume_workload = self.trogdor.create_task("consume_workload", consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_single_partition(self):
"""
Run a ConsumeBench against a single partition
"""
active_topics = {"consume_bench_topic": {"numPartitions": 2, "replicationFactor": 3}}
self.produce_messages(active_topics, 5000)
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2500,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
active_topics=["consume_bench_topic:1"])
consume_workload = self.trogdor.create_task("consume_workload", consume_spec)
consume_workload.wait_for_done(timeout_sec=180)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_multiple_consumers_random_group_topics(self):
"""
Runs multiple consumers group to read messages from topics.
Since a consumerGroup isn't specified, each consumer should read from all topics independently
"""
self.produce_messages(self.active_topics, max_messages=5000)
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=5000, # all should read exactly 5k messages
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=5,
active_topics=["consume_bench_topic[0-5]"])
consume_workload = self.trogdor.create_task("consume_workload", consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_two_consumers_specified_group_topics(self):
"""
Runs two consumers in the same consumer group to read messages from topics.
Since a consumerGroup is specified, each consumer should dynamically get assigned a partition from group
"""
self.produce_messages(self.active_topics)
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2000, # both should read at least 2k messages
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=2,
consumer_group="testGroup",
active_topics=["consume_bench_topic[0-5]"])
consume_workload = self.trogdor.create_task("consume_workload", consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_multiple_consumers_random_group_partitions(self):
"""
Runs multiple consumers in to read messages from specific partitions.
Since a consumerGroup isn't specified, each consumer will get assigned a random group
and consume from all partitions
"""
self.produce_messages(self.active_topics, max_messages=20000)
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=4,
active_topics=["consume_bench_topic1:[0-4]"])
consume_workload = self.trogdor.create_task("consume_workload", consume_spec)
consume_workload.wait_for_done(timeout_sec=360)
self.logger.debug("Consume workload finished")
tasks = self.trogdor.tasks()
self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
def test_multiple_consumers_specified_group_partitions_should_raise(self):
"""
Runs multiple consumers in the same group to read messages from specific partitions.
It is an invalid configuration to provide a consumer group and specific partitions.
"""
expected_error_msg = 'explicit partition assignment'
self.produce_messages(self.active_topics, max_messages=20000)
consume_spec = ConsumeBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS,
self.consumer_workload_service.consumer_node,
self.consumer_workload_service.bootstrap_servers,
target_messages_per_sec=1000,
max_messages=2000,
consumer_conf={},
admin_client_conf={},
common_client_conf={},
threads_per_worker=4,
consumer_group="fail_group",
active_topics=["consume_bench_topic1:[0-4]"])
consume_workload = self.trogdor.create_task("consume_workload", consume_spec)
try:
consume_workload.wait_for_done(timeout_sec=360)
raise Exception("Should have raised an exception due to an invalid configuration")
except RuntimeError as e:
if expected_error_msg not in str(e):
raise RuntimeError("Unexpected Exception - " + str(e))
self.logger.info(e)