def __init__(self, test_context):
super(TestBounce, self).__init__(test_context)
quorum_size_arg_name = 'quorum_size'
default_quorum_size = 1
quorum_size = default_quorum_size if not test_context.injected_args else test_context.injected_args.get(
quorum_size_arg_name, default_quorum_size)
if quorum_size < 1:
raise Exception("Illegal %s value provided for the test: %s" %
(quorum_size_arg_name, quorum_size))
self.topic = "topic"
self.zk = ZookeeperService(test_context,
num_nodes=quorum_size) if quorum.for_test(
test_context) == quorum.zk else None
num_kafka_nodes = quorum_size if quorum.for_test(
test_context) == quorum.colocated_kraft else 1
self.kafka = KafkaService(
test_context,
num_nodes=num_kafka_nodes,
zk=self.zk,
topics={self.topic: {
"partitions": 1,
"replication-factor": 1
}},
controller_num_nodes_override=quorum_size)
self.num_messages = 1000
def remote_quorum_nodes(self):
if quorum.for_test(self.test_context) == quorum.zk:
return self.zk.nodes
elif quorum.for_test(self.test_context) == quorum.remote_kraft:
return self.kafka.controller_quorum.nodes
else: # co-located case, which we currently don't test but handle here for completeness in case we do test it
return []
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ClientCompatibilityFeaturesTest,
self).__init__(test_context=test_context)
self.zk = ZookeeperService(test_context,
num_nodes=3) if quorum.for_test(
test_context) == quorum.zk else None
# Generate a unique topic name
topic_name = "client_compat_features_topic_%d%d" % (int(
time.time()), randint(0, 2147483647))
self.topics = {
topic_name: {
"partitions":
1, # Use only one partition to avoid worrying about ordering
"replication-factor": 3
}
}
self.kafka = KafkaService(test_context,
num_nodes=3,
zk=self.zk,
topics=self.topics)
# Always use the latest version of org.apache.kafka.tools.ClientCompatibilityTest
# so store away the path to the DEV version before we set the Kafka version
self.dev_script_path = self.kafka.path.script("kafka-run-class.sh",
self.kafka.nodes[0])
def __init__(self, test_context):
super(PerformanceServiceTest, self).__init__(test_context)
self.record_size = 100
self.num_records = 10000
self.topic = "topic"
self.zk = ZookeeperService(test_context, 1) if quorum.for_test(test_context) == quorum.zk else None
def setUp(self):
# Start the cluster and ensure that a snapshot is generated
self.logger.info(
"Starting the cluster and running until snapshot creation")
assert quorum.for_test(self.test_context) in quorum.all_kraft, \
"Snapshot test should be run Kraft Modes only"
self.kafka.start()
topic_count = 10
self.topics_created += self.create_n_topics(topic_count)
if self.kafka.remote_controller_quorum:
self.controller_nodes = self.kafka.remote_controller_quorum.nodes
else:
self.controller_nodes = self.kafka.nodes[:self.kafka.
num_nodes_controller_role]
# Waiting for snapshot creation and first log segment
# cleanup on all controller nodes
for node in self.controller_nodes:
self.logger.debug("Waiting for snapshot on: %s" %
self.kafka.who_am_i(node))
self.wait_for_log_segment_delete(node)
self.wait_for_snapshot(node)
self.logger.debug("Verified Snapshots exist on controller nodes")
def __init__(self, test_context):
super(FetchFromFollowerTest, self).__init__(test_context=test_context)
self.jmx_tool = JmxTool(test_context, jmx_poll_ms=100)
self.topic = "test_topic"
self.zk = ZookeeperService(test_context,
num_nodes=1) if quorum.for_test(
test_context) == quorum.zk else None
self.kafka = KafkaService(test_context,
num_nodes=3,
zk=self.zk,
topics={
self.topic: {
"partitions": 1,
"replication-factor": 3,
"configs": {
"min.insync.replicas": 1
}
},
},
server_prop_overrides=[[
"replica.selector.class",
self.RACK_AWARE_REPLICA_SELECTOR
]],
per_node_server_prop_overrides={
1: [("broker.rack", "rack-a")],
2: [("broker.rack", "rack-b")],
3: [("broker.rack", "rack-c")]
},
controller_num_nodes_override=1)
self.producer_throughput = 1000
self.num_producers = 1
self.num_consumers = 1
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(TransactionsTest, self).__init__(test_context=test_context)
self.input_topic = "input-topic"
self.output_topic = "output-topic"
self.num_brokers = 3
# Test parameters
self.num_input_partitions = 2
self.num_output_partitions = 3
self.num_seed_messages = 100000
self.transaction_size = 750
# The transaction timeout should be lower than the progress timeout, but at
# least as high as the request timeout (which is 30s by default). When the
# client is hard-bounced, progress may depend on the previous transaction
# being aborted. When the broker is hard-bounced, we may have to wait as
# long as the request timeout to get a `Produce` response and we do not
# want the coordinator timing out the transaction.
self.transaction_timeout = 40000
self.progress_timeout_sec = 60
self.consumer_group = "transactions-test-consumer-group"
self.zk = ZookeeperService(test_context,
num_nodes=1) if quorum.for_test(
test_context) == quorum.zk else None
self.kafka = KafkaService(test_context,
num_nodes=self.num_brokers,
zk=self.zk,
controller_num_nodes_override=1)
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ConsumeBenchTest, self).__init__(test_context)
self.zk = ZookeeperService(test_context,
num_nodes=3) if quorum.for_test(
test_context) == quorum.zk else None
self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk)
self.producer_workload_service = ProduceBenchWorkloadService(
test_context, self.kafka)
self.consumer_workload_service = ConsumeBenchWorkloadService(
test_context, self.kafka)
self.consumer_workload_service_2 = ConsumeBenchWorkloadService(
test_context, self.kafka)
self.active_topics = {
"consume_bench_topic[0-5]": {
"numPartitions": 5,
"replicationFactor": 3
}
}
self.trogdor = TrogdorService(context=self.test_context,
client_services=[
self.kafka,
self.producer_workload_service,
self.consumer_workload_service,
self.consumer_workload_service_2
])
def __init__(self, test_context):
super(ConsumerGroupCommandTest, self).__init__(test_context)
self.num_zk = 1
self.num_brokers = 1
self.topics = {
TOPIC: {'partitions': 1, 'replication-factor': 1}
}
self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None
def __init__(self, test_context):
super(ConnectStandaloneFileTest, self).__init__(test_context)
self.num_zk = 1
self.num_brokers = 1
self.topics = {'test': {'partitions': 1, 'replication-factor': 1}}
self.zk = ZookeeperService(test_context,
self.num_zk) if quorum.for_test(
test_context) == quorum.zk else None
def __init__(self, test_context):
super(TestBounce, self).__init__(test_context)
self.topic = "topic"
self.zk = ZookeeperService(test_context,
num_nodes=1) if quorum.for_test(
test_context) == quorum.zk else None
self.kafka = KafkaService(
test_context,
num_nodes=1,
zk=self.zk,
topics={self.topic: {
"partitions": 1,
"replication-factor": 1
}},
controller_num_nodes_override=3
if quorum.for_test(test_context) == quorum.remote_raft else 1)
self.num_messages = 1000
def __init__(self, test_context):
super(Log4jAppenderTest, self).__init__(test_context)
self.num_zk = 1
self.num_brokers = 1
self.messages_received_count = 0
self.topics = {
TOPIC: {'partitions': 1, 'replication-factor': 1}
}
self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None
def __init__(self, test_context):
super(LogCompactionTest, self).__init__(test_context)
self.num_zk = 1
self.num_brokers = 1
self.zk = ZookeeperService(test_context,
self.num_zk) if quorum.for_test(
test_context) == quorum.zk else None
self.kafka = None
self.compaction_verifier = None
def __init__(self, test_context):
super(ReplicaScaleTest, self).__init__(test_context=test_context)
self.test_context = test_context
self.zk = ZookeeperService(test_context,
num_nodes=1) if quorum.for_test(
test_context) == quorum.zk else None
self.kafka = KafkaService(self.test_context,
num_nodes=8,
zk=self.zk,
controller_num_nodes_override=1)
def __init__(self, test_context):
super(TestVerifiableProducer, self).__init__(test_context)
self.topic = "topic"
self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test(test_context) == quorum.zk else None
self.kafka = KafkaService(test_context, num_nodes=1, zk=self.zk,
topics={self.topic: {"partitions": 1, "replication-factor": 1}})
self.num_messages = 1000
# This will produce to source kafka cluster
self.producer = VerifiableProducer(test_context, num_nodes=1, kafka=self.kafka, topic=self.topic,
max_messages=self.num_messages, throughput=self.num_messages // 10)
def setUp(self):
self.topic = "test_topic"
self.zk = ZookeeperService(self.test_context, num_nodes=1) if quorum.for_test(self.test_context) == quorum.zk else None
if self.zk:
self.zk.start()
# Producer and consumer
self.producer_throughput = 10000
self.num_producers = 1
self.num_consumers = 1
self.messages_per_producer = 1000
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(RoundTripFaultTest, self).__init__(test_context)
self.zk = ZookeeperService(test_context,
num_nodes=3) if quorum.for_test(
test_context) == quorum.zk else None
self.kafka = KafkaService(test_context, num_nodes=4, zk=self.zk)
self.workload_service = RoundTripWorkloadService(
test_context, self.kafka)
if quorum.for_test(test_context) == quorum.zk:
trogdor_client_services = [
self.zk, self.kafka, self.workload_service
]
elif quorum.for_test(test_context) == quorum.remote_kraft:
trogdor_client_services = [
self.kafka.controller_quorum, self.kafka, self.workload_service
]
else: #co-located case, which we currently don't test but handle here for completeness in case we do test it
trogdor_client_services = [self.kafka, self.workload_service]
self.trogdor = TrogdorService(context=self.test_context,
client_services=trogdor_client_services)
topic_name = "round_trip_topic%d" % RoundTripFaultTest.topic_name_index
RoundTripFaultTest.topic_name_index = RoundTripFaultTest.topic_name_index + 1
# note that the broker.id values will be 1..num_nodes
active_topics = {
topic_name: {
"partitionAssignments": {
"0": [1, 2, 3]
}
}
}
self.round_trip_spec = RoundTripWorkloadSpec(
0,
TaskSpec.MAX_DURATION_MS,
self.workload_service.client_node,
self.workload_service.bootstrap_servers,
target_messages_per_sec=10000,
max_messages=100000,
active_topics=active_topics)
def __init__(self, test_context):
super(ReplicaVerificationToolTest, self).__init__(test_context)
self.num_zk = 1
self.num_brokers = 2
self.messages_received_count = 0
self.topics = {
TOPIC: {'partitions': 1, 'replication-factor': 2}
}
self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None
self.kafka = None
self.producer = None
self.replica_verifier = None
def __init__(self, test_context, num_zk, num_brokers, topics=None):
super(KafkaTest, self).__init__(test_context)
self.num_zk = num_zk
self.num_brokers = num_brokers
self.topics = topics
self.zk = ZookeeperService(test_context,
self.num_zk) if quorum.for_test(
test_context) == quorum.zk else None
self.kafka = KafkaService(test_context,
self.num_brokers,
self.zk,
topics=self.topics,
controller_num_nodes_override=self.num_zk)
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ClientCompatibilityProduceConsumeTest, self).__init__(test_context=test_context)
self.topic = "test_topic"
self.zk = ZookeeperService(test_context, num_nodes=3) if quorum.for_test(test_context) == quorum.zk else None
self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics={self.topic:{
"partitions": 10,
"replication-factor": 2}})
self.num_partitions = 10
self.timeout_sec = 60
self.producer_throughput = 1000
self.num_producers = 2
self.messages_per_producer = 1000
self.num_consumers = 1
def __init__(self, test_context):
super(GetOffsetShellTest, self).__init__(test_context)
self.num_zk = 1
self.num_brokers = 1
self.messages_received_count = 0
self.topics = {
TOPIC: {
'partitions': NUM_PARTITIONS,
'replication-factor': REPLICATION_FACTOR
}
}
self.zk = ZookeeperService(test_context,
self.num_zk) if quorum.for_test(
test_context) == quorum.zk else None
def test_quorum_ssl_endpoint_validation_failure(self,
metadata_quorum=quorum.zk):
"""
Test that invalid hostname in ZooKeeper or Raft Controller results in broker inability to start.
"""
# Start ZooKeeper/Raft-based Controller with valid hostnames in the certs' SANs
# so that we can start Kafka
SecurityConfig.ssl_stores = TestSslStores(
self.test_context.local_scratch_dir, valid_hostname=True)
self.create_zookeeper_if_necessary(
num_nodes=1,
zk_client_port=False,
zk_client_secure_port=True,
zk_tls_encrypt_only=True,
)
if self.zk:
self.zk.start()
self.create_kafka(
num_nodes=1,
interbroker_security_protocol=
'SSL', # also sets the broker-to-raft-controller security protocol for the Raft case
zk_client_secure=True, # ignored if we aren't using ZooKeeper
)
self.kafka.start()
# now stop the Kafka broker
# and set the cert for ZooKeeper/Raft-based Controller to have an invalid hostname
# so we can restart Kafka and ensure it is unable to start
self.kafka.stop_node(self.kafka.nodes[0])
SecurityConfig.ssl_stores.valid_hostname = False
if quorum.for_test(self.test_context) == quorum.zk:
self.kafka.zk.restart_cluster()
else:
self.kafka.remote_controller_quorum.restart_cluster()
try:
self.kafka.start_node(self.kafka.nodes[0], timeout_sec=30)
raise RuntimeError(
"Kafka restarted successfully but should not have!"
" Endpoint validation did not fail with invalid hostname")
except TimeoutError:
# expected
pass
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ClientCompatibilityFeaturesTest,
self).__init__(test_context=test_context)
self.zk = ZookeeperService(test_context,
num_nodes=3) if quorum.for_test(
test_context) == quorum.zk else None
# Generate a unique topic name
topic_name = "client_compat_features_topic_%d%d" % (int(
time.time()), randint(0, 2147483647))
self.topics = {
topic_name: {
"partitions":
1, # Use only one partition to avoid worrying about ordering
"replication-factor": 3
}
}
self.kafka = KafkaService(test_context,
num_nodes=3,
zk=self.zk,
topics=self.topics)
def __init__(self, test_context):
self.num_zk = 1
""":type test_context: ducktape.tests.test.TestContext"""
super(ReassignPartitionsTest, self).__init__(test_context=test_context)
self.topic = "test_topic"
self.num_partitions = 20
self.zk = ZookeeperService(test_context,
self.num_zk) if quorum.for_test(
test_context) == quorum.zk else None
# We set the min.insync.replicas to match the replication factor because
# it makes the test more stringent. If min.isr = 2 and
# replication.factor=3, then the test would tolerate the failure of
# reassignment for upto one replica per partition, which is not
# desirable for this test in particular.
self.kafka = KafkaService(
test_context,
num_nodes=4,
zk=self.zk,
server_prop_overrides=[[
config_property.LOG_ROLL_TIME_MS, "5000"
], [config_property.LOG_RETENTION_CHECK_INTERVAL_MS, "5000"]],
topics={
self.topic: {
"partitions": self.num_partitions,
"replication-factor": 3,
'configs': {
"min.insync.replicas": 3,
}
}
},
controller_num_nodes_override=self.num_zk)
self.timeout_sec = 60
self.producer_throughput = 1000
self.num_producers = 1
self.num_consumers = 1
def create_zookeeper_if_necessary(self, num_nodes=1, **kwargs):
self.zk = ZookeeperService(
self.test_context, num_nodes=num_nodes, **
kwargs) if quorum.for_test(
self.test_context) == quorum.zk else None
