def __init__(self, test_context): super(TestBounce, self).__init__(test_context) quorum_size_arg_name = 'quorum_size' default_quorum_size = 1 quorum_size = default_quorum_size if not test_context.injected_args else test_context.injected_args.get( quorum_size_arg_name, default_quorum_size) if quorum_size < 1: raise Exception("Illegal %s value provided for the test: %s" % (quorum_size_arg_name, quorum_size)) self.topic = "topic" self.zk = ZookeeperService(test_context, num_nodes=quorum_size) if quorum.for_test( test_context) == quorum.zk else None num_kafka_nodes = quorum_size if quorum.for_test( test_context) == quorum.colocated_kraft else 1 self.kafka = KafkaService( test_context, num_nodes=num_kafka_nodes, zk=self.zk, topics={self.topic: { "partitions": 1, "replication-factor": 1 }}, controller_num_nodes_override=quorum_size) self.num_messages = 1000
def remote_quorum_nodes(self): if quorum.for_test(self.test_context) == quorum.zk: return self.zk.nodes elif quorum.for_test(self.test_context) == quorum.remote_kraft: return self.kafka.controller_quorum.nodes else: # co-located case, which we currently don't test but handle here for completeness in case we do test it return []
def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ClientCompatibilityFeaturesTest, self).__init__(test_context=test_context) self.zk = ZookeeperService(test_context, num_nodes=3) if quorum.for_test( test_context) == quorum.zk else None # Generate a unique topic name topic_name = "client_compat_features_topic_%d%d" % (int( time.time()), randint(0, 2147483647)) self.topics = { topic_name: { "partitions": 1, # Use only one partition to avoid worrying about ordering "replication-factor": 3 } } self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics=self.topics) # Always use the latest version of org.apache.kafka.tools.ClientCompatibilityTest # so store away the path to the DEV version before we set the Kafka version self.dev_script_path = self.kafka.path.script("kafka-run-class.sh", self.kafka.nodes[0])
def __init__(self, test_context): super(PerformanceServiceTest, self).__init__(test_context) self.record_size = 100 self.num_records = 10000 self.topic = "topic" self.zk = ZookeeperService(test_context, 1) if quorum.for_test(test_context) == quorum.zk else None
def setUp(self): # Start the cluster and ensure that a snapshot is generated self.logger.info( "Starting the cluster and running until snapshot creation") assert quorum.for_test(self.test_context) in quorum.all_kraft, \ "Snapshot test should be run Kraft Modes only" self.kafka.start() topic_count = 10 self.topics_created += self.create_n_topics(topic_count) if self.kafka.remote_controller_quorum: self.controller_nodes = self.kafka.remote_controller_quorum.nodes else: self.controller_nodes = self.kafka.nodes[:self.kafka. num_nodes_controller_role] # Waiting for snapshot creation and first log segment # cleanup on all controller nodes for node in self.controller_nodes: self.logger.debug("Waiting for snapshot on: %s" % self.kafka.who_am_i(node)) self.wait_for_log_segment_delete(node) self.wait_for_snapshot(node) self.logger.debug("Verified Snapshots exist on controller nodes")
def __init__(self, test_context): super(FetchFromFollowerTest, self).__init__(test_context=test_context) self.jmx_tool = JmxTool(test_context, jmx_poll_ms=100) self.topic = "test_topic" self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test( test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics={ self.topic: { "partitions": 1, "replication-factor": 3, "configs": { "min.insync.replicas": 1 } }, }, server_prop_overrides=[[ "replica.selector.class", self.RACK_AWARE_REPLICA_SELECTOR ]], per_node_server_prop_overrides={ 1: [("broker.rack", "rack-a")], 2: [("broker.rack", "rack-b")], 3: [("broker.rack", "rack-c")] }, controller_num_nodes_override=1) self.producer_throughput = 1000 self.num_producers = 1 self.num_consumers = 1
def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(TransactionsTest, self).__init__(test_context=test_context) self.input_topic = "input-topic" self.output_topic = "output-topic" self.num_brokers = 3 # Test parameters self.num_input_partitions = 2 self.num_output_partitions = 3 self.num_seed_messages = 100000 self.transaction_size = 750 # The transaction timeout should be lower than the progress timeout, but at # least as high as the request timeout (which is 30s by default). When the # client is hard-bounced, progress may depend on the previous transaction # being aborted. When the broker is hard-bounced, we may have to wait as # long as the request timeout to get a `Produce` response and we do not # want the coordinator timing out the transaction. self.transaction_timeout = 40000 self.progress_timeout_sec = 60 self.consumer_group = "transactions-test-consumer-group" self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test( test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=self.num_brokers, zk=self.zk, controller_num_nodes_override=1)
def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ConsumeBenchTest, self).__init__(test_context) self.zk = ZookeeperService(test_context, num_nodes=3) if quorum.for_test( test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk) self.producer_workload_service = ProduceBenchWorkloadService( test_context, self.kafka) self.consumer_workload_service = ConsumeBenchWorkloadService( test_context, self.kafka) self.consumer_workload_service_2 = ConsumeBenchWorkloadService( test_context, self.kafka) self.active_topics = { "consume_bench_topic[0-5]": { "numPartitions": 5, "replicationFactor": 3 } } self.trogdor = TrogdorService(context=self.test_context, client_services=[ self.kafka, self.producer_workload_service, self.consumer_workload_service, self.consumer_workload_service_2 ])
def __init__(self, test_context): super(ConsumerGroupCommandTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.topics = { TOPIC: {'partitions': 1, 'replication-factor': 1} } self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None
def __init__(self, test_context): super(ConnectStandaloneFileTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.topics = {'test': {'partitions': 1, 'replication-factor': 1}} self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test( test_context) == quorum.zk else None
def __init__(self, test_context): super(TestBounce, self).__init__(test_context) self.topic = "topic" self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test( test_context) == quorum.zk else None self.kafka = KafkaService( test_context, num_nodes=1, zk=self.zk, topics={self.topic: { "partitions": 1, "replication-factor": 1 }}, controller_num_nodes_override=3 if quorum.for_test(test_context) == quorum.remote_raft else 1) self.num_messages = 1000
def __init__(self, test_context): super(Log4jAppenderTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.messages_received_count = 0 self.topics = { TOPIC: {'partitions': 1, 'replication-factor': 1} } self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None
def __init__(self, test_context): super(LogCompactionTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test( test_context) == quorum.zk else None self.kafka = None self.compaction_verifier = None
def __init__(self, test_context): super(ReplicaScaleTest, self).__init__(test_context=test_context) self.test_context = test_context self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test( test_context) == quorum.zk else None self.kafka = KafkaService(self.test_context, num_nodes=8, zk=self.zk, controller_num_nodes_override=1)
def __init__(self, test_context): super(TestVerifiableProducer, self).__init__(test_context) self.topic = "topic" self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=1, zk=self.zk, topics={self.topic: {"partitions": 1, "replication-factor": 1}}) self.num_messages = 1000 # This will produce to source kafka cluster self.producer = VerifiableProducer(test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, max_messages=self.num_messages, throughput=self.num_messages // 10)
def setUp(self): self.topic = "test_topic" self.zk = ZookeeperService(self.test_context, num_nodes=1) if quorum.for_test(self.test_context) == quorum.zk else None if self.zk: self.zk.start() # Producer and consumer self.producer_throughput = 10000 self.num_producers = 1 self.num_consumers = 1 self.messages_per_producer = 1000
def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(RoundTripFaultTest, self).__init__(test_context) self.zk = ZookeeperService(test_context, num_nodes=3) if quorum.for_test( test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=4, zk=self.zk) self.workload_service = RoundTripWorkloadService( test_context, self.kafka) if quorum.for_test(test_context) == quorum.zk: trogdor_client_services = [ self.zk, self.kafka, self.workload_service ] elif quorum.for_test(test_context) == quorum.remote_kraft: trogdor_client_services = [ self.kafka.controller_quorum, self.kafka, self.workload_service ] else: #co-located case, which we currently don't test but handle here for completeness in case we do test it trogdor_client_services = [self.kafka, self.workload_service] self.trogdor = TrogdorService(context=self.test_context, client_services=trogdor_client_services) topic_name = "round_trip_topic%d" % RoundTripFaultTest.topic_name_index RoundTripFaultTest.topic_name_index = RoundTripFaultTest.topic_name_index + 1 # note that the broker.id values will be 1..num_nodes active_topics = { topic_name: { "partitionAssignments": { "0": [1, 2, 3] } } } self.round_trip_spec = RoundTripWorkloadSpec( 0, TaskSpec.MAX_DURATION_MS, self.workload_service.client_node, self.workload_service.bootstrap_servers, target_messages_per_sec=10000, max_messages=100000, active_topics=active_topics)
def __init__(self, test_context): super(ReplicaVerificationToolTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 2 self.messages_received_count = 0 self.topics = { TOPIC: {'partitions': 1, 'replication-factor': 2} } self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None self.kafka = None self.producer = None self.replica_verifier = None
def __init__(self, test_context, num_zk, num_brokers, topics=None): super(KafkaTest, self).__init__(test_context) self.num_zk = num_zk self.num_brokers = num_brokers self.topics = topics self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test( test_context) == quorum.zk else None self.kafka = KafkaService(test_context, self.num_brokers, self.zk, topics=self.topics, controller_num_nodes_override=self.num_zk)
def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ClientCompatibilityProduceConsumeTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.zk = ZookeeperService(test_context, num_nodes=3) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics={self.topic:{ "partitions": 10, "replication-factor": 2}}) self.num_partitions = 10 self.timeout_sec = 60 self.producer_throughput = 1000 self.num_producers = 2 self.messages_per_producer = 1000 self.num_consumers = 1
def __init__(self, test_context): super(GetOffsetShellTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.messages_received_count = 0 self.topics = { TOPIC: { 'partitions': NUM_PARTITIONS, 'replication-factor': REPLICATION_FACTOR } } self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test( test_context) == quorum.zk else None
def test_quorum_ssl_endpoint_validation_failure(self, metadata_quorum=quorum.zk): """ Test that invalid hostname in ZooKeeper or Raft Controller results in broker inability to start. """ # Start ZooKeeper/Raft-based Controller with valid hostnames in the certs' SANs # so that we can start Kafka SecurityConfig.ssl_stores = TestSslStores( self.test_context.local_scratch_dir, valid_hostname=True) self.create_zookeeper_if_necessary( num_nodes=1, zk_client_port=False, zk_client_secure_port=True, zk_tls_encrypt_only=True, ) if self.zk: self.zk.start() self.create_kafka( num_nodes=1, interbroker_security_protocol= 'SSL', # also sets the broker-to-raft-controller security protocol for the Raft case zk_client_secure=True, # ignored if we aren't using ZooKeeper ) self.kafka.start() # now stop the Kafka broker # and set the cert for ZooKeeper/Raft-based Controller to have an invalid hostname # so we can restart Kafka and ensure it is unable to start self.kafka.stop_node(self.kafka.nodes[0]) SecurityConfig.ssl_stores.valid_hostname = False if quorum.for_test(self.test_context) == quorum.zk: self.kafka.zk.restart_cluster() else: self.kafka.remote_controller_quorum.restart_cluster() try: self.kafka.start_node(self.kafka.nodes[0], timeout_sec=30) raise RuntimeError( "Kafka restarted successfully but should not have!" " Endpoint validation did not fail with invalid hostname") except TimeoutError: # expected pass
def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ClientCompatibilityFeaturesTest, self).__init__(test_context=test_context) self.zk = ZookeeperService(test_context, num_nodes=3) if quorum.for_test( test_context) == quorum.zk else None # Generate a unique topic name topic_name = "client_compat_features_topic_%d%d" % (int( time.time()), randint(0, 2147483647)) self.topics = { topic_name: { "partitions": 1, # Use only one partition to avoid worrying about ordering "replication-factor": 3 } } self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics=self.topics)
def __init__(self, test_context): self.num_zk = 1 """:type test_context: ducktape.tests.test.TestContext""" super(ReassignPartitionsTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.num_partitions = 20 self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test( test_context) == quorum.zk else None # We set the min.insync.replicas to match the replication factor because # it makes the test more stringent. If min.isr = 2 and # replication.factor=3, then the test would tolerate the failure of # reassignment for upto one replica per partition, which is not # desirable for this test in particular. self.kafka = KafkaService( test_context, num_nodes=4, zk=self.zk, server_prop_overrides=[[ config_property.LOG_ROLL_TIME_MS, "5000" ], [config_property.LOG_RETENTION_CHECK_INTERVAL_MS, "5000"]], topics={ self.topic: { "partitions": self.num_partitions, "replication-factor": 3, 'configs': { "min.insync.replicas": 3, } } }, controller_num_nodes_override=self.num_zk) self.timeout_sec = 60 self.producer_throughput = 1000 self.num_producers = 1 self.num_consumers = 1
def create_zookeeper_if_necessary(self, num_nodes=1, **kwargs): self.zk = ZookeeperService( self.test_context, num_nodes=num_nodes, ** kwargs) if quorum.for_test( self.test_context) == quorum.zk else None