class ProduceBenchTest(Test): def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ProduceBenchTest, self).__init__(test_context) self.zk = ZookeeperService(test_context, num_nodes=3) self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk) self.workload_service = ProduceBenchWorkloadService(test_context, self.kafka) self.trogdor = TrogdorService(context=self.test_context, client_services=[self.kafka, self.workload_service]) def setUp(self): self.trogdor.start() self.zk.start() self.kafka.start() def teardown(self): self.trogdor.stop() self.kafka.stop() self.zk.stop() def test_produce_bench(self): active_topics={"produce_bench_topic[0-1]":{"numPartitions":1, "replicationFactor":3}} inactive_topics={"produce_bench_topic[2-9]":{"numPartitions":1, "replicationFactor":3}} spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS, self.workload_service.producer_node, self.workload_service.bootstrap_servers, target_messages_per_sec=1000, max_messages=100000, producer_conf={}, inactive_topics=inactive_topics, active_topics=active_topics) workload1 = self.trogdor.create_task("workload1", spec) workload1.wait_for_done(timeout_sec=360) tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2))
class CamusTest(Test): def __init__(self, test_context, num_zk, num_brokers, num_hadoop, num_schema_registry, num_rest, hadoop_distro='cdh', hadoop_version=2, topics=None): super(CamusTest, self).__init__(test_context) self.num_zk = num_zk self.num_brokers = num_brokers self.num_hadoop = num_hadoop self.num_schema_registry = num_schema_registry self.num_rest = num_rest self.topics = topics self.hadoop_distro = hadoop_distro self.hadoop_version = hadoop_version self.zk = ZookeeperService(test_context, self.num_zk) self.kafka = KafkaService(test_context, self.num_brokers, self.zk, topics=self.topics) self.hadoop = create_hadoop_service(test_context, self.num_hadoop, self.hadoop_distro, self.hadoop_version) self.schema_registry = SchemaRegistryService(test_context, self.num_schema_registry, self.zk, self.kafka) self.rest = KafkaRestService(test_context, self.num_rest, self.zk, self.kafka, self.schema_registry) def setUp(self): self.zk.start() self.kafka.start() self.hadoop.start() self.schema_registry.start() self.rest.start()
class KafkaVersionTest(Test): """Sanity checks on kafka versioning.""" def __init__(self, test_context): super(KafkaVersionTest, self).__init__(test_context) self.topic = "topic" self.zk = ZookeeperService(test_context, num_nodes=1) def setUp(self): self.zk.start() @cluster(num_nodes=2) def test_0_8_2(self): """Test kafka service node-versioning api - verify that we can bring up a single-node 0.8.2.X cluster.""" self.kafka = KafkaService(self.test_context, num_nodes=1, zk=self.zk, topics={self.topic: {"partitions": 1, "replication-factor": 1}}) node = self.kafka.nodes[0] node.version = LATEST_0_8_2 self.kafka.start() assert is_version(node, [LATEST_0_8_2], logger=self.logger) @cluster(num_nodes=3) def test_multi_version(self): """Test kafka service node-versioning api - ensure we can bring up a 2-node cluster, one on version 0.8.2.X, the other on the current development branch.""" self.kafka = KafkaService(self.test_context, num_nodes=2, zk=self.zk, topics={self.topic: {"partitions": 1, "replication-factor": 2}}) self.kafka.nodes[1].version = LATEST_0_8_2 self.kafka.nodes[1].config[config_property.INTER_BROKER_PROTOCOL_VERSION] = "0.8.2.X" self.kafka.start() assert is_version(self.kafka.nodes[0], [DEV_BRANCH.vstring], logger=self.logger) assert is_version(self.kafka.nodes[1], [LATEST_0_8_2], logger=self.logger)
class GetOffsetShellTest(Test): """ Tests GetOffsetShell tool """ def __init__(self, test_context): super(GetOffsetShellTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.messages_received_count = 0 self.topics = { TOPIC: {'partitions': NUM_PARTITIONS, 'replication-factor': REPLICATION_FACTOR} } self.zk = ZookeeperService(test_context, self.num_zk) def setUp(self): self.zk.start() def start_kafka(self, security_protocol, interbroker_security_protocol): self.kafka = KafkaService( self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, interbroker_security_protocol=interbroker_security_protocol, topics=self.topics) self.kafka.start() def start_producer(self): # This will produce to kafka cluster self.producer = VerifiableProducer(self.test_context, num_nodes=1, kafka=self.kafka, topic=TOPIC, throughput=1000, max_messages=MAX_MESSAGES) self.producer.start() current_acked = self.producer.num_acked wait_until(lambda: self.producer.num_acked >= current_acked + MAX_MESSAGES, timeout_sec=10, err_msg="Timeout awaiting messages to be produced and acked") def start_consumer(self): self.consumer = ConsoleConsumer(self.test_context, num_nodes=self.num_brokers, kafka=self.kafka, topic=TOPIC, consumer_timeout_ms=1000) self.consumer.start() @cluster(num_nodes=4) def test_get_offset_shell(self, security_protocol='PLAINTEXT'): """ Tests if GetOffsetShell is getting offsets correctly :return: None """ self.start_kafka(security_protocol, security_protocol) self.start_producer() # Assert that offset fetched without any consumers consuming is 0 assert self.kafka.get_offset_shell(TOPIC, None, 1000, 1, -1), "%s:%s:%s" % (TOPIC, NUM_PARTITIONS - 1, 0) self.start_consumer() node = self.consumer.nodes[0] wait_until(lambda: self.consumer.alive(node), timeout_sec=20, backoff_sec=.2, err_msg="Consumer was too slow to start") # Assert that offset is correctly indicated by GetOffsetShell tool wait_until(lambda: "%s:%s:%s" % (TOPIC, NUM_PARTITIONS - 1, MAX_MESSAGES) in self.kafka.get_offset_shell(TOPIC, None, 1000, 1, -1), timeout_sec=10, err_msg="Timed out waiting to reach expected offset.")
class CompressionTest(ProduceConsumeValidateTest): """ These tests validate produce / consume for compressed topics. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(CompressionTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(test_context, num_nodes=1, zk=self.zk, topics={self.topic: { "partitions": 10, "replication-factor": 1}}) self.num_partitions = 10 self.timeout_sec = 60 self.producer_throughput = 1000 self.num_producers = 4 self.messages_per_producer = 1000 self.num_consumers = 1 def setUp(self): self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super(CompressionTest, self).min_cluster_size() + self.num_producers + self.num_consumers @parametrize(compression_types=["snappy","gzip","lz4","none"], new_consumer=True) @parametrize(compression_types=["snappy","gzip","lz4","none"], new_consumer=False) def test_compressed_topic(self, compression_types, new_consumer): """Test produce => consume => validate for compressed topics Setup: 1 zk, 1 kafka node, 1 topic with partitions=10, replication-factor=1 compression_types parameter gives a list of compression types (or no compression if "none"). Each producer in a VerifiableProducer group (num_producers = 4) will use a compression type from the list based on producer's index in the group. - Produce messages in the background - Consume messages in the background - Stop producing, and finish consuming - Validate that every acked message was consumed """ self.kafka.security_protocol = "PLAINTEXT" self.kafka.interbroker_security_protocol = self.kafka.security_protocol self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, message_validator=is_int_with_prefix, compression_types=compression_types) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, new_consumer=new_consumer, consumer_timeout_ms=60000, message_validator=is_int_with_prefix) self.kafka.start() self.run_produce_consume_validate(lambda: wait_until( lambda: self.producer.each_produced_at_least(self.messages_per_producer) == True, timeout_sec=120, backoff_sec=1, err_msg="Producer did not produce all messages in reasonable amount of time"))
class ClientCompatibilityTestNewBroker(ProduceConsumeValidateTest): def __init__(self, test_context): super(ClientCompatibilityTestNewBroker, self).__init__(test_context=test_context) def setUp(self): self.topic = "test_topic" self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() # Producer and consumer self.producer_throughput = 10000 self.num_producers = 1 self.num_consumers = 1 self.messages_per_producer = 1000 @cluster(num_nodes=6) @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(DEV_BRANCH), compression_types=["snappy"], timestamp_type=str("LogAppendTime")) @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(DEV_BRANCH), compression_types=["none"], timestamp_type=str("LogAppendTime")) @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(LATEST_0_9), compression_types=["none"], new_consumer=False, timestamp_type=None) @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(LATEST_0_9), compression_types=["snappy"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_1_1), consumer_version=str(LATEST_1_1), compression_types=["lz4"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_1_0), consumer_version=str(LATEST_1_0), compression_types=["none"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_0_11_0), consumer_version=str(LATEST_0_11_0), compression_types=["gzip"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_0_10_2), consumer_version=str(LATEST_0_10_2), compression_types=["lz4"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_0_10_1), consumer_version=str(LATEST_0_10_1), compression_types=["snappy"], timestamp_type=str("LogAppendTime")) @parametrize(producer_version=str(LATEST_0_10_0), consumer_version=str(LATEST_0_10_0), compression_types=["snappy"], timestamp_type=str("LogAppendTime")) @parametrize(producer_version=str(LATEST_0_9), consumer_version=str(DEV_BRANCH), compression_types=["none"], timestamp_type=None) @parametrize(producer_version=str(LATEST_0_9), consumer_version=str(DEV_BRANCH), compression_types=["snappy"], timestamp_type=None) @parametrize(producer_version=str(LATEST_0_9), consumer_version=str(LATEST_0_9), compression_types=["snappy"], timestamp_type=str("LogAppendTime")) @parametrize(producer_version=str(LATEST_0_8_2), consumer_version=str(LATEST_0_8_2), compression_types=["none"], new_consumer=False, timestamp_type=None) def test_compatibility(self, producer_version, consumer_version, compression_types, new_consumer=True, timestamp_type=None): self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, version=DEV_BRANCH, topics={self.topic: { "partitions": 3, "replication-factor": 3, 'configs': {"min.insync.replicas": 2}}}) for node in self.kafka.nodes: if timestamp_type is not None: node.config[config_property.MESSAGE_TIMESTAMP_TYPE] = timestamp_type self.kafka.start() self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, message_validator=is_int, compression_types=compression_types, version=KafkaVersion(producer_version)) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=30000, new_consumer=new_consumer, message_validator=is_int, version=KafkaVersion(consumer_version)) self.run_produce_consume_validate(lambda: wait_until( lambda: self.producer.each_produced_at_least(self.messages_per_producer) == True, timeout_sec=120, backoff_sec=1, err_msg="Producer did not produce all messages in reasonable amount of time"))
class Log4jAppenderTest(Test): """ Tests KafkaLog4jAppender using VerifiableKafkaLog4jAppender that appends increasing ints to a Kafka topic """ def __init__(self, test_context): super(Log4jAppenderTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.topics = { TOPIC: {'partitions': 1, 'replication-factor': 1} } self.zk = ZookeeperService(test_context, self.num_zk) def setUp(self): self.zk.start() def start_kafka(self, security_protocol, interbroker_security_protocol): self.kafka = KafkaService( self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, interbroker_security_protocol=interbroker_security_protocol, topics=self.topics) self.kafka.start() def start_appender(self, security_protocol): self.appender = KafkaLog4jAppender(self.test_context, self.num_brokers, self.kafka, TOPIC, MAX_MESSAGES, security_protocol=security_protocol) self.appender.start() def start_consumer(self, security_protocol): enable_new_consumer = security_protocol == SecurityConfig.SSL self.consumer = ConsoleConsumer(self.test_context, num_nodes=self.num_brokers, kafka=self.kafka, topic=TOPIC, consumer_timeout_ms=1000, new_consumer=enable_new_consumer) self.consumer.start() @matrix(security_protocol=['PLAINTEXT', 'SSL']) def test_log4j_appender(self, security_protocol='PLAINTEXT'): """ Tests if KafkaLog4jAppender is producing to Kafka topic :return: None """ self.start_kafka(security_protocol, security_protocol) self.start_appender(security_protocol) self.appender.wait() self.start_consumer(security_protocol) node = self.consumer.nodes[0] wait_until(lambda: self.consumer.alive(node), timeout_sec=10, backoff_sec=.2, err_msg="Consumer was too slow to start") # Verify consumed messages count expected_lines_count = MAX_MESSAGES * 2 # two times to account for new lines introduced by log4j wait_until(lambda: len(self.consumer.messages_consumed[1]) == expected_lines_count, timeout_sec=10, err_msg="Timed out waiting to consume expected number of messages.") self.consumer.stop()
class MiniTest(Test): def __init__(self, test_context): super(MiniTest, self).__init__(test_context=test_context) self.zk = ZookeeperService(test_context, 1) self.kafka = KafkaService(test_context, 1, self.zk) def test(self): self.zk.start() self.kafka.start()
class ClientCompatibilityFeaturesTest(Test): """ Tests clients for the presence or absence of specific features when communicating with brokers with various versions. Relies on ClientCompatibilityTest.java for much of the functionality. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ClientCompatibilityFeaturesTest, self).__init__(test_context=test_context) self.zk = ZookeeperService(test_context, num_nodes=3) # Generate a unique topic name topic_name = "client_compat_features_topic_%d%d" % (int(time.time()), randint(0, 2147483647)) self.topics = { topic_name: { "partitions": 1, # Use only one partition to avoid worrying about ordering "replication-factor": 3 }} self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics=self.topics) def invoke_compatibility_program(self, features): # Run the compatibility test on the first Kafka node. node = self.zk.nodes[0] cmd = ("%s org.apache.kafka.tools.ClientCompatibilityTest " "--bootstrap-server %s " "--num-cluster-nodes %d " "--topic %s " % (self.zk.path.script("kafka-run-class.sh", node), self.kafka.bootstrap_servers(), len(self.kafka.nodes), self.topics.keys()[0])) for k, v in features.iteritems(): cmd = cmd + ("--%s %s " % (k, v)) results_dir = TestContext.results_dir(self.test_context, 0) os.makedirs(results_dir) ssh_log_file = "%s/%s" % (results_dir, "client_compatibility_test_output.txt") try: self.logger.info("Running %s" % cmd) run_command(node, cmd, ssh_log_file) except Exception as e: self.logger.info("** Command failed. See %s for log messages." % ssh_log_file) raise @parametrize(broker_version=str(DEV_BRANCH)) @parametrize(broker_version=str(LATEST_0_10_0)) @parametrize(broker_version=str(LATEST_0_10_1)) @parametrize(broker_version=str(LATEST_0_10_2)) @parametrize(broker_version=str(LATEST_0_11_0)) @parametrize(broker_version=str(LATEST_1_0)) @parametrize(broker_version=str(LATEST_1_1)) def run_compatibility_test(self, broker_version): self.zk.start() self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() features = get_broker_features(broker_version) self.invoke_compatibility_program(features)
class TestUpgrade(ProduceConsumeValidateTest): def __init__(self, test_context): super(TestUpgrade, self).__init__(test_context=test_context) def setUp(self): self.topic = "test_topic" self.zk = ZookeeperService(self.test_context, num_nodes=1) self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, version=LATEST_0_8_2, topics={self.topic: { "partitions": 3, "replication-factor": 3, 'configs': {"min.insync.replicas": 2}}}) self.zk.start() self.kafka.start() # Producer and consumer self.producer_throughput = 10000 self.num_producers = 1 self.num_consumers = 1 self.producer = VerifiableProducer( self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, version=LATEST_0_8_2) # TODO - reduce the timeout self.consumer = ConsoleConsumer( self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=30000, message_validator=is_int, version=LATEST_0_8_2) def perform_upgrade(self): self.logger.info("First pass bounce - rolling upgrade") for node in self.kafka.nodes: self.kafka.stop_node(node) node.version = TRUNK node.config[config_property.INTER_BROKER_PROTOCOL_VERSION] = "0.8.2.X" self.kafka.start_node(node) self.logger.info("Second pass bounce - remove inter.broker.protocol.version config") for node in self.kafka.nodes: self.kafka.stop_node(node) del node.config[config_property.INTER_BROKER_PROTOCOL_VERSION] self.kafka.start_node(node) def test_upgrade(self): """Test upgrade of Kafka broker cluster from 0.8.2 to 0.9.0 - Start 3 node broker cluster on version 0.8.2 - Start producer and consumer in the background - Perform two-phase rolling upgrade - First phase: upgrade brokers to 0.9.0 with inter.broker.protocol.version set to 0.8.2.X - Second phase: remove inter.broker.protocol.version config with rolling bounce - Finally, validate that every message acked by the producer was consumed by the consumer """ self.run_produce_consume_validate(core_test_action=self.perform_upgrade)
class TestVerifiableProducer(Test): """Sanity checks on verifiable producer service class.""" def __init__(self, test_context): super(TestVerifiableProducer, self).__init__(test_context) self.topic = "topic" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(test_context, num_nodes=1, zk=self.zk, topics={self.topic: {"partitions": 1, "replication-factor": 1}}) self.num_messages = 1000 # This will produce to source kafka cluster self.producer = VerifiableProducer(test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, max_messages=self.num_messages, throughput=self.num_messages/5) def setUp(self): self.zk.start() self.kafka.start() @cluster(num_nodes=3) @parametrize(producer_version=str(LATEST_0_8_2)) @parametrize(producer_version=str(LATEST_0_9)) @parametrize(producer_version=str(LATEST_0_10_0)) @parametrize(producer_version=str(LATEST_0_10_1)) @parametrize(producer_version=str(DEV_BRANCH)) def test_simple_run(self, producer_version=DEV_BRANCH): """ Test that we can start VerifiableProducer on the current branch snapshot version or against the 0.8.2 jar, and verify that we can produce a small number of messages. """ node = self.producer.nodes[0] node.version = KafkaVersion(producer_version) self.producer.start() wait_until(lambda: self.producer.num_acked > 5, timeout_sec=5, err_msg="Producer failed to start in a reasonable amount of time.") # using version.vstring (distutils.version.LooseVersion) is a tricky way of ensuring # that this check works with DEV_BRANCH # When running VerifiableProducer 0.8.X, both the current branch version and 0.8.X should show up because of the # way verifiable producer pulls in some development directories into its classpath # # If the test fails here because 'ps .. | grep' couldn't find the process it means # the login and grep that is_version() performs is slower than # the time it takes the producer to produce its messages. # Easy fix is to decrease throughput= above, the good fix is to make the producer # not terminate until explicitly killed in this case. if node.version <= LATEST_0_8_2: assert is_version(node, [node.version.vstring, DEV_BRANCH.vstring], logger=self.logger) else: assert is_version(node, [node.version.vstring], logger=self.logger) self.producer.wait() num_produced = self.producer.num_acked assert num_produced == self.num_messages, "num_produced: %d, num_messages: %d" % (num_produced, self.num_messages)
class ClientCompatibilityProduceConsumeTest(ProduceConsumeValidateTest): """ These tests validate that we can use a new client to produce and consume from older brokers. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ClientCompatibilityProduceConsumeTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.zk = ZookeeperService(test_context, num_nodes=3) self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics={self.topic:{ "partitions": 10, "replication-factor": 2}}) self.num_partitions = 10 self.timeout_sec = 60 self.producer_throughput = 1000 self.num_producers = 2 self.messages_per_producer = 1000 self.num_consumers = 1 def setUp(self): self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super(ClientCompatibilityProduceConsumeTest, self).min_cluster_size() + self.num_producers + self.num_consumers @parametrize(broker_version=str(DEV_BRANCH)) @parametrize(broker_version=str(LATEST_0_10_0)) @parametrize(broker_version=str(LATEST_0_10_1)) @parametrize(broker_version=str(LATEST_0_10_2)) @parametrize(broker_version=str(LATEST_0_11_0)) @parametrize(broker_version=str(LATEST_1_0)) @parametrize(broker_version=str(LATEST_1_1)) def test_produce_consume(self, broker_version): print("running producer_consumer_compat with broker_version = %s" % broker_version) self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.security_protocol = "PLAINTEXT" self.kafka.interbroker_security_protocol = self.kafka.security_protocol self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, message_validator=is_int_with_prefix) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=60000, message_validator=is_int_with_prefix) self.kafka.start() self.run_produce_consume_validate(lambda: wait_until( lambda: self.producer.each_produced_at_least(self.messages_per_producer) == True, timeout_sec=120, backoff_sec=1, err_msg="Producer did not produce all messages in reasonable amount of time"))
class SimpleConsumerShellTest(Test): """ Tests SimpleConsumerShell tool """ def __init__(self, test_context): super(SimpleConsumerShellTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.messages_received_count = 0 self.topics = {TOPIC: {"partitions": NUM_PARTITIONS, "replication-factor": REPLICATION_FACTOR}} self.zk = ZookeeperService(test_context, self.num_zk) def setUp(self): self.zk.start() def start_kafka(self): self.kafka = KafkaService(self.test_context, self.num_brokers, self.zk, topics=self.topics) self.kafka.start() def run_producer(self): # This will produce to kafka cluster self.producer = VerifiableProducer( self.test_context, num_nodes=1, kafka=self.kafka, topic=TOPIC, throughput=1000, max_messages=MAX_MESSAGES ) self.producer.start() wait_until( lambda: self.producer.num_acked == MAX_MESSAGES, timeout_sec=10, err_msg="Timeout awaiting messages to be produced and acked", ) def start_simple_consumer_shell(self): self.simple_consumer_shell = SimpleConsumerShell(self.test_context, 1, self.kafka, TOPIC) self.simple_consumer_shell.start() def test_simple_consumer_shell(self): """ Tests if SimpleConsumerShell is fetching expected records :return: None """ self.start_kafka() self.run_producer() self.start_simple_consumer_shell() # Assert that SimpleConsumerShell is fetching expected number of messages wait_until( lambda: self.simple_consumer_shell.get_output().count("\n") == (MAX_MESSAGES + 1), timeout_sec=10, err_msg="Timed out waiting to receive expected number of messages.", )
def __init__(self, test_context): super(ConnectStandaloneFileTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.topics = {"test": {"partitions": 1, "replication-factor": 1}} self.zk = ZookeeperService(test_context, self.num_zk)
def __init__(self, test_context): super(DelegationTokenTest, self).__init__(test_context) self.test_context = test_context self.topic = "topic" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(self.test_context, num_nodes=1, zk=self.zk, zk_chroot="/kafka", topics={self.topic: {"partitions": 1, "replication-factor": 1}}, server_prop_overides=[ [config_property.DELEGATION_TOKEN_MAX_LIFETIME_MS, "604800000"], [config_property.DELEGATION_TOKEN_EXPIRY_TIME_MS, "86400000"], [config_property.DELEGATION_TOKEN_MASTER_KEY, "test12345"], [config_property.SASL_ENABLED_MECHANISMS, "GSSAPI,SCRAM-SHA-256"] ]) self.jaas_deleg_conf_path = "/tmp/jaas_deleg.conf" self.jaas_deleg_conf = "" self.client_properties_content = """ security.protocol=SASL_PLAINTEXT sasl.mechanism=SCRAM-SHA-256 sasl.kerberos.service.name=kafka client.id=console-consumer """ self.client_kafka_opts=' -Djava.security.auth.login.config=' + self.jaas_deleg_conf_path self.producer = VerifiableProducer(self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, max_messages=1, throughput=1, kafka_opts_override=self.client_kafka_opts, client_prop_file_override=self.client_properties_content) self.consumer = ConsoleConsumer(self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, kafka_opts_override=self.client_kafka_opts, client_prop_file_override=self.client_properties_content) self.kafka.security_protocol = 'SASL_PLAINTEXT' self.kafka.client_sasl_mechanism = 'GSSAPI,SCRAM-SHA-256' self.kafka.interbroker_sasl_mechanism = 'GSSAPI'
def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ReassignPartitionsTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.num_partitions = 20 self.zk = ZookeeperService(test_context, num_nodes=1) # We set the min.insync.replicas to match the replication factor because # it makes the test more stringent. If min.isr = 2 and # replication.factor=3, then the test would tolerate the failure of # reassignment for upto one replica per partition, which is not # desirable for this test in particular. self.kafka = KafkaService(test_context, num_nodes=4, zk=self.zk, server_prop_overides=[ [config_property.LOG_ROLL_TIME_MS, "5000"], [config_property.LOG_RETENTION_CHECK_INTERVAL_MS, "5000"] ], topics={self.topic: { "partitions": self.num_partitions, "replication-factor": 3, 'configs': { "min.insync.replicas": 3, }} }) self.timeout_sec = 60 self.producer_throughput = 1000 self.num_producers = 1 self.num_consumers = 1
def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(LogDirFailureTest, self).__init__(test_context=test_context) self.topic1 = "test_topic_1" self.topic2 = "test_topic_2" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics={ self.topic1: {"partitions": 1, "replication-factor": 3, "configs": {"min.insync.replicas": 1}}, self.topic2: {"partitions": 1, "replication-factor": 3, "configs": {"min.insync.replicas": 2}} }, # Set log.roll.ms to 3 seconds so that broker will detect disk error sooner when it creates log segment # Otherwise broker will still be able to read/write the log file even if the log directory is inaccessible. server_prop_overides=[ [config_property.OFFSETS_TOPIC_NUM_PARTITIONS, "1"], [config_property.LOG_FLUSH_INTERVAL_MESSAGE, "5"], [config_property.REPLICA_HIGHWATERMARK_CHECKPOINT_INTERVAL_MS, "60000"], [config_property.LOG_ROLL_TIME_MS, "3000"] ]) self.producer_throughput = 1000 self.num_producers = 1 self.num_consumers = 1
def __init__(self, test_context): super(PerformanceServiceTest, self).__init__(test_context) self.record_size = 100 self.num_records = 10000 self.topic = "topic" self.zk = ZookeeperService(test_context, 1)
def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(QuotaTest, self).__init__(test_context=test_context) self.topic = 'test_topic' self.logger.info('use topic ' + self.topic) # quota related parameters self.quota_config = {'quota_producer_default': 2500000, 'quota_consumer_default': 2000000, 'quota_producer_bytes_per_second_overrides': 'overridden_id=3750000', 'quota_consumer_bytes_per_second_overrides': 'overridden_id=3000000'} self.maximum_client_deviation_percentage = 100.0 self.maximum_broker_deviation_percentage = 5.0 self.num_records = 100000 self.record_size = 3000 self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(test_context, num_nodes=1, zk=self.zk, security_protocol='PLAINTEXT', interbroker_security_protocol='PLAINTEXT', topics={self.topic: {'partitions': 6, 'replication-factor': 1, 'configs': {'min.insync.replicas': 1}}}, quota_config=self.quota_config, jmx_object_names=['kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec', 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec'], jmx_attributes=['OneMinuteRate']) self.num_producers = 1 self.num_consumers = 2
def __init__(self, test_context): super(LogCompactionTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.zk = ZookeeperService(test_context, self.num_zk) self.kafka = None self.compaction_verifier = None
def __init__(self, test_context): super(ConsumerGroupCommandTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.topics = { TOPIC: {'partitions': 1, 'replication-factor': 1} } self.zk = ZookeeperService(test_context, self.num_zk)
def __init__(self, test_context): super(ConsoleConsumerTest, self).__init__(test_context) self.topic = "topic" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(self.test_context, num_nodes=1, zk=self.zk, zk_chroot="/kafka", topics={self.topic: {"partitions": 1, "replication-factor": 1}}) self.consumer = ConsoleConsumer(self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic)
def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ProduceBenchTest, self).__init__(test_context) self.zk = ZookeeperService(test_context, num_nodes=3) self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk) self.workload_service = ProduceBenchWorkloadService(test_context, self.kafka) self.trogdor = TrogdorService(context=self.test_context, client_services=[self.kafka, self.workload_service])
def __init__(self, test_context): super(SimpleConsumerShellTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.messages_received_count = 0 self.topics = {TOPIC: {"partitions": NUM_PARTITIONS, "replication-factor": REPLICATION_FACTOR}} self.zk = ZookeeperService(test_context, self.num_zk)
class TestVerifiableProducer(Test): """Sanity checks on verifiable producer service class.""" def __init__(self, test_context): super(TestVerifiableProducer, self).__init__(test_context) self.topic = "topic" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(test_context, num_nodes=1, zk=self.zk, topics={self.topic: {"partitions": 1, "replication-factor": 1}}) self.num_messages = 1000 # This will produce to source kafka cluster self.producer = VerifiableProducer(test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, max_messages=self.num_messages, throughput=1000) def setUp(self): self.zk.start() self.kafka.start() @parametrize(producer_version=str(LATEST_0_8_2)) @parametrize(producer_version=str(LATEST_0_9)) @parametrize(producer_version=str(TRUNK)) def test_simple_run(self, producer_version=TRUNK): """ Test that we can start VerifiableProducer on trunk or against the 0.8.2 jar, and verify that we can produce a small number of messages. """ node = self.producer.nodes[0] node.version = KafkaVersion(producer_version) self.producer.start() wait_until(lambda: self.producer.num_acked > 5, timeout_sec=5, err_msg="Producer failed to start in a reasonable amount of time.") # using version.vstring (distutils.version.LooseVersion) is a tricky way of ensuring # that this check works with TRUNK # When running VerifiableProducer 0.8.X, both trunk version and 0.8.X should show up because of the way # verifiable producer pulls in some trunk directories into its classpath if node.version <= LATEST_0_8_2: assert is_version(node, [node.version.vstring, TRUNK.vstring]) else: assert is_version(node, [node.version.vstring]) self.producer.wait() num_produced = self.producer.num_acked assert num_produced == self.num_messages, "num_produced: %d, num_messages: %d" % (num_produced, self.num_messages)
def __init__(self, test_context): super(Log4jAppenderTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.topics = { TOPIC: {'partitions': 1, 'replication-factor': 1} } self.zk = ZookeeperService(test_context, self.num_zk)
def setUp(self): self.topic = "test_topic" self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() # Producer and consumer self.producer_throughput = 10000 self.num_producers = 1 self.num_consumers = 1
def test_version_probing_upgrade(self): """ Starts 3 KafkaStreams instances, and upgrades one-by-one to "future version" """ self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() self.kafka = KafkaService(self.test_context, num_nodes=1, zk=self.zk, topics=self.topics) self.kafka.start() self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) self.driver.disable_auto_terminate() self.processor1 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) self.processor2 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) self.processor3 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) self.driver.start() self.start_all_nodes_with("") # run with TRUNK self.processors = [self.processor1, self.processor2, self.processor3] self.old_processors = [self.processor1, self.processor2, self.processor3] self.upgraded_processors = [] for p in self.processors: self.leader_counter[p] = 2 self.update_leader() for p in self.processors: self.leader_counter[p] = 0 self.leader_counter[self.leader] = 3 counter = 1 current_generation = 3 random.seed() random.shuffle(self.processors) for p in self.processors: p.CLEAN_NODE_ENABLED = False current_generation = self.do_rolling_bounce(p, counter, current_generation) counter = counter + 1 # shutdown self.driver.stop() self.driver.wait() random.shuffle(self.processors) for p in self.processors: node = p.node with node.account.monitor_log(p.STDOUT_FILE) as monitor: p.stop() monitor.wait_until("UPGRADE-TEST-CLIENT-CLOSED", timeout_sec=60, err_msg="Never saw output 'UPGRADE-TEST-CLIENT-CLOSED' on" + str(node.account)) self.driver.stop()
def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ProduceBenchTest, self).__init__(test_context) self.zk = ZookeeperService(test_context, num_nodes=3) self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk) self.workload_service = ProduceBenchWorkloadService(test_context, self.kafka) self.trogdor = TrogdorService(context=self.test_context, client_services=[self.kafka, self.workload_service]) self.active_topics = {"produce_bench_topic[0-1]": {"numPartitions": 1, "replicationFactor": 3}} self.inactive_topics = {"produce_bench_topic[2-9]": {"numPartitions": 1, "replicationFactor": 3}}
def __init__(self, test_context): super(TestMirrorMakerService, self).__init__(test_context) self.topic = "topic" self.source_zk = ZookeeperService(test_context, num_nodes=1) self.target_zk = ZookeeperService(test_context, num_nodes=1) self.source_kafka = KafkaService(test_context, num_nodes=1, zk=self.source_zk, topics={self.topic: {"partitions": 1, "replication-factor": 1}}) self.target_kafka = KafkaService(test_context, num_nodes=1, zk=self.target_zk, topics={self.topic: {"partitions": 1, "replication-factor": 1}}) # This will produce to source kafka cluster self.producer = VerifiableProducer(test_context, num_nodes=1, kafka=self.source_kafka, topic=self.topic, throughput=1000) self.mirror_maker = MirrorMaker(test_context, num_nodes=1, source=self.source_kafka, target=self.target_kafka, whitelist=self.topic, offset_commit_interval_ms=1000) # This will consume from target kafka cluster self.consumer = ConsoleConsumer(test_context, num_nodes=1, kafka=self.target_kafka, topic=self.topic, message_validator=is_int, consumer_timeout_ms=60000)
class ConsoleConsumerTest(Test): """Sanity checks on console consumer service class.""" def __init__(self, test_context): super(ConsoleConsumerTest, self).__init__(test_context) self.topic = "topic" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(self.test_context, num_nodes=1, zk=self.zk, zk_chroot="/kafka", topics={self.topic: {"partitions": 1, "replication-factor": 1}}) self.consumer = ConsoleConsumer(self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic) def setUp(self): self.zk.start() @cluster(num_nodes=3) @matrix(security_protocol=['PLAINTEXT', 'SSL']) @cluster(num_nodes=4) @matrix(security_protocol=['SASL_SSL'], sasl_mechanism=['PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512']) @matrix(security_protocol=['SASL_PLAINTEXT', 'SASL_SSL']) def test_lifecycle(self, security_protocol, sasl_mechanism='GSSAPI'): """Check that console consumer starts/stops properly, and that we are capturing log output.""" self.kafka.security_protocol = security_protocol self.kafka.client_sasl_mechanism = sasl_mechanism self.kafka.interbroker_sasl_mechanism = sasl_mechanism self.kafka.start() self.consumer.security_protocol = security_protocol t0 = time.time() self.consumer.start() node = self.consumer.nodes[0] wait_until(lambda: self.consumer.alive(node), timeout_sec=20, backoff_sec=.2, err_msg="Consumer was too slow to start") self.logger.info("consumer started in %s seconds " % str(time.time() - t0)) # Verify that log output is happening wait_until(lambda: file_exists(node, ConsoleConsumer.LOG_FILE), timeout_sec=10, err_msg="Timed out waiting for consumer log file to exist.") wait_until(lambda: line_count(node, ConsoleConsumer.LOG_FILE) > 0, timeout_sec=1, backoff_sec=.25, err_msg="Timed out waiting for log entries to start.") # Verify no consumed messages assert line_count(node, ConsoleConsumer.STDOUT_CAPTURE) == 0 self.consumer.stop_node(node) @cluster(num_nodes=4) def test_version(self): """Check that console consumer v0.8.2.X successfully starts and consumes messages.""" self.kafka.start() num_messages = 1000 self.producer = VerifiableProducer(self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, max_messages=num_messages, throughput=1000) self.producer.start() self.producer.wait() self.consumer.nodes[0].version = LATEST_0_8_2 self.consumer.new_consumer = False self.consumer.consumer_timeout_ms = 1000 self.consumer.start() self.consumer.wait() num_consumed = len(self.consumer.messages_consumed[1]) num_produced = self.producer.num_acked assert num_produced == num_consumed, "num_produced: %d, num_consumed: %d" % (num_produced, num_consumed)
class QuotaTest(Test): """ These tests verify that quota provides expected functionality -- they run producer, broker, and consumer with different clientId and quota configuration and check that the observed throughput is close to the value we expect. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(QuotaTest, self).__init__(test_context=test_context) self.topic = 'test_topic' self.logger.info('use topic ' + self.topic) # quota related parameters self.quota_config = { 'quota_producer_default': 2500000, 'quota_consumer_default': 2000000, 'quota_producer_bytes_per_second_overrides': 'overridden_id=3750000', 'quota_consumer_bytes_per_second_overrides': 'overridden_id=3000000' } self.maximum_client_deviation_percentage = 100.0 self.maximum_broker_deviation_percentage = 5.0 self.num_records = 100000 self.record_size = 3000 self.security_protocol = 'PLAINTEXT' self.interbroker_security_protocol = 'PLAINTEXT' self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService( test_context, num_nodes=1, zk=self.zk, security_protocol=self.security_protocol, interbroker_security_protocol=self.interbroker_security_protocol, topics={ self.topic: { 'partitions': 6, 'replication-factor': 1, 'min.insync.replicas': 1 } }, quota_config=self.quota_config, jmx_object_names=[ 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec', 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec' ], jmx_attributes=['OneMinuteRate']) self.num_producers = 1 self.num_consumers = 2 def setUp(self): self.zk.start() self.kafka.start() def min_cluster_size(self): """Override this since we're adding services outside of the constructor""" return super( QuotaTest, self).min_cluster_size() + self.num_producers + self.num_consumers @parametrize(producer_id='default_id', producer_num=1, consumer_id='default_id', consumer_num=1) @parametrize(producer_id='overridden_id', producer_num=1, consumer_id='overridden_id', consumer_num=1) @parametrize(producer_id='overridden_id', producer_num=1, consumer_id='overridden_id', consumer_num=2) def test_quota(self, producer_id='default_id', producer_num=1, consumer_id='default_id', consumer_num=1): # Produce all messages producer = ProducerPerformanceService( self.test_context, producer_num, self.kafka, security_protocol=self.security_protocol, topic=self.topic, num_records=self.num_records, record_size=self.record_size, throughput=-1, client_id=producer_id, jmx_object_names=[ 'kafka.producer:type=producer-metrics,client-id=%s' % producer_id ], jmx_attributes=['outgoing-byte-rate']) producer.run() # Consume all messages consumer = ConsoleConsumer( self.test_context, consumer_num, self.kafka, self.topic, security_protocol=self.security_protocol, new_consumer=False, consumer_timeout_ms=60000, client_id=consumer_id, jmx_object_names=[ 'kafka.consumer:type=ConsumerTopicMetrics,name=BytesPerSec,clientId=%s' % consumer_id ], jmx_attributes=['OneMinuteRate']) consumer.run() for idx, messages in consumer.messages_consumed.iteritems(): assert len( messages ) > 0, "consumer %d didn't consume any message before timeout" % idx success, msg = self.validate(self.kafka, producer, consumer) assert success, msg def validate(self, broker, producer, consumer): """ For each client_id we validate that: 1) number of consumed messages equals number of produced messages 2) maximum_producer_throughput <= producer_quota * (1 + maximum_client_deviation_percentage/100) 3) maximum_broker_byte_in_rate <= producer_quota * (1 + maximum_broker_deviation_percentage/100) 4) maximum_consumer_throughput <= consumer_quota * (1 + maximum_client_deviation_percentage/100) 5) maximum_broker_byte_out_rate <= consumer_quota * (1 + maximum_broker_deviation_percentage/100) """ success = True msg = '' self.kafka.read_jmx_output_all_nodes() # validate that number of consumed messages equals number of produced messages produced_num = sum([value['records'] for value in producer.results]) consumed_num = sum( [len(value) for value in consumer.messages_consumed.values()]) self.logger.info('producer produced %d messages' % produced_num) self.logger.info('consumer consumed %d messages' % consumed_num) if produced_num != consumed_num: success = False msg += "number of produced messages %d doesn't equal number of consumed messages %d" % ( produced_num, consumed_num) # validate that maximum_producer_throughput <= producer_quota * (1 + maximum_client_deviation_percentage/100) producer_attribute_name = 'kafka.producer:type=producer-metrics,client-id=%s:outgoing-byte-rate' % producer.client_id producer_maximum_bps = producer.maximum_jmx_value[ producer_attribute_name] producer_quota_bps = self.get_producer_quota(producer.client_id) self.logger.info( 'producer has maximum throughput %.2f bps with producer quota %.2f bps' % (producer_maximum_bps, producer_quota_bps)) if producer_maximum_bps > producer_quota_bps * ( self.maximum_client_deviation_percentage / 100 + 1): success = False msg += 'maximum producer throughput %.2f bps exceeded producer quota %.2f bps by more than %.1f%%' % \ (producer_maximum_bps, producer_quota_bps, self.maximum_client_deviation_percentage) # validate that maximum_broker_byte_in_rate <= producer_quota * (1 + maximum_broker_deviation_percentage/100) broker_byte_in_attribute_name = 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec:OneMinuteRate' broker_maximum_byte_in_bps = broker.maximum_jmx_value[ broker_byte_in_attribute_name] self.logger.info( 'broker has maximum byte-in rate %.2f bps with producer quota %.2f bps' % (broker_maximum_byte_in_bps, producer_quota_bps)) if broker_maximum_byte_in_bps > producer_quota_bps * ( self.maximum_broker_deviation_percentage / 100 + 1): success = False msg += 'maximum broker byte-in rate %.2f bps exceeded producer quota %.2f bps by more than %.1f%%' % \ (broker_maximum_byte_in_bps, producer_quota_bps, self.maximum_broker_deviation_percentage) # validate that maximum_consumer_throughput <= consumer_quota * (1 + maximum_client_deviation_percentage/100) consumer_attribute_name = 'kafka.consumer:type=ConsumerTopicMetrics,name=BytesPerSec,clientId=%s:OneMinuteRate' % consumer.client_id consumer_maximum_bps = consumer.maximum_jmx_value[ consumer_attribute_name] consumer_quota_bps = self.get_consumer_quota(consumer.client_id) self.logger.info( 'consumer has maximum throughput %.2f bps with consumer quota %.2f bps' % (consumer_maximum_bps, consumer_quota_bps)) if consumer_maximum_bps > consumer_quota_bps * ( self.maximum_client_deviation_percentage / 100 + 1): success = False msg += 'maximum consumer throughput %.2f bps exceeded consumer quota %.2f bps by more than %.1f%%' % \ (consumer_maximum_bps, consumer_quota_bps, self.maximum_client_deviation_percentage) # validate that maximum_broker_byte_out_rate <= consumer_quota * (1 + maximum_broker_deviation_percentage/100) broker_byte_out_attribute_name = 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec:OneMinuteRate' broker_maximum_byte_out_bps = broker.maximum_jmx_value[ broker_byte_out_attribute_name] self.logger.info( 'broker has maximum byte-out rate %.2f bps with consumer quota %.2f bps' % (broker_maximum_byte_out_bps, consumer_quota_bps)) if broker_maximum_byte_out_bps > consumer_quota_bps * ( self.maximum_broker_deviation_percentage / 100 + 1): success = False msg += 'maximum broker byte-out rate %.2f bps exceeded consumer quota %.2f bps by more than %.1f%%' % \ (broker_maximum_byte_out_bps, consumer_quota_bps, self.maximum_broker_deviation_percentage) return success, msg def get_producer_quota(self, client_id): overridden_quotas = { value.split('=')[0]: value.split('=')[1] for value in self. quota_config['quota_producer_bytes_per_second_overrides'].split( ',') } if client_id in overridden_quotas: return float(overridden_quotas[client_id]) return self.quota_config['quota_producer_default'] def get_consumer_quota(self, client_id): overridden_quotas = { value.split('=')[0]: value.split('=')[1] for value in self. quota_config['quota_consumer_bytes_per_second_overrides'].split( ',') } if client_id in overridden_quotas: return float(overridden_quotas[client_id]) return self.quota_config['quota_consumer_default']
class StreamsBrokerCompatibility(Test): """ These tests validates that - Streams works for older brokers 0.11 (or newer) - Streams w/ EOS-alpha works for older brokers 0.11 (or newer) - Streams w/ EOS-beta works for older brokers 2.5 (or newer) - Streams fails fast for older brokers 0.10.0, 0.10.2, and 0.10.1 - Streams w/ EOS-beta fails fast for older brokers 2.4 or older """ input = "brokerCompatibilitySourceTopic" output = "brokerCompatibilitySinkTopic" def __init__(self, test_context): super(StreamsBrokerCompatibility, self).__init__(test_context=test_context) self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService( test_context, num_nodes=1, zk=self.zk, topics={ self.input: { 'partitions': 1, 'replication-factor': 1 }, self.output: { 'partitions': 1, 'replication-factor': 1 } }, server_prop_overrides=[[ "transaction.state.log.replication.factor", "1" ], ["transaction.state.log.min.isr", "1"]]) self.consumer = VerifiableConsumer( test_context, 1, self.kafka, self.output, "stream-broker-compatibility-verify-consumer") def setUp(self): self.zk.start() @cluster(num_nodes=4) @parametrize(broker_version=str(LATEST_2_4)) @parametrize(broker_version=str(LATEST_2_3)) @parametrize(broker_version=str(LATEST_2_2)) @parametrize(broker_version=str(LATEST_2_1)) @parametrize(broker_version=str(LATEST_2_0)) @parametrize(broker_version=str(LATEST_1_1)) @parametrize(broker_version=str(LATEST_1_0)) @parametrize(broker_version=str(LATEST_0_11_0)) def test_compatible_brokers_eos_disabled(self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, "at_least_once") processor.start() self.consumer.start() processor.wait() wait_until( lambda: self.consumer.total_consumed() > 0, timeout_sec=30, err_msg= "Did expect to read a message but got none within 30 seconds.") self.consumer.stop() self.kafka.stop() @cluster(num_nodes=4) @parametrize(broker_version=str(LATEST_2_6)) @parametrize(broker_version=str(LATEST_2_5)) @parametrize(broker_version=str(LATEST_2_4)) @parametrize(broker_version=str(LATEST_2_3)) @parametrize(broker_version=str(LATEST_2_2)) @parametrize(broker_version=str(LATEST_2_1)) @parametrize(broker_version=str(LATEST_2_0)) @parametrize(broker_version=str(LATEST_1_1)) @parametrize(broker_version=str(LATEST_1_0)) @parametrize(broker_version=str(LATEST_0_11_0)) def test_compatible_brokers_eos_alpha_enabled(self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, "exactly_once") processor.start() self.consumer.start() processor.wait() wait_until( lambda: self.consumer.total_consumed() > 0, timeout_sec=30, err_msg= "Did expect to read a message but got none within 30 seconds.") self.consumer.stop() self.kafka.stop() # TODO enable after 2.5 is released # @parametrize(broker_version=str(LATEST_2_5)) # def test_compatible_brokers_eos_beta_enabled(self, broker_version): # self.kafka.set_version(KafkaVersion(broker_version)) # self.kafka.start() # # processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, "exactly_once_beta") # processor.start() # # self.consumer.start() # # processor.wait() # # wait_until(lambda: self.consumer.total_consumed() > 0, timeout_sec=30, err_msg="Did expect to read a message but got none within 30 seconds.") # # self.consumer.stop() # self.kafka.stop() @cluster(num_nodes=4) @parametrize(broker_version=str(LATEST_0_10_2)) @parametrize(broker_version=str(LATEST_0_10_1)) @parametrize(broker_version=str(LATEST_0_10_0)) def test_fail_fast_on_incompatible_brokers(self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, "at_least_once") with processor.node.account.monitor_log( processor.STDERR_FILE) as monitor: processor.start() monitor.wait_until( 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException', timeout_sec=60, err_msg= "Never saw 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException " + str(processor.node.account)) self.kafka.stop() @cluster(num_nodes=4) @parametrize(broker_version=str(LATEST_2_4)) @parametrize(broker_version=str(LATEST_2_3)) @parametrize(broker_version=str(LATEST_2_2)) @parametrize(broker_version=str(LATEST_2_1)) @parametrize(broker_version=str(LATEST_2_0)) @parametrize(broker_version=str(LATEST_1_1)) @parametrize(broker_version=str(LATEST_1_0)) @parametrize(broker_version=str(LATEST_0_11_0)) def test_fail_fast_on_incompatible_brokers_if_eos_beta_enabled( self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, "exactly_once_beta") with processor.node.account.monitor_log( processor.STDERR_FILE) as monitor: with processor.node.account.monitor_log(processor.LOG_FILE) as log: processor.start() log.wait_until( 'Shutting down because the Kafka cluster seems to be on a too old version. Setting processing\.guarantee="exactly_once_beta" requires broker version 2\.5 or higher\.', timeout_sec=60, err_msg= "Never saw 'Shutting down, because the Kafka cluster seems to be on a too old version. Setting `processing.guarantee=\"exaclty_once_beta\"` requires broker version 2.5 or higher.' log message " + str(processor.node.account)) monitor.wait_until( 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException', timeout_sec=60, err_msg= "Never saw 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException' error message " + str(processor.node.account)) self.kafka.stop()
class MessageFormatChangeTest(ProduceConsumeValidateTest): def __init__(self, test_context): super(MessageFormatChangeTest, self).__init__(test_context=test_context) def setUp(self): self.topic = "test_topic" self.zk = ZookeeperService( self.test_context, num_nodes=1) if quorum.for_test( self.test_context) == quorum.zk else None if self.zk: self.zk.start() # Producer and consumer self.producer_throughput = 10000 self.num_producers = 1 self.num_consumers = 1 self.messages_per_producer = 100 def produce_and_consume(self, producer_version, consumer_version, group): self.producer = VerifiableProducer( self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, message_validator=is_int, version=KafkaVersion(producer_version)) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=30000, message_validator=is_int, version=KafkaVersion(consumer_version)) self.consumer.group_id = group self.run_produce_consume_validate(lambda: wait_until( lambda: self.producer.each_produced_at_least( self.messages_per_producer) == True, timeout_sec=120, backoff_sec=1, err_msg= "Producer did not produce all messages in reasonable amount of time" )) @cluster(num_nodes=12) @matrix(producer_version=[str(DEV_BRANCH)], consumer_version=[str(DEV_BRANCH)], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_0_10)], consumer_version=[str(LATEST_0_10)], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_0_9)], consumer_version=[str(LATEST_0_9)], metadata_quorum=quorum.all_non_upgrade) def test_compatibility(self, producer_version, consumer_version, metadata_quorum=quorum.zk): """ This tests performs the following checks: The workload is a mix of 0.9.x, 0.10.x and 0.11.x producers and consumers that produce to and consume from a DEV_BRANCH cluster 1. initially the topic is using message format 0.9.0 2. change the message format version for topic to 0.10.0 on the fly. 3. change the message format version for topic to 0.11.0 on the fly. 4. change the message format version for topic back to 0.10.0 on the fly (only if the client version is 0.11.0 or newer) - The producers and consumers should not have any issue. Note regarding step number 4. Downgrading the message format version is generally unsupported as it breaks older clients. More concretely, if we downgrade a topic from 0.11.0 to 0.10.0 after it contains messages with version 0.11.0, we will return the 0.11.0 messages without down conversion due to an optimisation in the handling of fetch requests. This will break any consumer that doesn't support 0.11.0. So, in practice, step 4 is similar to step 2 and it didn't seem worth it to increase the cluster size to in order to add a step 5 that would change the message format version for the topic back to 0.9.0.0. """ self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, version=DEV_BRANCH, topics={ self.topic: { "partitions": 3, "replication-factor": 3, 'configs': { "min.insync.replicas": 2 } } }, controller_num_nodes_override=1) self.kafka.start() self.logger.info("First format change to 0.9.0") self.kafka.alter_message_format(self.topic, str(LATEST_0_9)) self.produce_and_consume(producer_version, consumer_version, "group1") self.logger.info("Second format change to 0.10.0") self.kafka.alter_message_format(self.topic, str(LATEST_0_10)) self.produce_and_consume(producer_version, consumer_version, "group2") self.logger.info("Third format change to 0.11.0") self.kafka.alter_message_format(self.topic, str(LATEST_0_11)) self.produce_and_consume(producer_version, consumer_version, "group3") if producer_version == str(DEV_BRANCH) and consumer_version == str( DEV_BRANCH): self.logger.info("Fourth format change back to 0.10.0") self.kafka.alter_message_format(self.topic, str(LATEST_0_10)) self.produce_and_consume(producer_version, consumer_version, "group4")
class MessageFormatChangeTest(ProduceConsumeValidateTest): def __init__(self, test_context): super(MessageFormatChangeTest, self).__init__(test_context=test_context) def setUp(self): self.topic = "test_topic" self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() # Producer and consumer self.producer_throughput = 10000 self.num_producers = 1 self.num_consumers = 1 self.messages_per_producer = 100 def produce_and_consume(self, producer_version, consumer_version, group): self.producer = VerifiableProducer( self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, message_validator=is_int, version=KafkaVersion(producer_version)) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=30000, message_validator=is_int, version=KafkaVersion(consumer_version)) self.consumer.group_id = group self.run_produce_consume_validate(lambda: wait_until( lambda: self.producer.each_produced_at_least( self.messages_per_producer) == True, timeout_sec=120, backoff_sec=1, err_msg= "Producer did not produce all messages in reasonable amount of time" )) @parametrize(producer_version=str(TRUNK), consumer_version=str(TRUNK)) @parametrize(producer_version=str(LATEST_0_9), consumer_version=str(LATEST_0_9)) def test_compatibility(self, producer_version, consumer_version): """ This tests performs the following checks: The workload is a mix of 0.9.x and 0.10.x producers and consumers that produce to and consume from a 0.10.x cluster 1. initially the topic is using message format 0.9.0 2. change the message format version for topic to 0.10.0 on the fly. 3. change the message format version for topic back to 0.9.0 on the fly. - The producers and consumers should not have any issue. - Note that for 0.9.x consumers/producers we only do steps 1 and 2 """ self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, version=TRUNK, topics={ self.topic: { "partitions": 3, "replication-factor": 3, 'configs': { "min.insync.replicas": 2 } } }) self.kafka.start() self.logger.info("First format change to 0.9.0") self.kafka.alter_message_format(self.topic, str(LATEST_0_9)) self.produce_and_consume(producer_version, consumer_version, "group1") self.logger.info("Second format change to 0.10.0") self.kafka.alter_message_format(self.topic, str(LATEST_0_10)) self.produce_and_consume(producer_version, consumer_version, "group2") if producer_version == str(TRUNK) and consumer_version == str(TRUNK): self.logger.info("Third format change back to 0.9.0") self.kafka.alter_message_format(self.topic, str(LATEST_0_9)) self.produce_and_consume(producer_version, consumer_version, "group3")
class SecurityTest(ProduceConsumeValidateTest): """ These tests validate security features. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(SecurityTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService( test_context, num_nodes=1, zk=self.zk, topics={self.topic: { "partitions": 2, "replication-factor": 1 }}) self.num_partitions = 2 self.timeout_sec = 10000 self.producer_throughput = 1000 self.num_producers = 1 self.num_consumers = 1 def setUp(self): self.zk.start() def producer_consumer_have_expected_error(self, error): try: for node in self.producer.nodes: node.account.ssh("grep %s %s" % (error, self.producer.LOG_FILE)) for node in self.consumer.nodes: node.account.ssh("grep %s %s" % (error, self.consumer.LOG_FILE)) except RemoteCommandError: return False return True @cluster(num_nodes=7) @parametrize(security_protocol='PLAINTEXT', interbroker_security_protocol='SSL') @parametrize(security_protocol='SSL', interbroker_security_protocol='PLAINTEXT') def test_client_ssl_endpoint_validation_failure( self, security_protocol, interbroker_security_protocol): """ Test that invalid hostname in certificate results in connection failures. When security_protocol=SSL, client SSL handshakes are expected to fail due to hostname verification failure. When security_protocol=PLAINTEXT and interbroker_security_protocol=SSL, controller connections fail with hostname verification failure. Hence clients are expected to fail with LEADER_NOT_AVAILABLE. """ self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = interbroker_security_protocol SecurityConfig.ssl_stores = TestSslStores( self.test_context.local_scratch_dir, valid_hostname=False) self.kafka.start() self.create_producer_and_consumer() self.producer.log_level = "TRACE" self.producer.start() self.consumer.start() try: wait_until(lambda: self.producer.num_acked > 0, timeout_sec=5) # Fail quickly if messages are successfully acked raise RuntimeError( "Messages published successfully but should not have!" " Endpoint validation did not fail with invalid hostname") except TimeoutError: # expected pass error = 'SSLHandshakeException' if security_protocol == 'SSL' else 'LEADER_NOT_AVAILABLE' wait_until(lambda: self.producer_consumer_have_expected_error(error), timeout_sec=5) self.producer.stop() self.consumer.stop() self.producer.log_level = "INFO" SecurityConfig.ssl_stores.valid_hostname = True for node in self.kafka.nodes: self.kafka.restart_node(node, clean_shutdown=True) self.create_producer_and_consumer() self.run_produce_consume_validate() def create_producer_and_consumer(self): self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=10000, message_validator=is_int)
class StreamsUpgradeTest(Test): """ Tests rolling upgrades and downgrades of the Kafka Streams library. """ def __init__(self, test_context): super(StreamsUpgradeTest, self).__init__(test_context) self.replication = 3 self.partitions = 1 self.isr = 2 self.topics = { 'echo' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr}}, 'data' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'min' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'max' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'sum' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'dif' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'cnt' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'avg' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'wcnt' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'tagg' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} } } def perform_streams_upgrade(self, to_version): self.logger.info("First pass bounce - rolling streams upgrade") # get the node running the streams app node = self.processor1.node self.processor1.stop() # change it's version. This will automatically make it pick up a different # JAR when it starts again node.version = KafkaVersion(to_version) self.processor1.start() def perform_broker_upgrade(self, to_version): self.logger.info("First pass bounce - rolling broker upgrade") for node in self.kafka.nodes: self.kafka.stop_node(node) node.version = KafkaVersion(to_version) self.kafka.start_node(node) @cluster(num_nodes=6) @parametrize(from_version=str(LATEST_0_10_1), to_version=str(DEV_BRANCH)) @parametrize(from_version=str(LATEST_0_10_2), to_version=str(DEV_BRANCH)) @parametrize(from_version=str(DEV_BRANCH), to_version=str(LATEST_0_10_2)) def test_upgrade_downgrade_streams(self, from_version, to_version): """ Start a smoke test client, then abort (kill -9) and restart it a few times. Ensure that all records are delivered. Note, that just like tests/core/upgrade_test.py, a prerequisite for this test to succeed if the inclusion of all parametrized versions of kafka in kafka/vagrant/base.sh (search for get_kafka()). For streams in particular, that means that someone has manually copies the kafka-stream-$version-test.jar in the right S3 bucket as shown in base.sh. """ # Setup phase self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() # number of nodes needs to be >= 3 for the smoke test self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, version=KafkaVersion(from_version), topics=self.topics) self.kafka.start() # allow some time for topics to be created time.sleep(10) self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) self.processor1 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka) self.driver.start() self.processor1.start() time.sleep(15) self.perform_streams_upgrade(to_version) time.sleep(15) self.driver.wait() self.driver.stop() self.processor1.stop() node = self.driver.node node.account.ssh("grep ALL-RECORDS-DELIVERED %s" % self.driver.STDOUT_FILE, allow_fail=False) self.processor1.node.account.ssh_capture("grep SMOKE-TEST-CLIENT-CLOSED %s" % self.processor1.STDOUT_FILE, allow_fail=False) @cluster(num_nodes=6) @parametrize(from_version=str(LATEST_0_10_2), to_version=str(DEV_BRANCH)) def test_upgrade_brokers(self, from_version, to_version): """ Start a smoke test client then perform rolling upgrades on the broker. """ # Setup phase self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() # number of nodes needs to be >= 3 for the smoke test self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, version=KafkaVersion(from_version), topics=self.topics) self.kafka.start() # allow some time for topics to be created time.sleep(10) self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) self.processor1 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka) self.driver.start() self.processor1.start() time.sleep(15) self.perform_broker_upgrade(to_version) time.sleep(15) self.driver.wait() self.driver.stop() self.processor1.stop() node = self.driver.node node.account.ssh("grep ALL-RECORDS-DELIVERED %s" % self.driver.STDOUT_FILE, allow_fail=False) self.processor1.node.account.ssh_capture("grep SMOKE-TEST-CLIENT-CLOSED %s" % self.processor1.STDOUT_FILE, allow_fail=False)
class CompressionTest(ProduceConsumeValidateTest): """ These tests validate produce / consume for compressed topics. """ COMPRESSION_TYPES = ["snappy", "gzip", "lz4", "zstd", "none"] def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(CompressionTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService( test_context, num_nodes=1, zk=self.zk, topics={self.topic: { "partitions": 10, "replication-factor": 1 }}) self.num_partitions = 10 self.timeout_sec = 60 self.producer_throughput = 1000 self.num_producers = len(self.COMPRESSION_TYPES) self.messages_per_producer = 1000 self.num_consumers = 1 def setUp(self): self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super( CompressionTest, self).min_cluster_size() + self.num_producers + self.num_consumers @cluster(num_nodes=8) @parametrize(compression_types=COMPRESSION_TYPES) def test_compressed_topic(self, compression_types): """Test produce => consume => validate for compressed topics Setup: 1 zk, 1 kafka node, 1 topic with partitions=10, replication-factor=1 compression_types parameter gives a list of compression types (or no compression if "none"). Each producer in a VerifiableProducer group (num_producers = number of compression types) will use a compression type from the list based on producer's index in the group. - Produce messages in the background - Consume messages in the background - Stop producing, and finish consuming - Validate that every acked message was consumed """ self.kafka.security_protocol = "PLAINTEXT" self.kafka.interbroker_security_protocol = self.kafka.security_protocol self.producer = VerifiableProducer( self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, message_validator=is_int_with_prefix, compression_types=compression_types) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=60000, message_validator=is_int_with_prefix) self.kafka.start() self.run_produce_consume_validate(lambda: wait_until( lambda: self.producer.each_produced_at_least( self.messages_per_producer) == True, timeout_sec=120, backoff_sec=1, err_msg= "Producer did not produce all messages in reasonable amount of time" ))
class TestUpgrade(ProduceConsumeValidateTest): def __init__(self, test_context): super(TestUpgrade, self).__init__(test_context=test_context) def setUp(self): self.topic = "test_topic" self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() # Producer and consumer self.producer_throughput = 10000 self.num_producers = 1 self.num_consumers = 1 def perform_upgrade(self, from_kafka_version, to_message_format_version=None): self.logger.info("First pass bounce - rolling upgrade") for node in self.kafka.nodes: self.kafka.stop_node(node) node.version = DEV_BRANCH node.config[config_property. INTER_BROKER_PROTOCOL_VERSION] = from_kafka_version node.config[ config_property.MESSAGE_FORMAT_VERSION] = from_kafka_version self.kafka.start_node(node) self.logger.info( "Second pass bounce - remove inter.broker.protocol.version config") for node in self.kafka.nodes: self.kafka.stop_node(node) del node.config[config_property.INTER_BROKER_PROTOCOL_VERSION] if to_message_format_version is None: del node.config[config_property.MESSAGE_FORMAT_VERSION] else: node.config[config_property. MESSAGE_FORMAT_VERSION] = to_message_format_version self.kafka.start_node(node) @cluster(num_nodes=6) @parametrize(from_kafka_version=str(LATEST_0_10_1), to_message_format_version=None, compression_types=["lz4"]) @parametrize(from_kafka_version=str(LATEST_0_10_1), to_message_format_version=None, compression_types=["snappy"], new_consumer=False) @parametrize(from_kafka_version=str(LATEST_0_10_0), to_message_format_version=None, compression_types=["snappy"], new_consumer=False) @parametrize(from_kafka_version=str(LATEST_0_10_0), to_message_format_version=None, compression_types=["snappy"]) @parametrize(from_kafka_version=str(LATEST_0_9), to_message_format_version=None, compression_types=["none"], new_consumer=False) @cluster(num_nodes=7) @parametrize(from_kafka_version=str(LATEST_0_9), to_message_format_version=None, compression_types=["none"], security_protocol="SASL_SSL") @cluster(num_nodes=6) @parametrize(from_kafka_version=str(LATEST_0_9), to_message_format_version=None, compression_types=["snappy"]) @parametrize(from_kafka_version=str(LATEST_0_9), to_message_format_version=None, compression_types=["lz4"], new_consumer=False) @parametrize(from_kafka_version=str(LATEST_0_9), to_message_format_version=None, compression_types=["lz4"]) @parametrize(from_kafka_version=str(LATEST_0_9), to_message_format_version=str(LATEST_0_9), compression_types=["none"], new_consumer=False) @parametrize(from_kafka_version=str(LATEST_0_9), to_message_format_version=str(LATEST_0_9), compression_types=["snappy"]) @parametrize(from_kafka_version=str(LATEST_0_9), to_message_format_version=str(LATEST_0_9), compression_types=["lz4"], new_consumer=False) @parametrize(from_kafka_version=str(LATEST_0_9), to_message_format_version=str(LATEST_0_9), compression_types=["lz4"]) @cluster(num_nodes=7) @parametrize(from_kafka_version=str(LATEST_0_8_2), to_message_format_version=None, compression_types=["none"], new_consumer=False) @parametrize(from_kafka_version=str(LATEST_0_8_2), to_message_format_version=None, compression_types=["snappy"], new_consumer=False) def test_upgrade(self, from_kafka_version, to_message_format_version, compression_types, new_consumer=True, security_protocol="PLAINTEXT"): """Test upgrade of Kafka broker cluster from 0.8.2, 0.9.0, 0.10.0, 0.10.1 to the current version from_kafka_version is a Kafka version to upgrade from: either 0.8.2.X, 0.9.0.x or 0.10.0.x If to_message_format_version is None, it means that we will upgrade to default (latest) message format version. It is possible to upgrade to 0.10 brokers but still use message format version 0.9 - Start 3 node broker cluster on version 'from_kafka_version' - Start producer and consumer in the background - Perform two-phase rolling upgrade - First phase: upgrade brokers to 0.10 with inter.broker.protocol.version set to from_kafka_version and log.message.format.version set to from_kafka_version - Second phase: remove inter.broker.protocol.version config with rolling bounce; if to_message_format_version is set to 0.9, set log.message.format.version to to_message_format_version, otherwise remove log.message.format.version config - Finally, validate that every message acked by the producer was consumed by the consumer """ self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, version=KafkaVersion(from_kafka_version), topics={ self.topic: { "partitions": 3, "replication-factor": 3, 'configs': { "min.insync.replicas": 2 } } }) self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = security_protocol self.kafka.start() self.producer = VerifiableProducer( self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, message_validator=is_int, compression_types=compression_types, version=KafkaVersion(from_kafka_version)) if from_kafka_version <= LATEST_0_10_0: assert self.zk.query("/cluster/id") is None # TODO - reduce the timeout self.consumer = ConsoleConsumer( self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=30000, new_consumer=new_consumer, message_validator=is_int, version=KafkaVersion(from_kafka_version)) self.run_produce_consume_validate( core_test_action=lambda: self.perform_upgrade( from_kafka_version, to_message_format_version)) cluster_id_json = self.zk.query("/cluster/id") assert cluster_id_json is not None try: cluster_id = json.loads(cluster_id_json) except: self.logger.debug( "Data in /cluster/id znode could not be parsed. Data = %s" % cluster_id_json) self.logger.debug("Cluster id [%s]", cluster_id) assert len(cluster_id["id"]) == 22
class Log4jAppenderTest(Test): """ Tests KafkaLog4jAppender using VerifiableKafkaLog4jAppender that appends increasing ints to a Kafka topic """ def __init__(self, test_context): super(Log4jAppenderTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.messages_received_count = 0 self.topics = { TOPIC: {'partitions': 1, 'replication-factor': 1} } self.zk = ZookeeperService(test_context, self.num_zk) def setUp(self): self.zk.start() def start_kafka(self, security_protocol, interbroker_security_protocol): self.kafka = KafkaService( self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, interbroker_security_protocol=interbroker_security_protocol, topics=self.topics) self.kafka.start() def start_appender(self, security_protocol): self.appender = KafkaLog4jAppender(self.test_context, self.num_brokers, self.kafka, TOPIC, MAX_MESSAGES, security_protocol=security_protocol) self.appender.start() def custom_message_validator(self, msg): if msg and "INFO : org.apache.kafka.tools.VerifiableLog4jAppender" in msg: self.logger.debug("Received message: %s" % msg) self.messages_received_count += 1 def start_consumer(self, security_protocol): enable_new_consumer = security_protocol != SecurityConfig.PLAINTEXT self.consumer = ConsoleConsumer(self.test_context, num_nodes=self.num_brokers, kafka=self.kafka, topic=TOPIC, consumer_timeout_ms=1000, new_consumer=enable_new_consumer, message_validator=self.custom_message_validator) self.consumer.start() @matrix(security_protocol=['PLAINTEXT', 'SSL', 'SASL_PLAINTEXT', 'SASL_SSL']) def test_log4j_appender(self, security_protocol='PLAINTEXT'): """ Tests if KafkaLog4jAppender is producing to Kafka topic :return: None """ self.start_kafka(security_protocol, security_protocol) self.start_appender(security_protocol) self.appender.wait() self.start_consumer(security_protocol) node = self.consumer.nodes[0] wait_until(lambda: self.consumer.alive(node), timeout_sec=10, backoff_sec=.2, err_msg="Consumer was too slow to start") # Verify consumed messages count wait_until(lambda: self.messages_received_count == MAX_MESSAGES, timeout_sec=10, err_msg="Timed out waiting to consume expected number of messages.") self.consumer.stop()
class TransactionsTest(Test): """Tests transactions by transactionally copying data from a source topic to a destination topic and killing the copy process as well as the broker randomly through the process. In the end we verify that the final output topic contains exactly one committed copy of each message in the input topic """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(TransactionsTest, self).__init__(test_context=test_context) self.input_topic = "input-topic" self.output_topic = "output-topic" self.num_brokers = 3 # Test parameters self.num_input_partitions = 2 self.num_output_partitions = 3 self.num_seed_messages = 100000 self.transaction_size = 750 self.consumer_group = "transactions-test-consumer-group" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(test_context, num_nodes=self.num_brokers, zk=self.zk) def setUp(self): self.zk.start() def seed_messages(self, topic, num_seed_messages): seed_timeout_sec = 10000 seed_producer = VerifiableProducer(context=self.test_context, num_nodes=1, kafka=self.kafka, topic=topic, message_validator=is_int, max_messages=num_seed_messages, enable_idempotence=True) seed_producer.start() wait_until(lambda: seed_producer.num_acked >= num_seed_messages, timeout_sec=seed_timeout_sec, err_msg="Producer failed to produce messages %d in %ds." %\ (self.num_seed_messages, seed_timeout_sec)) return seed_producer.acked def get_messages_from_topic(self, topic, num_messages): consumer = self.start_consumer(topic, group_id="verifying_consumer") return self.drain_consumer(consumer, num_messages) def bounce_brokers(self, clean_shutdown): for node in self.kafka.nodes: if clean_shutdown: self.kafka.restart_node(node, clean_shutdown = True) else: self.kafka.stop_node(node, clean_shutdown = False) wait_until(lambda: len(self.kafka.pids(node)) == 0 and not self.kafka.is_registered(node), timeout_sec=self.kafka.zk_session_timeout + 5, err_msg="Failed to see timely deregistration of \ hard-killed broker %s" % str(node.account)) self.kafka.start_node(node) def create_and_start_message_copier(self, input_topic, input_partition, output_topic, transactional_id): message_copier = TransactionalMessageCopier( context=self.test_context, num_nodes=1, kafka=self.kafka, transactional_id=transactional_id, consumer_group=self.consumer_group, input_topic=input_topic, input_partition=input_partition, output_topic=output_topic, max_messages=-1, transaction_size=self.transaction_size ) message_copier.start() wait_until(lambda: message_copier.alive(message_copier.nodes[0]), timeout_sec=10, err_msg="Message copier failed to start after 10 s") return message_copier def bounce_copiers(self, copiers, clean_shutdown): for _ in range(3): for copier in copiers: wait_until(lambda: copier.progress_percent() >= 20.0, timeout_sec=30, err_msg="%s : Message copier didn't make enough progress in 30s. Current progress: %s" \ % (copier.transactional_id, str(copier.progress_percent()))) self.logger.info("%s - progress: %s" % (copier.transactional_id, str(copier.progress_percent()))) copier.restart(clean_shutdown) def create_and_start_copiers(self, input_topic, output_topic, num_copiers): copiers = [] for i in range(0, num_copiers): copiers.append(self.create_and_start_message_copier( input_topic=input_topic, output_topic=output_topic, input_partition=i, transactional_id="copier-" + str(i) )) return copiers def start_consumer(self, topic_to_read, group_id): consumer = ConsoleConsumer(context=self.test_context, num_nodes=1, kafka=self.kafka, topic=topic_to_read, group_id=group_id, new_consumer=True, message_validator=is_int, from_beginning=True, isolation_level="read_committed") consumer.start() # ensure that the consumer is up. wait_until(lambda: (len(consumer.messages_consumed[1]) > 0) == True, timeout_sec=60, err_msg="Consumer failed to consume any messages for %ds" %\ 60) return consumer def drain_consumer(self, consumer, num_messages): # wait until we read at least the expected number of messages. # This is a safe check because both failure modes will be caught: # 1. If we have 'num_seed_messages' but there are duplicates, then # this is checked for later. # # 2. If we never reach 'num_seed_messages', then this will cause the # test to fail. wait_until(lambda: len(consumer.messages_consumed[1]) >= num_messages, timeout_sec=90, err_msg="Consumer consumed only %d out of %d messages in %ds" %\ (len(consumer.messages_consumed[1]), num_messages, 90)) consumer.stop() return consumer.messages_consumed[1] def copy_messages_transactionally(self, failure_mode, bounce_target, input_topic, output_topic, num_copiers, num_messages_to_copy): """Copies messages transactionally from the seeded input topic to the output topic, either bouncing brokers or clients in a hard and soft way as it goes. This method also consumes messages in read_committed mode from the output topic while the bounces and copy is going on. It returns the concurrently consumed messages. """ copiers = self.create_and_start_copiers(input_topic=input_topic, output_topic=output_topic, num_copiers=num_copiers) concurrent_consumer = self.start_consumer(output_topic, group_id="concurrent_consumer") clean_shutdown = False if failure_mode == "clean_bounce": clean_shutdown = True if bounce_target == "brokers": self.bounce_brokers(clean_shutdown) elif bounce_target == "clients": self.bounce_copiers(copiers, clean_shutdown) for copier in copiers: wait_until(lambda: copier.is_done, timeout_sec=120, err_msg="%s - Failed to copy all messages in %ds." %\ (copier.transactional_id, 120)) self.logger.info("finished copying messages") return self.drain_consumer(concurrent_consumer, num_messages_to_copy) def setup_topics(self): self.kafka.topics = { self.input_topic: { "partitions": self.num_input_partitions, "replication-factor": 3, "configs": { "min.insync.replicas": 2 } }, self.output_topic: { "partitions": self.num_output_partitions, "replication-factor": 3, "configs": { "min.insync.replicas": 2 } } } @cluster(num_nodes=9) @matrix(failure_mode=["hard_bounce", "clean_bounce"], bounce_target=["brokers", "clients"], check_order=[True, False]) def test_transactions(self, failure_mode, bounce_target, check_order): security_protocol = 'PLAINTEXT' self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = security_protocol self.kafka.logs["kafka_data_1"]["collect_default"] = True self.kafka.logs["kafka_data_2"]["collect_default"] = True self.kafka.logs["kafka_operational_logs_debug"]["collect_default"] = True if check_order: # To check ordering, we simply create input and output topics # with a single partition. # We reduce the number of seed messages to copy to account for the fewer output # partitions, and thus lower parallelism. This helps keep the test # time shorter. self.num_seed_messages = self.num_seed_messages / 3 self.num_input_partitions = 1 self.num_output_partitions = 1 self.setup_topics() self.kafka.start() input_messages = self.seed_messages(self.input_topic, self.num_seed_messages) concurrently_consumed_messages = self.copy_messages_transactionally( failure_mode, bounce_target, input_topic=self.input_topic, output_topic=self.output_topic, num_copiers=self.num_input_partitions, num_messages_to_copy=self.num_seed_messages) output_messages = self.get_messages_from_topic(self.output_topic, self.num_seed_messages) concurrently_consumed_message_set = set(concurrently_consumed_messages) output_message_set = set(output_messages) input_message_set = set(input_messages) num_dups = abs(len(output_messages) - len(output_message_set)) num_dups_in_concurrent_consumer = abs(len(concurrently_consumed_messages) - len(concurrently_consumed_message_set)) assert num_dups == 0, "Detected %d duplicates in the output stream" % num_dups assert input_message_set == output_message_set, "Input and output message sets are not equal. Num input messages %d. Num output messages %d" %\ (len(input_message_set), len(output_message_set)) assert num_dups_in_concurrent_consumer == 0, "Detected %d dups in concurrently consumed messages" % num_dups_in_concurrent_consumer assert input_message_set == concurrently_consumed_message_set, \ "Input and concurrently consumed output message sets are not equal. Num input messages: %d. Num concurrently_consumed_messages: %d" %\ (len(input_message_set), len(concurrently_consumed_message_set)) if check_order: assert input_messages == sorted(input_messages), "The seed messages themselves were not in order" assert output_messages == input_messages, "Output messages are not in order" assert concurrently_consumed_messages == output_messages, "Concurrently consumed messages are not in order"
class ConnectStandaloneFileTest(Test): """ Simple test of Kafka Connect that produces data from a file in one standalone process and consumes it on another, validating the output is identical to the input. """ FILE_SOURCE_CONNECTOR = 'org.apache.kafka.connect.file.FileStreamSourceConnector' FILE_SINK_CONNECTOR = 'org.apache.kafka.connect.file.FileStreamSinkConnector' INPUT_FILE = "/mnt/connect.input" OUTPUT_FILE = "/mnt/connect.output" OFFSETS_FILE = "/mnt/connect.offsets" TOPIC = "${file:/mnt/connect/connect-file-external.properties:topic.external}" TOPIC_TEST = "test" FIRST_INPUT_LIST = ["foo", "bar", "baz"] FIRST_INPUT = "\n".join(FIRST_INPUT_LIST) + "\n" SECOND_INPUT_LIST = ["razz", "ma", "tazz"] SECOND_INPUT = "\n".join(SECOND_INPUT_LIST) + "\n" SCHEMA = {"type": "string", "optional": False} def __init__(self, test_context): super(ConnectStandaloneFileTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.topics = {'test': {'partitions': 1, 'replication-factor': 1}} self.zk = ZookeeperService(test_context, self.num_zk) @cluster(num_nodes=5) @parametrize(converter="org.apache.kafka.connect.json.JsonConverter", schemas=True) @parametrize(converter="org.apache.kafka.connect.json.JsonConverter", schemas=False) @parametrize(converter="org.apache.kafka.connect.storage.StringConverter", schemas=None) @parametrize(security_protocol=SecurityConfig.PLAINTEXT) @cluster(num_nodes=6) @parametrize(security_protocol=SecurityConfig.SASL_SSL) def test_file_source_and_sink( self, converter="org.apache.kafka.connect.json.JsonConverter", schemas=True, security_protocol='PLAINTEXT'): """ Validates basic end-to-end functionality of Connect standalone using the file source and sink converters. Includes parameterizations to test different converters (which also test per-connector converter overrides), schema/schemaless modes, and security support. """ assert converter != None, "converter type must be set" # Template parameters. Note that we don't set key/value.converter. These default to JsonConverter and we validate # converter overrides via the connector configuration. if converter != "org.apache.kafka.connect.json.JsonConverter": self.override_key_converter = converter self.override_value_converter = converter self.schemas = schemas self.kafka = KafkaService( self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, interbroker_security_protocol=security_protocol, topics=self.topics) self.source = ConnectStandaloneService( self.test_context, self.kafka, [self.INPUT_FILE, self.OFFSETS_FILE]) self.sink = ConnectStandaloneService( self.test_context, self.kafka, [self.OUTPUT_FILE, self.OFFSETS_FILE]) self.consumer_validator = ConsoleConsumer(self.test_context, 1, self.kafka, self.TOPIC_TEST, consumer_timeout_ms=10000) self.zk.start() self.kafka.start() source_external_props = os.path.join( self.source.PERSISTENT_ROOT, "connect-file-external.properties") self.source.node.account.create_file( source_external_props, self.render('connect-file-external.properties')) self.source.set_configs( lambda node: self.render("connect-standalone.properties", node=node), [self.render("connect-file-source.properties")]) sink_external_props = os.path.join(self.sink.PERSISTENT_ROOT, "connect-file-external.properties") self.sink.node.account.create_file( sink_external_props, self.render('connect-file-external.properties')) self.sink.set_configs( lambda node: self.render("connect-standalone.properties", node=node), [self.render("connect-file-sink.properties")]) self.source.start() self.sink.start() # Generating data on the source node should generate new records and create new output on the sink node self.source.node.account.ssh("echo -e -n " + repr(self.FIRST_INPUT) + " >> " + self.INPUT_FILE) wait_until( lambda: self.validate_output(self.FIRST_INPUT), timeout_sec=60, err_msg= "Data added to input file was not seen in the output file in a reasonable amount of time." ) # Restarting both should result in them picking up where they left off, # only processing new data. self.source.restart() self.sink.restart() self.source.node.account.ssh("echo -e -n " + repr(self.SECOND_INPUT) + " >> " + self.INPUT_FILE) wait_until( lambda: self.validate_output(self.FIRST_INPUT + self.SECOND_INPUT), timeout_sec=60, err_msg= "Sink output file never converged to the same state as the input file" ) # Validate the format of the data in the Kafka topic self.consumer_validator.run() expected = json.dumps([ line if not self.schemas else { "schema": self.SCHEMA, "payload": line } for line in self.FIRST_INPUT_LIST + self.SECOND_INPUT_LIST ]) decoder = (json.loads if converter.endswith("JsonConverter") else str) actual = json.dumps( [decoder(x) for x in self.consumer_validator.messages_consumed[1]]) assert expected == actual, "Expected %s but saw %s in Kafka" % ( expected, actual) def validate_output(self, value): try: output_hash = list( self.sink.node.account.ssh_capture( "md5sum " + self.OUTPUT_FILE))[0].strip().split()[0] return output_hash == hashlib.md5(value).hexdigest() except RemoteCommandError: return False
def create_zookeeper(self, num_nodes=1, **kwargs): self.zk = ZookeeperService(self.test_context, num_nodes=num_nodes, **kwargs)
class LogDirFailureTest(ProduceConsumeValidateTest): """ Note that consuming is a bit tricky, at least with console consumer. The goal is to consume all messages (foreach partition) in the topic. In this case, waiting for the last message may cause the consumer to stop too soon since console consumer is consuming multiple partitions from a single thread and therefore we lose ordering guarantees. Waiting on a count of consumed messages can be unreliable: if we stop consuming when num_consumed == num_acked, we might exit early if some messages are duplicated (though not an issue here since producer retries==0) Therefore rely here on the consumer.timeout.ms setting which times out on the interval between successively consumed messages. Since we run the producer to completion before running the consumer, this is a reliable indicator that nothing is left to consume. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(LogDirFailureTest, self).__init__(test_context=test_context) self.topic1 = "test_topic_1" self.topic2 = "test_topic_2" self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics={ self.topic1: {"partitions": 1, "replication-factor": 3, "configs": {"min.insync.replicas": 1}}, self.topic2: {"partitions": 1, "replication-factor": 3, "configs": {"min.insync.replicas": 2}} }, # Set log.roll.ms to 3 seconds so that broker will detect disk error sooner when it creates log segment # Otherwise broker will still be able to read/write the log file even if the log directory is inaccessible. server_prop_overides=[ [config_property.OFFSETS_TOPIC_NUM_PARTITIONS, "1"], [config_property.LOG_FLUSH_INTERVAL_MESSAGE, "5"], [config_property.REPLICA_HIGHWATERMARK_CHECKPOINT_INTERVAL_MS, "60000"], [config_property.LOG_ROLL_TIME_MS, "3000"], ["confluent.support.metrics.enable", "false"] ]) self.producer_throughput = 1000 self.num_producers = 1 self.num_consumers = 1 def setUp(self): self.zk.start() def min_cluster_size(self): """Override this since we're adding services outside of the constructor""" return super(LogDirFailureTest, self).min_cluster_size() + self.num_producers * 2 + self.num_consumers * 2 @cluster(num_nodes=9) @matrix(bounce_broker=[False, True], broker_type=["leader", "follower"], security_protocol=["PLAINTEXT"]) def test_replication_with_disk_failure(self, bounce_broker, security_protocol, broker_type): """Replication tests. These tests verify that replication provides simple durability guarantees by checking that data acked by brokers is still available for consumption in the face of various failure scenarios. Setup: 1 zk, 3 kafka nodes, 1 topic with partitions=3, replication-factor=3, and min.insync.replicas=2 and another topic with partitions=3, replication-factor=3, and min.insync.replicas=1 - Produce messages in the background - Consume messages in the background - Drive broker failures (shutdown, or bounce repeatedly with kill -15 or kill -9) - When done driving failures, stop producing, and finish consuming - Validate that every acked message was consumed """ self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = security_protocol self.kafka.start() try: # Initialize producer/consumer for topic2 self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic2, throughput=self.producer_throughput) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic2, group_id="test-consumer-group-1", consumer_timeout_ms=60000, message_validator=is_int) self.start_producer_and_consumer() # Get a replica of the partition of topic2 and make its log directory offline by changing the log dir's permission. # We assume that partition of topic2 is created in the second log directory of respective brokers. broker_node = select_node(self, broker_type, self.topic2) broker_idx = self.kafka.idx(broker_node) assert broker_idx in self.kafka.isr_idx_list(self.topic2), \ "Broker %d should be in isr set %s" % (broker_idx, str(self.kafka.isr_idx_list(self.topic2))) # Verify that topic1 and the consumer offset topic is in the first log directory and topic2 is in the second log directory topic_1_partition_0 = KafkaService.DATA_LOG_DIR_1 + "/test_topic_1-0" topic_2_partition_0 = KafkaService.DATA_LOG_DIR_2 + "/test_topic_2-0" offset_topic_partition_0 = KafkaService.DATA_LOG_DIR_1 + "/__consumer_offsets-0" for path in [topic_1_partition_0, topic_2_partition_0, offset_topic_partition_0]: assert path_exists(broker_node, path), "%s should exist" % path self.logger.debug("Making log dir %s inaccessible" % (KafkaService.DATA_LOG_DIR_2)) cmd = "chmod a-w %s -R" % (KafkaService.DATA_LOG_DIR_2) broker_node.account.ssh(cmd, allow_fail=False) if bounce_broker: self.kafka.restart_node(broker_node, clean_shutdown=True) # Verify the following: # 1) The broker with offline log directory is not the leader of the partition of topic2 # 2) The broker with offline log directory is not in the ISR # 3) The broker with offline log directory is still online # 4) Messages can still be produced and consumed from topic2 wait_until(lambda: self.kafka.leader(self.topic2, partition=0) != broker_node, timeout_sec=60, err_msg="Broker %d should not be leader of topic %s and partition 0" % (broker_idx, self.topic2)) assert self.kafka.alive(broker_node), "Broker %d should be still online" % (broker_idx) wait_until(lambda: broker_idx not in self.kafka.isr_idx_list(self.topic2), timeout_sec=60, err_msg="Broker %d should not be in isr set %s" % (broker_idx, str(self.kafka.isr_idx_list(self.topic2)))) self.stop_producer_and_consumer() self.validate() # Shutdown all other brokers so that the broker with offline log dir is the only online broker offline_nodes = [] for node in self.kafka.nodes: if broker_node != node: offline_nodes.append(node) self.logger.debug("Hard shutdown broker %d" % (self.kafka.idx(node))) self.kafka.stop_node(node) # Verify the following: # 1) The broker with offline directory is the only in-sync broker of the partition of topic1 # 2) Messages can still be produced and consumed from topic1 self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic1, throughput=self.producer_throughput, offline_nodes=offline_nodes) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic1, group_id="test-consumer-group-2", consumer_timeout_ms=90000, message_validator=is_int) self.consumer_start_timeout_sec = 90 self.start_producer_and_consumer() assert self.kafka.isr_idx_list(self.topic1) == [broker_idx], \ "In-sync replicas of topic %s and partition 0 should be %s" % (self.topic1, str([broker_idx])) self.stop_producer_and_consumer() self.validate() except BaseException as e: for s in self.test_context.services: self.mark_for_collect(s) raise
def test_upgrade_downgrade_brokers(self, from_version, to_version): """ Start a smoke test client then perform rolling upgrades on the broker. """ if from_version == to_version: return self.replication = 3 self.num_kafka_nodes = 3 self.partitions = 1 self.isr = 2 self.topics = { 'echo': { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': { "min.insync.replicas": self.isr } }, 'data': { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': { "min.insync.replicas": self.isr } }, 'min': { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': { "min.insync.replicas": self.isr } }, 'max': { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': { "min.insync.replicas": self.isr } }, 'sum': { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': { "min.insync.replicas": self.isr } }, 'dif': { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': { "min.insync.replicas": self.isr } }, 'cnt': { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': { "min.insync.replicas": self.isr } }, 'avg': { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': { "min.insync.replicas": self.isr } }, 'wcnt': { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': { "min.insync.replicas": self.isr } }, 'tagg': { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': { "min.insync.replicas": self.isr } } } # Setup phase self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() # number of nodes needs to be >= 3 for the smoke test self.kafka = KafkaService(self.test_context, num_nodes=self.num_kafka_nodes, zk=self.zk, version=KafkaVersion(from_version), topics=self.topics) self.kafka.start() # allow some time for topics to be created wait_until(lambda: self.confirm_topics_on_all_brokers( set(self.topics.keys())), timeout_sec=60, err_msg="Broker did not create all topics in 60 seconds ") self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) processor = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka) with self.driver.node.account.monitor_log( self.driver.STDOUT_FILE) as driver_monitor: self.driver.start() with processor.node.account.monitor_log( processor.STDOUT_FILE) as monitor: processor.start() monitor.wait_until( self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(processor.node)) connected_message = "Discovered group coordinator" with processor.node.account.monitor_log( processor.LOG_FILE) as log_monitor: with processor.node.account.monitor_log( processor.STDOUT_FILE) as stdout_monitor: self.perform_broker_upgrade(to_version) log_monitor.wait_until( connected_message, timeout_sec=120, err_msg=("Never saw output '%s' on " % connected_message) + str(processor.node.account)) stdout_monitor.wait_until( self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on" % self.processed_msg + str(processor.node.account)) # SmokeTestDriver allows up to 6 minutes to consume all # records for the verification step so this timeout is set to # 6 minutes (360 seconds) for consuming of verification records # and a very conservative additional 2 minutes (120 seconds) to process # the records in the verification step driver_monitor.wait_until( 'ALL-RECORDS-DELIVERED\|PROCESSED-MORE-THAN-GENERATED', timeout_sec=480, err_msg="Never saw output '%s' on" % 'ALL-RECORDS-DELIVERED|PROCESSED-MORE-THAN-GENERATED' + str(self.driver.node.account)) self.driver.stop() processor.stop() processor.node.account.ssh_capture("grep SMOKE-TEST-CLIENT-CLOSED %s" % processor.STDOUT_FILE, allow_fail=False)
class Benchmark(Test): """A benchmark of Kafka producer/consumer performance. This replicates the test run here: https://engineering.linkedin.com/kafka/benchmarking-apache-kafka-2-million-writes-second-three-cheap-machines """ def __init__(self, test_context): super(Benchmark, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 3 self.topics = { TOPIC_REP_ONE: { 'partitions': 6, 'replication-factor': 1 }, TOPIC_REP_THREE: { 'partitions': 6, 'replication-factor': 3 } } self.zk = ZookeeperService(test_context, self.num_zk) self.msgs_large = 10000000 self.batch_size = 8 * 1024 self.buffer_memory = 64 * 1024 * 1024 self.msg_sizes = [10, 100, 1000, 10000, 100000] self.target_data_size = 128 * 1024 * 1024 self.target_data_size_gb = self.target_data_size / float( 1024 * 1024 * 1024) def setUp(self): self.zk.start() def start_kafka(self, security_protocol, interbroker_security_protocol, version): self.kafka = KafkaService( self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, interbroker_security_protocol=interbroker_security_protocol, topics=self.topics, version=version) self.kafka.log_level = "INFO" # We don't DEBUG logging here self.kafka.start() @parametrize(acks=1, topic=TOPIC_REP_ONE) @parametrize(acks=1, topic=TOPIC_REP_THREE) @parametrize(acks=-1, topic=TOPIC_REP_THREE) @parametrize(acks=1, topic=TOPIC_REP_THREE, num_producers=3) @matrix(acks=[1], topic=[TOPIC_REP_THREE], message_size=[10, 100, 1000, 10000, 100000], compression_type=["none", "snappy"], security_protocol=['PLAINTEXT', 'SSL']) def test_producer_throughput(self, acks, topic, num_producers=1, message_size=DEFAULT_RECORD_SIZE, compression_type="none", security_protocol='PLAINTEXT', client_version=str(TRUNK), broker_version=str(TRUNK)): """ Setup: 1 node zk + 3 node kafka cluster Produce ~128MB worth of messages to a topic with 6 partitions. Required acks, topic replication factor, security protocol and message size are varied depending on arguments injected into this test. Collect and return aggregate throughput statistics after all messages have been acknowledged. (This runs ProducerPerformance.java under the hood) """ client_version = KafkaVersion(client_version) broker_version = KafkaVersion(broker_version) self.validate_versions(client_version, broker_version) self.start_kafka(security_protocol, security_protocol, broker_version) # Always generate the same total amount of data nrecords = int(self.target_data_size / message_size) self.producer = ProducerPerformanceService(self.test_context, num_producers, self.kafka, topic=topic, num_records=nrecords, record_size=message_size, throughput=-1, version=client_version, settings={ 'acks': acks, 'compression.type': compression_type, 'batch.size': self.batch_size, 'buffer.memory': self.buffer_memory }) self.producer.run() return compute_aggregate_throughput(self.producer) @parametrize(security_protocol='SSL', interbroker_security_protocol='PLAINTEXT') @matrix(security_protocol=['PLAINTEXT', 'SSL'], compression_type=["none", "snappy"]) def test_long_term_producer_throughput(self, compression_type="none", security_protocol='PLAINTEXT', interbroker_security_protocol=None, client_version=str(TRUNK), broker_version=str(TRUNK)): """ Setup: 1 node zk + 3 node kafka cluster Produce 10e6 100 byte messages to a topic with 6 partitions, replication-factor 3, and acks=1. Collect and return aggregate throughput statistics after all messages have been acknowledged. (This runs ProducerPerformance.java under the hood) """ client_version = KafkaVersion(client_version) broker_version = KafkaVersion(broker_version) self.validate_versions(client_version, broker_version) if interbroker_security_protocol is None: interbroker_security_protocol = security_protocol self.start_kafka(security_protocol, interbroker_security_protocol, broker_version) self.producer = ProducerPerformanceService( self.test_context, 1, self.kafka, topic=TOPIC_REP_THREE, num_records=self.msgs_large, record_size=DEFAULT_RECORD_SIZE, throughput=-1, version=client_version, settings={ 'acks': 1, 'compression.type': compression_type, 'batch.size': self.batch_size, 'buffer.memory': self.buffer_memory }, intermediate_stats=True) self.producer.run() summary = ["Throughput over long run, data > memory:"] data = {} # FIXME we should be generating a graph too # Try to break it into 5 blocks, but fall back to a smaller number if # there aren't even 5 elements block_size = max(len(self.producer.stats[0]) / 5, 1) nblocks = len(self.producer.stats[0]) / block_size for i in range(nblocks): subset = self.producer.stats[0][i * block_size:min( (i + 1) * block_size, len(self.producer.stats[0]))] if len(subset) == 0: summary.append(" Time block %d: (empty)" % i) data[i] = None else: records_per_sec = sum( [stat['records_per_sec'] for stat in subset]) / float(len(subset)) mb_per_sec = sum([stat['mbps'] for stat in subset]) / float(len(subset)) summary.append(" Time block %d: %f rec/sec (%f MB/s)" % (i, records_per_sec, mb_per_sec)) data[i] = throughput(records_per_sec, mb_per_sec) self.logger.info("\n".join(summary)) return data @parametrize(security_protocol='SSL', interbroker_security_protocol='PLAINTEXT') @matrix( security_protocol=['PLAINTEXT', 'SSL', 'SASL_PLAINTEXT', 'SASL_SSL'], compression_type=["none", "snappy"]) def test_end_to_end_latency(self, compression_type="none", security_protocol="PLAINTEXT", interbroker_security_protocol=None, client_version=str(TRUNK), broker_version=str(TRUNK)): """ Setup: 1 node zk + 3 node kafka cluster Produce (acks = 1) and consume 10e3 messages to a topic with 6 partitions and replication-factor 3, measuring the latency between production and consumption of each message. Return aggregate latency statistics. (Under the hood, this simply runs EndToEndLatency.scala) """ client_version = KafkaVersion(client_version) broker_version = KafkaVersion(broker_version) self.validate_versions(client_version, broker_version) if interbroker_security_protocol is None: interbroker_security_protocol = security_protocol self.start_kafka(security_protocol, interbroker_security_protocol, broker_version) self.logger.info("BENCHMARK: End to end latency") self.perf = EndToEndLatencyService(self.test_context, 1, self.kafka, topic=TOPIC_REP_THREE, num_records=10000, compression_type=compression_type, version=client_version) self.perf.run() return latency(self.perf.results[0]['latency_50th_ms'], self.perf.results[0]['latency_99th_ms'], self.perf.results[0]['latency_999th_ms']) @parametrize(security_protocol='PLAINTEXT', new_consumer=False) @parametrize(security_protocol='SSL', interbroker_security_protocol='PLAINTEXT') @matrix(security_protocol=['PLAINTEXT', 'SSL'], compression_type=["none", "snappy"]) def test_producer_and_consumer(self, compression_type="none", security_protocol="PLAINTEXT", interbroker_security_protocol=None, new_consumer=True, client_version=str(TRUNK), broker_version=str(TRUNK)): """ Setup: 1 node zk + 3 node kafka cluster Concurrently produce and consume 10e6 messages with a single producer and a single consumer, using new consumer if new_consumer == True Return aggregate throughput statistics for both producer and consumer. (Under the hood, this runs ProducerPerformance.java, and ConsumerPerformance.scala) """ client_version = KafkaVersion(client_version) broker_version = KafkaVersion(broker_version) self.validate_versions(client_version, broker_version) if interbroker_security_protocol is None: interbroker_security_protocol = security_protocol self.start_kafka(security_protocol, interbroker_security_protocol, broker_version) num_records = 10 * 1000 * 1000 # 10e6 self.producer = ProducerPerformanceService( self.test_context, 1, self.kafka, topic=TOPIC_REP_THREE, num_records=num_records, record_size=DEFAULT_RECORD_SIZE, throughput=-1, version=client_version, settings={ 'acks': 1, 'compression.type': compression_type, 'batch.size': self.batch_size, 'buffer.memory': self.buffer_memory }) self.consumer = ConsumerPerformanceService(self.test_context, 1, self.kafka, topic=TOPIC_REP_THREE, new_consumer=new_consumer, messages=num_records) Service.run_parallel(self.producer, self.consumer) data = { "producer": compute_aggregate_throughput(self.producer), "consumer": compute_aggregate_throughput(self.consumer) } summary = ["Producer + consumer:", str(data)] self.logger.info("\n".join(summary)) return data @parametrize(security_protocol='PLAINTEXT', new_consumer=False) @parametrize(security_protocol='SSL', interbroker_security_protocol='PLAINTEXT') @matrix(security_protocol=['PLAINTEXT', 'SSL'], compression_type=["none", "snappy"]) def test_consumer_throughput(self, compression_type="none", security_protocol="PLAINTEXT", interbroker_security_protocol=None, new_consumer=True, num_consumers=1, client_version=str(TRUNK), broker_version=str(TRUNK)): """ Consume 10e6 100-byte messages with 1 or more consumers from a topic with 6 partitions (using new consumer iff new_consumer == True), and report throughput. """ client_version = KafkaVersion(client_version) broker_version = KafkaVersion(broker_version) self.validate_versions(client_version, broker_version) if interbroker_security_protocol is None: interbroker_security_protocol = security_protocol self.start_kafka(security_protocol, interbroker_security_protocol, broker_version) num_records = 10 * 1000 * 1000 # 10e6 # seed kafka w/messages self.producer = ProducerPerformanceService( self.test_context, 1, self.kafka, topic=TOPIC_REP_THREE, num_records=num_records, record_size=DEFAULT_RECORD_SIZE, throughput=-1, version=client_version, settings={ 'acks': 1, 'compression.type': compression_type, 'batch.size': self.batch_size, 'buffer.memory': self.buffer_memory }) self.producer.run() # consume self.consumer = ConsumerPerformanceService(self.test_context, num_consumers, self.kafka, topic=TOPIC_REP_THREE, new_consumer=new_consumer, messages=num_records) self.consumer.group = "test-consumer-group" self.consumer.run() return compute_aggregate_throughput(self.consumer) def validate_versions(self, client_version, broker_version): assert client_version <= broker_version, "Client version %s should be <= than broker version %s" ( client_version, broker_version)
class GroupModeTransactionsTest(Test): """Essentially testing the same functionality as TransactionsTest by transactionally copying data from a source topic to a destination topic and killing the copy process as well as the broker randomly through the process. The major difference is that we choose to work as a collaborated group with same topic subscription instead of individual copiers. In the end we verify that the final output topic contains exactly one committed copy of each message from the original producer. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(GroupModeTransactionsTest, self).__init__(test_context=test_context) self.input_topic = "input-topic" self.output_topic = "output-topic" self.num_brokers = 3 # Test parameters self.num_input_partitions = 9 self.num_output_partitions = 9 self.num_copiers = 3 self.num_seed_messages = 100000 self.transaction_size = 750 # The transaction timeout should be lower than the progress timeout, but at # least as high as the request timeout (which is 30s by default). When the # client is hard-bounced, progress may depend on the previous transaction # being aborted. When the broker is hard-bounced, we may have to wait as # long as the request timeout to get a `Produce` response and we do not # want the coordinator timing out the transaction. self.transaction_timeout = 40000 self.progress_timeout_sec = 60 self.consumer_group = "grouped-transactions-test-consumer-group" self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test( test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=self.num_brokers, zk=self.zk, controller_num_nodes_override=1) def setUp(self): if self.zk: self.zk.start() def seed_messages(self, topic, num_seed_messages): seed_timeout_sec = 10000 seed_producer = VerifiableProducer( context=self.test_context, num_nodes=1, kafka=self.kafka, topic=topic, message_validator=is_int, max_messages=num_seed_messages, enable_idempotence=True, repeating_keys=self.num_input_partitions) seed_producer.start() wait_until(lambda: seed_producer.num_acked >= num_seed_messages, timeout_sec=seed_timeout_sec, err_msg="Producer failed to produce messages %d in %ds." % \ (self.num_seed_messages, seed_timeout_sec)) return seed_producer.acked_by_partition def get_messages_from_topic(self, topic, num_messages): consumer = self.start_consumer(topic, group_id="verifying_consumer") return self.drain_consumer(consumer, num_messages) def bounce_brokers(self, clean_shutdown): for node in self.kafka.nodes: if clean_shutdown: self.kafka.restart_node(node, clean_shutdown=True) else: self.kafka.stop_node(node, clean_shutdown=False) gracePeriodSecs = 5 if self.zk: wait_until( lambda: len(self.kafka.pids( node)) == 0 and not self.kafka.is_registered(node), timeout_sec=self.kafka.zk_session_timeout + gracePeriodSecs, err_msg= "Failed to see timely deregistration of hard-killed broker %s" % str(node.account)) else: brokerSessionTimeoutSecs = 18 wait_until( lambda: len(self.kafka.pids(node)) == 0, timeout_sec=brokerSessionTimeoutSecs + gracePeriodSecs, err_msg= "Failed to see timely disappearance of process for hard-killed broker %s" % str(node.account)) time.sleep(brokerSessionTimeoutSecs + gracePeriodSecs) self.kafka.start_node(node) def create_and_start_message_copier(self, input_topic, output_topic, transactional_id): message_copier = TransactionalMessageCopier( context=self.test_context, num_nodes=1, kafka=self.kafka, transactional_id=transactional_id, consumer_group=self.consumer_group, input_topic=input_topic, input_partition=-1, output_topic=output_topic, max_messages=-1, transaction_size=self.transaction_size, transaction_timeout=self.transaction_timeout, use_group_metadata=True, group_mode=True) message_copier.start() wait_until(lambda: message_copier.alive(message_copier.nodes[0]), timeout_sec=10, err_msg="Message copier failed to start after 10 s") return message_copier def bounce_copiers(self, copiers, clean_shutdown, timeout_sec=240): for _ in range(3): for copier in copiers: wait_until(lambda: copier.progress_percent() >= 20.0, timeout_sec=self.progress_timeout_sec, err_msg="%s : Message copier didn't make enough progress in %ds. Current progress: %s" \ % (copier.transactional_id, self.progress_timeout_sec, str(copier.progress_percent()))) self.logger.info( "%s - progress: %s" % (copier.transactional_id, str(copier.progress_percent()))) copier.restart(clean_shutdown) def create_and_start_copiers(self, input_topic, output_topic, num_copiers): copiers = [] for i in range(0, num_copiers): copiers.append( self.create_and_start_message_copier( input_topic=input_topic, output_topic=output_topic, transactional_id="copier-" + str(i))) return copiers @staticmethod def valid_value_and_partition(msg): """Method used to check whether the given message is a valid tab separated value + partition return value and partition as a size-two array represented tuple: [value, partition] """ try: splitted_msg = msg.split('\t') value = int(splitted_msg[1]) partition = int(splitted_msg[0].split(":")[1]) return [value, partition] except ValueError: raise Exception( "Unexpected message format (expected a tab separated [value, partition] tuple). Message: %s" % (msg)) def start_consumer(self, topic_to_read, group_id): consumer = ConsoleConsumer( context=self.test_context, num_nodes=1, kafka=self.kafka, topic=topic_to_read, group_id=group_id, message_validator=self.valid_value_and_partition, from_beginning=True, print_partition=True, isolation_level="read_committed") consumer.start() # ensure that the consumer is up. wait_until(lambda: (len(consumer.messages_consumed[1]) > 0) == True, timeout_sec=60, err_msg="Consumer failed to consume any messages for %ds" % \ 60) return consumer @staticmethod def split_by_partition(messages_consumed): messages_by_partition = {} for msg in messages_consumed: partition = msg[1] if partition not in messages_by_partition: messages_by_partition[partition] = [] messages_by_partition[partition].append(msg[0]) return messages_by_partition def drain_consumer(self, consumer, num_messages): # wait until we read at least the expected number of messages. # This is a safe check because both failure modes will be caught: # 1. If we have 'num_seed_messages' but there are duplicates, then # this is checked for later. # # 2. If we never reach 'num_seed_messages', then this will cause the # test to fail. wait_until(lambda: len(consumer.messages_consumed[1]) >= num_messages, timeout_sec=90, err_msg="Consumer consumed only %d out of %d messages in %ds" % \ (len(consumer.messages_consumed[1]), num_messages, 90)) consumer.stop() return self.split_by_partition(consumer.messages_consumed[1]) def copy_messages_transactionally(self, failure_mode, bounce_target, input_topic, output_topic, num_copiers, num_messages_to_copy): """Copies messages transactionally from the seeded input topic to the output topic, either bouncing brokers or clients in a hard and soft way as it goes. This method also consumes messages in read_committed mode from the output topic while the bounces and copy is going on. It returns the concurrently consumed messages. """ copiers = self.create_and_start_copiers(input_topic=input_topic, output_topic=output_topic, num_copiers=num_copiers) concurrent_consumer = self.start_consumer( output_topic, group_id="concurrent_consumer") clean_shutdown = False if failure_mode == "clean_bounce": clean_shutdown = True if bounce_target == "brokers": self.bounce_brokers(clean_shutdown) elif bounce_target == "clients": self.bounce_copiers(copiers, clean_shutdown) copier_timeout_sec = 240 for copier in copiers: wait_until(lambda: copier.is_done, timeout_sec=copier_timeout_sec, err_msg="%s - Failed to copy all messages in %ds." % \ (copier.transactional_id, copier_timeout_sec)) self.logger.info("finished copying messages") return self.drain_consumer(concurrent_consumer, num_messages_to_copy) def setup_topics(self): self.kafka.topics = { self.input_topic: { "partitions": self.num_input_partitions, "replication-factor": 3, "configs": { "min.insync.replicas": 2 } }, self.output_topic: { "partitions": self.num_output_partitions, "replication-factor": 3, "configs": { "min.insync.replicas": 2 } } } @cluster(num_nodes=10) @matrix(failure_mode=["hard_bounce", "clean_bounce"], bounce_target=["brokers", "clients"], metadata_quorum=quorum.all_non_upgrade) def test_transactions(self, failure_mode, bounce_target, metadata_quorum=quorum.zk): security_protocol = 'PLAINTEXT' self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = security_protocol self.kafka.logs["kafka_data_1"]["collect_default"] = True self.kafka.logs["kafka_data_2"]["collect_default"] = True self.kafka.logs["kafka_operational_logs_debug"][ "collect_default"] = True self.setup_topics() self.kafka.start() input_messages_by_partition = self.seed_messages( self.input_topic, self.num_seed_messages) concurrently_consumed_message_by_partition = self.copy_messages_transactionally( failure_mode, bounce_target, input_topic=self.input_topic, output_topic=self.output_topic, num_copiers=self.num_copiers, num_messages_to_copy=self.num_seed_messages) output_messages_by_partition = self.get_messages_from_topic( self.output_topic, self.num_seed_messages) assert len(input_messages_by_partition) == \ len(concurrently_consumed_message_by_partition), "The lengths of partition count doesn't match: " \ "input partitions count %d, " \ "concurrently consumed partitions count %d" % \ (len(input_messages_by_partition), len(concurrently_consumed_message_by_partition)) assert len(input_messages_by_partition) == \ len(output_messages_by_partition), "The lengths of partition count doesn't match: " \ "input partitions count %d, " \ "output partitions count %d" % \ (len(input_messages_by_partition), len(concurrently_consumed_message_by_partition)) for p in range(self.num_input_partitions): if p not in input_messages_by_partition: continue assert p in output_messages_by_partition, "Partition %d not in output messages" assert p in concurrently_consumed_message_by_partition, "Partition %d not in concurrently consumed messages" output_message_set = set(output_messages_by_partition[p]) input_message_set = set(input_messages_by_partition[p]) concurrently_consumed_message_set = set( concurrently_consumed_message_by_partition[p]) num_dups = abs(len(output_messages) - len(output_message_set)) num_dups_in_concurrent_consumer = abs( len(concurrently_consumed_messages) - len(concurrently_consumed_message_set)) assert num_dups == 0, "Detected %d duplicates in the output stream" % num_dups assert input_message_set == output_message_set, "Input and output message sets are not equal. Num input messages %d. Num output messages %d" % \ (len(input_message_set), len(output_message_set)) assert num_dups_in_concurrent_consumer == 0, "Detected %d dups in concurrently consumed messages" % num_dups_in_concurrent_consumer assert input_message_set == concurrently_consumed_message_set, \ "Input and concurrently consumed output message sets are not equal. Num input messages: %d. Num concurrently_consumed_messages: %d" % \ (len(input_message_set), len(concurrently_consumed_message_set)) assert input_messages == sorted( input_messages ), "The seed messages themselves were not in order" assert output_messages == input_messages, "Output messages are not in order" assert concurrently_consumed_messages == output_messages, "Concurrently consumed messages are not in order"
class StreamsBrokerBounceTest(Test): """ Simple test of Kafka Streams with brokers failing """ def __init__(self, test_context): super(StreamsBrokerBounceTest, self).__init__(test_context) self.replication = 3 self.partitions = 3 self.topics = { 'echo' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": 2}}, 'data' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": 2} }, 'min' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": 2} }, 'max' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": 2} }, 'sum' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": 2} }, 'dif' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": 2} }, 'cnt' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": 2} }, 'avg' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": 2} }, 'wcnt' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": 2} }, 'tagg' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": 2} }, '__consumer_offsets' : { 'partitions': 50, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": 2} } } def fail_broker_type(self, failure_mode, broker_type): # Pick a random topic and bounce it's leader topic_index = randint(0, len(self.topics.keys()) - 1) topic = self.topics.keys()[topic_index] failures[failure_mode](self, topic, broker_type) def fail_many_brokers(self, failure_mode, num_failures): sig = signal.SIGTERM if (failure_mode == "clean_shutdown"): sig = signal.SIGTERM else: sig = signal.SIGKILL for num in range(0, num_failures - 1): signal_node(self, self.kafka.nodes[num], sig) def confirm_topics_on_all_brokers(self, expected_topic_set): for node in self.kafka.nodes: match_count = 0 # need to iterate over topic_list_generator as kafka.list_topics() # returns a python generator so values are fetched lazily # so we can't just compare directly we must iterate over what's returned topic_list_generator = self.kafka.list_topics("placeholder", node) for topic in topic_list_generator: if topic in expected_topic_set: match_count += 1 if len(expected_topic_set) != match_count: return False return True def setup_system(self, start_processor=True): # Setup phase self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() self.kafka = KafkaService(self.test_context, num_nodes=self.replication, zk=self.zk, topics=self.topics) self.kafka.start() # allow some time for topics to be created wait_until(lambda: self.confirm_topics_on_all_brokers(set(self.topics.keys())), timeout_sec=60, err_msg="Broker did not create all topics in 60 seconds ") # Start test harness self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) self.processor1 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka) self.driver.start() if (start_processor): self.processor1.start() def collect_results(self, sleep_time_secs): data = {} # End test self.driver.wait() self.driver.stop() self.processor1.stop() node = self.driver.node # Success is declared if streams does not crash when sleep time > 0 # It should give an exception when sleep time is 0 since we kill the brokers immediately # and the topic manager cannot create internal topics with the desired replication factor if (sleep_time_secs == 0): output_streams = self.processor1.node.account.ssh_capture("grep SMOKE-TEST-CLIENT-EXCEPTION %s" % self.processor1.STDOUT_FILE, allow_fail=False) else: output_streams = self.processor1.node.account.ssh_capture("grep SMOKE-TEST-CLIENT-CLOSED %s" % self.processor1.STDOUT_FILE, allow_fail=False) for line in output_streams: data["Client closed"] = line # Currently it is hard to guarantee anything about Kafka since we don't have exactly once. # With exactly once in place, success will be defined as ALL-RECORDS-DELIEVERD and SUCCESS output = node.account.ssh_capture("grep -E 'ALL-RECORDS-DELIVERED|PROCESSED-MORE-THAN-GENERATED|PROCESSED-LESS-THAN-GENERATED' %s" % self.driver.STDOUT_FILE, allow_fail=False) for line in output: data["Records Delivered"] = line output = node.account.ssh_capture("grep -E 'SUCCESS|FAILURE' %s" % self.driver.STDOUT_FILE, allow_fail=False) for line in output: data["Logic Success/Failure"] = line return data @cluster(num_nodes=7) @matrix(failure_mode=["clean_shutdown", "hard_shutdown", "clean_bounce", "hard_bounce"], broker_type=["leader", "controller"], sleep_time_secs=[120]) def test_broker_type_bounce(self, failure_mode, broker_type, sleep_time_secs): """ Start a smoke test client, then kill one particular broker and ensure data is still received Record if records are delivered. """ self.setup_system() # Sleep to allow test to run for a bit time.sleep(sleep_time_secs) # Fail brokers self.fail_broker_type(failure_mode, broker_type) return self.collect_results(sleep_time_secs) @ignore @cluster(num_nodes=7) @matrix(failure_mode=["clean_shutdown"], broker_type=["controller"], sleep_time_secs=[0]) def test_broker_type_bounce_at_start(self, failure_mode, broker_type, sleep_time_secs): """ Start a smoke test client, then kill one particular broker immediately before streams stats Streams should throw an exception since it cannot create topics with the desired replication factor of 3 """ self.setup_system(start_processor=False) # Sleep to allow test to run for a bit time.sleep(sleep_time_secs) # Fail brokers self.fail_broker_type(failure_mode, broker_type) self.processor1.start() return self.collect_results(sleep_time_secs) @cluster(num_nodes=7) @matrix(failure_mode=["clean_shutdown", "hard_shutdown", "clean_bounce", "hard_bounce"], num_failures=[2]) def test_many_brokers_bounce(self, failure_mode, num_failures): """ Start a smoke test client, then kill a few brokers and ensure data is still received Record if records are delivered """ self.setup_system() # Sleep to allow test to run for a bit time.sleep(120) # Fail brokers self.fail_many_brokers(failure_mode, num_failures) return self.collect_results(120) @cluster(num_nodes=7) @matrix(failure_mode=["clean_bounce", "hard_bounce"], num_failures=[3]) def test_all_brokers_bounce(self, failure_mode, num_failures): """ Start a smoke test client, then kill a few brokers and ensure data is still received Record if records are delivered """ self.setup_system() # Sleep to allow test to run for a bit time.sleep(120) # Fail brokers self.fail_many_brokers(failure_mode, num_failures) return self.collect_results(120)
class ClientCompatibilityTestNewBroker(ProduceConsumeValidateTest): def __init__(self, test_context): super(ClientCompatibilityTestNewBroker, self).__init__(test_context=test_context) def setUp(self): self.topic = "test_topic" self.zk = ZookeeperService( self.test_context, num_nodes=1) if quorum.for_test( self.test_context) == quorum.zk else None if self.zk: self.zk.start() # Producer and consumer self.producer_throughput = 10000 self.num_producers = 1 self.num_consumers = 1 self.messages_per_producer = 1000 @cluster(num_nodes=6) @matrix(producer_version=[str(DEV_BRANCH)], consumer_version=[str(DEV_BRANCH)], compression_types=[["snappy"]], timestamp_type=[str("LogAppendTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(DEV_BRANCH)], consumer_version=[str(DEV_BRANCH)], compression_types=[["none"]], timestamp_type=[str("LogAppendTime")], metadata_quorum=quorum.all_non_upgrade) @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(LATEST_0_9), compression_types=["none"], new_consumer=False, timestamp_type=None) @matrix(producer_version=[str(DEV_BRANCH)], consumer_version=[str(LATEST_0_9)], compression_types=[["snappy"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_2_2)], consumer_version=[str(LATEST_2_2)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_2_3)], consumer_version=[str(LATEST_2_3)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_2_4)], consumer_version=[str(LATEST_2_4)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_2_5)], consumer_version=[str(LATEST_2_5)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_2_6)], consumer_version=[str(LATEST_2_6)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_2_7)], consumer_version=[str(LATEST_2_7)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_2_8)], consumer_version=[str(LATEST_2_8)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_2_1)], consumer_version=[str(LATEST_2_1)], compression_types=[["zstd"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_2_0)], consumer_version=[str(LATEST_2_0)], compression_types=[["snappy"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_1_1)], consumer_version=[str(LATEST_1_1)], compression_types=[["lz4"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_1_0)], consumer_version=[str(LATEST_1_0)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_0_11_0)], consumer_version=[str(LATEST_0_11_0)], compression_types=[["gzip"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_0_10_2)], consumer_version=[str(LATEST_0_10_2)], compression_types=[["lz4"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_0_10_1)], consumer_version=[str(LATEST_0_10_1)], compression_types=[["snappy"]], timestamp_type=[str("LogAppendTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_0_10_0)], consumer_version=[str(LATEST_0_10_0)], compression_types=[["snappy"]], timestamp_type=[str("LogAppendTime")], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_0_9)], consumer_version=[str(DEV_BRANCH)], compression_types=[["none"]], timestamp_type=[None], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_0_9)], consumer_version=[str(DEV_BRANCH)], compression_types=[["snappy"]], timestamp_type=[None], metadata_quorum=quorum.all_non_upgrade) @matrix(producer_version=[str(LATEST_0_9)], consumer_version=[str(LATEST_0_9)], compression_types=[["snappy"]], timestamp_type=[str("LogAppendTime")], metadata_quorum=quorum.all_non_upgrade) @parametrize(producer_version=str(LATEST_0_8_2), consumer_version=str(LATEST_0_8_2), compression_types=["none"], new_consumer=False, timestamp_type=None) def test_compatibility(self, producer_version, consumer_version, compression_types, new_consumer=True, timestamp_type=None, metadata_quorum=quorum.zk): if not new_consumer and metadata_quorum != quorum.zk: raise Exception( "ZooKeeper-based consumers are not supported when using a KRaft metadata quorum" ) self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, version=DEV_BRANCH, topics={ self.topic: { "partitions": 3, "replication-factor": 3, 'configs': { "min.insync.replicas": 2 } } }, controller_num_nodes_override=1) for node in self.kafka.nodes: if timestamp_type is not None: node.config[ config_property.MESSAGE_TIMESTAMP_TYPE] = timestamp_type self.kafka.start() self.producer = VerifiableProducer( self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, message_validator=is_int, compression_types=compression_types, version=KafkaVersion(producer_version)) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=30000, new_consumer=new_consumer, message_validator=is_int, version=KafkaVersion(consumer_version)) self.run_produce_consume_validate(lambda: wait_until( lambda: self.producer.each_produced_at_least( self.messages_per_producer) == True, timeout_sec=120, backoff_sec=1, err_msg= "Producer did not produce all messages in reasonable amount of time" ))
class ClientCompatibilityProduceConsumeTest(ProduceConsumeValidateTest): """ These tests validate that we can use a new client to produce and consume from older brokers. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ClientCompatibilityProduceConsumeTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.zk = ZookeeperService(test_context, num_nodes=3) if quorum.for_test( test_context) == quorum.zk else None self.kafka = KafkaService( test_context, num_nodes=3, zk=self.zk, topics={self.topic: { "partitions": 10, "replication-factor": 2 }}) self.num_partitions = 10 self.timeout_sec = 60 self.producer_throughput = 1000 self.num_producers = 2 self.messages_per_producer = 1000 self.num_consumers = 1 def setUp(self): if self.zk: self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super( ClientCompatibilityProduceConsumeTest, self).min_cluster_size() + self.num_producers + self.num_consumers @cluster(num_nodes=9) @matrix(broker_version=[str(DEV_BRANCH)], metadata_quorum=quorum.all_non_upgrade) @parametrize(broker_version=str(LATEST_0_10_0)) @parametrize(broker_version=str(LATEST_0_10_1)) @parametrize(broker_version=str(LATEST_0_10_2)) @parametrize(broker_version=str(LATEST_0_11_0)) @parametrize(broker_version=str(LATEST_1_0)) @parametrize(broker_version=str(LATEST_1_1)) @parametrize(broker_version=str(LATEST_2_0)) @parametrize(broker_version=str(LATEST_2_1)) @parametrize(broker_version=str(LATEST_2_2)) @parametrize(broker_version=str(LATEST_2_3)) @parametrize(broker_version=str(LATEST_2_4)) @parametrize(broker_version=str(LATEST_2_5)) @parametrize(broker_version=str(LATEST_2_6)) @parametrize(broker_version=str(LATEST_2_7)) @parametrize(broker_version=str(LATEST_2_8)) @parametrize(broker_version=str(LATEST_3_0)) @parametrize(broker_version=str(LATEST_3_1)) @parametrize(broker_version=str(LATEST_3_2)) def test_produce_consume(self, broker_version, metadata_quorum=quorum.zk): print("running producer_consumer_compat with broker_version = %s" % broker_version, flush=True) self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.security_protocol = "PLAINTEXT" self.kafka.interbroker_security_protocol = self.kafka.security_protocol self.producer = VerifiableProducer( self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, message_validator=is_int_with_prefix) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=60000, message_validator=is_int_with_prefix) self.kafka.start() self.run_produce_consume_validate(lambda: wait_until( lambda: self.producer.each_produced_at_least( self.messages_per_producer) == True, timeout_sec=120, backoff_sec=1, err_msg= "Producer did not produce all messages in reasonable amount of time" ))
class TestSecurityRollingUpgrade(ProduceConsumeValidateTest): """Tests a rolling upgrade from PLAINTEXT to a secured cluster """ def __init__(self, test_context): super(TestSecurityRollingUpgrade, self).__init__(test_context=test_context) def setUp(self): self.acls = ACLs(self.test_context) self.topic = "test_topic" self.group = "group" self.producer_throughput = 100 self.num_producers = 1 self.num_consumers = 1 self.zk = ZookeeperService(self.test_context, num_nodes=1) self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, topics={ self.topic: { "partitions": 3, "replication-factor": 3, 'configs': { "min.insync.replicas": 2 } } }) self.zk.start() def create_producer_and_consumer(self): self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=60000, message_validator=is_int) self.consumer.group_id = "group" def bounce(self): self.kafka.start_minikdc() for node in self.kafka.nodes: self.kafka.stop_node(node) self.kafka.start_node(node) time.sleep(10) def roll_in_secured_settings(self, client_protocol, broker_protocol): # Roll cluster to include inter broker security protocol. self.kafka.setup_interbroker_listener(broker_protocol) self.bounce() # Roll cluster to disable PLAINTEXT port self.kafka.close_port(SecurityConfig.PLAINTEXT) self.set_authorizer_and_bounce(client_protocol, broker_protocol) def set_authorizer_and_bounce(self, client_protocol, broker_protocol): self.kafka.authorizer_class_name = KafkaService.SIMPLE_AUTHORIZER self.acls.set_acls(client_protocol, self.kafka, self.topic, self.group) self.acls.set_acls(broker_protocol, self.kafka, self.topic, self.group) self.bounce() def open_secured_port(self, client_protocol): self.kafka.security_protocol = client_protocol self.kafka.open_port(client_protocol) self.kafka.start_minikdc() self.bounce() def add_sasl_mechanism(self, new_client_sasl_mechanism): self.kafka.client_sasl_mechanism = new_client_sasl_mechanism self.kafka.start_minikdc() self.bounce() def roll_in_sasl_mechanism(self, security_protocol, new_sasl_mechanism): # Roll cluster to update inter-broker SASL mechanism. This disables the old mechanism. self.kafka.interbroker_sasl_mechanism = new_sasl_mechanism self.bounce() # Bounce again with ACLs for new mechanism self.set_authorizer_and_bounce(security_protocol, security_protocol) def add_separate_broker_listener(self, broker_security_protocol, broker_sasl_mechanism): self.kafka.setup_interbroker_listener(broker_security_protocol, True) self.kafka.interbroker_sasl_mechanism = broker_sasl_mechanism # kafka opens interbroker port automatically in start() but not in bounce() self.kafka.open_port(self.kafka.INTERBROKER_LISTENER_NAME) self.bounce() def remove_separate_broker_listener(self, client_security_protocol, client_sasl_mechanism): # separate interbroker listener port will be closed automatically in setup_interbroker_listener # if not using separate interbroker listener self.kafka.setup_interbroker_listener(client_security_protocol, False) self.kafka.interbroker_sasl_mechanism = client_sasl_mechanism self.bounce() @cluster(num_nodes=8) @matrix(client_protocol=[SecurityConfig.SSL]) @cluster(num_nodes=9) @matrix(client_protocol=[ SecurityConfig.SASL_PLAINTEXT, SecurityConfig.SASL_SSL ]) def test_rolling_upgrade_phase_one(self, client_protocol): """ Start with a PLAINTEXT cluster, open a SECURED port, via a rolling upgrade, ensuring we could produce and consume throughout over PLAINTEXT. Finally check we can produce and consume the new secured port. """ self.kafka.setup_interbroker_listener(SecurityConfig.PLAINTEXT) self.kafka.security_protocol = SecurityConfig.PLAINTEXT self.kafka.start() # Create PLAINTEXT producer and consumer self.create_producer_and_consumer() # Rolling upgrade, opening a secure protocol, ensuring the Plaintext producer/consumer continues to run self.run_produce_consume_validate(self.open_secured_port, client_protocol) # Now we can produce and consume via the secured port self.kafka.security_protocol = client_protocol self.create_producer_and_consumer() self.run_produce_consume_validate(lambda: time.sleep(1)) @cluster(num_nodes=8) @matrix(client_protocol=[ SecurityConfig.SASL_SSL, SecurityConfig.SSL, SecurityConfig.SASL_PLAINTEXT ], broker_protocol=[ SecurityConfig.SASL_SSL, SecurityConfig.SSL, SecurityConfig.SASL_PLAINTEXT ]) def test_rolling_upgrade_phase_two(self, client_protocol, broker_protocol): """ Start with a PLAINTEXT cluster with a second Secured port open (i.e. result of phase one). A third secure port is also open if inter-broker and client protocols are different. Start a Producer and Consumer via the SECURED client port Incrementally upgrade to add inter-broker be the secure broker protocol Incrementally upgrade again to add ACLs as well as disabling the PLAINTEXT port Ensure the producer and consumer ran throughout """ #Given we have a broker that has both secure and PLAINTEXT ports open self.kafka.security_protocol = client_protocol self.kafka.setup_interbroker_listener(SecurityConfig.PLAINTEXT, use_separate_listener=False) self.kafka.open_port(broker_protocol) self.kafka.start() #Create Secured Producer and Consumer self.create_producer_and_consumer() #Roll in the security protocol. Disable Plaintext. Ensure we can produce and Consume throughout self.run_produce_consume_validate(self.roll_in_secured_settings, client_protocol, broker_protocol) @cluster(num_nodes=9) @matrix(new_client_sasl_mechanism=[SecurityConfig.SASL_MECHANISM_PLAIN]) def test_rolling_upgrade_sasl_mechanism_phase_one( self, new_client_sasl_mechanism): """ Start with a SASL/GSSAPI cluster, add new SASL mechanism, via a rolling upgrade, ensuring we could produce and consume throughout over SASL/GSSAPI. Finally check we can produce and consume using new mechanism. """ self.kafka.setup_interbroker_listener(SecurityConfig.SASL_SSL, use_separate_listener=False) self.kafka.security_protocol = SecurityConfig.SASL_SSL self.kafka.client_sasl_mechanism = SecurityConfig.SASL_MECHANISM_GSSAPI self.kafka.interbroker_sasl_mechanism = SecurityConfig.SASL_MECHANISM_GSSAPI self.kafka.start() # Create SASL/GSSAPI producer and consumer self.create_producer_and_consumer() # Rolling upgrade, adding new SASL mechanism, ensuring the GSSAPI producer/consumer continues to run self.run_produce_consume_validate(self.add_sasl_mechanism, new_client_sasl_mechanism) # Now we can produce and consume using the new SASL mechanism self.kafka.client_sasl_mechanism = new_client_sasl_mechanism self.create_producer_and_consumer() self.run_produce_consume_validate(lambda: time.sleep(1)) @cluster(num_nodes=8) @matrix(new_sasl_mechanism=[SecurityConfig.SASL_MECHANISM_PLAIN]) def test_rolling_upgrade_sasl_mechanism_phase_two(self, new_sasl_mechanism): """ Start with a SASL cluster with GSSAPI for inter-broker and a second mechanism for clients (i.e. result of phase one). Start Producer and Consumer using the second mechanism Incrementally upgrade to set inter-broker to the second mechanism and disable GSSAPI Incrementally upgrade again to add ACLs Ensure the producer and consumer run throughout """ #Start with a broker that has GSSAPI for inter-broker and a second mechanism for clients self.kafka.security_protocol = SecurityConfig.SASL_SSL self.kafka.setup_interbroker_listener(SecurityConfig.SASL_SSL, use_separate_listener=False) self.kafka.client_sasl_mechanism = new_sasl_mechanism self.kafka.interbroker_sasl_mechanism = SecurityConfig.SASL_MECHANISM_GSSAPI self.kafka.start() #Create Producer and Consumer using second mechanism self.create_producer_and_consumer() #Roll in the second SASL mechanism for inter-broker, disabling first mechanism. Ensure we can produce and consume throughout self.run_produce_consume_validate(self.roll_in_sasl_mechanism, self.kafka.security_protocol, new_sasl_mechanism) @cluster(num_nodes=9) def test_enable_separate_interbroker_listener(self): """ Start with a cluster that has a single PLAINTEXT listener. Start producing/consuming on PLAINTEXT port. While doing that, do a rolling restart to enable separate secured interbroker port """ self.kafka.security_protocol = SecurityConfig.PLAINTEXT self.kafka.setup_interbroker_listener(SecurityConfig.PLAINTEXT, use_separate_listener=False) self.kafka.start() self.create_producer_and_consumer() self.run_produce_consume_validate(self.add_separate_broker_listener, SecurityConfig.SASL_SSL, SecurityConfig.SASL_MECHANISM_PLAIN) @cluster(num_nodes=9) def test_disable_separate_interbroker_listener(self): """ Start with a cluster that has two listeners, one on SSL (clients), another on SASL_SSL (broker-to-broker). Start producer and consumer on SSL listener. Close dedicated interbroker listener via rolling restart. Ensure we can produce and consume via SSL listener throughout. """ client_protocol = SecurityConfig.SSL client_sasl_mechanism = SecurityConfig.SASL_MECHANISM_GSSAPI self.kafka.security_protocol = client_protocol self.kafka.client_sasl_mechanism = client_sasl_mechanism self.kafka.setup_interbroker_listener(SecurityConfig.SASL_SSL, use_separate_listener=True) self.kafka.interbroker_sasl_mechanism = SecurityConfig.SASL_MECHANISM_GSSAPI self.kafka.start() # create producer and consumer via client security protocol self.create_producer_and_consumer() # run produce/consume/validate loop while disabling a separate interbroker listener via rolling restart self.run_produce_consume_validate(self.remove_separate_broker_listener, client_protocol, client_sasl_mechanism)
class StreamsUpgradeTest(Test): """ Test upgrading Kafka Streams (all version combination) If metadata was changes, upgrade is more difficult Metadata version was bumped in 0.10.1.0 and subsequently bumped in 2.0.0 """ def __init__(self, test_context): super(StreamsUpgradeTest, self).__init__(test_context) self.topics = { 'echo' : { 'partitions': 5 }, 'data' : { 'partitions': 5 }, } processed_msg = "processed [0-9]* records" base_version_number = str(DEV_VERSION).split("-")[0] def perform_broker_upgrade(self, to_version): self.logger.info("First pass bounce - rolling broker upgrade") for node in self.kafka.nodes: self.kafka.stop_node(node) node.version = KafkaVersion(to_version) self.kafka.start_node(node) @cluster(num_nodes=6) @matrix(from_version=smoke_test_versions, to_version=dev_version, bounce_type=["full"]) def test_app_upgrade(self, from_version, to_version, bounce_type): """ Starts 3 KafkaStreams instances with <old_version>, and upgrades one-by-one to <new_version> """ if from_version == to_version: return self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() self.kafka = KafkaService(self.test_context, num_nodes=1, zk=self.zk, topics={ 'echo' : { 'partitions': 5, 'replication-factor': 1 }, 'data' : { 'partitions': 5, 'replication-factor': 1 }, 'min' : { 'partitions': 5, 'replication-factor': 1 }, 'min-suppressed' : { 'partitions': 5, 'replication-factor': 1 }, 'min-raw' : { 'partitions': 5, 'replication-factor': 1 }, 'max' : { 'partitions': 5, 'replication-factor': 1 }, 'sum' : { 'partitions': 5, 'replication-factor': 1 }, 'sws-raw' : { 'partitions': 5, 'replication-factor': 1 }, 'sws-suppressed' : { 'partitions': 5, 'replication-factor': 1 }, 'dif' : { 'partitions': 5, 'replication-factor': 1 }, 'cnt' : { 'partitions': 5, 'replication-factor': 1 }, 'avg' : { 'partitions': 5, 'replication-factor': 1 }, 'wcnt' : { 'partitions': 5, 'replication-factor': 1 }, 'tagg' : { 'partitions': 5, 'replication-factor': 1 } }) self.kafka.start() self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) self.driver.disable_auto_terminate() self.processor1 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka, processing_guarantee = "at_least_once", replication_factor = 1) self.processor2 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka, processing_guarantee = "at_least_once", replication_factor = 1) self.processor3 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka, processing_guarantee = "at_least_once", replication_factor = 1) self.purge_state_dir(self.processor1) self.purge_state_dir(self.processor2) self.purge_state_dir(self.processor3) self.driver.start() self.start_all_nodes_with(from_version) self.processors = [self.processor1, self.processor2, self.processor3] if bounce_type == "rolling": counter = 1 random.seed() # upgrade one-by-one via rolling bounce random.shuffle(self.processors) for p in self.processors: p.CLEAN_NODE_ENABLED = False self.do_stop_start_bounce(p, None, to_version, counter) counter = counter + 1 elif bounce_type == "full": self.restart_all_nodes_with(to_version) else: raise Exception("Unrecognized bounce_type: " + str(bounce_type)) # shutdown self.driver.stop() # Ideally, we would actually verify the expected results. # See KAFKA-10202 random.shuffle(self.processors) for p in self.processors: node = p.node with node.account.monitor_log(p.STDOUT_FILE) as monitor: p.stop() monitor.wait_until("SMOKE-TEST-CLIENT-CLOSED", timeout_sec=60, err_msg="Never saw output 'SMOKE-TEST-CLIENT-CLOSED' on " + str(node.account)) def start_all_nodes_with(self, version): self.set_version(self.processor1, version) self.set_version(self.processor2, version) self.set_version(self.processor3, version) self.processor1.start() self.processor2.start() self.processor3.start() # double-check the version kafka_version_str = self.get_version_string(version) self.wait_for_verification(self.processor1, kafka_version_str, self.processor1.LOG_FILE) self.wait_for_verification(self.processor2, kafka_version_str, self.processor2.LOG_FILE) self.wait_for_verification(self.processor3, kafka_version_str, self.processor3.LOG_FILE) # wait for the members to join self.wait_for_verification(self.processor1, "SMOKE-TEST-CLIENT-STARTED", self.processor1.STDOUT_FILE) self.wait_for_verification(self.processor2, "SMOKE-TEST-CLIENT-STARTED", self.processor2.STDOUT_FILE) self.wait_for_verification(self.processor3, "SMOKE-TEST-CLIENT-STARTED", self.processor3.STDOUT_FILE) # make sure they've processed something self.wait_for_verification(self.processor1, self.processed_msg, self.processor1.STDOUT_FILE) self.wait_for_verification(self.processor2, self.processed_msg, self.processor2.STDOUT_FILE) self.wait_for_verification(self.processor3, self.processed_msg, self.processor3.STDOUT_FILE) def restart_all_nodes_with(self, version): self.processor1.stop_node(self.processor1.node) self.processor2.stop_node(self.processor2.node) self.processor3.stop_node(self.processor3.node) # make sure the members have stopped self.wait_for_verification(self.processor1, "SMOKE-TEST-CLIENT-CLOSED", self.processor1.STDOUT_FILE) self.wait_for_verification(self.processor2, "SMOKE-TEST-CLIENT-CLOSED", self.processor2.STDOUT_FILE) self.wait_for_verification(self.processor3, "SMOKE-TEST-CLIENT-CLOSED", self.processor3.STDOUT_FILE) self.roll_logs(self.processor1, ".1-1") self.roll_logs(self.processor2, ".1-1") self.roll_logs(self.processor3, ".1-1") self.set_version(self.processor1, version) self.set_version(self.processor2, version) self.set_version(self.processor3, version) self.processor1.start_node(self.processor1.node) self.processor2.start_node(self.processor2.node) self.processor3.start_node(self.processor3.node) # double-check the version kafka_version_str = self.get_version_string(version) self.wait_for_verification(self.processor1, kafka_version_str, self.processor1.LOG_FILE) self.wait_for_verification(self.processor2, kafka_version_str, self.processor2.LOG_FILE) self.wait_for_verification(self.processor3, kafka_version_str, self.processor3.LOG_FILE) # wait for the members to join self.wait_for_verification(self.processor1, "SMOKE-TEST-CLIENT-STARTED", self.processor1.STDOUT_FILE) self.wait_for_verification(self.processor2, "SMOKE-TEST-CLIENT-STARTED", self.processor2.STDOUT_FILE) self.wait_for_verification(self.processor3, "SMOKE-TEST-CLIENT-STARTED", self.processor3.STDOUT_FILE) # make sure they've processed something self.wait_for_verification(self.processor1, self.processed_msg, self.processor1.STDOUT_FILE) self.wait_for_verification(self.processor2, self.processed_msg, self.processor2.STDOUT_FILE) self.wait_for_verification(self.processor3, self.processed_msg, self.processor3.STDOUT_FILE) def get_version_string(self, version): if version.startswith("0") or version.startswith("1") \ or version.startswith("2.0") or version.startswith("2.1"): return "Kafka version : " + version elif "SNAPSHOT" in version: return "Kafka version.*" + self.base_version_number + ".*SNAPSHOT" else: return "Kafka version: " + version def wait_for_verification(self, processor, message, file, num_lines=1): wait_until(lambda: self.verify_from_file(processor, message, file) >= num_lines, timeout_sec=60, err_msg="Did expect to read '%s' from %s" % (message, processor.node.account)) def verify_from_file(self, processor, message, file): result = processor.node.account.ssh_output("grep -E '%s' %s | wc -l" % (message, file), allow_fail=False) try: return int(result) except ValueError: self.logger.warn("Command failed with ValueError: " + result) return 0 def set_version(self, processor, version): if version == str(DEV_VERSION): processor.set_version("") # set to TRUNK else: processor.set_version(version) def purge_state_dir(self, processor): processor.node.account.ssh("rm -rf " + processor.PERSISTENT_ROOT, allow_fail=False) def do_stop_start_bounce(self, processor, upgrade_from, new_version, counter): kafka_version_str = self.get_version_string(new_version) first_other_processor = None second_other_processor = None for p in self.processors: if p != processor: if first_other_processor is None: first_other_processor = p else: second_other_processor = p node = processor.node first_other_node = first_other_processor.node second_other_node = second_other_processor.node # stop processor and wait for rebalance of others with first_other_node.account.monitor_log(first_other_processor.STDOUT_FILE) as first_other_monitor: with second_other_node.account.monitor_log(second_other_processor.STDOUT_FILE) as second_other_monitor: processor.stop_node(processor.node) first_other_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(first_other_node.account)) second_other_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(second_other_node.account)) node.account.ssh_capture("grep SMOKE-TEST-CLIENT-CLOSED %s" % processor.STDOUT_FILE, allow_fail=False) if upgrade_from is None: # upgrade disabled -- second round of rolling bounces roll_counter = ".1-" # second round of rolling bounces else: roll_counter = ".0-" # first round of rolling bounces self.roll_logs(processor, roll_counter + str(counter)) self.set_version(processor, new_version) processor.set_upgrade_from(upgrade_from) grep_metadata_error = "grep \"org.apache.kafka.streams.errors.TaskAssignmentException: unable to decode subscription data: version=2\" " with node.account.monitor_log(processor.STDOUT_FILE) as monitor: with node.account.monitor_log(processor.LOG_FILE) as log_monitor: with first_other_node.account.monitor_log(first_other_processor.STDOUT_FILE) as first_other_monitor: with second_other_node.account.monitor_log(second_other_processor.STDOUT_FILE) as second_other_monitor: processor.start_node(processor.node) log_monitor.wait_until(kafka_version_str, timeout_sec=60, err_msg="Could not detect Kafka Streams version " + new_version + " on " + str(node.account)) first_other_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(first_other_node.account)) found = list(first_other_node.account.ssh_capture(grep_metadata_error + first_other_processor.STDERR_FILE, allow_fail=True)) if len(found) > 0: raise Exception("Kafka Streams failed with 'unable to decode subscription data: version=2'") second_other_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(second_other_node.account)) found = list(second_other_node.account.ssh_capture(grep_metadata_error + second_other_processor.STDERR_FILE, allow_fail=True)) if len(found) > 0: raise Exception("Kafka Streams failed with 'unable to decode subscription data: version=2'") monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(node.account)) def roll_logs(self, processor, roll_suffix): processor.node.account.ssh("mv " + processor.STDOUT_FILE + " " + processor.STDOUT_FILE + roll_suffix, allow_fail=False) processor.node.account.ssh("mv " + processor.STDERR_FILE + " " + processor.STDERR_FILE + roll_suffix, allow_fail=False) processor.node.account.ssh("mv " + processor.LOG_FILE + " " + processor.LOG_FILE + roll_suffix, allow_fail=False) processor.node.account.ssh("mv " + processor.CONFIG_FILE + " " + processor.CONFIG_FILE + roll_suffix, allow_fail=False)
def create_zookeeper_if_necessary(self, num_nodes=1, **kwargs): self.zk = ZookeeperService( self.test_context, num_nodes=num_nodes, ** kwargs) if quorum.for_test( self.test_context) == quorum.zk else None
def test_app_upgrade(self, from_version, to_version, bounce_type): """ Starts 3 KafkaStreams instances with <old_version>, and upgrades one-by-one to <new_version> """ if from_version == to_version: return self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() self.kafka = KafkaService(self.test_context, num_nodes=1, zk=self.zk, topics={ 'echo' : { 'partitions': 5, 'replication-factor': 1 }, 'data' : { 'partitions': 5, 'replication-factor': 1 }, 'min' : { 'partitions': 5, 'replication-factor': 1 }, 'min-suppressed' : { 'partitions': 5, 'replication-factor': 1 }, 'min-raw' : { 'partitions': 5, 'replication-factor': 1 }, 'max' : { 'partitions': 5, 'replication-factor': 1 }, 'sum' : { 'partitions': 5, 'replication-factor': 1 }, 'sws-raw' : { 'partitions': 5, 'replication-factor': 1 }, 'sws-suppressed' : { 'partitions': 5, 'replication-factor': 1 }, 'dif' : { 'partitions': 5, 'replication-factor': 1 }, 'cnt' : { 'partitions': 5, 'replication-factor': 1 }, 'avg' : { 'partitions': 5, 'replication-factor': 1 }, 'wcnt' : { 'partitions': 5, 'replication-factor': 1 }, 'tagg' : { 'partitions': 5, 'replication-factor': 1 } }) self.kafka.start() self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) self.driver.disable_auto_terminate() self.processor1 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka, processing_guarantee = "at_least_once", replication_factor = 1) self.processor2 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka, processing_guarantee = "at_least_once", replication_factor = 1) self.processor3 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka, processing_guarantee = "at_least_once", replication_factor = 1) self.purge_state_dir(self.processor1) self.purge_state_dir(self.processor2) self.purge_state_dir(self.processor3) self.driver.start() self.start_all_nodes_with(from_version) self.processors = [self.processor1, self.processor2, self.processor3] if bounce_type == "rolling": counter = 1 random.seed() # upgrade one-by-one via rolling bounce random.shuffle(self.processors) for p in self.processors: p.CLEAN_NODE_ENABLED = False self.do_stop_start_bounce(p, None, to_version, counter) counter = counter + 1 elif bounce_type == "full": self.restart_all_nodes_with(to_version) else: raise Exception("Unrecognized bounce_type: " + str(bounce_type)) # shutdown self.driver.stop() # Ideally, we would actually verify the expected results. # See KAFKA-10202 random.shuffle(self.processors) for p in self.processors: node = p.node with node.account.monitor_log(p.STDOUT_FILE) as monitor: p.stop() monitor.wait_until("SMOKE-TEST-CLIENT-CLOSED", timeout_sec=60, err_msg="Never saw output 'SMOKE-TEST-CLIENT-CLOSED' on " + str(node.account))
class QuotaTest(Test): """ These tests verify that quota provides expected functionality -- they run producer, broker, and consumer with different clientId and quota configuration and check that the observed throughput is close to the value we expect. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(QuotaTest, self).__init__(test_context=test_context) self.topic = 'test_topic' self.logger.info('use topic ' + self.topic) self.maximum_client_deviation_percentage = 100.0 self.maximum_broker_deviation_percentage = 5.0 self.num_records = 50000 self.record_size = 3000 self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService( test_context, num_nodes=1, zk=self.zk, security_protocol='SSL', authorizer_class_name='', interbroker_security_protocol='SSL', topics={ self.topic: { 'partitions': 6, 'replication-factor': 1, 'configs': { 'min.insync.replicas': 1 } } }, jmx_object_names=[ 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec', 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec' ], jmx_attributes=['OneMinuteRate']) self.num_producers = 1 self.num_consumers = 2 def setUp(self): self.zk.start() self.kafka.start() def min_cluster_size(self): """Override this since we're adding services outside of the constructor""" return super( QuotaTest, self).min_cluster_size() + self.num_producers + self.num_consumers @cluster(num_nodes=5) @matrix(quota_type=[ QuotaConfig.CLIENT_ID, QuotaConfig.USER, QuotaConfig.USER_CLIENT ], override_quota=[True, False]) @parametrize(quota_type=QuotaConfig.CLIENT_ID, consumer_num=2) def test_quota(self, quota_type, override_quota=True, producer_num=1, consumer_num=1): self.quota_config = QuotaConfig(quota_type, override_quota, self.kafka) producer_client_id = self.quota_config.client_id consumer_client_id = self.quota_config.client_id # Produce all messages producer = ProducerPerformanceService(self.test_context, producer_num, self.kafka, topic=self.topic, num_records=self.num_records, record_size=self.record_size, throughput=-1, client_id=producer_client_id) producer.run() # Consume all messages consumer = ConsoleConsumer( self.test_context, consumer_num, self.kafka, self.topic, consumer_timeout_ms=60000, client_id=consumer_client_id, jmx_object_names=[ 'kafka.consumer:type=consumer-fetch-manager-metrics,client-id=%s' % consumer_client_id ], jmx_attributes=['bytes-consumed-rate']) consumer.run() for idx, messages in consumer.messages_consumed.iteritems(): assert len( messages ) > 0, "consumer %d didn't consume any message before timeout" % idx success, msg = self.validate(self.kafka, producer, consumer) assert success, msg def validate(self, broker, producer, consumer): """ For each client_id we validate that: 1) number of consumed messages equals number of produced messages 2) maximum_producer_throughput <= producer_quota * (1 + maximum_client_deviation_percentage/100) 3) maximum_broker_byte_in_rate <= producer_quota * (1 + maximum_broker_deviation_percentage/100) 4) maximum_consumer_throughput <= consumer_quota * (1 + maximum_client_deviation_percentage/100) 5) maximum_broker_byte_out_rate <= consumer_quota * (1 + maximum_broker_deviation_percentage/100) """ success = True msg = '' self.kafka.read_jmx_output_all_nodes() # validate that number of consumed messages equals number of produced messages produced_num = sum([value['records'] for value in producer.results]) consumed_num = sum( [len(value) for value in consumer.messages_consumed.values()]) self.logger.info('producer produced %d messages' % produced_num) self.logger.info('consumer consumed %d messages' % consumed_num) if produced_num != consumed_num: success = False msg += "number of produced messages %d doesn't equal number of consumed messages %d" % ( produced_num, consumed_num) # validate that maximum_producer_throughput <= producer_quota * (1 + maximum_client_deviation_percentage/100) producer_maximum_bps = max( metric.value for k, metrics in producer.metrics(group='producer-metrics', name='outgoing-byte-rate', client_id=producer.client_id) for metric in metrics) producer_quota_bps = self.quota_config.producer_quota self.logger.info( 'producer has maximum throughput %.2f bps with producer quota %.2f bps' % (producer_maximum_bps, producer_quota_bps)) if producer_maximum_bps > producer_quota_bps * ( self.maximum_client_deviation_percentage / 100 + 1): success = False msg += 'maximum producer throughput %.2f bps exceeded producer quota %.2f bps by more than %.1f%%' % \ (producer_maximum_bps, producer_quota_bps, self.maximum_client_deviation_percentage) # validate that maximum_broker_byte_in_rate <= producer_quota * (1 + maximum_broker_deviation_percentage/100) broker_byte_in_attribute_name = 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec:OneMinuteRate' broker_maximum_byte_in_bps = broker.maximum_jmx_value[ broker_byte_in_attribute_name] self.logger.info( 'broker has maximum byte-in rate %.2f bps with producer quota %.2f bps' % (broker_maximum_byte_in_bps, producer_quota_bps)) if broker_maximum_byte_in_bps > producer_quota_bps * ( self.maximum_broker_deviation_percentage / 100 + 1): success = False msg += 'maximum broker byte-in rate %.2f bps exceeded producer quota %.2f bps by more than %.1f%%' % \ (broker_maximum_byte_in_bps, producer_quota_bps, self.maximum_broker_deviation_percentage) # validate that maximum_consumer_throughput <= consumer_quota * (1 + maximum_client_deviation_percentage/100) consumer_attribute_name = 'kafka.consumer:type=consumer-fetch-manager-metrics,client-id=%s:bytes-consumed-rate' % consumer.client_id consumer_maximum_bps = consumer.maximum_jmx_value[ consumer_attribute_name] consumer_quota_bps = self.quota_config.consumer_quota self.logger.info( 'consumer has maximum throughput %.2f bps with consumer quota %.2f bps' % (consumer_maximum_bps, consumer_quota_bps)) if consumer_maximum_bps > consumer_quota_bps * ( self.maximum_client_deviation_percentage / 100 + 1): success = False msg += 'maximum consumer throughput %.2f bps exceeded consumer quota %.2f bps by more than %.1f%%' % \ (consumer_maximum_bps, consumer_quota_bps, self.maximum_client_deviation_percentage) # validate that maximum_broker_byte_out_rate <= consumer_quota * (1 + maximum_broker_deviation_percentage/100) broker_byte_out_attribute_name = 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec:OneMinuteRate' broker_maximum_byte_out_bps = broker.maximum_jmx_value[ broker_byte_out_attribute_name] self.logger.info( 'broker has maximum byte-out rate %.2f bps with consumer quota %.2f bps' % (broker_maximum_byte_out_bps, consumer_quota_bps)) if broker_maximum_byte_out_bps > consumer_quota_bps * ( self.maximum_broker_deviation_percentage / 100 + 1): success = False msg += 'maximum broker byte-out rate %.2f bps exceeded consumer quota %.2f bps by more than %.1f%%' % \ (broker_maximum_byte_out_bps, consumer_quota_bps, self.maximum_broker_deviation_percentage) return success, msg
class ThrottlingTest(ProduceConsumeValidateTest): """Tests throttled partition reassignment. This is essentially similar to the reassign_partitions_test, except that we throttle the reassignment and verify that it takes a sensible amount of time given the throttle and the amount of data being moved. Since the correctness is time dependent, this test also simplifies the cluster topology. In particular, we fix the number of brokers, the replication-factor, the number of partitions, the partition size, and the number of partitions being moved so that we can accurately predict the time throttled reassignment should take. """ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ThrottlingTest, self).__init__(test_context=test_context) self.topic = "test_topic" self.zk = ZookeeperService(test_context, num_nodes=1) # Because we are starting the producer/consumer/validate cycle _after_ # seeding the cluster with big data (to test throttling), we need to # Start the consumer from the end of the stream. further, we need to # ensure that the consumer is fully started before the producer starts # so that we don't miss any messages. This timeout ensures the sufficient # condition. self.consumer_init_timeout_sec = 10 self.num_brokers = 6 self.num_partitions = 3 self.kafka = KafkaService(test_context, num_nodes=self.num_brokers, zk=self.zk, topics={ self.topic: { "partitions": self.num_partitions, "replication-factor": 2, "configs": { "segment.bytes": 64 * 1024 * 1024 } } }) self.producer_throughput = 1000 self.timeout_sec = 400 self.num_records = 2000 self.record_size = 4096 * 100 # 400 KB # 1 MB per partition on average. self.partition_size = (self.num_records * self.record_size) / self.num_partitions self.num_producers = 2 self.num_consumers = 1 self.throttle = 4 * 1024 * 1024 # 4 MB/s def setUp(self): self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super(ThrottlingTest, self).min_cluster_size() +\ self.num_producers + self.num_consumers def clean_bounce_some_brokers(self): """Bounce every other broker""" for node in self.kafka.nodes[::2]: self.kafka.restart_node(node, clean_shutdown=True) def reassign_partitions(self, bounce_brokers, throttle): """This method reassigns partitions using a throttle. It makes an assertion about the minimum amount of time the reassignment should take given the value of the throttle, the number of partitions being moved, and the size of each partition. """ partition_info = self.kafka.parse_describe_topic( self.kafka.describe_topic(self.topic)) self.logger.debug("Partitions before reassignment:" + str(partition_info)) max_num_moves = 0 for i in range(0, self.num_partitions): old_replicas = set(partition_info["partitions"][i]["replicas"]) new_part = (i + 1) % self.num_partitions new_replicas = set( partition_info["partitions"][new_part]["replicas"]) max_num_moves = max(len(new_replicas - old_replicas), max_num_moves) partition_info["partitions"][i]["partition"] = new_part self.logger.debug("Jumbled partitions: " + str(partition_info)) self.kafka.execute_reassign_partitions(partition_info, throttle=throttle) start = time.time() if bounce_brokers: # bounce a few brokers at the same time self.clean_bounce_some_brokers() # Wait until finished or timeout size_per_broker = max_num_moves * self.partition_size self.logger.debug("Max amount of data transfer per broker: %fb", size_per_broker) estimated_throttled_time = math.ceil( float(size_per_broker) / self.throttle) estimated_time_with_buffer = estimated_throttled_time * 2 self.logger.debug("Waiting %ds for the reassignment to complete", estimated_time_with_buffer) wait_until( lambda: self.kafka.verify_reassign_partitions(partition_info), timeout_sec=estimated_time_with_buffer, backoff_sec=.5) stop = time.time() time_taken = stop - start self.logger.debug("Transfer took %d second. Estimated time : %ds", time_taken, estimated_throttled_time) assert time_taken >= estimated_throttled_time, \ ("Expected rebalance to take at least %ds, but it took %ds" % ( estimated_throttled_time, time_taken)) @parametrize(bounce_brokers=False) @parametrize(bounce_brokers=True) def test_throttled_reassignment(self, bounce_brokers): security_protocol = 'PLAINTEXT' self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = security_protocol producer_id = 'bulk_producer' bulk_producer = ProducerPerformanceService( context=self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, num_records=self.num_records, record_size=self.record_size, throughput=-1, client_id=producer_id, jmx_object_names=[ 'kafka.producer:type=producer-metrics,client-id=%s' % producer_id ], jmx_attributes=['outgoing-byte-rate']) self.producer = VerifiableProducer(context=self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, message_validator=is_int, throughput=self.producer_throughput) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=60000, message_validator=is_int, from_beginning=False) self.kafka.start() bulk_producer.run() self.run_produce_consume_validate( core_test_action=lambda: self.reassign_partitions( bounce_brokers, self.throttle))
class ZooKeeperSecurityUpgradeTest(ProduceConsumeValidateTest): """Tests a rolling upgrade for zookeeper. """ def __init__(self, test_context): super(ZooKeeperSecurityUpgradeTest, self).__init__(test_context=test_context) def setUp(self): self.topic = "test_topic" self.group = "group" self.producer_throughput = 100 self.num_producers = 1 self.num_consumers = 1 self.acls = ACLs(self.test_context) self.zk = ZookeeperService(self.test_context, num_nodes=3) self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, topics={ self.topic: { "partitions": 3, "replication-factor": 3, 'configs': { "min.insync.replicas": 2 } } }) def create_producer_and_consumer(self): self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput) self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, consumer_timeout_ms=60000, message_validator=is_int) self.consumer.group_id = self.group @property def no_sasl(self): return self.kafka.security_protocol == "PLAINTEXT" or self.kafka.security_protocol == "SSL" @property def is_secure(self): return self.kafka.security_protocol == "SASL_PLAINTEXT" \ or self.kafka.security_protocol == "SSL" \ or self.kafka.security_protocol == "SASL_SSL" def run_zk_migration(self): # change zk config (auth provider + jaas login) self.zk.kafka_opts = self.zk.security_system_properties self.zk.zk_sasl = True if self.no_sasl: self.kafka.start_minikdc(self.zk.zk_principals) # restart zk for node in self.zk.nodes: self.zk.stop_node(node) self.zk.start_node(node) # restart broker with jaas login for node in self.kafka.nodes: self.kafka.stop_node(node) self.kafka.start_node(node) # run migration tool for node in self.zk.nodes: self.zk.zookeeper_migration(node, "secure") # restart broker with zookeeper.set.acl=true and acls self.kafka.zk_set_acl = True for node in self.kafka.nodes: self.kafka.stop_node(node) self.kafka.start_node(node) @ignore @cluster(num_nodes=9) @matrix( security_protocol=["PLAINTEXT", "SSL", "SASL_SSL", "SASL_PLAINTEXT"]) def test_zk_security_upgrade(self, security_protocol): self.zk.start() self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = security_protocol # set acls if self.is_secure: self.kafka.authorizer_class_name = KafkaService.SIMPLE_AUTHORIZER self.acls.set_acls(security_protocol, self.kafka, self.zk, self.topic, self.group) if self.no_sasl: self.kafka.start() else: self.kafka.start(self.zk.zk_principals) #Create Producer and Consumer self.create_producer_and_consumer() #Run upgrade self.run_produce_consume_validate(self.run_zk_migration)
class ConnectDistributedTest(Test): """ Simple test of Kafka Connect in distributed mode, producing data from files on one cluster and consuming it on another, validating the total output is identical to the input. """ FILE_SOURCE_CONNECTOR = 'org.apache.kafka.connect.file.FileStreamSourceConnector' FILE_SINK_CONNECTOR = 'org.apache.kafka.connect.file.FileStreamSinkConnector' INPUT_FILE = "/mnt/connect.input" OUTPUT_FILE = "/mnt/connect.output" TOPIC = "test" OFFSETS_TOPIC = "connect-offsets" OFFSETS_REPLICATION_FACTOR = "1" OFFSETS_PARTITIONS = "1" CONFIG_TOPIC = "connect-configs" CONFIG_REPLICATION_FACTOR = "1" STATUS_TOPIC = "connect-status" STATUS_REPLICATION_FACTOR = "1" STATUS_PARTITIONS = "1" SCHEDULED_REBALANCE_MAX_DELAY_MS = "60000" CONNECT_PROTOCOL = "sessioned" # Since tasks can be assigned to any node and we're testing with files, we need to make sure the content is the same # across all nodes. FIRST_INPUT_LIST = ["foo", "bar", "baz"] FIRST_INPUTS = "\n".join(FIRST_INPUT_LIST) + "\n" SECOND_INPUT_LIST = ["razz", "ma", "tazz"] SECOND_INPUTS = "\n".join(SECOND_INPUT_LIST) + "\n" SCHEMA = {"type": "string", "optional": False} def __init__(self, test_context): super(ConnectDistributedTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.topics = {self.TOPIC: {'partitions': 1, 'replication-factor': 1}} self.zk = ZookeeperService(test_context, self.num_zk) self.key_converter = "org.apache.kafka.connect.json.JsonConverter" self.value_converter = "org.apache.kafka.connect.json.JsonConverter" self.schemas = True def setup_services(self, security_protocol=SecurityConfig.PLAINTEXT, timestamp_type=None, broker_version=DEV_BRANCH, auto_create_topics=False): self.kafka = KafkaService( self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, interbroker_security_protocol=security_protocol, topics=self.topics, version=broker_version, server_prop_overides=[[ "auto.create.topics.enable", str(auto_create_topics) ]]) if timestamp_type is not None: for node in self.kafka.nodes: node.config[ config_property.MESSAGE_TIMESTAMP_TYPE] = timestamp_type self.cc = ConnectDistributedService( self.test_context, 3, self.kafka, [self.INPUT_FILE, self.OUTPUT_FILE]) self.cc.log_level = "DEBUG" self.zk.start() self.kafka.start() def _start_connector(self, config_file): connector_props = self.render(config_file) connector_config = dict([ line.strip().split('=', 1) for line in connector_props.split('\n') if line.strip() and not line.strip().startswith('#') ]) self.cc.create_connector(connector_config) def _connector_status(self, connector, node=None): try: return self.cc.get_connector_status(connector, node) except ConnectRestError: return None def _connector_has_state(self, status, state): return status is not None and status['connector']['state'] == state def _task_has_state(self, task_id, status, state): if not status: return False tasks = status['tasks'] if not tasks: return False for task in tasks: if task['id'] == task_id: return task['state'] == state return False def _all_tasks_have_state(self, status, task_count, state): if status is None: return False tasks = status['tasks'] if len(tasks) != task_count: return False return reduce(operator.and_, [task['state'] == state for task in tasks], True) def is_running(self, connector, node=None): status = self._connector_status(connector.name, node) return self._connector_has_state( status, 'RUNNING') and self._all_tasks_have_state( status, connector.tasks, 'RUNNING') def is_paused(self, connector, node=None): status = self._connector_status(connector.name, node) return self._connector_has_state( status, 'PAUSED') and self._all_tasks_have_state( status, connector.tasks, 'PAUSED') def connector_is_running(self, connector, node=None): status = self._connector_status(connector.name, node) return self._connector_has_state(status, 'RUNNING') def connector_is_failed(self, connector, node=None): status = self._connector_status(connector.name, node) return self._connector_has_state(status, 'FAILED') def task_is_failed(self, connector, task_id, node=None): status = self._connector_status(connector.name, node) return self._task_has_state(task_id, status, 'FAILED') def task_is_running(self, connector, task_id, node=None): status = self._connector_status(connector.name, node) return self._task_has_state(task_id, status, 'RUNNING') @cluster(num_nodes=5) @matrix(connect_protocol=['sessioned', 'compatible', 'eager']) def test_restart_failed_connector(self, connect_protocol): self.CONNECT_PROTOCOL = connect_protocol self.setup_services() self.cc.set_configs(lambda node: self.render( "connect-distributed.properties", node=node)) self.cc.start() self.sink = MockSink(self.cc, self.topics.keys(), mode='connector-failure', delay_sec=5) self.sink.start() wait_until( lambda: self.connector_is_failed(self.sink), timeout_sec=15, err_msg="Failed to see connector transition to the FAILED state") self.cc.restart_connector(self.sink.name) wait_until( lambda: self.connector_is_running(self.sink), timeout_sec=10, err_msg="Failed to see connector transition to the RUNNING state") @cluster(num_nodes=5) @matrix(connector_type=['source', 'sink'], connect_protocol=['sessioned', 'compatible', 'eager']) def test_restart_failed_task(self, connector_type, connect_protocol): self.CONNECT_PROTOCOL = connect_protocol self.setup_services() self.cc.set_configs(lambda node: self.render( "connect-distributed.properties", node=node)) self.cc.start() connector = None if connector_type == "sink": connector = MockSink(self.cc, self.topics.keys(), mode='task-failure', delay_sec=5) else: connector = MockSource(self.cc, mode='task-failure', delay_sec=5) connector.start() task_id = 0 wait_until(lambda: self.task_is_failed(connector, task_id), timeout_sec=20, err_msg="Failed to see task transition to the FAILED state") self.cc.restart_task(connector.name, task_id) wait_until( lambda: self.task_is_running(connector, task_id), timeout_sec=10, err_msg="Failed to see task transition to the RUNNING state") @cluster(num_nodes=5) @matrix(connect_protocol=['sessioned', 'compatible', 'eager']) def test_pause_and_resume_source(self, connect_protocol): """ Verify that source connectors stop producing records when paused and begin again after being resumed. """ self.CONNECT_PROTOCOL = connect_protocol self.setup_services() self.cc.set_configs(lambda node: self.render( "connect-distributed.properties", node=node)) self.cc.start() self.source = VerifiableSource(self.cc, topic=self.TOPIC) self.source.start() wait_until( lambda: self.is_running(self.source), timeout_sec=30, err_msg="Failed to see connector transition to the RUNNING state") self.cc.pause_connector(self.source.name) # wait until all nodes report the paused transition for node in self.cc.nodes: wait_until( lambda: self.is_paused(self.source, node), timeout_sec=30, err_msg="Failed to see connector transition to the PAUSED state" ) # verify that we do not produce new messages while paused num_messages = len(self.source.sent_messages()) time.sleep(10) assert num_messages == len(self.source.sent_messages( )), "Paused source connector should not produce any messages" self.cc.resume_connector(self.source.name) for node in self.cc.nodes: wait_until( lambda: self.is_running(self.source, node), timeout_sec=30, err_msg= "Failed to see connector transition to the RUNNING state") # after resuming, we should see records produced again wait_until( lambda: len(self.source.sent_messages()) > num_messages, timeout_sec=30, err_msg="Failed to produce messages after resuming source connector" ) @cluster(num_nodes=5) @matrix(connect_protocol=['sessioned', 'compatible', 'eager']) def test_pause_and_resume_sink(self, connect_protocol): """ Verify that sink connectors stop consuming records when paused and begin again after being resumed. """ self.CONNECT_PROTOCOL = connect_protocol self.setup_services() self.cc.set_configs(lambda node: self.render( "connect-distributed.properties", node=node)) self.cc.start() # use the verifiable source to produce a steady stream of messages self.source = VerifiableSource(self.cc, topic=self.TOPIC) self.source.start() wait_until( lambda: len(self.source.committed_messages()) > 0, timeout_sec=30, err_msg= "Timeout expired waiting for source task to produce a message") self.sink = VerifiableSink(self.cc, topics=[self.TOPIC]) self.sink.start() wait_until( lambda: self.is_running(self.sink), timeout_sec=30, err_msg="Failed to see connector transition to the RUNNING state") self.cc.pause_connector(self.sink.name) # wait until all nodes report the paused transition for node in self.cc.nodes: wait_until( lambda: self.is_paused(self.sink, node), timeout_sec=30, err_msg="Failed to see connector transition to the PAUSED state" ) # verify that we do not consume new messages while paused num_messages = len(self.sink.received_messages()) time.sleep(10) assert num_messages == len(self.sink.received_messages( )), "Paused sink connector should not consume any messages" self.cc.resume_connector(self.sink.name) for node in self.cc.nodes: wait_until( lambda: self.is_running(self.sink, node), timeout_sec=30, err_msg= "Failed to see connector transition to the RUNNING state") # after resuming, we should see records consumed again wait_until( lambda: len(self.sink.received_messages()) > num_messages, timeout_sec=30, err_msg="Failed to consume messages after resuming sink connector") @cluster(num_nodes=5) @matrix(connect_protocol=['sessioned', 'compatible', 'eager']) def test_pause_state_persistent(self, connect_protocol): """ Verify that paused state is preserved after a cluster restart. """ self.CONNECT_PROTOCOL = connect_protocol self.setup_services() self.cc.set_configs(lambda node: self.render( "connect-distributed.properties", node=node)) self.cc.start() self.source = VerifiableSource(self.cc, topic=self.TOPIC) self.source.start() wait_until( lambda: self.is_running(self.source), timeout_sec=30, err_msg="Failed to see connector transition to the RUNNING state") self.cc.pause_connector(self.source.name) self.cc.restart() if connect_protocol == 'compatible': timeout_sec = 120 else: timeout_sec = 30 # we should still be paused after restarting for node in self.cc.nodes: wait_until( lambda: self.is_paused(self.source, node), timeout_sec=timeout_sec, err_msg="Failed to see connector startup in PAUSED state") @cluster(num_nodes=6) @matrix( security_protocol=[SecurityConfig.PLAINTEXT, SecurityConfig.SASL_SSL], connect_protocol=['sessioned', 'compatible', 'eager']) def test_file_source_and_sink(self, security_protocol, connect_protocol): """ Tests that a basic file connector works across clean rolling bounces. This validates that the connector is correctly created, tasks instantiated, and as nodes restart the work is rebalanced across nodes. """ self.CONNECT_PROTOCOL = connect_protocol self.setup_services(security_protocol=security_protocol) self.cc.set_configs(lambda node: self.render( "connect-distributed.properties", node=node)) self.cc.start() self.logger.info("Creating connectors") self._start_connector("connect-file-source.properties") self._start_connector("connect-file-sink.properties") # Generating data on the source node should generate new records and create new output on the sink node. Timeouts # here need to be more generous than they are for standalone mode because a) it takes longer to write configs, # do rebalancing of the group, etc, and b) without explicit leave group support, rebalancing takes awhile for node in self.cc.nodes: node.account.ssh("echo -e -n " + repr(self.FIRST_INPUTS) + " >> " + self.INPUT_FILE) wait_until( lambda: self._validate_file_output(self.FIRST_INPUT_LIST), timeout_sec=70, err_msg= "Data added to input file was not seen in the output file in a reasonable amount of time." ) # Restarting both should result in them picking up where they left off, # only processing new data. self.cc.restart() if connect_protocol == 'compatible': timeout_sec = 150 else: timeout_sec = 70 for node in self.cc.nodes: node.account.ssh("echo -e -n " + repr(self.SECOND_INPUTS) + " >> " + self.INPUT_FILE) wait_until( lambda: self._validate_file_output(self.FIRST_INPUT_LIST + self. SECOND_INPUT_LIST), timeout_sec=timeout_sec, err_msg= "Sink output file never converged to the same state as the input file" ) @cluster(num_nodes=6) @matrix(clean=[True, False], connect_protocol=['sessioned', 'compatible', 'eager']) def test_bounce(self, clean, connect_protocol): """ Validates that source and sink tasks that run continuously and produce a predictable sequence of messages run correctly and deliver messages exactly once when Kafka Connect workers undergo clean rolling bounces. """ num_tasks = 3 self.CONNECT_PROTOCOL = connect_protocol self.setup_services() self.cc.set_configs(lambda node: self.render( "connect-distributed.properties", node=node)) self.cc.start() self.source = VerifiableSource(self.cc, topic=self.TOPIC, tasks=num_tasks, throughput=100) self.source.start() self.sink = VerifiableSink(self.cc, tasks=num_tasks, topics=[self.TOPIC]) self.sink.start() for _ in range(3): for node in self.cc.nodes: started = time.time() self.logger.info("%s bouncing Kafka Connect on %s", clean and "Clean" or "Hard", str(node.account)) self.cc.stop_node(node, clean_shutdown=clean) with node.account.monitor_log(self.cc.LOG_FILE) as monitor: self.cc.start_node(node) monitor.wait_until( "Starting connectors and tasks using config offset", timeout_sec=90, err_msg= "Kafka Connect worker didn't successfully join group and start work" ) self.logger.info( "Bounced Kafka Connect on %s and rejoined in %f seconds", node.account, time.time() - started) # Give additional time for the consumer groups to recover. Even if it is not a hard bounce, there are # some cases where a restart can cause a rebalance to take the full length of the session timeout # (e.g. if the client shuts down before it has received the memberId from its initial JoinGroup). # If we don't give enough time for the group to stabilize, the next bounce may cause consumers to # be shut down before they have any time to process data and we can end up with zero data making it # through the test. time.sleep(15) self.source.stop() self.sink.stop() self.cc.stop() # Validate at least once delivery of everything that was reported as written since we should have flushed and # cleanly exited. Currently this only tests at least once delivery because the sink task may not have consumed # all the messages generated by the source task. This needs to be done per-task since seqnos are not unique across # tasks. success = True errors = [] allow_dups = not clean src_messages = self.source.committed_messages() sink_messages = self.sink.flushed_messages() for task in range(num_tasks): # Validate source messages src_seqnos = [ msg['seqno'] for msg in src_messages if msg['task'] == task ] # Every seqno up to the largest one we ever saw should appear. Each seqno should only appear once because clean # bouncing should commit on rebalance. src_seqno_max = max(src_seqnos) self.logger.debug("Max source seqno: %d", src_seqno_max) src_seqno_counts = Counter(src_seqnos) missing_src_seqnos = sorted( set(range(src_seqno_max)).difference(set(src_seqnos))) duplicate_src_seqnos = sorted([ seqno for seqno, count in src_seqno_counts.iteritems() if count > 1 ]) if missing_src_seqnos: self.logger.error("Missing source sequence numbers for task " + str(task)) errors.append( "Found missing source sequence numbers for task %d: %s" % (task, missing_src_seqnos)) success = False if not allow_dups and duplicate_src_seqnos: self.logger.error( "Duplicate source sequence numbers for task " + str(task)) errors.append( "Found duplicate source sequence numbers for task %d: %s" % (task, duplicate_src_seqnos)) success = False # Validate sink messages sink_seqnos = [ msg['seqno'] for msg in sink_messages if msg['task'] == task ] # Every seqno up to the largest one we ever saw should appear. Each seqno should only appear once because # clean bouncing should commit on rebalance. sink_seqno_max = max(sink_seqnos) self.logger.debug("Max sink seqno: %d", sink_seqno_max) sink_seqno_counts = Counter(sink_seqnos) missing_sink_seqnos = sorted( set(range(sink_seqno_max)).difference(set(sink_seqnos))) duplicate_sink_seqnos = sorted([ seqno for seqno, count in sink_seqno_counts.iteritems() if count > 1 ]) if missing_sink_seqnos: self.logger.error("Missing sink sequence numbers for task " + str(task)) errors.append( "Found missing sink sequence numbers for task %d: %s" % (task, missing_sink_seqnos)) success = False if not allow_dups and duplicate_sink_seqnos: self.logger.error("Duplicate sink sequence numbers for task " + str(task)) errors.append( "Found duplicate sink sequence numbers for task %d: %s" % (task, duplicate_sink_seqnos)) success = False # Validate source and sink match if sink_seqno_max > src_seqno_max: self.logger.error( "Found sink sequence number greater than any generated sink sequence number for task %d: %d > %d", task, sink_seqno_max, src_seqno_max) errors.append( "Found sink sequence number greater than any generated sink sequence number for task %d: %d > %d" % (task, sink_seqno_max, src_seqno_max)) success = False if src_seqno_max < 1000 or sink_seqno_max < 1000: errors.append( "Not enough messages were processed: source:%d sink:%d" % (src_seqno_max, sink_seqno_max)) success = False if not success: self.mark_for_collect(self.cc) # Also collect the data in the topic to aid in debugging consumer_validator = ConsoleConsumer(self.test_context, 1, self.kafka, self.source.topic, consumer_timeout_ms=1000, print_key=True) consumer_validator.run() self.mark_for_collect(consumer_validator, "consumer_stdout") assert success, "Found validation errors:\n" + "\n ".join(errors) @cluster(num_nodes=6) @matrix(connect_protocol=['sessioned', 'compatible', 'eager']) def test_transformations(self, connect_protocol): self.CONNECT_PROTOCOL = connect_protocol self.setup_services(timestamp_type='CreateTime') self.cc.set_configs(lambda node: self.render( "connect-distributed.properties", node=node)) self.cc.start() ts_fieldname = 'the_timestamp' NamedConnector = namedtuple('Connector', ['name']) source_connector = NamedConnector(name='file-src') self.cc.create_connector({ 'name': source_connector.name, 'connector.class': 'org.apache.kafka.connect.file.FileStreamSourceConnector', 'tasks.max': 1, 'file': self.INPUT_FILE, 'topic': self.TOPIC, 'transforms': 'hoistToStruct,insertTimestampField', 'transforms.hoistToStruct.type': 'org.apache.kafka.connect.transforms.HoistField$Value', 'transforms.hoistToStruct.field': 'content', 'transforms.insertTimestampField.type': 'org.apache.kafka.connect.transforms.InsertField$Value', 'transforms.insertTimestampField.timestamp.field': ts_fieldname, }) wait_until( lambda: self.connector_is_running(source_connector), timeout_sec=30, err_msg='Failed to see connector transition to the RUNNING state') for node in self.cc.nodes: node.account.ssh("echo -e -n " + repr(self.FIRST_INPUTS) + " >> " + self.INPUT_FILE) consumer = ConsoleConsumer(self.test_context, 1, self.kafka, self.TOPIC, consumer_timeout_ms=15000, print_timestamp=True) consumer.run() assert len(consumer.messages_consumed[1]) == len(self.FIRST_INPUT_LIST) expected_schema = { 'type': 'struct', 'fields': [ { 'field': 'content', 'type': 'string', 'optional': False }, { 'field': ts_fieldname, 'name': 'org.apache.kafka.connect.data.Timestamp', 'type': 'int64', 'version': 1, 'optional': True }, ], 'optional': False } for msg in consumer.messages_consumed[1]: (ts_info, value) = msg.split('\t') assert ts_info.startswith('CreateTime:') ts = int(ts_info[len('CreateTime:'):]) obj = json.loads(value) assert obj['schema'] == expected_schema assert obj['payload']['content'] in self.FIRST_INPUT_LIST assert obj['payload'][ts_fieldname] == ts @cluster(num_nodes=5) @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned') @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned') @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned') @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned') @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned') @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') @parametrize(broker_version=str(LATEST_2_3), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') @parametrize(broker_version=str(LATEST_2_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') @parametrize(broker_version=str(LATEST_2_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') @parametrize(broker_version=str(LATEST_2_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') @parametrize(broker_version=str(LATEST_1_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') @parametrize(broker_version=str(LATEST_1_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') @parametrize(broker_version=str(LATEST_2_3), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') @parametrize(broker_version=str(LATEST_2_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') @parametrize(broker_version=str(LATEST_2_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') @parametrize(broker_version=str(LATEST_2_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') @parametrize(broker_version=str(LATEST_1_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') @parametrize(broker_version=str(LATEST_1_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') def test_broker_compatibility(self, broker_version, auto_create_topics, security_protocol, connect_protocol): """ Verify that Connect will start up with various broker versions with various configurations. When Connect distributed starts up, it either creates internal topics (v0.10.1.0 and after) or relies upon the broker to auto-create the topics (v0.10.0.x and before). """ self.CONNECT_PROTOCOL = connect_protocol self.setup_services(broker_version=KafkaVersion(broker_version), auto_create_topics=auto_create_topics, security_protocol=security_protocol) self.cc.set_configs(lambda node: self.render( "connect-distributed.properties", node=node)) self.cc.start() self.logger.info("Creating connectors") self._start_connector("connect-file-source.properties") self._start_connector("connect-file-sink.properties") # Generating data on the source node should generate new records and create new output on the sink node. Timeouts # here need to be more generous than they are for standalone mode because a) it takes longer to write configs, # do rebalancing of the group, etc, and b) without explicit leave group support, rebalancing takes awhile for node in self.cc.nodes: node.account.ssh("echo -e -n " + repr(self.FIRST_INPUTS) + " >> " + self.INPUT_FILE) wait_until( lambda: self._validate_file_output(self.FIRST_INPUT_LIST), timeout_sec=70, err_msg= "Data added to input file was not seen in the output file in a reasonable amount of time." ) def _validate_file_output(self, input): input_set = set(input) # Output needs to be collected from all nodes because we can't be sure where the tasks will be scheduled. # Between the first and second rounds, we might even end up with half the data on each node. output_set = set( itertools.chain(*[[ line.strip() for line in self._file_contents(node, self.OUTPUT_FILE) ] for node in self.cc.nodes])) return input_set == output_set def _file_contents(self, node, file): try: # Convert to a list here or the RemoteCommandError may be returned during a call to the generator instead of # immediately return list(node.account.ssh_capture("cat " + file)) except RemoteCommandError: return []
class StreamsUpgradeTest(Test): """ Test upgrading Kafka Streams (all version combination) If metadata was changes, upgrade is more difficult Metadata version was bumped in 0.10.1.0 and subsequently bumped in 2.0.0 """ def __init__(self, test_context): super(StreamsUpgradeTest, self).__init__(test_context) self.topics = { 'echo' : { 'partitions': 5 }, 'data' : { 'partitions': 5 }, } processed_msg = "processed [0-9]* records" base_version_number = str(DEV_VERSION).split("-")[0] def perform_broker_upgrade(self, to_version): self.logger.info("First pass bounce - rolling broker upgrade") for node in self.kafka.nodes: self.kafka.stop_node(node) node.version = KafkaVersion(to_version) self.kafka.start_node(node) @ignore @cluster(num_nodes=6) @matrix(from_version=broker_upgrade_versions, to_version=broker_upgrade_versions) def test_upgrade_downgrade_brokers(self, from_version, to_version): """ Start a smoke test client then perform rolling upgrades on the broker. """ if from_version == to_version: return self.replication = 3 self.num_kafka_nodes = 3 self.partitions = 1 self.isr = 2 self.topics = { 'echo' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr}}, 'data' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'min' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'max' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'sum' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'dif' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'cnt' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'avg' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'wcnt' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} }, 'tagg' : { 'partitions': self.partitions, 'replication-factor': self.replication, 'configs': {"min.insync.replicas": self.isr} } } # Setup phase self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() # number of nodes needs to be >= 3 for the smoke test self.kafka = KafkaService(self.test_context, num_nodes=self.num_kafka_nodes, zk=self.zk, version=KafkaVersion(from_version), topics=self.topics) self.kafka.start() # allow some time for topics to be created wait_until(lambda: self.confirm_topics_on_all_brokers(set(self.topics.keys())), timeout_sec=60, err_msg="Broker did not create all topics in 60 seconds ") self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) processor = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka, "at_least_once") with self.driver.node.account.monitor_log(self.driver.STDOUT_FILE) as driver_monitor: self.driver.start() with processor.node.account.monitor_log(processor.STDOUT_FILE) as monitor: processor.start() monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(processor.node)) connected_message = "Discovered group coordinator" with processor.node.account.monitor_log(processor.LOG_FILE) as log_monitor: with processor.node.account.monitor_log(processor.STDOUT_FILE) as stdout_monitor: self.perform_broker_upgrade(to_version) log_monitor.wait_until(connected_message, timeout_sec=120, err_msg=("Never saw output '%s' on " % connected_message) + str(processor.node.account)) stdout_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on" % self.processed_msg + str(processor.node.account)) # SmokeTestDriver allows up to 6 minutes to consume all # records for the verification step so this timeout is set to # 6 minutes (360 seconds) for consuming of verification records # and a very conservative additional 2 minutes (120 seconds) to process # the records in the verification step driver_monitor.wait_until('ALL-RECORDS-DELIVERED\|PROCESSED-MORE-THAN-GENERATED', timeout_sec=480, err_msg="Never saw output '%s' on" % 'ALL-RECORDS-DELIVERED|PROCESSED-MORE-THAN-GENERATED' + str(self.driver.node.account)) self.driver.stop() processor.stop() processor.node.account.ssh_capture("grep SMOKE-TEST-CLIENT-CLOSED %s" % processor.STDOUT_FILE, allow_fail=False) @cluster(num_nodes=6) @matrix(from_version=metadata_1_versions, to_version=[str(DEV_VERSION)]) @matrix(from_version=metadata_2_versions, to_version=[str(DEV_VERSION)]) def test_metadata_upgrade(self, from_version, to_version): """ Starts 3 KafkaStreams instances with version <from_version> and upgrades one-by-one to <to_version> """ self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() self.kafka = KafkaService(self.test_context, num_nodes=1, zk=self.zk, topics=self.topics) self.kafka.start() self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) self.driver.disable_auto_terminate() self.processor1 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) self.processor2 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) self.processor3 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) self.driver.start() self.start_all_nodes_with(from_version) self.processors = [self.processor1, self.processor2, self.processor3] counter = 1 random.seed() # first rolling bounce random.shuffle(self.processors) for p in self.processors: p.CLEAN_NODE_ENABLED = False self.do_stop_start_bounce(p, from_version[:-2], to_version, counter) counter = counter + 1 # second rolling bounce random.shuffle(self.processors) for p in self.processors: self.do_stop_start_bounce(p, None, to_version, counter) counter = counter + 1 # shutdown self.driver.stop() random.shuffle(self.processors) for p in self.processors: node = p.node with node.account.monitor_log(p.STDOUT_FILE) as monitor: p.stop() monitor.wait_until("UPGRADE-TEST-CLIENT-CLOSED", timeout_sec=60, err_msg="Never saw output 'UPGRADE-TEST-CLIENT-CLOSED' on" + str(node.account)) @cluster(num_nodes=6) def test_version_probing_upgrade(self): """ Starts 3 KafkaStreams instances, and upgrades one-by-one to "future version" """ self.zk = ZookeeperService(self.test_context, num_nodes=1) self.zk.start() self.kafka = KafkaService(self.test_context, num_nodes=1, zk=self.zk, topics=self.topics) self.kafka.start() self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) self.driver.disable_auto_terminate() self.processor1 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) self.processor2 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) self.processor3 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) self.driver.start() self.start_all_nodes_with("") # run with TRUNK self.processors = [self.processor1, self.processor2, self.processor3] self.old_processors = [self.processor1, self.processor2, self.processor3] self.upgraded_processors = [] counter = 1 current_generation = 3 random.seed() random.shuffle(self.processors) for p in self.processors: p.CLEAN_NODE_ENABLED = False current_generation = self.do_rolling_bounce(p, counter, current_generation) counter = counter + 1 # shutdown self.driver.stop() random.shuffle(self.processors) for p in self.processors: node = p.node with node.account.monitor_log(p.STDOUT_FILE) as monitor: p.stop() monitor.wait_until("UPGRADE-TEST-CLIENT-CLOSED", timeout_sec=60, err_msg="Never saw output 'UPGRADE-TEST-CLIENT-CLOSED' on" + str(node.account)) def get_version_string(self, version): if version.startswith("0") or version.startswith("1") \ or version.startswith("2.0") or version.startswith("2.1"): return "Kafka version : " + version elif "SNAPSHOT" in version: return "Kafka version.*" + self.base_version_number + ".*SNAPSHOT" else: return "Kafka version: " + version def start_all_nodes_with(self, version): kafka_version_str = self.get_version_string(version) # start first with <version> self.prepare_for(self.processor1, version) node1 = self.processor1.node with node1.account.monitor_log(self.processor1.STDOUT_FILE) as monitor: with node1.account.monitor_log(self.processor1.LOG_FILE) as log_monitor: self.processor1.start() log_monitor.wait_until(kafka_version_str, timeout_sec=60, err_msg="Could not detect Kafka Streams version " + version + " " + str(node1.account)) monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(node1.account)) # start second with <version> self.prepare_for(self.processor2, version) node2 = self.processor2.node with node1.account.monitor_log(self.processor1.STDOUT_FILE) as first_monitor: with node2.account.monitor_log(self.processor2.STDOUT_FILE) as second_monitor: with node2.account.monitor_log(self.processor2.LOG_FILE) as log_monitor: self.processor2.start() log_monitor.wait_until(kafka_version_str, timeout_sec=60, err_msg="Could not detect Kafka Streams version " + version + " on " + str(node2.account)) first_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(node1.account)) second_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(node2.account)) # start third with <version> self.prepare_for(self.processor3, version) node3 = self.processor3.node with node1.account.monitor_log(self.processor1.STDOUT_FILE) as first_monitor: with node2.account.monitor_log(self.processor2.STDOUT_FILE) as second_monitor: with node3.account.monitor_log(self.processor3.STDOUT_FILE) as third_monitor: with node3.account.monitor_log(self.processor3.LOG_FILE) as log_monitor: self.processor3.start() log_monitor.wait_until(kafka_version_str, timeout_sec=60, err_msg="Could not detect Kafka Streams version " + version + " on " + str(node3.account)) first_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(node1.account)) second_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(node2.account)) third_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(node3.account)) @staticmethod def prepare_for(processor, version): processor.node.account.ssh("rm -rf " + processor.PERSISTENT_ROOT, allow_fail=False) if version == str(DEV_VERSION): processor.set_version("") # set to TRUNK else: processor.set_version(version) def do_stop_start_bounce(self, processor, upgrade_from, new_version, counter): kafka_version_str = self.get_version_string(new_version) first_other_processor = None second_other_processor = None for p in self.processors: if p != processor: if first_other_processor is None: first_other_processor = p else: second_other_processor = p node = processor.node first_other_node = first_other_processor.node second_other_node = second_other_processor.node # stop processor and wait for rebalance of others with first_other_node.account.monitor_log(first_other_processor.STDOUT_FILE) as first_other_monitor: with second_other_node.account.monitor_log(second_other_processor.STDOUT_FILE) as second_other_monitor: processor.stop() first_other_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(first_other_node.account)) second_other_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(second_other_node.account)) node.account.ssh_capture("grep UPGRADE-TEST-CLIENT-CLOSED %s" % processor.STDOUT_FILE, allow_fail=False) if upgrade_from is None: # upgrade disabled -- second round of rolling bounces roll_counter = ".1-" # second round of rolling bounces else: roll_counter = ".0-" # first round of rolling bounces node.account.ssh("mv " + processor.STDOUT_FILE + " " + processor.STDOUT_FILE + roll_counter + str(counter), allow_fail=False) node.account.ssh("mv " + processor.STDERR_FILE + " " + processor.STDERR_FILE + roll_counter + str(counter), allow_fail=False) node.account.ssh("mv " + processor.LOG_FILE + " " + processor.LOG_FILE + roll_counter + str(counter), allow_fail=False) if new_version == str(DEV_VERSION): processor.set_version("") # set to TRUNK else: processor.set_version(new_version) processor.set_upgrade_from(upgrade_from) grep_metadata_error = "grep \"org.apache.kafka.streams.errors.TaskAssignmentException: unable to decode subscription data: version=2\" " with node.account.monitor_log(processor.STDOUT_FILE) as monitor: with node.account.monitor_log(processor.LOG_FILE) as log_monitor: with first_other_node.account.monitor_log(first_other_processor.STDOUT_FILE) as first_other_monitor: with second_other_node.account.monitor_log(second_other_processor.STDOUT_FILE) as second_other_monitor: processor.start() log_monitor.wait_until(kafka_version_str, timeout_sec=60, err_msg="Could not detect Kafka Streams version " + new_version + " on " + str(node.account)) first_other_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(first_other_node.account)) found = list(first_other_node.account.ssh_capture(grep_metadata_error + first_other_processor.STDERR_FILE, allow_fail=True)) if len(found) > 0: raise Exception("Kafka Streams failed with 'unable to decode subscription data: version=2'") second_other_monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(second_other_node.account)) found = list(second_other_node.account.ssh_capture(grep_metadata_error + second_other_processor.STDERR_FILE, allow_fail=True)) if len(found) > 0: raise Exception("Kafka Streams failed with 'unable to decode subscription data: version=2'") monitor.wait_until(self.processed_msg, timeout_sec=60, err_msg="Never saw output '%s' on " % self.processed_msg + str(node.account)) def do_rolling_bounce(self, processor, counter, current_generation): first_other_processor = None second_other_processor = None for p in self.processors: if p != processor: if first_other_processor is None: first_other_processor = p else: second_other_processor = p node = processor.node first_other_node = first_other_processor.node second_other_node = second_other_processor.node with first_other_node.account.monitor_log(first_other_processor.LOG_FILE) as first_other_monitor: with second_other_node.account.monitor_log(second_other_processor.LOG_FILE) as second_other_monitor: # stop processor processor.stop() node.account.ssh_capture("grep UPGRADE-TEST-CLIENT-CLOSED %s" % processor.STDOUT_FILE, allow_fail=False) node.account.ssh("mv " + processor.STDOUT_FILE + " " + processor.STDOUT_FILE + "." + str(counter), allow_fail=False) node.account.ssh("mv " + processor.STDERR_FILE + " " + processor.STDERR_FILE + "." + str(counter), allow_fail=False) node.account.ssh("mv " + processor.LOG_FILE + " " + processor.LOG_FILE + "." + str(counter), allow_fail=False) with node.account.monitor_log(processor.LOG_FILE) as log_monitor: processor.set_upgrade_to("future_version") processor.start() self.old_processors.remove(processor) self.upgraded_processors.append(processor) # checking for the dev version which should be the only SNAPSHOT log_monitor.wait_until("Kafka version.*" + self.base_version_number + ".*SNAPSHOT", timeout_sec=60, err_msg="Could not detect Kafka Streams version " + str(DEV_VERSION) + " in " + str(node.account)) log_monitor.offset = 5 log_monitor.wait_until("partition\.assignment\.strategy = \[org\.apache\.kafka\.streams\.tests\.StreamsUpgradeTest$FutureStreamsPartitionAssignor\]", timeout_sec=60, err_msg="Could not detect FutureStreamsPartitionAssignor in " + str(node.account)) monitors = {} monitors[processor] = log_monitor monitors[first_other_processor] = first_other_monitor monitors[second_other_processor] = second_other_monitor highest_version = 10 version_probing_message = "Sent a version " + str(highest_version + 1) + " subscription and got version " + str(highest_version) + " assignment back (successful version probing). Downgrade subscription metadata to commonly supported version " + str(highest_version) + " and trigger new rebalance." end_of_upgrade_message = "Sent a version " + str(highest_version) + " subscription and group.s latest commonly supported version is " + str(highest_version + 1) + " (successful version probing and end of rolling upgrade). Upgrading subscription metadata version to " + str(highest_version + 1) + " for next rebalance." end_of_upgrade_error_message = "Could not detect 'successful version probing and end of rolling upgrade' at upgraded node " followup_rebalance_message = "Triggering the followup rebalance scheduled for 0 ms." followup_rebalance_error_message = "Could not detect 'Triggering followup rebalance' at node " if len(self.old_processors) > 0: log_monitor.wait_until(version_probing_message, timeout_sec=60, err_msg="Could not detect 'successful version probing' at upgrading node " + str(node.account)) log_monitor.wait_until(followup_rebalance_message, timeout_sec=60, err_msg=followup_rebalance_error_message + str(node.account)) else: first_other_monitor.wait_until(end_of_upgrade_message, timeout_sec=60, err_msg=end_of_upgrade_error_message + str(first_other_node.account)) first_other_monitor.wait_until(followup_rebalance_message, timeout_sec=60, err_msg=followup_rebalance_error_message + str(node.account)) second_other_monitor.wait_until(end_of_upgrade_message, timeout_sec=60, err_msg=end_of_upgrade_error_message + str(second_other_node.account)) second_other_monitor.wait_until(followup_rebalance_message, timeout_sec=60, err_msg=followup_rebalance_error_message + str(node.account)) # version probing should trigger second rebalance # now we check that after consecutive rebalances we have synchronized generation generation_synchronized = False retries = 0 while retries < 10: processor_found = extract_generation_from_logs(processor) first_other_processor_found = extract_generation_from_logs(first_other_processor) second_other_processor_found = extract_generation_from_logs(second_other_processor) if len(processor_found) > 0 and len(first_other_processor_found) > 0 and len(second_other_processor_found) > 0: self.logger.info("processor: " + str(processor_found)) self.logger.info("first other processor: " + str(first_other_processor_found)) self.logger.info("second other processor: " + str(second_other_processor_found)) processor_generation = self.extract_highest_generation(processor_found) first_other_processor_generation = self.extract_highest_generation(first_other_processor_found) second_other_processor_generation = self.extract_highest_generation(second_other_processor_found) if processor_generation == first_other_processor_generation and processor_generation == second_other_processor_generation: current_generation = processor_generation generation_synchronized = True break time.sleep(5) retries = retries + 1 if generation_synchronized == False: raise Exception("Never saw all three processors have the synchronized generation number") if len(self.old_processors) > 0: self.verify_metadata_no_upgraded_yet(end_of_upgrade_message) return current_generation def extract_highest_generation(self, found_generations): return extract_generation_id(found_generations[-1]) def verify_metadata_no_upgraded_yet(self, end_of_upgrade_message): for p in self.processors: found = list(p.node.account.ssh_capture("grep \"" + end_of_upgrade_message + "\" " + p.LOG_FILE, allow_fail=True)) if len(found) > 0: raise Exception("Kafka Streams failed with 'group member upgraded to metadata 8 too early'") def confirm_topics_on_all_brokers(self, expected_topic_set): for node in self.kafka.nodes: match_count = 0 # need to iterate over topic_list_generator as kafka.list_topics() # returns a python generator so values are fetched lazily # so we can't just compare directly we must iterate over what's returned topic_list_generator = self.kafka.list_topics(node=node) for topic in topic_list_generator: if topic in expected_topic_set: match_count += 1 if len(expected_topic_set) != match_count: return False return True
class RoundTripFaultTest(Test): topic_name_index = 0 def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(RoundTripFaultTest, self).__init__(test_context) self.zk = ZookeeperService(test_context, num_nodes=3) self.kafka = KafkaService(test_context, num_nodes=4, zk=self.zk) self.workload_service = RoundTripWorkloadService( test_context, self.kafka) self.trogdor = TrogdorService( context=self.test_context, client_services=[self.zk, self.kafka, self.workload_service]) topic_name = "round_trip_topic%d" % RoundTripFaultTest.topic_name_index RoundTripFaultTest.topic_name_index = RoundTripFaultTest.topic_name_index + 1 active_topics = { topic_name: { "partitionAssignments": { "0": [0, 1, 2] } } } self.round_trip_spec = RoundTripWorkloadSpec( 0, TaskSpec.MAX_DURATION_MS, self.workload_service.client_node, self.workload_service.bootstrap_servers, target_messages_per_sec=10000, max_messages=100000, active_topics=active_topics) def setUp(self): self.zk.start() self.kafka.start() self.trogdor.start() def teardown(self): self.trogdor.stop() self.kafka.stop() self.zk.stop() def test_round_trip_workload(self): workload1 = self.trogdor.create_task("workload1", self.round_trip_spec) workload1.wait_for_done(timeout_sec=600) def test_round_trip_workload_with_broker_partition(self): workload1 = self.trogdor.create_task("workload1", self.round_trip_spec) time.sleep(2) part1 = [self.kafka.nodes[0]] part2 = self.kafka.nodes[1:] + [self.workload_service.nodes[0] ] + self.zk.nodes partition1_spec = NetworkPartitionFaultSpec(0, TaskSpec.MAX_DURATION_MS, [part1, part2]) partition1 = self.trogdor.create_task("partition1", partition1_spec) workload1.wait_for_done(timeout_sec=600) partition1.stop() partition1.wait_for_done() def test_produce_consume_with_broker_pause(self): workload1 = self.trogdor.create_task("workload1", self.round_trip_spec) time.sleep(2) stop1_spec = ProcessStopFaultSpec(0, TaskSpec.MAX_DURATION_MS, [self.kafka.nodes[0]], self.kafka.java_class_name()) stop1 = self.trogdor.create_task("stop1", stop1_spec) workload1.wait_for_done(timeout_sec=600) stop1.stop() stop1.wait_for_done() self.kafka.stop_node(self.kafka.nodes[0], False) def test_produce_consume_with_client_partition(self): workload1 = self.trogdor.create_task("workload1", self.round_trip_spec) time.sleep(2) part1 = [self.workload_service.nodes[0]] part2 = self.kafka.nodes + self.zk.nodes partition1_spec = NetworkPartitionFaultSpec(0, 60000, [part1, part2]) stop1 = self.trogdor.create_task("stop1", partition1_spec) workload1.wait_for_done(timeout_sec=600) stop1.stop() stop1.wait_for_done()