class ClientCompatibilityFeaturesTest(Test):
"""
Tests clients for the presence or absence of specific features when communicating with brokers with various
versions. Relies on ClientCompatibilityTest.java for much of the functionality.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ClientCompatibilityFeaturesTest,
self).__init__(test_context=test_context)
self.zk = ZookeeperService(test_context, num_nodes=3)
# Generate a unique topic name
topic_name = "client_compat_features_topic_%d%d" % (int(
time.time()), randint(0, 2147483647))
self.topics = {
topic_name: {
"partitions":
1, # Use only one partition to avoid worrying about ordering
"replication-factor": 3
}
}
self.kafka = KafkaService(test_context,
num_nodes=3,
zk=self.zk,
topics=self.topics)
def invoke_compatibility_program(self, features):
# Run the compatibility test on the first Kafka node.
node = self.zk.nodes[0]
cmd = ("%s org.apache.kafka.tools.ClientCompatibilityTest "
"--bootstrap-server %s "
"--num-cluster-nodes %d "
"--topic %s " % (self.zk.path.script(
"kafka-run-class.sh", node), self.kafka.bootstrap_servers(),
len(self.kafka.nodes), self.topics.keys()[0]))
for k, v in features.iteritems():
cmd = cmd + ("--%s %s " % (k, v))
results_dir = TestContext.results_dir(self.test_context, 0)
os.makedirs(results_dir)
ssh_log_file = "%s/%s" % (results_dir,
"client_compatibility_test_output.txt")
try:
self.logger.info("Running %s" % cmd)
run_command(node, cmd, ssh_log_file)
except Exception as e:
self.logger.info("** Command failed. See %s for log messages." %
ssh_log_file)
raise
@parametrize(broker_version=str(DEV_BRANCH))
@parametrize(broker_version=str(LATEST_0_10_0))
@parametrize(broker_version=str(LATEST_0_10_1))
@parametrize(broker_version=str(LATEST_0_10_2))
def run_compatibility_test(self, broker_version):
self.zk.start()
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
features = get_broker_features(broker_version)
self.invoke_compatibility_program(features)
class ClientCompatibilityProduceConsumeTest(ProduceConsumeValidateTest):
"""
These tests validate that we can use a new client to produce and consume from older brokers.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ClientCompatibilityProduceConsumeTest, self).__init__(test_context=test_context)
self.topic = "test_topic"
self.zk = ZookeeperService(test_context, num_nodes=3)
self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics={self.topic:{
"partitions": 10,
"replication-factor": 2}})
self.num_partitions = 10
self.timeout_sec = 60
self.producer_throughput = 1000
self.num_producers = 2
self.messages_per_producer = 1000
self.num_consumers = 1
def setUp(self):
self.zk.start()
def min_cluster_size(self):
# Override this since we're adding services outside of the constructor
return super(ClientCompatibilityProduceConsumeTest, self).min_cluster_size() + self.num_producers + self.num_consumers
@parametrize(broker_version=str(DEV_BRANCH))
@parametrize(broker_version=str(LATEST_0_10_0))
@parametrize(broker_version=str(LATEST_0_10_1))
@parametrize(broker_version=str(LATEST_0_10_2))
@parametrize(broker_version=str(LATEST_0_11_0))
@parametrize(broker_version=str(LATEST_1_0))
@parametrize(broker_version=str(LATEST_1_1))
@parametrize(broker_version=str(LATEST_2_0))
@parametrize(broker_version=str(LATEST_2_1))
@parametrize(broker_version=str(LATEST_2_2))
@parametrize(broker_version=str(LATEST_2_3))
@parametrize(broker_version=str(LATEST_2_4))
@parametrize(broker_version=str(LATEST_2_5))
def test_produce_consume(self, broker_version):
print("running producer_consumer_compat with broker_version = %s" % broker_version, flush=True)
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.security_protocol = "PLAINTEXT"
self.kafka.interbroker_security_protocol = self.kafka.security_protocol
self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka,
self.topic, throughput=self.producer_throughput,
message_validator=is_int_with_prefix)
self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic,
consumer_timeout_ms=60000,
message_validator=is_int_with_prefix)
self.kafka.start()
self.run_produce_consume_validate(lambda: wait_until(
lambda: self.producer.each_produced_at_least(self.messages_per_producer) == True,
timeout_sec=120, backoff_sec=1,
err_msg="Producer did not produce all messages in reasonable amount of time"))
class ClientCompatibilityFeaturesTest(Test):
"""
Tests clients for the presence or absence of specific features when communicating with brokers with various
versions. Relies on ClientCompatibilityTest.java for much of the functionality.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ClientCompatibilityFeaturesTest, self).__init__(test_context=test_context)
self.zk = ZookeeperService(test_context, num_nodes=3)
# Generate a unique topic name
topic_name = "client_compat_features_topic_%d%d" % (int(time.time()), randint(0, 2147483647))
self.topics = { topic_name: {
"partitions": 1, # Use only one partition to avoid worrying about ordering
"replication-factor": 3
}}
self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics=self.topics)
def invoke_compatibility_program(self, features):
# Run the compatibility test on the first Kafka node.
node = self.zk.nodes[0]
cmd = ("%s org.apache.kafka.tools.ClientCompatibilityTest "
"--bootstrap-server %s "
"--num-cluster-nodes %d "
"--topic %s " % (self.zk.path.script("kafka-run-class.sh", node),
self.kafka.bootstrap_servers(),
len(self.kafka.nodes),
self.topics.keys()[0]))
for k, v in features.iteritems():
cmd = cmd + ("--%s %s " % (k, v))
results_dir = TestContext.results_dir(self.test_context, 0)
os.makedirs(results_dir)
ssh_log_file = "%s/%s" % (results_dir, "client_compatibility_test_output.txt")
try:
self.logger.info("Running %s" % cmd)
run_command(node, cmd, ssh_log_file)
except Exception as e:
self.logger.info("** Command failed. See %s for log messages." % ssh_log_file)
raise
@parametrize(broker_version=str(DEV_BRANCH))
@parametrize(broker_version=str(LATEST_0_10_0))
@parametrize(broker_version=str(LATEST_0_10_1))
@parametrize(broker_version=str(LATEST_0_10_2))
@parametrize(broker_version=str(LATEST_0_11_0))
@parametrize(broker_version=str(LATEST_1_0))
@parametrize(broker_version=str(LATEST_1_1))
def run_compatibility_test(self, broker_version):
self.zk.start()
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
features = get_broker_features(broker_version)
self.invoke_compatibility_program(features)
class ClientCompatibilityProduceConsumeTest(ProduceConsumeValidateTest):
"""
These tests validate that we can use a new client to produce and consume from older brokers.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ClientCompatibilityProduceConsumeTest, self).__init__(test_context=test_context)
self.topic = "test_topic"
self.zk = ZookeeperService(test_context, num_nodes=3)
self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics={self.topic:{
"partitions": 10,
"replication-factor": 2}})
self.num_partitions = 10
self.timeout_sec = 60
self.producer_throughput = 1000
self.num_producers = 2
self.messages_per_producer = 1000
self.num_consumers = 1
def setUp(self):
self.zk.start()
def min_cluster_size(self):
# Override this since we're adding services outside of the constructor
return super(ClientCompatibilityProduceConsumeTest, self).min_cluster_size() + self.num_producers + self.num_consumers
@parametrize(broker_version=str(DEV_BRANCH))
@parametrize(broker_version=str(LATEST_0_10_0))
@parametrize(broker_version=str(LATEST_0_10_1))
@parametrize(broker_version=str(LATEST_0_10_2))
@parametrize(broker_version=str(LATEST_0_11_0))
@parametrize(broker_version=str(LATEST_1_0))
@parametrize(broker_version=str(LATEST_1_1))
def test_produce_consume(self, broker_version):
print("running producer_consumer_compat with broker_version = %s" % broker_version)
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.security_protocol = "PLAINTEXT"
self.kafka.interbroker_security_protocol = self.kafka.security_protocol
self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka,
self.topic, throughput=self.producer_throughput,
message_validator=is_int_with_prefix)
self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic,
consumer_timeout_ms=60000,
message_validator=is_int_with_prefix)
self.kafka.start()
self.run_produce_consume_validate(lambda: wait_until(
lambda: self.producer.each_produced_at_least(self.messages_per_producer) == True,
timeout_sec=120, backoff_sec=1,
err_msg="Producer did not produce all messages in reasonable amount of time"))
class StreamsBrokerCompatibility(Test):
"""
These tests validates that
- Streams 0.11+ w/ EOS fails fast for older brokers 0.10.2 and 0.10.1
- Streams 0.11+ w/o EOS works for older brokers 0.10.2 and 0.10.1
- Streams fails fast for 0.10.0 brokers
- Streams times-out for pre-0.10.0 brokers
"""
input = "brokerCompatibilitySourceTopic"
output = "brokerCompatibilitySinkTopic"
def __init__(self, test_context):
super(StreamsBrokerCompatibility,
self).__init__(test_context=test_context)
self.zk = ZookeeperService(test_context, num_nodes=1)
self.kafka = KafkaService(test_context,
num_nodes=1,
zk=self.zk,
topics={
self.input: {
'partitions': 1,
'replication-factor': 1
},
self.output: {
'partitions': 1,
'replication-factor': 1
}
})
self.consumer = VerifiableConsumer(
test_context, 1, self.kafka, self.output,
"stream-broker-compatibility-verify-consumer")
def setUp(self):
self.zk.start()
@parametrize(broker_version=str(LATEST_0_10_2))
@parametrize(broker_version=str(LATEST_0_10_1))
def test_fail_fast_on_incompatible_brokers_if_eos_enabled(
self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
processor = StreamsBrokerCompatibilityService(self.test_context,
self.kafka, True)
processor.start()
processor.node.account.ssh(processor.start_cmd(processor.node))
with processor.node.account.monitor_log(
processor.STDERR_FILE) as monitor:
monitor.wait_until(
'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support LIST_OFFSETS ',
timeout_sec=60,
err_msg=
"Never saw 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support LIST_OFFSETS ' error message "
+ str(processor.node.account))
self.kafka.stop()
@parametrize(broker_version=str(LATEST_0_11_0))
@parametrize(broker_version=str(LATEST_0_10_2))
@parametrize(broker_version=str(LATEST_0_10_1))
def test_compatible_brokers_eos_disabled(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
processor = StreamsBrokerCompatibilityService(self.test_context,
self.kafka, False)
processor.start()
self.consumer.start()
processor.wait()
wait_until(
lambda: self.consumer.total_consumed() > 0,
timeout_sec=30,
err_msg=
"Did expect to read a message but got none within 30 seconds.")
self.consumer.stop()
self.kafka.stop()
@parametrize(broker_version=str(LATEST_0_10_0))
def test_fail_fast_on_incompatible_brokers(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
processor = StreamsBrokerCompatibilityService(self.test_context,
self.kafka, False)
processor.start()
processor.node.account.ssh(processor.start_cmd(processor.node))
with processor.node.account.monitor_log(
processor.STDERR_FILE) as monitor:
monitor.wait_until(
'FATAL: An unexpected exception org.apache.kafka.streams.errors.StreamsException: Could not create internal topics.',
timeout_sec=60,
err_msg=
"Never saw 'FATAL: An unexpected exception org.apache.kafka.streams.errors.StreamsException: Could not create internal topics.' error message "
+ str(processor.node.account))
self.kafka.stop()
@ignore
@parametrize(broker_version=str(LATEST_0_9))
@parametrize(broker_version=str(LATEST_0_8_2))
def test_timeout_on_pre_010_brokers(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
processor = StreamsBrokerCompatibilityService(self.test_context,
self.kafka, False)
processor.start()
processor.node.account.ssh(processor.start_cmd(processor.node))
with processor.node.account.monitor_log(
processor.STDERR_FILE) as monitor:
monitor.wait_until(
'Exception in thread "main" org.apache.kafka.streams.errors.BrokerNotFoundException: Could not find any available broker.',
timeout_sec=60,
err_msg="Never saw 'no available brokers' error message " +
str(processor.node.account))
self.kafka.stop()
class ClientCompatibilityFeaturesTest(Test):
"""
Tests clients for the presence or absence of specific features when communicating with brokers with various
versions. Relies on ClientCompatibilityTest.java for much of the functionality.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(ClientCompatibilityFeaturesTest,
self).__init__(test_context=test_context)
self.zk = ZookeeperService(test_context,
num_nodes=3) if quorum.for_test(
test_context) == quorum.zk else None
# Generate a unique topic name
topic_name = "client_compat_features_topic_%d%d" % (int(
time.time()), randint(0, 2147483647))
self.topics = {
topic_name: {
"partitions":
1, # Use only one partition to avoid worrying about ordering
"replication-factor": 3
}
}
self.kafka = KafkaService(test_context,
num_nodes=3,
zk=self.zk,
topics=self.topics)
# Always use the latest version of org.apache.kafka.tools.ClientCompatibilityTest
# so store away the path to the DEV version before we set the Kafka version
self.dev_script_path = self.kafka.path.script("kafka-run-class.sh",
self.kafka.nodes[0])
def invoke_compatibility_program(self, features):
# Run the compatibility test on the first Kafka node.
node = self.kafka.nodes[0]
cmd = ("%s org.apache.kafka.tools.ClientCompatibilityTest "
"--bootstrap-server %s "
"--num-cluster-nodes %d "
"--topic %s " %
(self.dev_script_path, self.kafka.bootstrap_servers(),
len(self.kafka.nodes), list(self.topics.keys())[0]))
for k, v in features.items():
cmd = cmd + ("--%s %s " % (k, v))
results_dir = TestContext.results_dir(self.test_context, 0)
try:
os.makedirs(results_dir)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(results_dir):
pass
else:
raise
ssh_log_file = "%s/%s" % (results_dir,
"client_compatibility_test_output.txt")
try:
self.logger.info("Running %s" % cmd)
run_command(node, cmd, ssh_log_file)
except Exception as e:
self.logger.info("** Command failed. See %s for log messages." %
ssh_log_file)
raise
@cluster(num_nodes=7)
@matrix(broker_version=[str(DEV_BRANCH)],
metadata_quorum=quorum.all_non_upgrade)
@parametrize(broker_version=str(LATEST_0_10_0))
@parametrize(broker_version=str(LATEST_0_10_1))
@parametrize(broker_version=str(LATEST_0_10_2))
@parametrize(broker_version=str(LATEST_0_11_0))
@parametrize(broker_version=str(LATEST_1_0))
@parametrize(broker_version=str(LATEST_1_1))
@parametrize(broker_version=str(LATEST_2_0))
@parametrize(broker_version=str(LATEST_2_1))
@parametrize(broker_version=str(LATEST_2_2))
@parametrize(broker_version=str(LATEST_2_3))
@parametrize(broker_version=str(LATEST_2_4))
@parametrize(broker_version=str(LATEST_2_5))
@parametrize(broker_version=str(LATEST_2_6))
@parametrize(broker_version=str(LATEST_2_7))
@parametrize(broker_version=str(LATEST_2_8))
@parametrize(broker_version=str(LATEST_3_0))
@parametrize(broker_version=str(LATEST_3_1))
def run_compatibility_test(self,
broker_version,
metadata_quorum=quorum.zk):
if self.zk:
self.zk.start()
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
features = get_broker_features(broker_version)
self.invoke_compatibility_program(features)
class StreamsBrokerCompatibility(Test):
"""
These tests validate that Streams v0.10.2+ can connect to older brokers v0.10.1+
and that Streams fails fast for pre-0.10.0 brokers
"""
input = "brokerCompatibilitySourceTopic"
output = "brokerCompatibilitySinkTopic"
def __init__(self, test_context):
super(StreamsBrokerCompatibility,
self).__init__(test_context=test_context)
self.zk = ZookeeperService(test_context, num_nodes=1)
self.kafka = KafkaService(test_context,
num_nodes=1,
zk=self.zk,
topics={
self.input: {
'partitions': 1,
'replication-factor': 1
},
self.output: {
'partitions': 1,
'replication-factor': 1
}
})
self.processor = StreamsBrokerCompatibilityService(
self.test_context, self.kafka)
self.consumer = VerifiableConsumer(
test_context, 1, self.kafka, self.output,
"stream-broker-compatibility-verify-consumer")
def setUp(self):
self.zk.start()
@parametrize(broker_version=str(DEV_BRANCH))
@parametrize(broker_version=str(LATEST_0_10_1))
def test_compatible_brokers(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
self.processor.start()
self.consumer.start()
self.processor.wait()
wait_until(
lambda: self.consumer.total_consumed() > 0,
timeout_sec=30,
err_msg=
"Did expect to read a message but got none within 30 seconds.")
self.consumer.stop()
self.kafka.stop()
@parametrize(broker_version=str(LATEST_0_10_0))
def test_fail_fast_on_incompatible_brokers(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
self.processor.start()
self.processor.node.account.ssh(
self.processor.start_cmd(self.processor.node))
with self.processor.node.account.monitor_log(
self.processor.STDERR_FILE) as monitor:
monitor.wait_until(
'Exception in thread "main" org.apache.kafka.streams.errors.StreamsException: Kafka Streams requires broker version 0.10.1.x or higher.',
timeout_sec=60,
err_msg="Never saw 'incompatible broker' error message " +
str(self.processor.node.account))
self.kafka.stop()
class StreamsBrokerCompatibility(Test):
"""
These tests validates that
- Streams works for older brokers 0.11 (or newer)
- Streams w/ EOS-alpha works for older brokers 0.11 (or newer)
- Streams w/ EOS-beta works for older brokers 2.5 (or newer)
- Streams fails fast for older brokers 0.10.0, 0.10.2, and 0.10.1
- Streams w/ EOS-beta fails fast for older brokers 2.4 or older
"""
input = "brokerCompatibilitySourceTopic"
output = "brokerCompatibilitySinkTopic"
def __init__(self, test_context):
super(StreamsBrokerCompatibility,
self).__init__(test_context=test_context)
self.zk = ZookeeperService(test_context, num_nodes=1)
self.kafka = KafkaService(
test_context,
num_nodes=1,
zk=self.zk,
topics={
self.input: {
'partitions': 1,
'replication-factor': 1
},
self.output: {
'partitions': 1,
'replication-factor': 1
}
},
server_prop_overides=[[
"transaction.state.log.replication.factor", "1"
], ["transaction.state.log.min.isr", "1"]])
self.consumer = VerifiableConsumer(
test_context, 1, self.kafka, self.output,
"stream-broker-compatibility-verify-consumer")
def setUp(self):
self.zk.start()
@parametrize(broker_version=str(LATEST_2_4))
@parametrize(broker_version=str(LATEST_2_3))
@parametrize(broker_version=str(LATEST_2_2))
@parametrize(broker_version=str(LATEST_2_1))
@parametrize(broker_version=str(LATEST_2_0))
@parametrize(broker_version=str(LATEST_1_1))
@parametrize(broker_version=str(LATEST_1_0))
@parametrize(broker_version=str(LATEST_0_11_0))
def test_compatible_brokers_eos_disabled(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
processor = StreamsBrokerCompatibilityService(self.test_context,
self.kafka,
"at_least_once")
processor.start()
self.consumer.start()
processor.wait()
wait_until(
lambda: self.consumer.total_consumed() > 0,
timeout_sec=30,
err_msg=
"Did expect to read a message but got none within 30 seconds.")
self.consumer.stop()
self.kafka.stop()
@parametrize(broker_version=str(LATEST_2_5))
@parametrize(broker_version=str(LATEST_2_4))
@parametrize(broker_version=str(LATEST_2_3))
@parametrize(broker_version=str(LATEST_2_2))
@parametrize(broker_version=str(LATEST_2_1))
@parametrize(broker_version=str(LATEST_2_0))
@parametrize(broker_version=str(LATEST_1_1))
@parametrize(broker_version=str(LATEST_1_0))
@parametrize(broker_version=str(LATEST_0_11_0))
def test_compatible_brokers_eos_alpha_enabled(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
processor = StreamsBrokerCompatibilityService(self.test_context,
self.kafka,
"exactly_once")
processor.start()
self.consumer.start()
processor.wait()
wait_until(
lambda: self.consumer.total_consumed() > 0,
timeout_sec=30,
err_msg=
"Did expect to read a message but got none within 30 seconds.")
self.consumer.stop()
self.kafka.stop()
# TODO enable after 2.5 is released
# @parametrize(broker_version=str(LATEST_2_5))
# def test_compatible_brokers_eos_beta_enabled(self, broker_version):
# self.kafka.set_version(KafkaVersion(broker_version))
# self.kafka.start()
#
# processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, "exactly_once_beta")
# processor.start()
#
# self.consumer.start()
#
# processor.wait()
#
# wait_until(lambda: self.consumer.total_consumed() > 0, timeout_sec=30, err_msg="Did expect to read a message but got none within 30 seconds.")
#
# self.consumer.stop()
# self.kafka.stop()
@parametrize(broker_version=str(LATEST_0_10_2))
@parametrize(broker_version=str(LATEST_0_10_1))
@parametrize(broker_version=str(LATEST_0_10_0))
def test_fail_fast_on_incompatible_brokers(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
processor = StreamsBrokerCompatibilityService(self.test_context,
self.kafka,
"at_least_once")
with processor.node.account.monitor_log(
processor.STDERR_FILE) as monitor:
processor.start()
monitor.wait_until(
'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException',
timeout_sec=60,
err_msg=
"Never saw 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException "
+ str(processor.node.account))
self.kafka.stop()
@parametrize(broker_version=str(LATEST_2_4))
@parametrize(broker_version=str(LATEST_2_3))
@parametrize(broker_version=str(LATEST_2_2))
@parametrize(broker_version=str(LATEST_2_1))
@parametrize(broker_version=str(LATEST_2_0))
@parametrize(broker_version=str(LATEST_1_1))
@parametrize(broker_version=str(LATEST_1_0))
@parametrize(broker_version=str(LATEST_0_11_0))
def test_fail_fast_on_incompatible_brokers_if_eos_beta_enabled(
self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
processor = StreamsBrokerCompatibilityService(self.test_context,
self.kafka,
"exactly_once_beta")
with processor.node.account.monitor_log(
processor.STDERR_FILE) as monitor:
with processor.node.account.monitor_log(processor.LOG_FILE) as log:
processor.start()
log.wait_until(
'Shutting down because the Kafka cluster seems to be on a too old version. Setting processing\.guarantee="exactly_once_beta" requires broker version 2\.5 or higher\.',
timeout_sec=60,
err_msg=
"Never saw 'Shutting down, because the Kafka cluster seems to be on a too old version. Setting `processing.guarantee=\"exaclty_once_beta\"` requires broker version 2.5 or higher.' log message "
+ str(processor.node.account))
monitor.wait_until(
'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException',
timeout_sec=60,
err_msg=
"Never saw 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException' error message "
+ str(processor.node.account))
self.kafka.stop()
class StreamsBrokerCompatibility(Test):
"""
These tests validate that Streams v0.10.2+ can connect to older brokers v0.10+
and that Streams fails fast for pre-0.10 brokers
"""
input = "brokerCompatibilitySourceTopic"
output = "brokerCompatibilitySinkTopic"
def __init__(self, test_context):
super(StreamsBrokerCompatibility, self).__init__(test_context=test_context)
self.zk = ZookeeperService(test_context, num_nodes=1)
self.kafka = KafkaService(test_context,
num_nodes=1,
zk=self.zk,
topics={
self.input: {'partitions': 1, 'replication-factor': 1},
self.output: {'partitions': 1, 'replication-factor': 1}
})
self.processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka)
self.consumer = VerifiableConsumer(test_context,
1,
self.kafka,
self.output,
"stream-broker-compatibility-verify-consumer")
def setUp(self):
self.zk.start()
@parametrize(broker_version=str(DEV_BRANCH))
@parametrize(broker_version=str(LATEST_0_10_1))
def test_compatible_brokers(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
self.processor.start()
self.consumer.start()
self.processor.wait()
num_consumed_mgs = self.consumer.total_consumed()
self.consumer.stop()
self.kafka.stop()
assert num_consumed_mgs == 1, \
"Did expect to read exactly one message but got %d" % num_consumed_mgs
@parametrize(broker_version=str(LATEST_0_10_0))
def test_fail_fast_on_incompatible_brokers(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
self.processor.start()
self.processor.node.account.ssh(self.processor.start_cmd(self.processor.node))
with self.processor.node.account.monitor_log(self.processor.STDERR_FILE) as monitor:
monitor.wait_until('Exception in thread "main" org.apache.kafka.streams.errors.StreamsException: Kafka Streams requires broker version 0.10.1.x or higher.',
timeout_sec=60,
err_msg="Never saw 'incompatible broker' error message " + str(self.processor.node.account))
self.kafka.stop()
class QuotaTest(Test):
"""
These tests verify that quota provides expected functionality -- they run
producer, broker, and consumer with different clientId and quota configuration and
check that the observed throughput is close to the value we expect.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(QuotaTest, self).__init__(test_context=test_context)
self.topic = 'test_topic'
self.logger.info('use topic ' + self.topic)
self.maximum_client_deviation_percentage = 100.0
self.maximum_broker_deviation_percentage = 5.0
self.num_records = 50000
self.record_size = 3000
self.zk = ZookeeperService(test_context, num_nodes=1)
self.kafka = KafkaService(
test_context,
num_nodes=1,
zk=self.zk,
security_protocol='SSL',
authorizer_class_name='',
interbroker_security_protocol='SSL',
topics={
self.topic: {
'partitions': 6,
'replication-factor': 1,
'configs': {
'min.insync.replicas': 1
}
}
},
jmx_object_names=[
'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec',
'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec'
],
jmx_attributes=['OneMinuteRate'])
self.num_producers = 1
self.num_consumers = 2
def setUp(self):
self.zk.start()
def min_cluster_size(self):
"""Override this since we're adding services outside of the constructor"""
return super(
QuotaTest,
self).min_cluster_size() + self.num_producers + self.num_consumers
@cluster(num_nodes=5)
@matrix(quota_type=[
QuotaConfig.CLIENT_ID, QuotaConfig.USER, QuotaConfig.USER_CLIENT
],
override_quota=[True, False])
@parametrize(quota_type=QuotaConfig.CLIENT_ID, consumer_num=2)
@parametrize(quota_type=QuotaConfig.CLIENT_ID,
old_broker_throttling_behavior=True)
@parametrize(quota_type=QuotaConfig.CLIENT_ID,
old_client_throttling_behavior=True)
def test_quota(self,
quota_type,
override_quota=True,
producer_num=1,
consumer_num=1,
old_broker_throttling_behavior=False,
old_client_throttling_behavior=False):
# Old (pre-2.0) throttling behavior for broker throttles before sending a response to the client.
if old_broker_throttling_behavior:
self.kafka.set_version(LATEST_1_1)
self.kafka.start()
self.quota_config = QuotaConfig(quota_type, override_quota, self.kafka)
producer_client_id = self.quota_config.client_id
consumer_client_id = self.quota_config.client_id
# Old (pre-2.0) throttling behavior for client does not throttle upon receiving a response with a non-zero throttle time.
if old_client_throttling_behavior:
client_version = LATEST_1_1
else:
client_version = DEV_BRANCH
# Produce all messages
producer = ProducerPerformanceService(self.test_context,
producer_num,
self.kafka,
topic=self.topic,
num_records=self.num_records,
record_size=self.record_size,
throughput=-1,
client_id=producer_client_id,
version=client_version)
producer.run()
# Consume all messages
consumer = ConsoleConsumer(
self.test_context,
consumer_num,
self.kafka,
self.topic,
consumer_timeout_ms=60000,
client_id=consumer_client_id,
jmx_object_names=[
'kafka.consumer:type=consumer-fetch-manager-metrics,client-id=%s'
% consumer_client_id
],
jmx_attributes=['bytes-consumed-rate'],
version=client_version)
consumer.run()
for idx, messages in consumer.messages_consumed.iteritems():
assert len(
messages
) > 0, "consumer %d didn't consume any message before timeout" % idx
success, msg = self.validate(self.kafka, producer, consumer)
assert success, msg
def validate(self, broker, producer, consumer):
"""
For each client_id we validate that:
1) number of consumed messages equals number of produced messages
2) maximum_producer_throughput <= producer_quota * (1 + maximum_client_deviation_percentage/100)
3) maximum_broker_byte_in_rate <= producer_quota * (1 + maximum_broker_deviation_percentage/100)
4) maximum_consumer_throughput <= consumer_quota * (1 + maximum_client_deviation_percentage/100)
5) maximum_broker_byte_out_rate <= consumer_quota * (1 + maximum_broker_deviation_percentage/100)
"""
success = True
msg = ''
self.kafka.read_jmx_output_all_nodes()
# validate that number of consumed messages equals number of produced messages
produced_num = sum([value['records'] for value in producer.results])
consumed_num = sum(
[len(value) for value in consumer.messages_consumed.values()])
self.logger.info('producer produced %d messages' % produced_num)
self.logger.info('consumer consumed %d messages' % consumed_num)
if produced_num != consumed_num:
success = False
msg += "number of produced messages %d doesn't equal number of consumed messages %d" % (
produced_num, consumed_num)
# validate that maximum_producer_throughput <= producer_quota * (1 + maximum_client_deviation_percentage/100)
producer_maximum_bps = max(
metric.value
for k, metrics in producer.metrics(group='producer-metrics',
name='outgoing-byte-rate',
client_id=producer.client_id)
for metric in metrics)
producer_quota_bps = self.quota_config.producer_quota
self.logger.info(
'producer has maximum throughput %.2f bps with producer quota %.2f bps'
% (producer_maximum_bps, producer_quota_bps))
if producer_maximum_bps > producer_quota_bps * (
self.maximum_client_deviation_percentage / 100 + 1):
success = False
msg += 'maximum producer throughput %.2f bps exceeded producer quota %.2f bps by more than %.1f%%' % \
(producer_maximum_bps, producer_quota_bps, self.maximum_client_deviation_percentage)
# validate that maximum_broker_byte_in_rate <= producer_quota * (1 + maximum_broker_deviation_percentage/100)
broker_byte_in_attribute_name = 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec:OneMinuteRate'
broker_maximum_byte_in_bps = broker.maximum_jmx_value[
broker_byte_in_attribute_name]
self.logger.info(
'broker has maximum byte-in rate %.2f bps with producer quota %.2f bps'
% (broker_maximum_byte_in_bps, producer_quota_bps))
if broker_maximum_byte_in_bps > producer_quota_bps * (
self.maximum_broker_deviation_percentage / 100 + 1):
success = False
msg += 'maximum broker byte-in rate %.2f bps exceeded producer quota %.2f bps by more than %.1f%%' % \
(broker_maximum_byte_in_bps, producer_quota_bps, self.maximum_broker_deviation_percentage)
# validate that maximum_consumer_throughput <= consumer_quota * (1 + maximum_client_deviation_percentage/100)
consumer_attribute_name = 'kafka.consumer:type=consumer-fetch-manager-metrics,client-id=%s:bytes-consumed-rate' % consumer.client_id
consumer_maximum_bps = consumer.maximum_jmx_value[
consumer_attribute_name]
consumer_quota_bps = self.quota_config.consumer_quota
self.logger.info(
'consumer has maximum throughput %.2f bps with consumer quota %.2f bps'
% (consumer_maximum_bps, consumer_quota_bps))
if consumer_maximum_bps > consumer_quota_bps * (
self.maximum_client_deviation_percentage / 100 + 1):
success = False
msg += 'maximum consumer throughput %.2f bps exceeded consumer quota %.2f bps by more than %.1f%%' % \
(consumer_maximum_bps, consumer_quota_bps, self.maximum_client_deviation_percentage)
# validate that maximum_broker_byte_out_rate <= consumer_quota * (1 + maximum_broker_deviation_percentage/100)
broker_byte_out_attribute_name = 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec:OneMinuteRate'
broker_maximum_byte_out_bps = broker.maximum_jmx_value[
broker_byte_out_attribute_name]
self.logger.info(
'broker has maximum byte-out rate %.2f bps with consumer quota %.2f bps'
% (broker_maximum_byte_out_bps, consumer_quota_bps))
if broker_maximum_byte_out_bps > consumer_quota_bps * (
self.maximum_broker_deviation_percentage / 100 + 1):
success = False
msg += 'maximum broker byte-out rate %.2f bps exceeded consumer quota %.2f bps by more than %.1f%%' % \
(broker_maximum_byte_out_bps, consumer_quota_bps, self.maximum_broker_deviation_percentage)
return success, msg
class StreamsBrokerCompatibility(Test):
"""
These tests validates that
- Streams 0.11+ w/ EOS fails fast for older brokers 0.10.2 and 0.10.1
- Streams 0.11+ w/o EOS works for older brokers 0.10.2 and 0.10.1
- Streams fails fast for 0.10.0 brokers
- Streams times-out for pre-0.10.0 brokers
"""
input = "brokerCompatibilitySourceTopic"
output = "brokerCompatibilitySinkTopic"
def __init__(self, test_context):
super(StreamsBrokerCompatibility, self).__init__(test_context=test_context)
self.zk = ZookeeperService(test_context, num_nodes=1)
self.kafka = KafkaService(test_context,
num_nodes=1,
zk=self.zk,
topics={
self.input: {'partitions': 1, 'replication-factor': 1},
self.output: {'partitions': 1, 'replication-factor': 1}
})
self.consumer = VerifiableConsumer(test_context,
1,
self.kafka,
self.output,
"stream-broker-compatibility-verify-consumer")
def setUp(self):
self.zk.start()
@parametrize(broker_version=str(LATEST_0_10_2))
@parametrize(broker_version=str(LATEST_0_10_1))
def test_fail_fast_on_incompatible_brokers_if_eos_enabled(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, True)
with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor:
processor.start()
monitor.wait_until('FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: Cannot create a v0 FindCoordinator request because we require features supported only in 1 or later.',
timeout_sec=60,
err_msg="Never saw 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: Cannot create a v0 FindCoordinator request because we require features supported only in 1 or later.' error message " + str(processor.node.account))
self.kafka.stop()
@parametrize(broker_version=str(LATEST_0_11_0))
@parametrize(broker_version=str(LATEST_0_10_2))
@parametrize(broker_version=str(LATEST_0_10_1))
def test_compatible_brokers_eos_disabled(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, False)
processor.start()
self.consumer.start()
processor.wait()
wait_until(lambda: self.consumer.total_consumed() > 0, timeout_sec=30, err_msg="Did expect to read a message but got none within 30 seconds.")
self.consumer.stop()
self.kafka.stop()
@parametrize(broker_version=str(LATEST_0_10_0))
def test_fail_fast_on_incompatible_brokers(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, False)
with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor:
processor.start()
monitor.wait_until('FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support CREATE_TOPICS',
timeout_sec=60,
err_msg="Never saw 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support CREATE_TOPICS' error message " + str(processor.node.account))
self.kafka.stop()
@ignore
@parametrize(broker_version=str(LATEST_0_9))
@parametrize(broker_version=str(LATEST_0_8_2))
def test_timeout_on_pre_010_brokers(self, broker_version):
self.kafka.set_version(KafkaVersion(broker_version))
self.kafka.start()
processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, False)
with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor:
processor.start()
monitor.wait_until('Exception in thread "main" org.apache.kafka.streams.errors.BrokerNotFoundException: Could not find any available broker.',
timeout_sec=60,
err_msg="Never saw 'no available brokers' error message " + str(processor.node.account))
self.kafka.stop()
class QuotaTest(Test):
"""
These tests verify that quota provides expected functionality -- they run
producer, broker, and consumer with different clientId and quota configuration and
check that the observed throughput is close to the value we expect.
"""
def __init__(self, test_context):
""":type test_context: ducktape.tests.test.TestContext"""
super(QuotaTest, self).__init__(test_context=test_context)
self.topic = 'test_topic'
self.logger.info('use topic ' + self.topic)
self.maximum_client_deviation_percentage = 100.0
self.maximum_broker_deviation_percentage = 5.0
self.num_records = 50000
self.record_size = 3000
self.zk = ZookeeperService(test_context, num_nodes=1)
self.kafka = KafkaService(test_context, num_nodes=1, zk=self.zk,
security_protocol='SSL', authorizer_class_name='',
interbroker_security_protocol='SSL',
topics={self.topic: {'partitions': 6, 'replication-factor': 1, 'configs': {'min.insync.replicas': 1}}},
jmx_object_names=['kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec',
'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec'],
jmx_attributes=['OneMinuteRate'])
self.num_producers = 1
self.num_consumers = 2
def setUp(self):
self.zk.start()
def min_cluster_size(self):
"""Override this since we're adding services outside of the constructor"""
return super(QuotaTest, self).min_cluster_size() + self.num_producers + self.num_consumers
@cluster(num_nodes=5)
@matrix(quota_type=[QuotaConfig.CLIENT_ID, QuotaConfig.USER, QuotaConfig.USER_CLIENT], override_quota=[True, False])
@parametrize(quota_type=QuotaConfig.CLIENT_ID, consumer_num=2)
@parametrize(quota_type=QuotaConfig.CLIENT_ID, old_broker_throttling_behavior=True)
@parametrize(quota_type=QuotaConfig.CLIENT_ID, old_client_throttling_behavior=True)
def test_quota(self, quota_type, override_quota=True, producer_num=1, consumer_num=1,
old_broker_throttling_behavior=False, old_client_throttling_behavior=False):
# Old (pre-2.0) throttling behavior for broker throttles before sending a response to the client.
if old_broker_throttling_behavior:
self.kafka.set_version(LATEST_1_1)
self.kafka.start()
self.quota_config = QuotaConfig(quota_type, override_quota, self.kafka)
producer_client_id = self.quota_config.client_id
consumer_client_id = self.quota_config.client_id
# Old (pre-2.0) throttling behavior for client does not throttle upon receiving a response with a non-zero throttle time.
if old_client_throttling_behavior:
client_version = LATEST_1_1
else:
client_version = DEV_BRANCH
# Produce all messages
producer = ProducerPerformanceService(
self.test_context, producer_num, self.kafka,
topic=self.topic, num_records=self.num_records, record_size=self.record_size, throughput=-1,
client_id=producer_client_id, version=client_version)
producer.run()
# Consume all messages
consumer = ConsoleConsumer(self.test_context, consumer_num, self.kafka, self.topic,
consumer_timeout_ms=60000, client_id=consumer_client_id,
jmx_object_names=['kafka.consumer:type=consumer-fetch-manager-metrics,client-id=%s' % consumer_client_id],
jmx_attributes=['bytes-consumed-rate'], version=client_version)
consumer.run()
for idx, messages in consumer.messages_consumed.iteritems():
assert len(messages) > 0, "consumer %d didn't consume any message before timeout" % idx
success, msg = self.validate(self.kafka, producer, consumer)
assert success, msg
def validate(self, broker, producer, consumer):
"""
For each client_id we validate that:
1) number of consumed messages equals number of produced messages
2) maximum_producer_throughput <= producer_quota * (1 + maximum_client_deviation_percentage/100)
3) maximum_broker_byte_in_rate <= producer_quota * (1 + maximum_broker_deviation_percentage/100)
4) maximum_consumer_throughput <= consumer_quota * (1 + maximum_client_deviation_percentage/100)
5) maximum_broker_byte_out_rate <= consumer_quota * (1 + maximum_broker_deviation_percentage/100)
"""
success = True
msg = ''
self.kafka.read_jmx_output_all_nodes()
# validate that number of consumed messages equals number of produced messages
produced_num = sum([value['records'] for value in producer.results])
consumed_num = sum([len(value) for value in consumer.messages_consumed.values()])
self.logger.info('producer produced %d messages' % produced_num)
self.logger.info('consumer consumed %d messages' % consumed_num)
if produced_num != consumed_num:
success = False
msg += "number of produced messages %d doesn't equal number of consumed messages %d" % (produced_num, consumed_num)
# validate that maximum_producer_throughput <= producer_quota * (1 + maximum_client_deviation_percentage/100)
producer_maximum_bps = max(
metric.value for k, metrics in producer.metrics(group='producer-metrics', name='outgoing-byte-rate', client_id=producer.client_id) for metric in metrics
)
producer_quota_bps = self.quota_config.producer_quota
self.logger.info('producer has maximum throughput %.2f bps with producer quota %.2f bps' % (producer_maximum_bps, producer_quota_bps))
if producer_maximum_bps > producer_quota_bps*(self.maximum_client_deviation_percentage/100+1):
success = False
msg += 'maximum producer throughput %.2f bps exceeded producer quota %.2f bps by more than %.1f%%' % \
(producer_maximum_bps, producer_quota_bps, self.maximum_client_deviation_percentage)
# validate that maximum_broker_byte_in_rate <= producer_quota * (1 + maximum_broker_deviation_percentage/100)
broker_byte_in_attribute_name = 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec:OneMinuteRate'
broker_maximum_byte_in_bps = broker.maximum_jmx_value[broker_byte_in_attribute_name]
self.logger.info('broker has maximum byte-in rate %.2f bps with producer quota %.2f bps' %
(broker_maximum_byte_in_bps, producer_quota_bps))
if broker_maximum_byte_in_bps > producer_quota_bps*(self.maximum_broker_deviation_percentage/100+1):
success = False
msg += 'maximum broker byte-in rate %.2f bps exceeded producer quota %.2f bps by more than %.1f%%' % \
(broker_maximum_byte_in_bps, producer_quota_bps, self.maximum_broker_deviation_percentage)
# validate that maximum_consumer_throughput <= consumer_quota * (1 + maximum_client_deviation_percentage/100)
consumer_attribute_name = 'kafka.consumer:type=consumer-fetch-manager-metrics,client-id=%s:bytes-consumed-rate' % consumer.client_id
consumer_maximum_bps = consumer.maximum_jmx_value[consumer_attribute_name]
consumer_quota_bps = self.quota_config.consumer_quota
self.logger.info('consumer has maximum throughput %.2f bps with consumer quota %.2f bps' % (consumer_maximum_bps, consumer_quota_bps))
if consumer_maximum_bps > consumer_quota_bps*(self.maximum_client_deviation_percentage/100+1):
success = False
msg += 'maximum consumer throughput %.2f bps exceeded consumer quota %.2f bps by more than %.1f%%' % \
(consumer_maximum_bps, consumer_quota_bps, self.maximum_client_deviation_percentage)
# validate that maximum_broker_byte_out_rate <= consumer_quota * (1 + maximum_broker_deviation_percentage/100)
broker_byte_out_attribute_name = 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec:OneMinuteRate'
broker_maximum_byte_out_bps = broker.maximum_jmx_value[broker_byte_out_attribute_name]
self.logger.info('broker has maximum byte-out rate %.2f bps with consumer quota %.2f bps' %
(broker_maximum_byte_out_bps, consumer_quota_bps))
if broker_maximum_byte_out_bps > consumer_quota_bps*(self.maximum_broker_deviation_percentage/100+1):
success = False
msg += 'maximum broker byte-out rate %.2f bps exceeded consumer quota %.2f bps by more than %.1f%%' % \
(broker_maximum_byte_out_bps, consumer_quota_bps, self.maximum_broker_deviation_percentage)
return success, msg
