def setup_node(self, node):
if self.has_ssl:
self.setup_ssl(node)
if self.has_sasl:
self.setup_sasl(node)
if java_version(node) <= 11 and self.properties.get('tls.version') == 'TLSv1.3':
self.properties.update({'tls.version': 'TLSv1.2'})
def fix_opts_for_new_jvm(node):
# Startup scripts for early versions of Kafka contains options
# that not supported on latest versions of JVM like -XX:+PrintGCDateStamps or -XX:UseParNewGC.
# When system test run on JVM that doesn't support these options
# we should setup environment variables with correct options.
java_ver = java_version(node)
if java_ver <= 9:
return ""
cmd = ""
if node.version == LATEST_0_8_2 or node.version == LATEST_0_9 or node.version == LATEST_0_10_0 or node.version == LATEST_0_10_1 or node.version == LATEST_0_10_2 or node.version == LATEST_0_11_0 or node.version == LATEST_1_0:
cmd += "export KAFKA_GC_LOG_OPTS=\"-Xlog:gc*:file=kafka-gc.log:time,tags:filecount=10,filesize=102400\"; "
cmd += "export KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -XX:MaxInlineLevel=15 -Djava.awt.headless=true\"; "
return cmd
def test_upgrade(self,
from_kafka_version,
to_message_format_version,
compression_types,
security_protocol="PLAINTEXT"):
"""Test upgrade of Kafka broker cluster from various versions to the current version
from_kafka_version is a Kafka version to upgrade from
If to_message_format_version is None, it means that we will upgrade to default (latest)
message format version. It is possible to upgrade to 0.10 brokers but still use message
format version 0.9
- Start 3 node broker cluster on version 'from_kafka_version'
- Start producer and consumer in the background
- Perform two-phase rolling upgrade
- First phase: upgrade brokers to 0.10 with inter.broker.protocol.version set to
from_kafka_version and log.message.format.version set to from_kafka_version
- Second phase: remove inter.broker.protocol.version config with rolling bounce; if
to_message_format_version is set to 0.9, set log.message.format.version to
to_message_format_version, otherwise remove log.message.format.version config
- Finally, validate that every message acked by the producer was consumed by the consumer
"""
self.zk = ZookeeperService(self.test_context,
num_nodes=1,
version=KafkaVersion(from_kafka_version))
self.kafka = KafkaService(self.test_context,
num_nodes=3,
zk=self.zk,
version=KafkaVersion(from_kafka_version),
topics={
self.topic: {
"partitions": self.partitions,
"replication-factor":
self.replication_factor,
'configs': {
"min.insync.replicas": 2
}
}
})
self.kafka.security_protocol = security_protocol
self.kafka.interbroker_security_protocol = security_protocol
jdk_version = java_version(self.kafka.nodes[0])
if jdk_version > 9 and from_kafka_version in new_jdk_not_supported:
self.logger.info("Test ignored! Kafka " + from_kafka_version +
" not support jdk " + str(jdk_version))
return
self.zk.start()
self.kafka.start()
old_id = self.kafka.topic_id(self.topic)
self.producer = VerifiableProducer(
self.test_context,
self.num_producers,
self.kafka,
self.topic,
throughput=self.producer_throughput,
message_validator=is_int,
compression_types=compression_types,
version=KafkaVersion(from_kafka_version))
if from_kafka_version <= LATEST_0_10_0:
assert self.kafka.cluster_id() is None
# With older message formats before KIP-101, message loss may occur due to truncation
# after leader change. Tolerate limited data loss for this case to avoid transient test failures.
self.may_truncate_acked_records = False if from_kafka_version >= V_0_11_0_0 else True
new_consumer = from_kafka_version.consumer_supports_bootstrap_server()
# TODO - reduce the timeout
self.consumer = ConsoleConsumer(
self.test_context,
self.num_consumers,
self.kafka,
self.topic,
new_consumer=new_consumer,
consumer_timeout_ms=30000,
message_validator=is_int,
version=KafkaVersion(from_kafka_version))
self.run_produce_consume_validate(
core_test_action=lambda: self.perform_upgrade(
from_kafka_version, to_message_format_version))
cluster_id = self.kafka.cluster_id()
assert cluster_id is not None
assert len(cluster_id) == 22
assert self.kafka.all_nodes_support_topic_ids()
new_id = self.kafka.topic_id(self.topic)
if from_kafka_version >= V_2_8_0:
assert old_id is not None
assert new_id is not None
assert old_id == new_id
else:
assert old_id is None
assert new_id is not None
assert self.kafka.check_protocol_errors(self)