def start_replica_verification_tool(self, security_protocol):
self.replica_verifier = ReplicaVerificationTool(
self.test_context,
1,
self.kafka,
TOPIC,
report_interval_ms=REPORT_INTERVAL_MS,
security_protocol=security_protocol)
self.replica_verifier.start()
def start_replica_verification_tool(self, security_protocol):
self.replica_verifier = ReplicaVerificationTool(self.test_context, 1, self.kafka, TOPIC, report_interval_ms=REPORT_INTERVAL_MS, security_protocol=security_protocol)
self.replica_verifier.start()
class ReplicaVerificationToolTest(Test):
"""
Tests ReplicaVerificationTool
"""
def __init__(self, test_context):
super(ReplicaVerificationToolTest, self).__init__(test_context)
self.num_zk = 1
self.num_brokers = 2
self.messages_received_count = 0
self.topics = {
TOPIC: {'partitions': 1, 'replication-factor': 2}
}
self.zk = ZookeeperService(test_context, self.num_zk)
self.kafka = None
self.producer = None
self.replica_verifier = None
def setUp(self):
self.zk.start()
def start_kafka(self, security_protocol, interbroker_security_protocol):
self.kafka = KafkaService(
self.test_context, self.num_brokers,
self.zk, security_protocol=security_protocol,
interbroker_security_protocol=interbroker_security_protocol, topics=self.topics)
self.kafka.start()
def start_replica_verification_tool(self, security_protocol):
self.replica_verifier = ReplicaVerificationTool(self.test_context, 1, self.kafka, TOPIC, report_interval_ms=REPORT_INTERVAL_MS, security_protocol=security_protocol)
self.replica_verifier.start()
def start_producer(self, max_messages, acks, timeout):
# This will produce to kafka cluster
self.producer = VerifiableProducer(self.test_context, num_nodes=1, kafka=self.kafka, topic=TOPIC, throughput=1000, acks=acks, max_messages=max_messages)
current_acked = self.producer.num_acked
self.logger.info("current_acked = %s" % current_acked)
self.producer.start()
wait_until(lambda: acks == 0 or self.producer.num_acked >= current_acked + max_messages, timeout_sec=timeout,
err_msg="Timeout awaiting messages to be produced and acked")
def stop_producer(self):
self.producer.stop()
def test_replica_lags(self, security_protocol='PLAINTEXT'):
"""
Tests ReplicaVerificationTool
:return: None
"""
self.start_kafka(security_protocol, security_protocol)
self.start_replica_verification_tool(security_protocol)
self.start_producer(max_messages=10, acks=-1, timeout=15)
# Verify that there is no lag in replicas and is correctly reported by ReplicaVerificationTool
wait_until(lambda: self.replica_verifier.get_lag_for_partition(TOPIC, 0) == 0, timeout_sec=10,
err_msg="Timed out waiting to reach zero replica lags.")
self.stop_producer()
self.start_producer(max_messages=1000, acks=0, timeout=5)
# Verify that there is lag in replicas and is correctly reported by ReplicaVerificationTool
wait_until(lambda: self.replica_verifier.get_lag_for_partition(TOPIC, 0) > 0, timeout_sec=10,
err_msg="Timed out waiting to reach non-zero number of replica lags.")
class ReplicaVerificationToolTest(Test):
"""
Tests ReplicaVerificationTool
"""
def __init__(self, test_context):
super(ReplicaVerificationToolTest, self).__init__(test_context)
self.num_zk = 1
self.num_brokers = 2
self.messages_received_count = 0
self.topics = {TOPIC: {'partitions': 1, 'replication-factor': 2}}
self.zk = ZookeeperService(test_context, self.num_zk)
self.kafka = None
self.producer = None
self.replica_verifier = None
def setUp(self):
self.zk.start()
def start_kafka(self, security_protocol, interbroker_security_protocol):
self.kafka = KafkaService(
self.test_context,
self.num_brokers,
self.zk,
security_protocol=security_protocol,
interbroker_security_protocol=interbroker_security_protocol,
topics=self.topics)
self.kafka.start()
def start_replica_verification_tool(self, security_protocol):
self.replica_verifier = ReplicaVerificationTool(
self.test_context,
1,
self.kafka,
TOPIC,
report_interval_ms=REPORT_INTERVAL_MS,
security_protocol=security_protocol)
self.replica_verifier.start()
def start_producer(self, max_messages, acks, timeout):
# This will produce to kafka cluster
current_acked = 0
self.producer = VerifiableProducer(self.test_context,
num_nodes=1,
kafka=self.kafka,
topic=TOPIC,
throughput=1000,
acks=acks,
max_messages=max_messages)
self.producer.start()
wait_until(
lambda: acks == 0 or self.producer.num_acked >= current_acked +
max_messages,
timeout_sec=timeout,
err_msg="Timeout awaiting messages to be produced and acked")
def stop_producer(self):
self.producer.stop()
@cluster(num_nodes=6)
def test_replica_lags(self, security_protocol='PLAINTEXT'):
"""
Tests ReplicaVerificationTool
:return: None
"""
self.start_kafka(security_protocol, security_protocol)
self.start_replica_verification_tool(security_protocol)
self.start_producer(max_messages=10, acks=-1, timeout=15)
# Verify that there is no lag in replicas and is correctly reported by ReplicaVerificationTool
wait_until(
lambda: self.replica_verifier.get_lag_for_partition(TOPIC, 0) == 0,
timeout_sec=10,
err_msg="Timed out waiting to reach zero replica lags.")
self.stop_producer()
self.start_producer(max_messages=10000, acks=0, timeout=5)
# Verify that there is lag in replicas and is correctly reported by ReplicaVerificationTool
wait_until(
lambda: self.replica_verifier.get_lag_for_partition(TOPIC, 0) > 0,
timeout_sec=10,
err_msg=
"Timed out waiting to reach non-zero number of replica lags.")
