def __init__(self, test_context):
super(StreamsBrokerCompatibility,
self).__init__(test_context=test_context)
self.zk = ZookeeperService(test_context, num_nodes=1)
self.kafka = KafkaService(test_context,
num_nodes=1,
zk=self.zk,
topics={
self.input: {
'partitions': 1,
'replication-factor': 1
},
self.output: {
'partitions': 1,
'replication-factor': 1
}
})
self.processor = StreamsBrokerCompatibilityService(
self.test_context, self.kafka)
self.consumer = VerifiableConsumer(
test_context, 1, self.kafka, self.output,
"stream-broker-compatibility-verify-consumer")
def get_consumer(self, num_messages):
return VerifiableConsumer(self.test_context,
1,
self.kafka,
self.outputTopic,
"stream-broker-resilience-verify-consumer",
max_messages=num_messages)
def get_consumer(self, client_id, topic, num_messages):
return VerifiableConsumer(self.test_context,
1,
self.kafka,
topic,
client_id,
max_messages=num_messages)
def create_consumer(self, num_nodes=1, group_id="test_group", **kwargs):
self.consumer = VerifiableConsumer(self.test_context,
num_nodes=num_nodes,
kafka=self.kafka,
topic=self.topic,
group_id=group_id,
on_record_consumed=self.on_record_consumed,
**kwargs)
def _setup_consumer(self, topic, enable_autocommit=False):
return VerifiableConsumer(self.test_context,
self.num_consumers,
self.kafka,
topic,
self.GROUP_ID,
session_timeout=self.session_timeout,
enable_autocommit=enable_autocommit)
def setup_consumer(
self,
topic,
enable_autocommit=False,
assignment_strategy="org.apache.kafka.clients.consumer.RangeAssignor"
):
return VerifiableConsumer(self.test_context,
self.num_consumers,
self.kafka,
topic,
self.group_id,
session_timeout_sec=self.session_timeout_sec,
assignment_strategy=assignment_strategy,
enable_autocommit=enable_autocommit)
def setup_consumer(
self,
topic,
static_membership=False,
enable_autocommit=False,
assignment_strategy="org.apache.kafka.clients.consumer.RangeAssignor",
**kwargs):
return VerifiableConsumer(self.test_context,
self.num_consumers,
self.kafka,
topic,
self.group_id,
static_membership=static_membership,
session_timeout_sec=self.session_timeout_sec,
assignment_strategy=assignment_strategy,
enable_autocommit=enable_autocommit,
log_level="TRACE",
**kwargs)
def __init__(self, test_context):
super(StreamsBrokerCompatibility, self).__init__(test_context=test_context)
self.zk = ZookeeperService(test_context, num_nodes=1)
self.kafka = KafkaService(test_context,
num_nodes=1,
zk=self.zk,
topics={
self.input: {'partitions': 1, 'replication-factor': 1},
self.output: {'partitions': 1, 'replication-factor': 1}
},
server_prop_overrides=[
["transaction.state.log.replication.factor", "1"],
["transaction.state.log.min.isr", "1"]
])
self.consumer = VerifiableConsumer(test_context,
1,
self.kafka,
self.output,
"stream-broker-compatibility-verify-consumer")
def _setup_consumer(self, topic):
return VerifiableConsumer(self.test_context, self.num_consumers, self.kafka,
topic, self.GROUP_ID, session_timeout=self.session_timeout)
def test_offset_truncate(self):
"""
Verify correct consumer behavior when the brokers are consecutively restarted.
Setup: single Kafka cluster with one producer writing messages to a single topic with one
partition, an a set of consumers in the same group reading from the same topic.
- Start a producer which continues producing new messages throughout the test.
- Start up the consumers and wait until they've joined the group.
- In a loop, restart each broker consecutively, waiting for the group to stabilize between
each broker restart.
- Verify delivery semantics according to the failure type and that the broker bounces
did not cause unexpected group rebalances.
"""
tp = TopicPartition(self.TOPIC, 0)
producer = self.setup_producer(self.TOPIC, throughput=10)
producer.start()
self.await_produced_messages(producer, min_messages=10)
consumer = self.setup_consumer(self.TOPIC, reset_policy="earliest", verify_offsets=False)
consumer.start()
self.await_all_members(consumer)
# Reduce ISR to one node
isr = self.kafka.isr_idx_list(self.TOPIC, 0)
node1 = self.kafka.get_node(isr[0])
self.kafka.stop_node(node1)
self.logger.info("Reduced ISR to one node, consumer is at %s", consumer.current_position(tp))
# Ensure remaining ISR member has a little bit of data
current_total = consumer.total_consumed()
wait_until(lambda: consumer.total_consumed() > current_total + 10,
timeout_sec=30,
err_msg="Timed out waiting for consumer to move ahead by 10 messages")
# Kill last ISR member
node2 = self.kafka.get_node(isr[1])
self.kafka.stop_node(node2)
self.logger.info("No members in ISR, consumer is at %s", consumer.current_position(tp))
# Keep consuming until we've caught up to HW
def none_consumed(this, consumer):
new_total = consumer.total_consumed()
if new_total == this.last_total:
return True
else:
this.last_total = new_total
return False
self.last_total = consumer.total_consumed()
wait_until(lambda: none_consumed(self, consumer),
timeout_sec=30,
err_msg="Timed out waiting for the consumer to catch up")
self.kafka.start_node(node1)
self.logger.info("Out of sync replica is online, but not electable. Consumer is at %s", consumer.current_position(tp))
pre_truncation_pos = consumer.current_position(tp)
self.kafka.set_unclean_leader_election(self.TOPIC)
self.logger.info("New unclean leader, consumer is at %s", consumer.current_position(tp))
# Wait for truncation to be detected
self.kafka.start_node(node2)
wait_until(lambda: consumer.current_position(tp) >= pre_truncation_pos,
timeout_sec=30,
err_msg="Timed out waiting for truncation")
# Make sure we didn't reset to beginning of log
total_records_consumed = len(self.all_values_consumed)
assert total_records_consumed == len(set(self.all_values_consumed)), "Received duplicate records"
consumer.stop()
producer.stop()
# Re-consume all the records
consumer2 = VerifiableConsumer(self.test_context, 1, self.kafka, self.TOPIC, group_id="group2",
reset_policy="earliest", verify_offsets=True)
consumer2.start()
self.await_all_members(consumer2)
wait_until(lambda: consumer2.total_consumed() > 0,
timeout_sec=30,
err_msg="Timed out waiting for consumer to consume at least 10 messages")
self.last_total = consumer2.total_consumed()
wait_until(lambda: none_consumed(self, consumer2),
timeout_sec=30,
err_msg="Timed out waiting for the consumer to fully consume data")
second_total_consumed = consumer2.total_consumed()
assert second_total_consumed < total_records_consumed, "Expected fewer records with new consumer since we truncated"
self.logger.info("Second consumer saw only %s, meaning %s were truncated",
second_total_consumed, total_records_consumed - second_total_consumed)