class GetOffsetShellTest(Test): """ Tests GetOffsetShell tool """ def __init__(self, test_context): super(GetOffsetShellTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.messages_received_count = 0 self.topics = { TOPIC: {'partitions': NUM_PARTITIONS, 'replication-factor': REPLICATION_FACTOR} } self.zk = ZookeeperService(test_context, self.num_zk) def setUp(self): self.zk.start() def start_kafka(self, security_protocol, interbroker_security_protocol): self.kafka = KafkaService( self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, interbroker_security_protocol=interbroker_security_protocol, topics=self.topics) self.kafka.start() def start_producer(self): # This will produce to kafka cluster self.producer = VerifiableProducer(self.test_context, num_nodes=1, kafka=self.kafka, topic=TOPIC, throughput=1000, max_messages=MAX_MESSAGES) self.producer.start() current_acked = self.producer.num_acked wait_until(lambda: self.producer.num_acked >= current_acked + MAX_MESSAGES, timeout_sec=10, err_msg="Timeout awaiting messages to be produced and acked") def start_consumer(self): self.consumer = ConsoleConsumer(self.test_context, num_nodes=self.num_brokers, kafka=self.kafka, topic=TOPIC, consumer_timeout_ms=1000) self.consumer.start() @cluster(num_nodes=4) def test_get_offset_shell(self, security_protocol='PLAINTEXT'): """ Tests if GetOffsetShell is getting offsets correctly :return: None """ self.start_kafka(security_protocol, security_protocol) self.start_producer() # Assert that offset fetched without any consumers consuming is 0 assert self.kafka.get_offset_shell(TOPIC, None, 1000, 1, -1), "%s:%s:%s" % (TOPIC, NUM_PARTITIONS - 1, 0) self.start_consumer() node = self.consumer.nodes[0] wait_until(lambda: self.consumer.alive(node), timeout_sec=20, backoff_sec=.2, err_msg="Consumer was too slow to start") # Assert that offset is correctly indicated by GetOffsetShell tool wait_until(lambda: "%s:%s:%s" % (TOPIC, NUM_PARTITIONS - 1, MAX_MESSAGES) in self.kafka.get_offset_shell(TOPIC, None, 1000, 1, -1), timeout_sec=10, err_msg="Timed out waiting to reach expected offset.")
class GetOffsetShellTest(Test): """ Tests GetOffsetShell tool """ def __init__(self, test_context): super(GetOffsetShellTest, self).__init__(test_context) self.num_zk = 1 self.num_brokers = 1 self.messages_received_count = 0 self.topics = { TOPIC_TEST_NAME: { 'partitions': NUM_PARTITIONS, 'replication-factor': REPLICATION_FACTOR }, TOPIC_TEST_PATTERN1: { 'partitions': 1, 'replication-factor': REPLICATION_FACTOR }, TOPIC_TEST_PATTERN2: { 'partitions': 1, 'replication-factor': REPLICATION_FACTOR }, TOPIC_TEST_PARTITIONS: { 'partitions': 2, 'replication-factor': REPLICATION_FACTOR }, TOPIC_TEST_INTERNAL_FILTER: { 'partitions': 1, 'replication-factor': REPLICATION_FACTOR }, TOPIC_TEST_TOPIC_PARTITIONS1: { 'partitions': 2, 'replication-factor': REPLICATION_FACTOR }, TOPIC_TEST_TOPIC_PARTITIONS2: { 'partitions': 2, 'replication-factor': REPLICATION_FACTOR } } self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test( test_context) == quorum.zk else None def setUp(self): if self.zk: self.zk.start() def start_kafka(self, security_protocol, interbroker_security_protocol): self.kafka = KafkaService( self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, interbroker_security_protocol=interbroker_security_protocol, topics=self.topics) self.kafka.start() def start_producer(self, topic): # This will produce to kafka cluster self.producer = VerifiableProducer(self.test_context, num_nodes=1, kafka=self.kafka, topic=topic, throughput=1000, max_messages=MAX_MESSAGES, repeating_keys=MAX_MESSAGES) self.producer.start() current_acked = self.producer.num_acked wait_until( lambda: self.producer.num_acked >= current_acked + MAX_MESSAGES, timeout_sec=10, err_msg="Timeout awaiting messages to be produced and acked") def start_consumer(self, topic): self.consumer = ConsoleConsumer(self.test_context, num_nodes=self.num_brokers, kafka=self.kafka, topic=topic, consumer_timeout_ms=1000) self.consumer.start() def check_message_count_sum_equals(self, message_count, **kwargs): sum = self.extract_message_count_sum(**kwargs) return sum == message_count def extract_message_count_sum(self, **kwargs): offsets = self.kafka.get_offset_shell(**kwargs).split("\n") sum = 0 for offset in offsets: if len(offset) == 0: continue sum += int(offset.split(":")[-1]) return sum @cluster(num_nodes=3) @matrix(metadata_quorum=quorum.all_non_upgrade) def test_get_offset_shell_topic_name(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if GetOffsetShell handles --topic argument with a simple name correctly :return: None """ self.start_kafka(security_protocol, security_protocol) self.start_producer(TOPIC_TEST_NAME) # Assert that offset is correctly indicated by GetOffsetShell tool wait_until(lambda: self.check_message_count_sum_equals( MAX_MESSAGES, topic=TOPIC_TEST_NAME), timeout_sec=10, err_msg="Timed out waiting to reach expected offset.") @cluster(num_nodes=4) @matrix(metadata_quorum=quorum.all_non_upgrade) def test_get_offset_shell_topic_pattern(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if GetOffsetShell handles --topic argument with a pattern correctly :return: None """ self.start_kafka(security_protocol, security_protocol) self.start_producer(TOPIC_TEST_PATTERN1) self.start_producer(TOPIC_TEST_PATTERN2) # Assert that offset is correctly indicated by GetOffsetShell tool wait_until(lambda: self.check_message_count_sum_equals( 2 * MAX_MESSAGES, topic=TOPIC_TEST_PATTERN_PATTERN), timeout_sec=10, err_msg="Timed out waiting to reach expected offset.") @cluster(num_nodes=3) @matrix(metadata_quorum=quorum.all_non_upgrade) def test_get_offset_shell_partitions(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if GetOffsetShell handles --partitions argument correctly :return: None """ self.start_kafka(security_protocol, security_protocol) self.start_producer(TOPIC_TEST_PARTITIONS) def fetch_and_sum_partitions_separately(): partition_count0 = self.extract_message_count_sum( topic=TOPIC_TEST_PARTITIONS, partitions="0") partition_count1 = self.extract_message_count_sum( topic=TOPIC_TEST_PARTITIONS, partitions="1") return partition_count0 + partition_count1 == MAX_MESSAGES # Assert that offset is correctly indicated when fetching partitions one by one wait_until(fetch_and_sum_partitions_separately, timeout_sec=10, err_msg="Timed out waiting to reach expected offset.") # Assert that offset is correctly indicated when fetching partitions together wait_until(lambda: self.check_message_count_sum_equals( MAX_MESSAGES, topic=TOPIC_TEST_PARTITIONS), timeout_sec=10, err_msg="Timed out waiting to reach expected offset.") @cluster(num_nodes=4) @matrix(metadata_quorum=quorum.all_non_upgrade) def test_get_offset_shell_topic_partitions(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if GetOffsetShell handles --topic-partitions argument correctly :return: None """ self.start_kafka(security_protocol, security_protocol) self.start_producer(TOPIC_TEST_TOPIC_PARTITIONS1) self.start_producer(TOPIC_TEST_TOPIC_PARTITIONS2) # Assert that a single topic pattern matches all 4 partitions wait_until(lambda: self.check_message_count_sum_equals( 2 * MAX_MESSAGES, topic_partitions=TOPIC_TEST_TOPIC_PARTITIONS_PATTERN), timeout_sec=10, err_msg="Timed out waiting to reach expected offset.") # Assert that a topic pattern with partition range matches all 4 partitions wait_until(lambda: self.check_message_count_sum_equals( 2 * MAX_MESSAGES, topic_partitions=TOPIC_TEST_TOPIC_PARTITIONS_PATTERN + ":0-2"), timeout_sec=10, err_msg="Timed out waiting to reach expected offset.") # Assert that 2 separate topic patterns match all 4 partitions wait_until(lambda: self.check_message_count_sum_equals( 2 * MAX_MESSAGES, topic_partitions=TOPIC_TEST_TOPIC_PARTITIONS1 + "," + TOPIC_TEST_TOPIC_PARTITIONS2), timeout_sec=10, err_msg="Timed out waiting to reach expected offset.") # Assert that 4 separate topic-partition patterns match all 4 partitions wait_until(lambda: self.check_message_count_sum_equals( 2 * MAX_MESSAGES, topic_partitions=TOPIC_TEST_TOPIC_PARTITIONS1 + ":0," + TOPIC_TEST_TOPIC_PARTITIONS1 + ":1," + TOPIC_TEST_TOPIC_PARTITIONS2 + ":0," + TOPIC_TEST_TOPIC_PARTITIONS2 + ":1"), timeout_sec=10, err_msg="Timed out waiting to reach expected offset.") # Assert that only partitions #0 are matched with topic pattern and fix partition number filtered_partitions = self.kafka.get_offset_shell( topic_partitions=TOPIC_TEST_TOPIC_PARTITIONS_PATTERN + ":0") assert 1 == filtered_partitions.count( "%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS1, 0)) assert 0 == filtered_partitions.count( "%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS1, 1)) assert 1 == filtered_partitions.count( "%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS2, 0)) assert 0 == filtered_partitions.count( "%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS2, 1)) # Assert that only partitions #1 are matched with topic pattern and partition lower bound filtered_partitions = self.kafka.get_offset_shell( topic_partitions=TOPIC_TEST_TOPIC_PARTITIONS_PATTERN + ":1-") assert 1 == filtered_partitions.count( "%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS1, 1)) assert 0 == filtered_partitions.count( "%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS1, 0)) assert 1 == filtered_partitions.count( "%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS2, 1)) assert 0 == filtered_partitions.count( "%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS2, 0)) # Assert that only partitions #0 are matched with topic pattern and partition upper bound filtered_partitions = self.kafka.get_offset_shell( topic_partitions=TOPIC_TEST_TOPIC_PARTITIONS_PATTERN + ":-1") assert 1 == filtered_partitions.count( "%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS1, 0)) assert 0 == filtered_partitions.count( "%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS1, 1)) assert 1 == filtered_partitions.count( "%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS2, 0)) assert 0 == filtered_partitions.count( "%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS2, 1)) @cluster(num_nodes=4) @matrix(metadata_quorum=quorum.all_non_upgrade) def test_get_offset_shell_internal_filter(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if GetOffsetShell handles --exclude-internal-topics flag correctly :return: None """ self.start_kafka(security_protocol, security_protocol) self.start_producer(TOPIC_TEST_INTERNAL_FILTER) # Create consumer and poll messages to create consumer offset record self.start_consumer(TOPIC_TEST_INTERNAL_FILTER) node = self.consumer.nodes[0] wait_until(lambda: self.consumer.alive(node), timeout_sec=20, backoff_sec=.2, err_msg="Consumer was too slow to start") # Assert that a single topic pattern matches all 4 partitions wait_until(lambda: self.check_message_count_sum_equals( MAX_MESSAGES, topic_partitions=TOPIC_TEST_INTERNAL_FILTER), timeout_sec=10, err_msg="Timed out waiting to reach expected offset.") # No filters # Assert that without exclusion, we can find both the test topic and the __consumer_offsets internal topic offset_output = self.kafka.get_offset_shell() assert "__consumer_offsets" in offset_output assert TOPIC_TEST_INTERNAL_FILTER in offset_output # Assert that with exclusion, we can find the test topic but not the __consumer_offsets internal topic offset_output = self.kafka.get_offset_shell( exclude_internal_topics=True) assert "__consumer_offsets" not in offset_output assert TOPIC_TEST_INTERNAL_FILTER in offset_output # Topic filter # Assert that without exclusion, we can find both the test topic and the __consumer_offsets internal topic offset_output = self.kafka.get_offset_shell(topic=".*consumer_offsets") assert "__consumer_offsets" in offset_output assert TOPIC_TEST_INTERNAL_FILTER in offset_output # Assert that with exclusion, we can find the test topic but not the __consumer_offsets internal topic offset_output = self.kafka.get_offset_shell( topic=".*consumer_offsets", exclude_internal_topics=True) assert "__consumer_offsets" not in offset_output assert TOPIC_TEST_INTERNAL_FILTER in offset_output # Topic-partition filter # Assert that without exclusion, we can find both the test topic and the __consumer_offsets internal topic offset_output = self.kafka.get_offset_shell( topic_partitions=".*consumer_offsets:0") assert "__consumer_offsets" in offset_output assert TOPIC_TEST_INTERNAL_FILTER in offset_output # Assert that with exclusion, we can find the test topic but not the __consumer_offsets internal topic offset_output = self.kafka.get_offset_shell( topic_partitions=".*consumer_offsets:0", exclude_internal_topics=True) assert "__consumer_offsets" not in offset_output assert TOPIC_TEST_INTERNAL_FILTER in offset_output