def test_retention_archival_coordination(self): """ Test that only archived segments can be evicted and that eviction restarts once the segments have been archived. """ self.kafka_tools.alter_topic_config( self.topic, { TopicSpec.PROPERTY_RETENTION_BYTES: 5 * self.log_segment_size, }, ) with firewall_blocked(self.redpanda.nodes, self._get_s3_endpoint_ip()): produce_until_segments(redpanda=self.redpanda, topic=self.topic, partition_idx=0, count=10) # Sleep some time sufficient for log eviction under normal conditions # and check that no segment has been evicted (because we can't upload # segments to the cloud storage). time.sleep(3 * self.log_compaction_interval_ms / 1000.0) counts = list( segments_count(self.redpanda, self.topic, partition_idx=0)) self.logger.info(f"node segment counts: {counts}") assert len(counts) == len(self.redpanda.nodes) assert all(c >= 10 for c in counts) # Check that eviction restarts after we restored the connection to cloud # storage. wait_for_segments_removal(redpanda=self.redpanda, topic=self.topic, partition_idx=0, count=6)
def test_fetch_after_committed_offset_was_removed(self, transactions_enabled): """ Test fetching when consumer offset was deleted by retention """ self.redpanda._extra_rp_conf[ "enable_transactions"] = transactions_enabled self.redpanda._extra_rp_conf[ "enable_idempotence"] = transactions_enabled self.redpanda.start() topic = TopicSpec(partition_count=1, replication_factor=3, cleanup_policy=TopicSpec.CLEANUP_DELETE) self.client().create_topic(topic) kafka_tools = KafkaCliTools(self.redpanda) # produce until segments have been compacted produce_until_segments( self.redpanda, topic=topic.name, partition_idx=0, count=10, ) consumer_group = 'test' rpk = RpkTool(self.redpanda) def consume(n=1): out = rpk.consume(topic.name, group=consumer_group, n=n) split = out.split('}') split = filter(lambda s: "{" in s, split) return map(lambda s: json.loads(s + "}"), split) #consume from the beggining msgs = consume(10) last = list(msgs).pop() offset = last['offset'] # change retention time kafka_tools.alter_topic_config( topic.name, { TopicSpec.PROPERTY_RETENTION_BYTES: 2 * self.segment_size, }) wait_for_segments_removal(self.redpanda, topic.name, partition_idx=0, count=5) partitions = list(rpk.describe_topic(topic.name)) p = partitions[0] assert p.start_offset > offset # consume from the offset that doesn't exists, # the one that was committed previously was already removed out = list(consume(1)) assert out[0]['offset'] == p.start_offset
def test_consume_from_blocked_s3(self): produce_until_segments(redpanda=self.redpanda, topic=self.s3_topic_name, partition_idx=0, count=5, acks=-1) self.rpk.alter_topic_config(self.s3_topic_name, TopicSpec.PROPERTY_RETENTION_BYTES, self.retention_bytes) wait_for_segments_removal(redpanda=self.redpanda, topic=self.s3_topic_name, partition_idx=0, count=4) """Disconnect redpanda from S3 and try to read starting with offset 0""" with firewall_blocked(self.redpanda.nodes, self._s3_port): try: out = self.rpk.consume(topic=self.s3_topic_name) except RpkException as e: assert 'timed out' in e.msg else: raise RuntimeError( f"RPK consume should have timed out, but ran with output: {out}" )
def test_changing_topic_retention(self, property, acks): """ Test changing topic retention duration for topics with data produced with ACKS=1 and ACKS=-1. This test produces data until 10 segments appear, then it changes retention topic property and waits for segments to be removed """ kafka_tools = KafkaCliTools(self.redpanda) # produce until segments have been compacted produce_until_segments( self.redpanda, topic=self.topic, partition_idx=0, count=10, acks=acks, ) # change retention time kafka_tools.alter_topic_config(self.topic, { property: 10000, }) wait_for_segments_removal(self.redpanda, self.topic, partition_idx=0, count=5)
def test_write(self): """Write at least 10 segments, set retention policy to leave only 5 segments, wait for segments removal, consume data and run validation, that everything that is acked is consumed.""" self.start_producer() produce_until_segments( redpanda=self.redpanda, topic=self.topic, partition_idx=0, count=10, ) self.kafka_tools.alter_topic_config( self.topic, { TopicSpec.PROPERTY_RETENTION_BYTES: 5 * EndToEndShadowIndexingTest.segment_size, }, ) wait_for_segments_removal(redpanda=self.redpanda, topic=self.topic, partition_idx=0, count=6) self.start_consumer() self.run_validation()
def test_write_with_node_failures(self): self.start_producer() produce_until_segments( redpanda=self.redpanda, topic=self.topic, partition_idx=0, count=10, ) self.kafka_tools.alter_topic_config( self.topic, { TopicSpec.PROPERTY_RETENTION_BYTES: 5 * EndToEndShadowIndexingTest.segment_size }, ) with random_process_kills(self.redpanda) as ctx: wait_for_segments_removal(redpanda=self.redpanda, topic=self.topic, partition_idx=0, count=6) self.start_consumer() self.run_validation() ctx.assert_actions_triggered()
def test_changing_topic_retention_with_restart(self): """ Test changing topic retention duration for topics with data produced with ACKS=1 and ACKS=-1. This test produces data until 10 segments appear, then it changes retention topic property and waits for some segmetnts to be removed """ segment_size = 1048576 # produce until segments have been compacted produce_until_segments( self.redpanda, topic=self.topic, partition_idx=0, count=20, acks=-1, ) # restart all nodes to force replicating raft configuration self.redpanda.restart_nodes(self.redpanda.nodes) kafka_tools = KafkaCliTools(self.redpanda) # Wait for controller, alter configs doesn't have a retry loop kafka_tools.describe_topic(self.topic) # change retention bytes to preserve 15 segments self.client().alter_topic_configs( self.topic, { TopicSpec.PROPERTY_RETENTION_BYTES: 15 * segment_size, }) wait_for_segments_removal(redpanda=self.redpanda, topic=self.topic, partition_idx=0, count=16) # change retention bytes again to preserve 10 segments self.client().alter_topic_configs( self.topic, { TopicSpec.PROPERTY_RETENTION_BYTES: 10 * segment_size, }) wait_for_segments_removal(redpanda=self.redpanda, topic=self.topic, partition_idx=0, count=11) # change retention bytes again to preserve 5 segments self.client().alter_topic_configs( self.topic, { TopicSpec.PROPERTY_RETENTION_BYTES: 4 * segment_size, }) wait_for_segments_removal(redpanda=self.redpanda, topic=self.topic, partition_idx=0, count=5)
def test_timequery_after_segments_eviction(self): """ Test checking if the offset returned by time based index is valid during applying log cleanup policy """ segment_size = 1048576 # produce until segments have been compacted produce_until_segments( self.redpanda, topic=self.topic, partition_idx=0, count=10, acks=-1, ) # restart all nodes to force replicating raft configuration self.redpanda.restart_nodes(self.redpanda.nodes) kafka_tools = KafkaCliTools(self.redpanda) # Wait for controller, alter configs doesn't have a retry loop kafka_tools.describe_topic(self.topic) # change retention bytes to preserve 15 segments self.client().alter_topic_configs( self.topic, { TopicSpec.PROPERTY_RETENTION_BYTES: 2 * segment_size, }) def validate_time_query_until_deleted(): def done(): kcat = KafkaCat(self.redpanda) ts = 1638748800 # 12.6.2021 - old timestamp, query first offset offset = kcat.query_offset(self.topic, 0, ts) # assert that offset is valid assert offset >= 0 topic_partitions = segments_count(self.redpanda, self.topic, 0) partitions = [] for p in topic_partitions: partitions.append(p <= 5) return all([p <= 5 for p in topic_partitions]) wait_until(done, timeout_sec=30, backoff_sec=5, err_msg="Segments were not removed") validate_time_query_until_deleted()
def test_querying_remote_partitions(self): topic = TopicSpec(redpanda_remote_read=True, redpanda_remote_write=True) epoch_offsets = {} rpk = RpkTool(self.redpanda) self.client().create_topic(topic) rpk.alter_topic_config(topic.name, "redpanda.remote.read", 'true') rpk.alter_topic_config(topic.name, "redpanda.remote.write", 'true') def wait_for_topic(): wait_until(lambda: len(list(rpk.describe_topic(topic.name))) > 0, 30, backoff_sec=2) # restart whole cluster 6 times to trigger term rolls for i in range(0, 6): wait_for_topic() produce_until_segments( redpanda=self.redpanda, topic=topic.name, partition_idx=0, count=2 * i, ) res = list(rpk.describe_topic(topic.name)) epoch_offsets[res[0].leader_epoch] = res[0].high_watermark self.redpanda.restart_nodes(self.redpanda.nodes) self.logger.info(f"ledear epoch high watermarks: {epoch_offsets}") wait_for_topic() rpk.alter_topic_config(topic.name, TopicSpec.PROPERTY_RETENTION_BYTES, OffsetForLeaderEpochArchivalTest.segment_size) wait_for_segments_removal(redpanda=self.redpanda, topic=topic.name, partition_idx=0, count=7) kcl = KCL(self.redpanda) for epoch, offset in epoch_offsets.items(): self.logger.info(f"querying partition epoch {epoch} end offsets") epoch_end_offset = kcl.offset_for_leader_epoch( topics=topic.name, leader_epoch=epoch)[0].epoch_end_offset self.logger.info( f"epoch {epoch} end_offset: {epoch_end_offset}, expected offset: {offset}" ) assert epoch_end_offset == offset