def test_retention_archival_coordination(self):
"""
Test that only archived segments can be evicted and that eviction
restarts once the segments have been archived.
"""
self.kafka_tools.alter_topic_config(
self.topic,
{
TopicSpec.PROPERTY_RETENTION_BYTES: 5 * self.log_segment_size,
},
)
with firewall_blocked(self.redpanda.nodes, self._get_s3_endpoint_ip()):
produce_until_segments(redpanda=self.redpanda,
topic=self.topic,
partition_idx=0,
count=10)
# Sleep some time sufficient for log eviction under normal conditions
# and check that no segment has been evicted (because we can't upload
# segments to the cloud storage).
time.sleep(3 * self.log_compaction_interval_ms / 1000.0)
counts = list(
segments_count(self.redpanda, self.topic, partition_idx=0))
self.logger.info(f"node segment counts: {counts}")
assert len(counts) == len(self.redpanda.nodes)
assert all(c >= 10 for c in counts)
# Check that eviction restarts after we restored the connection to cloud
# storage.
wait_for_segments_removal(redpanda=self.redpanda,
topic=self.topic,
partition_idx=0,
count=6)
def test_fetch_after_committed_offset_was_removed(self,
transactions_enabled):
"""
Test fetching when consumer offset was deleted by retention
"""
self.redpanda._extra_rp_conf[
"enable_transactions"] = transactions_enabled
self.redpanda._extra_rp_conf[
"enable_idempotence"] = transactions_enabled
self.redpanda.start()
topic = TopicSpec(partition_count=1,
replication_factor=3,
cleanup_policy=TopicSpec.CLEANUP_DELETE)
self.client().create_topic(topic)
kafka_tools = KafkaCliTools(self.redpanda)
# produce until segments have been compacted
produce_until_segments(
self.redpanda,
topic=topic.name,
partition_idx=0,
count=10,
)
consumer_group = 'test'
rpk = RpkTool(self.redpanda)
def consume(n=1):
out = rpk.consume(topic.name, group=consumer_group, n=n)
split = out.split('}')
split = filter(lambda s: "{" in s, split)
return map(lambda s: json.loads(s + "}"), split)
#consume from the beggining
msgs = consume(10)
last = list(msgs).pop()
offset = last['offset']
# change retention time
kafka_tools.alter_topic_config(
topic.name, {
TopicSpec.PROPERTY_RETENTION_BYTES: 2 * self.segment_size,
})
wait_for_segments_removal(self.redpanda,
topic.name,
partition_idx=0,
count=5)
partitions = list(rpk.describe_topic(topic.name))
p = partitions[0]
assert p.start_offset > offset
# consume from the offset that doesn't exists,
# the one that was committed previously was already removed
out = list(consume(1))
assert out[0]['offset'] == p.start_offset
def test_consume_from_blocked_s3(self):
produce_until_segments(redpanda=self.redpanda,
topic=self.s3_topic_name,
partition_idx=0,
count=5,
acks=-1)
self.rpk.alter_topic_config(self.s3_topic_name,
TopicSpec.PROPERTY_RETENTION_BYTES,
self.retention_bytes)
wait_for_segments_removal(redpanda=self.redpanda,
topic=self.s3_topic_name,
partition_idx=0,
count=4)
"""Disconnect redpanda from S3 and try to read starting with offset 0"""
with firewall_blocked(self.redpanda.nodes, self._s3_port):
try:
out = self.rpk.consume(topic=self.s3_topic_name)
except RpkException as e:
assert 'timed out' in e.msg
else:
raise RuntimeError(
f"RPK consume should have timed out, but ran with output: {out}"
)
def test_changing_topic_retention(self, property, acks):
"""
Test changing topic retention duration for topics with data produced
with ACKS=1 and ACKS=-1. This test produces data until 10 segments
appear, then it changes retention topic property and waits for
segments to be removed
"""
kafka_tools = KafkaCliTools(self.redpanda)
# produce until segments have been compacted
produce_until_segments(
self.redpanda,
topic=self.topic,
partition_idx=0,
count=10,
acks=acks,
)
# change retention time
kafka_tools.alter_topic_config(self.topic, {
property: 10000,
})
wait_for_segments_removal(self.redpanda,
self.topic,
partition_idx=0,
count=5)
def test_write(self):
"""Write at least 10 segments, set retention policy to leave only 5
segments, wait for segments removal, consume data and run validation,
that everything that is acked is consumed."""
self.start_producer()
produce_until_segments(
redpanda=self.redpanda,
topic=self.topic,
partition_idx=0,
count=10,
)
self.kafka_tools.alter_topic_config(
self.topic,
{
TopicSpec.PROPERTY_RETENTION_BYTES:
5 * EndToEndShadowIndexingTest.segment_size,
},
)
wait_for_segments_removal(redpanda=self.redpanda,
topic=self.topic,
partition_idx=0,
count=6)
self.start_consumer()
self.run_validation()
def test_write_with_node_failures(self):
self.start_producer()
produce_until_segments(
redpanda=self.redpanda,
topic=self.topic,
partition_idx=0,
count=10,
)
self.kafka_tools.alter_topic_config(
self.topic,
{
TopicSpec.PROPERTY_RETENTION_BYTES:
5 * EndToEndShadowIndexingTest.segment_size
},
)
with random_process_kills(self.redpanda) as ctx:
wait_for_segments_removal(redpanda=self.redpanda,
topic=self.topic,
partition_idx=0,
count=6)
self.start_consumer()
self.run_validation()
ctx.assert_actions_triggered()
def test_changing_topic_retention_with_restart(self):
"""
Test changing topic retention duration for topics with data produced
with ACKS=1 and ACKS=-1. This test produces data until 10 segments
appear, then it changes retention topic property and waits for some
segmetnts to be removed
"""
segment_size = 1048576
# produce until segments have been compacted
produce_until_segments(
self.redpanda,
topic=self.topic,
partition_idx=0,
count=20,
acks=-1,
)
# restart all nodes to force replicating raft configuration
self.redpanda.restart_nodes(self.redpanda.nodes)
kafka_tools = KafkaCliTools(self.redpanda)
# Wait for controller, alter configs doesn't have a retry loop
kafka_tools.describe_topic(self.topic)
# change retention bytes to preserve 15 segments
self.client().alter_topic_configs(
self.topic, {
TopicSpec.PROPERTY_RETENTION_BYTES: 15 * segment_size,
})
wait_for_segments_removal(redpanda=self.redpanda,
topic=self.topic,
partition_idx=0,
count=16)
# change retention bytes again to preserve 10 segments
self.client().alter_topic_configs(
self.topic, {
TopicSpec.PROPERTY_RETENTION_BYTES: 10 * segment_size,
})
wait_for_segments_removal(redpanda=self.redpanda,
topic=self.topic,
partition_idx=0,
count=11)
# change retention bytes again to preserve 5 segments
self.client().alter_topic_configs(
self.topic, {
TopicSpec.PROPERTY_RETENTION_BYTES: 4 * segment_size,
})
wait_for_segments_removal(redpanda=self.redpanda,
topic=self.topic,
partition_idx=0,
count=5)
def test_timequery_after_segments_eviction(self):
"""
Test checking if the offset returned by time based index is
valid during applying log cleanup policy
"""
segment_size = 1048576
# produce until segments have been compacted
produce_until_segments(
self.redpanda,
topic=self.topic,
partition_idx=0,
count=10,
acks=-1,
)
# restart all nodes to force replicating raft configuration
self.redpanda.restart_nodes(self.redpanda.nodes)
kafka_tools = KafkaCliTools(self.redpanda)
# Wait for controller, alter configs doesn't have a retry loop
kafka_tools.describe_topic(self.topic)
# change retention bytes to preserve 15 segments
self.client().alter_topic_configs(
self.topic, {
TopicSpec.PROPERTY_RETENTION_BYTES: 2 * segment_size,
})
def validate_time_query_until_deleted():
def done():
kcat = KafkaCat(self.redpanda)
ts = 1638748800 # 12.6.2021 - old timestamp, query first offset
offset = kcat.query_offset(self.topic, 0, ts)
# assert that offset is valid
assert offset >= 0
topic_partitions = segments_count(self.redpanda, self.topic, 0)
partitions = []
for p in topic_partitions:
partitions.append(p <= 5)
return all([p <= 5 for p in topic_partitions])
wait_until(done,
timeout_sec=30,
backoff_sec=5,
err_msg="Segments were not removed")
validate_time_query_until_deleted()
def test_querying_remote_partitions(self):
topic = TopicSpec(redpanda_remote_read=True,
redpanda_remote_write=True)
epoch_offsets = {}
rpk = RpkTool(self.redpanda)
self.client().create_topic(topic)
rpk.alter_topic_config(topic.name, "redpanda.remote.read", 'true')
rpk.alter_topic_config(topic.name, "redpanda.remote.write", 'true')
def wait_for_topic():
wait_until(lambda: len(list(rpk.describe_topic(topic.name))) > 0,
30,
backoff_sec=2)
# restart whole cluster 6 times to trigger term rolls
for i in range(0, 6):
wait_for_topic()
produce_until_segments(
redpanda=self.redpanda,
topic=topic.name,
partition_idx=0,
count=2 * i,
)
res = list(rpk.describe_topic(topic.name))
epoch_offsets[res[0].leader_epoch] = res[0].high_watermark
self.redpanda.restart_nodes(self.redpanda.nodes)
self.logger.info(f"ledear epoch high watermarks: {epoch_offsets}")
wait_for_topic()
rpk.alter_topic_config(topic.name, TopicSpec.PROPERTY_RETENTION_BYTES,
OffsetForLeaderEpochArchivalTest.segment_size)
wait_for_segments_removal(redpanda=self.redpanda,
topic=topic.name,
partition_idx=0,
count=7)
kcl = KCL(self.redpanda)
for epoch, offset in epoch_offsets.items():
self.logger.info(f"querying partition epoch {epoch} end offsets")
epoch_end_offset = kcl.offset_for_leader_epoch(
topics=topic.name, leader_epoch=epoch)[0].epoch_end_offset
self.logger.info(
f"epoch {epoch} end_offset: {epoch_end_offset}, expected offset: {offset}"
)
assert epoch_end_offset == offset