def alter_broker_config(self,
values: dict[str, typing.Any],
incremental: bool,
*,
broker: typing.Optional[int] = None):
kcl = KCL(self._redpanda)
return kcl.alter_broker_config(values, incremental, broker)
def test_incremental_alter_configs(self):
"""
Central config can also be accessed via Kafka API -- exercise that
using `kcl`.
:param incremental: whether to use incremental kafka config API or
legacy config API.
"""
kcl = KCL(self.redpanda)
# Redpanda only support incremental config changes: the legacy
# AlterConfig API is a bad user experience
incremental = True
# Set a property by its redpanda name
out = kcl.alter_broker_config(
{"log_message_timestamp_type": "CreateTime"}, incremental)
# kcl does not set an error exist status when config set fails, so must
# read its output text to validate that calls are successful
assert 'OK' in out
out = kcl.alter_broker_config(
{"log_message_timestamp_type": "LogAppendTime"}, incremental)
assert 'OK' in out
if incremental:
kcl.delete_broker_config(["log_message_timestamp_type"],
incremental)
assert 'OK' in out
# Set a property by its Kafka-interop names and values
kafka_props = {
"log.message.timestamp.type": ["CreateTime", "LogAppendTime"],
"log.cleanup.policy": ["compact", "delete"],
"log.compression.type": ["gzip", "snappy", "lz4", "zstd"],
}
for property, value_list in kafka_props.items():
for value in value_list:
out = kcl.alter_broker_config({property: value}, incremental)
assert 'OK' in out
# Set a nonexistent property
out = kcl.alter_broker_config({"does_not_exist": "avalue"},
incremental)
assert 'INVALID_CONFIG' in out
# Set a malformed property
out = kcl.alter_broker_config(
{"log_message_timestamp_type": "BadValue"}, incremental)
assert 'INVALID_CONFIG' in out
# Set a property on a named broker: should fail because this
# interface is only for cluster-wide properties
out = kcl.alter_broker_config(
{"log_message_timestamp_type": "CreateTime"},
incremental,
broker="1")
assert 'INVALID_CONFIG' in out
assert "Setting broker properties on named brokers is unsupported" in out
def is_topic_present(name):
kcl = KCL(self.redpanda)
lines = kcl.list_topics().splitlines()
self.redpanda.logger.debug(
f"checking if topic {name} is present in {lines}")
for l in lines:
if l.startswith(name):
return True
return False
def test_list_groups(self):
kcl = KCL(self.redpanda)
kcl.produce(self.topic, "msg\n")
kcl.consume(self.topic, n=1, group="g0")
kcl.list_groups()
out = kcl.list_groups()
assert "COORDINATOR_LOAD_IN_PROGRESS" not in out
def test_alter_configs(self):
"""
We only support incremental config changes. Check that AlterConfigs requests
are correctly handled with an 'unsupported' response.
"""
kcl = KCL(self.redpanda)
out = kcl.alter_broker_config(
{"log_message_timestamp_type": "CreateTime"}, incremental=False)
self.logger.info("AlterConfigs output: {out}")
assert 'INVALID_CONFIG' in out
assert "changing broker properties isn't supported via this API" in out
def test_querying_remote_partitions(self):
topic = TopicSpec(redpanda_remote_read=True,
redpanda_remote_write=True)
epoch_offsets = {}
rpk = RpkTool(self.redpanda)
self.client().create_topic(topic)
rpk.alter_topic_config(topic.name, "redpanda.remote.read", 'true')
rpk.alter_topic_config(topic.name, "redpanda.remote.write", 'true')
def wait_for_topic():
wait_until(lambda: len(list(rpk.describe_topic(topic.name))) > 0,
30,
backoff_sec=2)
# restart whole cluster 6 times to trigger term rolls
for i in range(0, 6):
wait_for_topic()
produce_until_segments(
redpanda=self.redpanda,
topic=topic.name,
partition_idx=0,
count=2 * i,
)
res = list(rpk.describe_topic(topic.name))
epoch_offsets[res[0].leader_epoch] = res[0].high_watermark
self.redpanda.restart_nodes(self.redpanda.nodes)
self.logger.info(f"ledear epoch high watermarks: {epoch_offsets}")
wait_for_topic()
rpk.alter_topic_config(topic.name, TopicSpec.PROPERTY_RETENTION_BYTES,
OffsetForLeaderEpochArchivalTest.segment_size)
wait_for_segments_removal(redpanda=self.redpanda,
topic=topic.name,
partition_idx=0,
count=7)
kcl = KCL(self.redpanda)
for epoch, offset in epoch_offsets.items():
self.logger.info(f"querying partition epoch {epoch} end offsets")
epoch_end_offset = kcl.offset_for_leader_epoch(
topics=topic.name, leader_epoch=epoch)[0].epoch_end_offset
self.logger.info(
f"epoch {epoch} end_offset: {epoch_end_offset}, expected offset: {offset}"
)
assert epoch_end_offset == offset
def test_list_groups_has_no_duplicates(self):
"""
Reproducer for:
https://github.com/vectorizedio/redpanda/issues/2528
"""
kcl = KCL(self.redpanda)
kcl.produce(self.topic, "msg\n")
# create 4 groups
kcl.consume(self.topic, n=1, group="g0")
kcl.consume(self.topic, n=1, group="g1")
kcl.consume(self.topic, n=1, group="g2")
kcl.consume(self.topic, n=1, group="g3")
# transfer kafka_internal/group/0 leadership across the nodes to trigger
# group state recovery on each node
for n in self.redpanda.nodes:
self._transfer_with_retry(namespace="kafka_internal",
topic="group",
partition=0,
target_id=self.redpanda.idx(n))
# assert that there are no duplicates in
def _list_groups():
out = kcl.list_groups()
groups = []
for l in out.splitlines():
# skip header line
if l.startswith("BROKER"):
continue
parts = l.split()
groups.append({'node_id': parts[0], 'group': parts[1]})
return groups
def _check_no_duplicates():
groups = _list_groups()
self.redpanda.logger.debug(f"groups: {groups}")
return len(groups) == 4
wait_until(_check_no_duplicates,
10,
backoff_sec=2,
err_msg="found persistent duplicates in groups listing")
def simple_fetch_handler_fairness_test(self, type):
"""
Testing if when fetching from single node all partitions are
returned in round robin fashion
"""
def multiple_topics(count):
return [
TopicSpec(partition_count=1, replication_factor=1)
for _ in range(count)
]
def multiple_partitions(count):
return [TopicSpec(partition_count=count, replication_factor=1)]
number_of_partitions = 32
records_per_partition = 100
topics = []
if type == 'multiple-topics':
topics = multiple_topics(number_of_partitions)
else:
topics = multiple_partitions(number_of_partitions)
# create topics
self.client().create_topic(specs=topics)
self.redpanda.logger.info(f"topics: {topics}")
rpk = RpkTool(self.redpanda)
# publish 10 messages to each topic/partition
for s in topics:
t = s.name
for p in range(0, s.partition_count):
for i in range(0, records_per_partition):
self.redpanda.logger.info(f"producing to : {t}/{p}")
rpk.produce(t,
f"k.{t}.{p}.{i}",
f"p.{t}.{p}.{i}",
partition=p)
# configure kcl to fetch at most 1 byte in single fetch request,
# this way we should receive exactly one record per each partition
consumed_frequency = defaultdict(lambda: 0)
kcl = KCL(self.redpanda)
expected_frequency = records_per_partition / 2
# issue single kcl command that will generate multiple fetch requests
# in single fetch session, it should include single partition in each
# fetch response.
to_consume = int(number_of_partitions * expected_frequency)
consumed = kcl.consume(topic="topic-.*",
n=to_consume,
regex=True,
fetch_max_bytes=1,
group="test-gr-1")
for c in consumed.split():
parts = c.split('.')
consumed_frequency[(parts[1], parts[2])] += 1
for p, freq in consumed_frequency.items():
self.redpanda.logger.info(f"consumed {freq} messages from: {p}")
# assert that frequency is in expected range
assert freq <= expected_frequency + 0.1 * expected_frequency
assert freq >= expected_frequency - 0.1 * expected_frequency
def test_offset_for_leader_epoch(self):
replication_factors = [1, 3, 5]
cleanup_policies = [
TopicSpec.CLEANUP_COMPACT, TopicSpec.CLEANUP_DELETE
]
topics = []
for i in range(0, 10):
topics.append(
TopicSpec(
partition_count=random.randint(1, 50),
replication_factor=random.choice(replication_factors),
cleanup_policy=random.choice(cleanup_policies)))
topic_names = [t.name for t in topics]
# create test topics
self.client().create_topic(topics)
kcl = KCL(self.redpanda)
for t in topics:
self._produce(t.name, 20)
def list_offsets_map():
offsets_map = {}
offsets = kcl.list_offsets(topic_names)
self.logger.info(f"offsets_list: {offsets}")
for p in offsets:
offsets_map[(p.topic, p.partition)] = int(p.end_offset)
self.logger.info(f"offsets_map: {offsets_map}")
return offsets_map
initial_offsets = list_offsets_map()
leader_epoch_offsets = kcl.offset_for_leader_epoch(topics=topic_names,
leader_epoch=1)
for o in leader_epoch_offsets:
assert initial_offsets[(o.topic,
o.partition)] == o.epoch_end_offset
# restart all the nodes to force leader election,
# increase start timeout as partition count may get large
self.redpanda.restart_nodes(self.redpanda.nodes, start_timeout=30)
# produce more data
for t in topics:
self._produce(t.name, 20)
# check epoch end offsets for term 1
leader_epoch_offsets = kcl.offset_for_leader_epoch(topics=topic_names,
leader_epoch=1)
for o in leader_epoch_offsets:
assert initial_offsets[(o.topic,
o.partition)] == o.epoch_end_offset
last_offsets = list_offsets_map()
rpk = RpkTool(self.redpanda)
for t in topics:
tp_desc = rpk.describe_topic(t.name)
for p in tp_desc:
for o in kcl.offset_for_leader_epoch(
topics=f"{t.name}:{p.id}",
leader_epoch=p.leader_epoch,
current_leader_epoch=p.leader_epoch):
assert last_offsets[(o.topic,
o.partition)] == o.epoch_end_offset
# test returning unknown leader epoch error, we use large leader epoch value
leader_epoch_offsets = kcl.offset_for_leader_epoch(
topics=topic_names, leader_epoch=1, current_leader_epoch=1000)
for o in leader_epoch_offsets:
assert o.error is not None and "UNKNOWN_LEADER_EPOCH" in o.error
def test_migrating_consume_offsets(self, failures, cpus):
'''
Validates correctness while executing consumer offsets migration
'''
# set redpanda logical version to value without __consumer_offsets support
self.redpanda = RedpandaService(
self.test_context,
5,
resource_settings=ResourceSettings(num_cpus=cpus),
extra_rp_conf={
"group_topic_partitions": 16,
"default_topic_replications": 3,
},
environment={"__REDPANDA_LOGICAL_VERSION": 1})
self.redpanda.start()
self._client = DefaultClient(self.redpanda)
# set of failure suppressed nodes - required to make restarts deterministic
suppressed = set()
def failure_injector_loop():
f_injector = FailureInjector(self.redpanda)
while failures:
f_type = random.choice(FailureSpec.FAILURE_TYPES)
length = 0
node = random.choice(self.redpanda.nodes)
while self.redpanda.idx(node) in suppressed:
node = random.choice(self.redpanda.nodes)
# allow suspending any node
if f_type == FailureSpec.FAILURE_SUSPEND:
length = random.randint(
1,
ConsumerOffsetsMigrationTest.max_suspend_duration_sec)
f_injector.inject_failure(
FailureSpec(node=node, type=f_type, length=length))
delay = random.randint(
ConsumerOffsetsMigrationTest.min_inter_failure_time_sec,
ConsumerOffsetsMigrationTest.max_inter_failure_time_sec)
self.redpanda.logger.info(
f"waiting {delay} seconds before next failure")
time.sleep(delay)
if failures:
finjector_thread = threading.Thread(target=failure_injector_loop,
args=())
finjector_thread.daemon = True
finjector_thread.start()
spec = TopicSpec(partition_count=6, replication_factor=3)
self.client().create_topic(spec)
self.topic = spec.name
self.start_producer(1, throughput=5000)
self.start_consumer(1)
self.await_startup()
def cluster_is_stable():
admin = Admin(self.redpanda)
brokers = admin.get_brokers()
if len(brokers) < 3:
return False
for b in brokers:
self.logger.debug(f"broker: {b}")
if not (b['is_alive'] and 'disk_space' in b):
return False
return True
kcl = KCL(self.redpanda)
def _group_present():
return len(kcl.list_groups().splitlines()) > 1
# make sure that group is there
wait_until(_group_present, 10, 1)
# check that consumer offsets topic is not present
topics = set(kcl.list_topics())
assert "__consumer_offsets" not in topics
# enable consumer offsets support
self.redpanda.set_environment({"__REDPANDA_LOGICAL_VERSION": 2})
for n in self.redpanda.nodes:
id = self.redpanda.idx(n)
suppressed.add(id)
self.redpanda.restart_nodes(n, stop_timeout=60)
suppressed.remove(id)
# wait for leader balancer to start evening out leadership
wait_until(cluster_is_stable, 90, backoff_sec=2)
def _consumer_offsets_present():
try:
partitions = list(
self.client().describe_topic("__consumer_offsets"))
return len(partitions) > 0
except:
return False
wait_until(_consumer_offsets_present, timeout_sec=90, backoff_sec=3)
self.run_validation(min_records=100000,
producer_timeout_sec=300,
consumer_timeout_sec=180)
def test_cluster_is_available_during_upgrade_without_group_topic(self):
'''
Validates that cluster is available and healthy during
upgrade when `kafka_internal::group` topic is not present
'''
# set redpanda logical version to value without __consumer_offsets support
self.redpanda = RedpandaService(
self.test_context,
5,
extra_rp_conf={
"group_topic_partitions": 16,
"default_topic_replications": 3,
},
environment={"__REDPANDA_LOGICAL_VERSION": 1})
self.redpanda.start()
self._client = DefaultClient(self.redpanda)
spec = TopicSpec(partition_count=6, replication_factor=3)
self.client().create_topic(spec)
self.topic = spec.name
def cluster_is_stable():
admin = Admin(self.redpanda)
brokers = admin.get_brokers()
if len(brokers) < 3:
return False
for b in brokers:
self.logger.debug(f"broker: {b}")
if not (b['is_alive'] and 'disk_space' in b):
return False
return True
def node_stopped(node_id):
admin = Admin(self.redpanda)
brokers = admin.get_brokers()
for b in brokers:
self.logger.debug(f"broker: {b}")
if b['node_id'] == node_id:
return b['is_alive'] == False
return False
kcl = KCL(self.redpanda)
# check that consumer offsets topic is not present
topics = set(kcl.list_topics())
assert "__consumer_offsets" not in topics
# enable consumer offsets support
self.redpanda.set_environment({"__REDPANDA_LOGICAL_VERSION": 2})
def get_raft0_follower():
ctrl = self.redpanda.controller
node = random.choice(self.redpanda.nodes)
while self.redpanda.idx(node) == self.redpanda.idx(ctrl):
node = random.choice(self.redpanda.nodes)
return node
# restart node that is not controller
n = get_raft0_follower()
self.logger.info(f"restarting node {n.account.hostname}")
self.redpanda.stop_node(n, timeout=60)
# wait for leader balancer to start evening out leadership
wait_until(lambda: node_stopped(self.redpanda.idx(n)),
90,
backoff_sec=2)
self.redpanda.start_node(n)
wait_until(cluster_is_stable, 90, backoff_sec=2)
def delete_broker_config(self, keys: list[str], incremental: bool):
kcl = KCL(self._redpanda)
return kcl.delete_broker_config(keys, incremental)