def __init__(self, context):
super(SchemaRegistryTest, self).__init__(
context,
num_brokers=3,
enable_pp=True,
enable_sr=True,
extra_rp_conf={"auto_create_topics_enabled": False},
resource_settings=ResourceSettings(num_cpus=1))
http.client.HTTPConnection.debuglevel = 1
http.client.print = lambda *args: self.logger.debug(" ".join(args))
self._ctx = context
def __init__(self, test_context):
resource_setting = ResourceSettings(num_cpus=1)
super(ConnectionRateLimitTest,
self).__init__(test_context=test_context,
num_brokers=1,
node_prealloc_count=1,
extra_rp_conf={"kafka_connection_rate_limit": 6},
resource_settings=resource_setting)
self._producer = FranzGoVerifiableProducer(test_context, self.redpanda,
self.topics[0],
self.MSG_SIZE,
self.PRODUCE_COUNT,
self.preallocated_nodes)
def _restart_with_num_cpus(self, node: ClusterNode, num_cpus: int,
expect_fail: bool):
self.redpanda.stop_node(node)
self.redpanda.set_resource_settings(
ResourceSettings(num_cpus=num_cpus))
self.logger.info(f"Restarting redpanda with num_cpus={num_cpus}")
if expect_fail:
try:
self.redpanda.start_node(node)
except Exception as e:
self.logger.info(
f"As expected, redpanda failed to start ({e})")
else:
raise RuntimeError(
"Redpanda started: it should have failed to start!")
else:
self.redpanda.start_node(node)
self.logger.info("As expected, redpanda started successfully")
def test_memory_limited(self):
"""
Check enforcement of the RAM-per-partition threshold
"""
self.redpanda.set_resource_settings(
ResourceSettings(memory_mb=1024, num_cpus=1))
self.redpanda.set_extra_rp_conf({
# Use a larger than default memory per partition, so that a 1GB system can be
# tested without creating 1000 partitions (which overwhelms debug redpanda
# builds because they're much slower than the real product)
'topic_memory_per_partition':
10 * 1024 * 1024,
})
self.redpanda.start()
rpk = RpkTool(self.redpanda)
# Three nodes, each with 1GB memory, replicas=3, should
# result in an effective limit of 1024 with the default
# threshold of 1MB per topic.
try:
rpk.create_topic("toobig", partitions=110, replicas=3)
except RpkException as e:
assert 'INVALID_PARTITIONS' in e.msg
else:
assert False
# Should succeed
rpk.create_topic("okay", partitions=55, replicas=3)
# Trying to grow the partition count in violation of the limit should fail
try:
rpk.add_topic_partitions("okay", 55)
except RpkException as e:
assert 'INVALID_PARTITIONS' in e.msg
else:
assert False
# Growing the partition count within the limit should succeed
rpk.add_topic_partitions("okay", 10)
def test_fd_limited(self):
self.redpanda.set_resource_settings(ResourceSettings(nfiles=1000))
self.redpanda.set_extra_rp_conf({
# Disable memory limit: on a test node the physical memory can easily
# be the limiting factor
'topic_memory_per_partition': None,
})
self.redpanda.start()
rpk = RpkTool(self.redpanda)
# Default 10 fds per partition, we set ulimit down to 1000, so 100 should be the limit
try:
rpk.create_topic("toobig", partitions=110, replicas=3)
except RpkException as e:
assert 'INVALID_PARTITIONS' in e.msg
else:
assert False
# Should succeed
rpk.create_topic("okay", partitions=90, replicas=3)
def test_cpu_limited(self):
"""
Check enforcement of the partitions-per-core
"""
self.redpanda.set_resource_settings(ResourceSettings(num_cpus=1))
self.redpanda.set_extra_rp_conf({
# Disable memory limit: on a test node the physical memory can easily
# be the limiting factor
'topic_memory_per_partition': None,
# Disable FD enforcement: tests running on workstations may have low ulimits
'topic_fds_per_partition': None
})
self.redpanda.start()
rpk = RpkTool(self.redpanda)
# Three nodes, each with 1 core, 7000 partition-replicas
# per core, so with replicas=3, 7000 partitions should be the limit
try:
rpk.create_topic("toobig", partitions=8000, replicas=3)
except RpkException as e:
assert 'INVALID_PARTITIONS' in e.msg
else:
assert False
try:
rpk.create_topic("okay", partitions=6000, replicas=3)
except RpkException as e:
# Because this many partitions will overwhelm a debug build
# of redpanda, we tolerate exceptions, as long as the exception
# isn't about the partition count specifically.
#
# It would be better to execute this part of the test conditionally
# on release builds only.
assert 'INVALID_PARTITIONS' not in e.msg
def test_many_partitions(self):
"""
Validate that redpanda works with partition counts close to its resource
limits.
This test should evolve over time as we improve efficiency and can reliably
run with higher partition counts. It should roughly track the values we
use for topic_memory_per_partition and topic_fds_per_partition.
* Check topic can be created.
* Check leadership election succeeds for all partitions.
* Write in enough data such that an unlimited size fetch
would exhaust ram (check enforcement of kafka_max_bytes_per_fetch).
* Consume all the data from the topic
* Restart nodes several times (check that recovery works, and that the additional
log segments created by rolling segments on restart do not cause us
to exhaust resources.
* Run a general produce+consume workload to check that the system remains in
a functional state.
"""
# This test requires dedicated system resources to run reliably.
#assert self.redpanda.dedicated_nodes
# Scale tests are not run on debug builds
assert not self.debug_mode
replication_factor = 3
node_count = len(self.redpanda.nodes)
# If we run on nodes with more memory than our HARD_PARTITION_LIMIT, then
# artificially throttle the nodes' memory to avoid the test being too easy.
# We are validating that the system works up to the limit, and that it works
# up to the limit within the default per-partition memory footprint.
node_memory = self.redpanda.get_node_memory_mb()
# HARD_PARTITION_LIMIT is for a 3 node cluster, adjust according to
# the number of nodes in this cluster.
partition_limit = HARD_PARTITION_LIMIT * (node_count / 3)
mb_per_partition = 1
# How much memory to reserve for internal partitions, such as
# id_allocator. This is intentionally higher than needed, to
# avoid having to update this test each time a new internal topic
# is added.
internal_partition_slack = 10
# Emulate seastar's policy for default reserved memory
reserved_memory = max(1536, int(0.07 * node_memory) + 1)
effective_node_memory = node_memory - reserved_memory
# TODO: calculate an appropriate segment size for the disk space divided
# by the partition count, then set an appropriate retention.bytes and
# enable compaction, so that during the final stress period of the test,
# we are exercising compaction.
# Clamp memory if nodes have more memory than should be required
# to exercise the partition limit.
if effective_node_memory > HARD_PARTITION_LIMIT / mb_per_partition:
clamp_memory = mb_per_partition * (
(HARD_PARTITION_LIMIT + internal_partition_slack) +
reserved_memory)
# Handy if hacking HARD_PARTITION_LIMIT to something low to run on a workstation
clamp_memory = max(clamp_memory, 500)
resource_settings = ResourceSettings(memory_mb=clamp_memory)
self.redpanda.set_resource_settings(resource_settings)
elif effective_node_memory < HARD_PARTITION_LIMIT / mb_per_partition:
raise RuntimeError(
f"Node memory is too small ({node_memory}MB - {reserved_memory}MB)"
)
# Run with one huge topic: this is the more stressful case for Redpanda, compared
# with multiple modestly-sized topics, so it's what we test to find the system's limits.
n_topics = 1
# Partitions per topic
n_partitions = int(partition_limit / n_topics)
self.logger.info(
f"Running partition scale test with {n_partitions} partitions on {n_topics} topics"
)
self.redpanda.start()
self.logger.info("Entering topic creation")
topic_names = [f"scale_{i:06d}" for i in range(0, n_topics)]
for tn in topic_names:
self.logger.info(
f"Creating topic {tn} with {n_partitions} partitions")
self.rpk.create_topic(tn,
partitions=n_partitions,
replicas=replication_factor)
self.logger.info(f"Awaiting elections...")
wait_until(lambda: self._all_elections_done(topic_names, n_partitions),
timeout_sec=60,
backoff_sec=5)
self.logger.info(f"Initial elections done.")
for node in self.redpanda.nodes:
files = self.redpanda.lsof_node(node)
file_count = sum(1 for _ in files)
self.logger.info(
f"Open files after initial selection on {node.name}: {file_count}"
)
# Assume fetches will be 10MB, the franz-go default
fetch_mb_per_partition = 10 * 1024 * 1024
# * Need enough data that if a consumer tried to fetch it all at once
# in a single request, it would run out of memory. OR the amount of
# data that would fill a 10MB max_bytes per partition in a fetch, whichever
# is lower (avoid writing excessive data for tests with fewer partitions).
# * Then apply a factor of two to make sure we have enough data to drive writes
# to disk during consumption, not just enough data to hold it all in the batch
# cache.
write_bytes_per_topic = min(
int((self.redpanda.get_node_memory_mb() * 1024 * 1024) / n_topics),
fetch_mb_per_partition * n_partitions) * 2
if self.scale.release:
# Release tests can be much longer running: 10x the amount of
# data we fire through the system
write_bytes_per_topic *= 10
msg_size = 128 * 1024
msg_count_per_topic = int((write_bytes_per_topic / msg_size))
# Approx time to write or read all messages, for timeouts
# Pessimistic bandwidth guess, accounting for the sub-disk bandwidth
# that a single-threaded consumer may see
expect_bandwidth = 50 * 1024 * 1024
expect_transmit_time = int(write_bytes_per_topic / expect_bandwidth)
expect_transmit_time = max(expect_transmit_time, 30)
self.logger.info("Entering initial produce")
for tn in topic_names:
t1 = time.time()
producer = FranzGoVerifiableProducer(
self.test_context,
self.redpanda,
tn,
msg_size,
msg_count_per_topic,
custom_node=self.preallocated_nodes)
producer.start()
producer.wait(timeout_sec=expect_transmit_time)
self.free_preallocated_nodes()
duration = time.time() - t1
self.logger.info(
f"Wrote {write_bytes_per_topic} bytes to {tn} in {duration}s, bandwidth {(write_bytes_per_topic / duration)/(1024 * 1024)}MB/s"
)
def get_fd_counts():
counts = {}
with concurrent.futures.ThreadPoolExecutor(
max_workers=node_count) as executor:
futs = {}
for node in self.redpanda.nodes:
futs[node.name] = executor.submit(
lambda: sum(1 for _ in self.redpanda.lsof_node(node)))
for node_name, fut in futs.items():
file_count = fut.result()
counts[node_name] = file_count
return counts
for node_name, file_count in get_fd_counts().items():
self.logger.info(
f"Open files before restarts on {node_name}: {file_count}")
# Over large partition counts, the startup time is linear with the
# amount of data we played in, because no one partition gets far
# enough to snapshot.
expect_start_time = expect_transmit_time
# Measure the impact of restarts on resource utilization on an idle system:
# at time of writing we know that the used FD count will go up substantially
# on each restart (https://github.com/redpanda-data/redpanda/issues/4057)
restart_count = 2
self.logger.info("Entering restart stress test")
for i in range(1, restart_count + 1):
self.logger.info(f"Cluster restart {i}/{restart_count}...")
# Normal restarts are rolling restarts, but because replay takes substantial time,
# on an idle system it is helpful to do a concurrent global restart rather than
# waiting for each node one by one.
with concurrent.futures.ThreadPoolExecutor(
max_workers=node_count) as executor:
futs = []
for node in self.redpanda.nodes:
futs.append(
executor.submit(self.redpanda.restart_nodes,
nodes=[node],
start_timeout=expect_start_time))
for f in futs:
# Raise on error
f.result()
self.logger.info(
f"Restart {i}/{restart_count} complete. Waiting for elections..."
)
wait_until(
lambda: self._all_elections_done(topic_names, n_partitions),
timeout_sec=60,
backoff_sec=5)
self.logger.info(f"Post-restart elections done.")
for node_name, file_count in get_fd_counts().items():
self.logger.info(
f"Open files after {i} restarts on {node_name}: {file_count}"
)
# With increased overhead from all those segment rolls during restart,
# check that consume still works.
self._consume_all(topic_names, msg_count_per_topic,
expect_transmit_time)
# Now that we've tested basic ability to form consensus and survive some
# restarts, move on to a more general stress test.
self.logger.info("Entering traffic stress test")
target_topic = topic_names[0]
stress_msg_size = 32768
stress_data_size = 1024 * 1024 * 1024 * 100
stress_msg_count = int(stress_data_size / stress_msg_size)
fast_producer = FranzGoVerifiableProducer(
self.test_context,
self.redpanda,
target_topic,
stress_msg_size,
stress_msg_count,
custom_node=self.preallocated_nodes)
fast_producer.start()
# Don't start consumers until the producer has written out its first
# checkpoint with valid ranges.
wait_until(lambda: fast_producer.produce_status.acked > 0,
timeout_sec=30,
backoff_sec=1.0)
rand_consumer = FranzGoVerifiableRandomConsumer(
self.test_context,
self.redpanda,
target_topic,
0,
100,
10,
nodes=self.preallocated_nodes)
rand_consumer.start(clean=False)
rand_consumer.shutdown()
rand_consumer.wait()
fast_producer.wait()
seq_consumer = FranzGoVerifiableSeqConsumer(self.test_context,
self.redpanda,
target_topic, 0,
self.preallocated_nodes)
seq_consumer.start(clean=False)
seq_consumer.shutdown()
seq_consumer.wait()
assert seq_consumer.consumer_status.invalid_reads == 0
assert seq_consumer.consumer_status.valid_reads == stress_msg_count + msg_count_per_topic
self.logger.info("Entering leader balancer stress test")
# Enable the leader balancer and check that the system remains stable
# under load. We do not leave the leader balancer on for most of the test, because
# it makes reads _much_ slower, because the consumer keeps stalling and waiting for
# elections: at any moment in a 10k partition topic, it's highly likely at least
# one partition is offline for a leadership migration.
self.redpanda.set_cluster_config({'enable_leader_balancer': True},
expect_restart=False)
lb_stress_period = 120
lb_stress_produce_bytes = expect_bandwidth * lb_stress_period
lb_stress_message_count = int(lb_stress_produce_bytes /
stress_msg_size)
fast_producer = FranzGoVerifiableProducer(
self.test_context,
self.redpanda,
target_topic,
stress_msg_size,
lb_stress_message_count,
custom_node=self.preallocated_nodes)
fast_producer.start()
rand_consumer.start()
time.sleep(lb_stress_period
) # Let the system receive traffic for a set time period
rand_consumer.shutdown()
rand_consumer.wait()
fast_producer.wait()
def test_migrating_consume_offsets(self, failures, cpus):
'''
Validates correctness while executing consumer offsets migration
'''
# set redpanda logical version to value without __consumer_offsets support
self.redpanda = RedpandaService(
self.test_context,
5,
resource_settings=ResourceSettings(num_cpus=cpus),
extra_rp_conf={
"group_topic_partitions": 16,
"default_topic_replications": 3,
},
environment={"__REDPANDA_LOGICAL_VERSION": 1})
self.redpanda.start()
self._client = DefaultClient(self.redpanda)
# set of failure suppressed nodes - required to make restarts deterministic
suppressed = set()
def failure_injector_loop():
f_injector = FailureInjector(self.redpanda)
while failures:
f_type = random.choice(FailureSpec.FAILURE_TYPES)
length = 0
node = random.choice(self.redpanda.nodes)
while self.redpanda.idx(node) in suppressed:
node = random.choice(self.redpanda.nodes)
# allow suspending any node
if f_type == FailureSpec.FAILURE_SUSPEND:
length = random.randint(
1,
ConsumerOffsetsMigrationTest.max_suspend_duration_sec)
f_injector.inject_failure(
FailureSpec(node=node, type=f_type, length=length))
delay = random.randint(
ConsumerOffsetsMigrationTest.min_inter_failure_time_sec,
ConsumerOffsetsMigrationTest.max_inter_failure_time_sec)
self.redpanda.logger.info(
f"waiting {delay} seconds before next failure")
time.sleep(delay)
if failures:
finjector_thread = threading.Thread(target=failure_injector_loop,
args=())
finjector_thread.daemon = True
finjector_thread.start()
spec = TopicSpec(partition_count=6, replication_factor=3)
self.client().create_topic(spec)
self.topic = spec.name
self.start_producer(1, throughput=5000)
self.start_consumer(1)
self.await_startup()
def cluster_is_stable():
admin = Admin(self.redpanda)
brokers = admin.get_brokers()
if len(brokers) < 3:
return False
for b in brokers:
self.logger.debug(f"broker: {b}")
if not (b['is_alive'] and 'disk_space' in b):
return False
return True
kcl = KCL(self.redpanda)
def _group_present():
return len(kcl.list_groups().splitlines()) > 1
# make sure that group is there
wait_until(_group_present, 10, 1)
# check that consumer offsets topic is not present
topics = set(kcl.list_topics())
assert "__consumer_offsets" not in topics
# enable consumer offsets support
self.redpanda.set_environment({"__REDPANDA_LOGICAL_VERSION": 2})
for n in self.redpanda.nodes:
id = self.redpanda.idx(n)
suppressed.add(id)
self.redpanda.restart_nodes(n, stop_timeout=60)
suppressed.remove(id)
# wait for leader balancer to start evening out leadership
wait_until(cluster_is_stable, 90, backoff_sec=2)
def _consumer_offsets_present():
try:
partitions = list(
self.client().describe_topic("__consumer_offsets"))
return len(partitions) > 0
except:
return False
wait_until(_consumer_offsets_present, timeout_sec=90, backoff_sec=3)
self.run_validation(min_records=100000,
producer_timeout_sec=300,
consumer_timeout_sec=180)
def __init__(self, *args, **kwargs):
super().__init__(
*args,
resource_settings=ResourceSettings(num_cpus=self.INITIAL_NUM_CPUS),
**kwargs)
from rptest.clients.types import TopicSpec
from rptest.tests.redpanda_test import RedpandaTest
from rptest.services.redpanda import ResourceSettings, RESTART_LOG_ALLOW_LIST
from rptest.services.cluster import cluster
from rptest.services.rpk_consumer import RpkConsumer
from ducktape.utils.util import wait_until
from rptest.services.producer_swarm import ProducerSwarm
resource_settings = ResourceSettings(
num_cpus=2,
# Set a low memory size, such that there is only ~100k of memory available
# for dealing with each client.
memory_mb=768,
# Test nodes and developer workstations may have slow fsync. For this test
# we need things to be consistently fast, so disable fsync (this test
# has nothing to do with verifying the correctness of the storage layer)
bypass_fsync=True)
class ManyClientsTest(RedpandaTest):
def __init__(self, *args, **kwargs):
# We will send huge numbers of messages, so tune down the log verbosity
# as this is just a "did we stay up?" test
kwargs['log_level'] = "info"
kwargs['resource_settings'] = resource_settings
super().__init__(*args, **kwargs)