def test_multiple_topics_and_partitions(self):
rpk = RpkTool(self.redpanda)
topics = self.topics
group = "g0"
for i in range(100):
payload = str(random.randint(0, 1000))
for topic_spec in topics:
for p in range(topic_spec.partition_count):
rpk.produce(topic_spec.name,
"",
payload,
partition=p,
timeout=5)
for topic_spec in topics:
rpk.consume(topic_spec.name,
group=group,
n=100 * topic_spec.partition_count)
metrics_offsets = self._get_offset_from_metrics(group)
partitions_amount = sum(map(lambda x: x.partition_count, topics))
assert len(metrics_offsets) == partitions_amount
for topic_spec in topics:
for i in range(topic_spec.partition_count):
assert (topic_spec.name, str(i)) in metrics_offsets
assert metrics_offsets[(topic_spec.name, str(i))] == 100
class TopicAutocreateTest(RedpandaTest):
"""
Verify that autocreation works, and that the settings of an autocreated
topic match those for a topic created by hand with rpk.
"""
def __init__(self, test_context):
super(TopicAutocreateTest, self).__init__(
test_context=test_context,
num_brokers=1,
extra_rp_conf={'auto_create_topics_enabled': False})
self.kafka_tools = KafkaCliTools(self.redpanda)
self.rpk = RpkTool(self.redpanda)
@cluster(num_nodes=1)
def topic_autocreate_test(self):
auto_topic = 'autocreated'
manual_topic = "manuallycreated"
# With autocreation disabled, producing to a nonexistent topic should not work.
try:
# Use rpk rather than kafka CLI because rpk errors out promptly
self.rpk.produce(auto_topic, "foo", "bar")
except Exception:
# The write failed, and shouldn't have created a topic
assert auto_topic not in self.kafka_tools.list_topics()
else:
assert False, "Producing to a nonexistent topic should fail"
# Enable autocreation
self.redpanda.restart_nodes(self.redpanda.nodes,
{'auto_create_topics_enabled': True})
# Auto create topic
assert auto_topic not in self.kafka_tools.list_topics()
self.kafka_tools.produce(auto_topic, 1, 4096)
assert auto_topic in self.kafka_tools.list_topics()
auto_topic_spec = self.kafka_tools.describe_topic(auto_topic)
assert auto_topic_spec.retention_ms is None
assert auto_topic_spec.retention_bytes is None
# Create topic by hand, compare its properties to the autocreated one
self.rpk.create_topic(manual_topic)
manual_topic_spec = self.kafka_tools.describe_topic(auto_topic)
assert manual_topic_spec.retention_ms == auto_topic_spec.retention_ms
assert manual_topic_spec.retention_bytes == auto_topic_spec.retention_bytes
# Clear name and compare the rest of the attributes
manual_topic_spec.name = auto_topic_spec.name = None
assert manual_topic_spec == auto_topic_spec
def _ping_pong(self):
kc = KafkaCat(self.redpanda)
rpk = RpkTool(self.redpanda)
payload = str(random.randint(0, 1000))
start = time.time()
offset = rpk.produce(self.topic, "tkey", payload, timeout=5)
consumed = kc.consume_one(self.topic, 0, offset)
latency = time.time() - start
self.logger.info(
f"_ping_pong produced '{payload}' consumed '{consumed}' in {(latency)*1000.0:.2f} ms"
)
if consumed['payload'] != payload:
raise RuntimeError(f"expected '{payload}' got '{consumed}'")
def fetch_long_poll_test(self):
"""
Test if redpanda is able to debounce fetches when consumer requests
to wait for data
"""
partition_count = 20
total_messages = 10
topic = TopicSpec(partition_count=partition_count,
replication_factor=3)
# create topic
self.client().create_topic(specs=topic)
rpk = RpkTool(self.redpanda)
consumer = KafkaCliConsumer(self.test_context,
self.redpanda,
topic=topic.name,
group='test-gr-1',
from_beginning=True,
consumer_properties={
'fetch.min.bytes': 1024,
'fetch.max.wait.ms': 1000,
})
consumer.start()
for p in range(0, total_messages + 1):
rpk.produce(topic.name,
f"k-{p}",
f"v-test-{p}",
partition=p % partition_count)
# sleep for 2 seconds every each message
# to prevent fetch handler from returning fast
sleep(2)
consumer.wait_for_messages(10)
consumer.stop()
consumer.wait()
def test_check_value(self):
rpk = RpkTool(self.redpanda)
topic = next(filter(lambda x: x.partition_count == 1,
self.topics)).name
for i in range(100):
payload = str(random.randint(0, 1000))
offset = rpk.produce(topic, "", payload, timeout=5)
group_1 = "g1"
metric_key = (topic, "0")
for i in range(10):
rpk.consume(topic, group=group_1, n=10)
metrics_offsets = self._get_offset_from_metrics(group_1)
assert metric_key in metrics_offsets
assert metrics_offsets[metric_key] == (i + 1) * 10
group_2 = "g2"
rpk.consume(topic, group=group_2, n=50)
gr_2_metrics_offsets = self._get_offset_from_metrics(group_2)
assert metric_key in gr_2_metrics_offsets
assert gr_2_metrics_offsets[metric_key] == 50
rpk.group_seek_to_group(group_1, group_2)
gr_1_metrics_offsets = self._get_offset_from_metrics(group_1)
assert metric_key in gr_1_metrics_offsets
assert gr_1_metrics_offsets[metric_key] == 50
rpk.group_seek_to(group_2, "start")
gr_2_metrics_offsets = self._get_offset_from_metrics(group_2)
assert metric_key in gr_2_metrics_offsets
assert gr_2_metrics_offsets[metric_key] == 0
self.client().delete_topic(topic)
def metrics_gone():
metrics_offsets = self._get_offset_from_metrics(group_1)
return metrics_offsets is None
wait_until(metrics_gone, timeout_sec=30, backoff_sec=5)
class RpkToolTest(RedpandaTest):
def __init__(self, ctx):
super(RpkToolTest, self).__init__(test_context=ctx)
self._ctx = ctx
self._rpk = RpkTool(self.redpanda)
@cluster(num_nodes=3)
def test_create_topic(self):
self._rpk.create_topic("topic")
wait_until(lambda: "topic" in self._rpk.list_topics(),
timeout_sec=10,
backoff_sec=1,
err_msg="Topic never appeared.")
@cluster(num_nodes=4)
def test_produce(self):
topic = 'topic'
message = 'message'
key = 'key'
h_key = 'h_key'
h_value = 'h_value'
headers = [h_key + ':' + h_value]
self._rpk.create_topic(topic)
self._rpk.produce(topic, key, message, headers)
c = RpkConsumer(self._ctx, self.redpanda, topic)
c.start()
def cond():
return c.messages is not None \
and len(c.messages) == 1 \
and c.messages[0]['value'] == message \
and c.messages[0]['key'] == key \
and c.messages[0]['headers'] == [
{'key': h_key, 'value': h_value},
]
wait_until(cond,
timeout_sec=120,
backoff_sec=30,
err_msg="Message didn't appear.")
@cluster(num_nodes=4)
def test_consume_as_group(self):
topic = 'topic_group'
message = 'message'
key = 'key'
h_key = 'h_key'
h_value = 'h_value'
headers = [h_key + ':' + h_value]
self._rpk.create_topic(topic)
c = RpkConsumer(self._ctx, self.redpanda, topic, group='group')
c.start()
def cond():
if c.error:
raise c.error
self._rpk.produce(topic, key, message, headers)
return c.messages \
and c.messages[0]['value'] == message \
and c.messages[0]['key'] == key \
and c.messages[0]['headers'] == [
{'key': h_key, 'value': h_value},
]
wait_until(cond,
timeout_sec=120,
backoff_sec=15,
err_msg="Message didn't appear.")
@cluster(num_nodes=4)
def test_consume_newest(self):
topic = 'topic_newest'
message = 'newest message'
key = 'key'
h_key = 'h_key'
h_value = 'h_value'
headers = [h_key + ':' + h_value]
self._rpk.create_topic(topic)
c = RpkConsumer(self._ctx, self.redpanda, topic, offset='newest')
c.start()
def cond():
if c.error:
raise c.error
self._rpk.produce(topic, key, message, headers)
return c.messages \
and c.messages[0]['value'] == message \
and c.messages[0]['key'] == key \
and c.messages[0]['headers'] == [
{'key': h_key, 'value': h_value},
]
wait_until(cond,
timeout_sec=150,
backoff_sec=30,
err_msg="Message didn't appear.")
@cluster(num_nodes=4)
def test_consume_oldest(self):
topic = 'topic'
n = random.randint(10, 100)
msgs = {}
for i in range(n):
msgs['key-' + str(i)] = 'message-' + str(i)
self._rpk.create_topic(topic)
# Produce messages
for k in msgs:
self._rpk.produce(topic, k, msgs[k])
c = RpkConsumer(self._ctx, self.redpanda, topic)
c.start()
def cond():
# Consume from the beginning
if len(c.messages) != len(msgs):
return False
for m in c.messages:
key = m['key']
if key is None:
return False
if m['value'] != msgs[key]:
return False
return True
wait_until(cond,
timeout_sec=60,
backoff_sec=20,
err_msg="Message didn't appear.")
@cluster(num_nodes=4)
def test_consume_from_partition(self):
topic = 'topic_partition'
n_parts = random.randint(3, 100)
self._rpk.create_topic(topic, partitions=n_parts)
n = random.randint(10, 30)
msgs = {}
for i in range(n):
msgs['key-' + str(i)] = 'message-' + str(i)
part = random.randint(0, n_parts - 1)
# Produce messages to a random partition
for k in msgs:
self._rpk.produce(topic, k, msgs[k], partition=part)
# Consume from the beginning
c = RpkConsumer(self._ctx,
self.redpanda,
topic,
offset='oldest',
partitions=[part])
c.start()
def cond():
if len(c.messages) != len(msgs):
return False
for m in c.messages:
key = m['key']
if key is None:
return False
if m['value'] != msgs[key]:
return False
return True
# timeout loop, but reset the timeout if we appear to be making progress
retries = 10
prev_msg_count = len(c.messages)
while retries > 0:
self.redpanda.logger.debug(
f"Message count {len(c.messages)} retries {retries}")
if cond():
return
if len(c.messages) > prev_msg_count:
prev_msg_count = len(c.messages)
retries = 10
time.sleep(1)
retries -= 1
raise ducktape.errors.TimeoutError("Message didn't appear")
def _produce(self, topic, msg_cnt):
rpk = RpkTool(self.redpanda)
for i in range(0, msg_cnt):
rpk.produce(topic, f"k-{i}", f"v-{i}")
class RpkToolTest(RedpandaTest):
def __init__(self, ctx):
super(RpkToolTest, self).__init__(test_context=ctx)
self._ctx = ctx
self._rpk = RpkTool(self.redpanda)
def test_create_topic(self):
self._rpk.create_topic("topic")
wait_until(lambda: "topic" in self._rpk.list_topics(),
timeout_sec=10,
backoff_sec=1,
err_msg="Topic never appeared.")
def test_produce(self):
topic = 'topic'
message = 'message'
key = 'key'
h_key = 'h_key'
h_value = 'h_value'
headers = [h_key + ':' + h_value]
self._rpk.create_topic(topic)
self._rpk.produce(topic, key, message, headers)
c = RpkConsumer(self._ctx, self.redpanda, topic)
c.start()
def cond():
return len(c.messages) == 1 \
and c.messages[0]['message'] == message \
and c.messages[0]['key'] == key \
and c.messages[0]['headers'] == [
{'key': h_key, 'value': h_value},
]
wait_until(cond,
timeout_sec=30,
backoff_sec=2,
err_msg="Message didn't appear.")
def test_consume_as_group(self):
topic = 'topic_group'
message = 'message'
key = 'key'
h_key = 'h_key'
h_value = 'h_value'
headers = [h_key + ':' + h_value]
self._rpk.create_topic(topic)
c = RpkConsumer(self._ctx, self.redpanda, topic, group='group')
c.start()
def cond():
if c.error:
raise c.error
self._rpk.produce(topic, key, message, headers)
return c.messages \
and c.messages[0]['message'] == message \
and c.messages[0]['key'] == key \
and c.messages[0]['headers'] == [
{'key': h_key, 'value': h_value},
]
wait_until(cond,
timeout_sec=30,
backoff_sec=8,
err_msg="Message didn't appear.")
def test_consume_newest(self):
topic = 'topic_newest'
message = 'message'
key = 'key'
h_key = 'h_key'
h_value = 'h_value'
headers = [h_key + ':' + h_value]
self._rpk.create_topic(topic)
# Gotta sleep to make sure the topic is replicated and the
# consumer doesn't fail.
time.sleep(5)
c = RpkConsumer(self._ctx, self.redpanda, topic, offset='newest')
c.start()
def cond():
if c.error:
raise c.error
self._rpk.produce(topic, key, message, headers)
return c.messages \
and c.messages[0]['message'] == message \
and c.messages[0]['key'] == key \
and c.messages[0]['headers'] == [
{'key': h_key, 'value': h_value},
]
wait_until(cond,
timeout_sec=30,
backoff_sec=8,
err_msg="Message didn't appear.")
def test_consume_oldest(self):
topic = 'topic'
n = random.randint(10, 100)
msgs = {}
for i in range(n):
msgs['key-' + str(i)] = 'message-' + str(i)
# Produce messages
for k in msgs:
self._rpk.produce(topic, k, msgs[k])
c = RpkConsumer(self._ctx, self.redpanda, topic)
c.start()
def cond():
# Consume from the beginning
if len(c.messages) != len(msgs):
return False
for m in c.messages:
key = m['key']
if key is None:
return False
if m['message'] != msgs[key]:
return False
return True
wait_until(cond,
timeout_sec=30,
backoff_sec=8,
err_msg="Message didn't appear.")
def test_consume_from_partition(self):
topic = 'topic_partition'
n_parts = random.randint(3, 100)
self._rpk.create_topic(topic, partitions=n_parts)
n = random.randint(10, 30)
msgs = {}
for i in range(n):
msgs['key-' + str(i)] = 'message-' + str(i)
part = random.randint(0, n_parts)
# Produce messages to a random partition
for k in msgs:
self._rpk.produce(topic, k, msgs[k], partition=part)
# Consume from the beginning
c = RpkConsumer(self._ctx,
self.redpanda,
topic,
offset='oldest',
partitions=[part])
c.start()
def cond():
if len(c.messages) != len(msgs):
return False
for m in c.messages:
key = m['key']
if key is None:
return False
if m['message'] != msgs[key]:
return False
return True
wait_until(cond,
timeout_sec=10,
backoff_sec=1,
err_msg="Message didn't appear.")
def simple_fetch_handler_fairness_test(self, type):
"""
Testing if when fetching from single node all partitions are
returned in round robin fashion
"""
def multiple_topics(count):
return [
TopicSpec(partition_count=1, replication_factor=1)
for _ in range(count)
]
def multiple_partitions(count):
return [TopicSpec(partition_count=count, replication_factor=1)]
number_of_partitions = 32
records_per_partition = 100
topics = []
if type == 'multiple-topics':
topics = multiple_topics(number_of_partitions)
else:
topics = multiple_partitions(number_of_partitions)
# create topics
self.client().create_topic(specs=topics)
self.redpanda.logger.info(f"topics: {topics}")
rpk = RpkTool(self.redpanda)
# publish 10 messages to each topic/partition
for s in topics:
t = s.name
for p in range(0, s.partition_count):
for i in range(0, records_per_partition):
self.redpanda.logger.info(f"producing to : {t}/{p}")
rpk.produce(t,
f"k.{t}.{p}.{i}",
f"p.{t}.{p}.{i}",
partition=p)
# configure kcl to fetch at most 1 byte in single fetch request,
# this way we should receive exactly one record per each partition
consumed_frequency = defaultdict(lambda: 0)
kcl = KCL(self.redpanda)
expected_frequency = records_per_partition / 2
# issue single kcl command that will generate multiple fetch requests
# in single fetch session, it should include single partition in each
# fetch response.
to_consume = int(number_of_partitions * expected_frequency)
consumed = kcl.consume(topic="topic-.*",
n=to_consume,
regex=True,
fetch_max_bytes=1,
group="test-gr-1")
for c in consumed.split():
parts = c.split('.')
consumed_frequency[(parts[1], parts[2])] += 1
for p, freq in consumed_frequency.items():
self.redpanda.logger.info(f"consumed {freq} messages from: {p}")
# assert that frequency is in expected range
assert freq <= expected_frequency + 0.1 * expected_frequency
assert freq >= expected_frequency - 0.1 * expected_frequency
