class RpkToolTest(RedpandaTest):
def __init__(self, ctx):
super(RpkToolTest, self).__init__(test_context=ctx)
self._ctx = ctx
self._rpk = RpkTool(self.redpanda)
def test_create_topic(self):
self._rpk.create_topic("topic")
wait_until(lambda: "topic" in self._rpk.list_topics(),
timeout_sec=10,
backoff_sec=1,
err_msg="Topic never appeared.")
def test_produce(self):
topic = 'topic'
message = 'message'
key = 'key'
h_key = 'h_key'
h_value = 'h_value'
headers = [h_key + ':' + h_value]
self._rpk.create_topic(topic)
self._rpk.produce(topic, key, message, headers)
c = RpkConsumer(self._ctx, self.redpanda, topic)
c.start()
def cond():
return len(c.messages) == 1 \
and c.messages[0]['message'] == message \
and c.messages[0]['key'] == key \
and c.messages[0]['headers'] == [
{'key': h_key, 'value': h_value},
]
wait_until(cond,
timeout_sec=30,
backoff_sec=2,
err_msg="Message didn't appear.")
def test_consume_as_group(self):
topic = 'topic_group'
message = 'message'
key = 'key'
h_key = 'h_key'
h_value = 'h_value'
headers = [h_key + ':' + h_value]
self._rpk.create_topic(topic)
c = RpkConsumer(self._ctx, self.redpanda, topic, group='group')
c.start()
def cond():
if c.error:
raise c.error
self._rpk.produce(topic, key, message, headers)
return c.messages \
and c.messages[0]['message'] == message \
and c.messages[0]['key'] == key \
and c.messages[0]['headers'] == [
{'key': h_key, 'value': h_value},
]
wait_until(cond,
timeout_sec=30,
backoff_sec=8,
err_msg="Message didn't appear.")
def test_consume_newest(self):
topic = 'topic_newest'
message = 'message'
key = 'key'
h_key = 'h_key'
h_value = 'h_value'
headers = [h_key + ':' + h_value]
self._rpk.create_topic(topic)
# Gotta sleep to make sure the topic is replicated and the
# consumer doesn't fail.
time.sleep(5)
c = RpkConsumer(self._ctx, self.redpanda, topic, offset='newest')
c.start()
def cond():
if c.error:
raise c.error
self._rpk.produce(topic, key, message, headers)
return c.messages \
and c.messages[0]['message'] == message \
and c.messages[0]['key'] == key \
and c.messages[0]['headers'] == [
{'key': h_key, 'value': h_value},
]
wait_until(cond,
timeout_sec=30,
backoff_sec=8,
err_msg="Message didn't appear.")
def test_consume_oldest(self):
topic = 'topic'
n = random.randint(10, 100)
msgs = {}
for i in range(n):
msgs['key-' + str(i)] = 'message-' + str(i)
# Produce messages
for k in msgs:
self._rpk.produce(topic, k, msgs[k])
c = RpkConsumer(self._ctx, self.redpanda, topic)
c.start()
def cond():
# Consume from the beginning
if len(c.messages) != len(msgs):
return False
for m in c.messages:
key = m['key']
if key is None:
return False
if m['message'] != msgs[key]:
return False
return True
wait_until(cond,
timeout_sec=30,
backoff_sec=8,
err_msg="Message didn't appear.")
def test_consume_from_partition(self):
topic = 'topic_partition'
n_parts = random.randint(3, 100)
self._rpk.create_topic(topic, partitions=n_parts)
n = random.randint(10, 30)
msgs = {}
for i in range(n):
msgs['key-' + str(i)] = 'message-' + str(i)
part = random.randint(0, n_parts)
# Produce messages to a random partition
for k in msgs:
self._rpk.produce(topic, k, msgs[k], partition=part)
# Consume from the beginning
c = RpkConsumer(self._ctx,
self.redpanda,
topic,
offset='oldest',
partitions=[part])
c.start()
def cond():
if len(c.messages) != len(msgs):
return False
for m in c.messages:
key = m['key']
if key is None:
return False
if m['message'] != msgs[key]:
return False
return True
wait_until(cond,
timeout_sec=10,
backoff_sec=1,
err_msg="Message didn't appear.")
class RpkToolTest(RedpandaTest):
def __init__(self, ctx):
super(RpkToolTest, self).__init__(test_context=ctx)
self._ctx = ctx
self._rpk = RpkTool(self.redpanda)
@cluster(num_nodes=3)
def test_create_topic(self):
self._rpk.create_topic("topic")
wait_until(lambda: "topic" in self._rpk.list_topics(),
timeout_sec=10,
backoff_sec=1,
err_msg="Topic never appeared.")
@cluster(num_nodes=4)
def test_produce(self):
topic = 'topic'
message = 'message'
key = 'key'
h_key = 'h_key'
h_value = 'h_value'
headers = [h_key + ':' + h_value]
self._rpk.create_topic(topic)
self._rpk.produce(topic, key, message, headers)
c = RpkConsumer(self._ctx, self.redpanda, topic)
c.start()
def cond():
return c.messages is not None \
and len(c.messages) == 1 \
and c.messages[0]['value'] == message \
and c.messages[0]['key'] == key \
and c.messages[0]['headers'] == [
{'key': h_key, 'value': h_value},
]
wait_until(cond,
timeout_sec=120,
backoff_sec=30,
err_msg="Message didn't appear.")
@cluster(num_nodes=4)
def test_consume_as_group(self):
topic = 'topic_group'
message = 'message'
key = 'key'
h_key = 'h_key'
h_value = 'h_value'
headers = [h_key + ':' + h_value]
self._rpk.create_topic(topic)
c = RpkConsumer(self._ctx, self.redpanda, topic, group='group')
c.start()
def cond():
if c.error:
raise c.error
self._rpk.produce(topic, key, message, headers)
return c.messages \
and c.messages[0]['value'] == message \
and c.messages[0]['key'] == key \
and c.messages[0]['headers'] == [
{'key': h_key, 'value': h_value},
]
wait_until(cond,
timeout_sec=120,
backoff_sec=15,
err_msg="Message didn't appear.")
@cluster(num_nodes=4)
def test_consume_newest(self):
topic = 'topic_newest'
message = 'newest message'
key = 'key'
h_key = 'h_key'
h_value = 'h_value'
headers = [h_key + ':' + h_value]
self._rpk.create_topic(topic)
c = RpkConsumer(self._ctx, self.redpanda, topic, offset='newest')
c.start()
def cond():
if c.error:
raise c.error
self._rpk.produce(topic, key, message, headers)
return c.messages \
and c.messages[0]['value'] == message \
and c.messages[0]['key'] == key \
and c.messages[0]['headers'] == [
{'key': h_key, 'value': h_value},
]
wait_until(cond,
timeout_sec=150,
backoff_sec=30,
err_msg="Message didn't appear.")
@cluster(num_nodes=4)
def test_consume_oldest(self):
topic = 'topic'
n = random.randint(10, 100)
msgs = {}
for i in range(n):
msgs['key-' + str(i)] = 'message-' + str(i)
self._rpk.create_topic(topic)
# Produce messages
for k in msgs:
self._rpk.produce(topic, k, msgs[k])
c = RpkConsumer(self._ctx, self.redpanda, topic)
c.start()
def cond():
# Consume from the beginning
if len(c.messages) != len(msgs):
return False
for m in c.messages:
key = m['key']
if key is None:
return False
if m['value'] != msgs[key]:
return False
return True
wait_until(cond,
timeout_sec=60,
backoff_sec=20,
err_msg="Message didn't appear.")
@cluster(num_nodes=4)
def test_consume_from_partition(self):
topic = 'topic_partition'
n_parts = random.randint(3, 100)
self._rpk.create_topic(topic, partitions=n_parts)
n = random.randint(10, 30)
msgs = {}
for i in range(n):
msgs['key-' + str(i)] = 'message-' + str(i)
part = random.randint(0, n_parts - 1)
# Produce messages to a random partition
for k in msgs:
self._rpk.produce(topic, k, msgs[k], partition=part)
# Consume from the beginning
c = RpkConsumer(self._ctx,
self.redpanda,
topic,
offset='oldest',
partitions=[part])
c.start()
def cond():
if len(c.messages) != len(msgs):
return False
for m in c.messages:
key = m['key']
if key is None:
return False
if m['value'] != msgs[key]:
return False
return True
# timeout loop, but reset the timeout if we appear to be making progress
retries = 10
prev_msg_count = len(c.messages)
while retries > 0:
self.redpanda.logger.debug(
f"Message count {len(c.messages)} retries {retries}")
if cond():
return
if len(c.messages) > prev_msg_count:
prev_msg_count = len(c.messages)
retries = 10
time.sleep(1)
retries -= 1
raise ducktape.errors.TimeoutError("Message didn't appear")
def test_overlapping_changes(self):
"""
Check that while a movement is in flight, rules about
overlapping operations are properly enforced.
"""
self.start_redpanda(num_nodes=4)
node_ids = {1, 2, 3, 4}
# Create topic with enough data that inter-node movement
# will take a while.
name = f"movetest"
spec = TopicSpec(name=name, partition_count=1, replication_factor=3)
self.client().create_topic(spec)
# Wait for the partition to have a leader (`rpk produce` errors
# out if it tries to write data before this)
def partition_ready():
return KafkaCat(self.redpanda).get_partition_leader(
name, 0)[0] is not None
wait_until(partition_ready, timeout_sec=10, backoff_sec=0.5)
# Write a substantial amount of data to the topic
msg_size = 512 * 1024
write_bytes = 512 * 1024 * 1024
producer = RpkProducer(self._ctx,
self.redpanda,
name,
msg_size=msg_size,
msg_count=int(write_bytes / msg_size))
t1 = time.time()
producer.start()
# This is an absurdly low expected throughput, but necessarily
# so to run reliably on current test runners, which share an EBS
# backend among many parallel tests. 10MB/s has been empirically
# shown to be too high an expectation.
expect_bps = 1 * 1024 * 1024
expect_runtime = write_bytes / expect_bps
producer.wait(timeout_sec=expect_runtime)
self.logger.info(
f"Write complete {write_bytes} in {time.time() - t1} seconds")
# - Admin API redirects writes but not reads. Because we want synchronous
# status after submitting operations, send all operations to the controller
# leader. This is not necessary for operations to work, just to simplify
# this test by letting it see synchronous status updates.
# - Because we will later verify that a 503 is sent in response to
# a move request to an in_progress topic, set retry_codes=[] to
# disable default retries on 503.
admin_node = self.redpanda.controller()
admin = Admin(self.redpanda, default_node=admin_node, retry_codes=[])
# Start an inter-node move, which should take some time
# to complete because of recovery network traffic
assignments = self._get_assignments(admin, name, 0)
new_node = list(node_ids - set([a['node_id'] for a in assignments]))[0]
self.logger.info(f"old assignments: {assignments}")
old_assignments = assignments
assignments = assignments[1:] + [{'node_id': new_node, 'core': 0}]
self.logger.info(f"new assignments: {assignments}")
r = admin.set_partition_replicas(name, 0, assignments)
r.raise_for_status()
assert admin.get_partitions(name, 0)['status'] == "in_progress"
# Another move should fail
assert admin.get_partitions(name, 0)['status'] == "in_progress"
try:
r = admin.set_partition_replicas(name, 0, old_assignments)
except requests.exceptions.HTTPError as e:
assert e.response.status_code == 503
else:
raise RuntimeError(f"Expected 503 but got {r.status_code}")
# An update to partition properties should succeed
# (issue https://github.com/vectorizedio/redpanda/issues/2300)
rpk = RpkTool(self.redpanda)
assert admin.get_partitions(name, 0)['status'] == "in_progress"
rpk.alter_topic_config(name, "retention.ms", "3600000")
# A deletion should succeed
assert name in rpk.list_topics()
assert admin.get_partitions(name, 0)['status'] == "in_progress"
rpk.delete_topic(name)
assert name not in rpk.list_topics()
