def test_produce_topic(self):
"""
Create a topic and verify that pandaproxy can produce to it.
"""
name = create_topic_names(1)[0]
data = '''
{
"records": [
{"value": "dmVjdG9yaXplZA==", "partition": 0},
{"value": "cGFuZGFwcm94eQ==", "partition": 1},
{"value": "bXVsdGlicm9rZXI=", "partition": 2}
]
}'''
self.logger.info(f"Producing to non-existant topic: {name}")
produce_result_raw = self._produce_topic(name, data)
assert produce_result_raw.status_code == requests.codes.ok
produce_result = produce_result_raw.json()
for o in produce_result["offsets"]:
assert o["error_code"] == 3
assert o["offset"] == -1
self.logger.info(f"Creating test topic: {name}")
self._create_topics([name], partitions=3)
self.logger.info(f"Producing to topic: {name}")
produce_result_raw = self._produce_topic(name, data)
assert produce_result_raw.status_code == requests.codes.ok
assert produce_result_raw.headers[
"Content-Type"] == "application/vnd.kafka.v2+json"
produce_result = produce_result_raw.json()
for o in produce_result["offsets"]:
assert o["offset"] == 0, f'error_code {o["error_code"]}'
self.logger.info(f"Consuming from topic: {name}")
kc = KafkaCat(self.redpanda)
assert kc.consume_one(name, 0, 0)["payload"] == "vectorized"
assert kc.consume_one(name, 1, 0)["payload"] == "pandaproxy"
assert kc.consume_one(name, 2, 0)["payload"] == "multibroker"
self.logger.info(f"Producing to topic without partition: {name}")
produce_result_raw = self._produce_topic(
name, '''
{
"records": [
{"value": "dmVjdG9yaXplZA=="},
{"value": "cGFuZGFwcm94eQ=="},
{"value": "bXVsdGlicm9rZXI="}
]
}''')
assert produce_result_raw.status_code == requests.codes.ok
produce_result = produce_result_raw.json()
for o in produce_result["offsets"]:
assert o["offset"] == 1, f'error_code {o["error_code"]}'
def test_produce_topic(self):
"""
Create a topic and verify that pandaproxy can produce to it.
"""
name = "pandaproxy-topic-{}".format(uuid.uuid4())
self.logger.debug("Topic name %s", name)
prev = set(self._get_topics())
self.logger.debug("Existing topics %s", prev)
assert prev.isdisjoint(name)
data = '{"records": [{"value": "dmVjdG9yaXplZA==", "partition": 0},{"value": "cGFuZGFwcm94eQ==", "partition": 1},{"value": "bXVsdGlicm9rZXI=", "partition": 2}]}'
self.logger.debug("Producing to non-existant topic")
produce_result = self._produce_topic(name, data)
for o in produce_result["offsets"]:
assert o["error_code"] == 3
assert o["offset"] == -1
kc = KafkaCat(self.redpanda)
self.logger.debug("Creating test topic")
kafka_tools = KafkaCliTools(self.redpanda)
kafka_tools.create_topic(
TopicSpec(name=name, replication_factor=1, partition_count=3))
self.logger.debug("Waiting for leaders to settle")
has_leaders = False
while not has_leaders:
topics = kc.metadata()["topics"]
maybe_leaders = True
for t in topics:
if t["topic"] == name:
for p in t["partitions"]:
if p["leader"] == -1:
maybe_leaders = False
has_leaders = maybe_leaders
# TODO:
# Despite the above test, Pandaproxy can still get back no leaders
# Query Pandaproxy metadata to see when leaders have settled
# The retry logic for produce should have sufficient time for this
# additional settle time.
self.logger.debug("Producing to topic")
produce_result = self._produce_topic(name, data)
self.logger.debug("Producing to topic: %s", produce_result)
for o in produce_result["offsets"]:
assert o["offset"] == 1, f'error_code {o["error_code"]}'
self.logger.debug(f"Consuming topic: {name}")
assert kc.consume_one(name, 0, 1)["payload"] == "vectorized"
assert kc.consume_one(name, 1, 1)["payload"] == "pandaproxy"
assert kc.consume_one(name, 2, 1)["payload"] == "multibroker"
def test_produce_topic(self):
"""
Create a topic and verify that pandaproxy can produce to it.
"""
name = create_topic_names(1)[0]
data = '''
{
"records": [
{"value": "dmVjdG9yaXplZA==", "partition": 0},
{"value": "cGFuZGFwcm94eQ==", "partition": 1},
{"value": "bXVsdGlicm9rZXI=", "partition": 2}
]
}'''
self.logger.info(f"Producing to non-existant topic: {name}")
produce_result = self._produce_topic(name, data)
for o in produce_result["offsets"]:
assert o["error_code"] == 3
assert o["offset"] == -1
kc = KafkaCat(self.redpanda)
self.logger.info(f"Creating test topic: {name}")
self._create_topics([name], partitions=3)
self.logger.debug("Waiting for leaders to settle")
has_leaders = False
while not has_leaders:
topics = kc.metadata()["topics"]
maybe_leaders = True
for t in topics:
if t["topic"] == name:
for p in t["partitions"]:
if p["leader"] == -1:
maybe_leaders = False
has_leaders = maybe_leaders
# TODO:
# Despite the above test, Pandaproxy can still get back no leaders
# Query Pandaproxy metadata to see when leaders have settled
# The retry logic for produce should have sufficient time for this
# additional settle time.
self.logger.info(f"Producing to topic: {name}")
produce_result = self._produce_topic(name, data)
for o in produce_result["offsets"]:
assert o["offset"] == 1, f'error_code {o["error_code"]}'
self.logger.info(f"Consuming from topic: {name}")
assert kc.consume_one(name, 0, 1)["payload"] == "vectorized"
assert kc.consume_one(name, 1, 1)["payload"] == "pandaproxy"
assert kc.consume_one(name, 2, 1)["payload"] == "multibroker"
def _ping_pong(self):
kc = KafkaCat(self.redpanda)
rpk = RpkTool(self.redpanda)
payload = str(random.randint(0, 1000))
start = time.time()
offset = rpk.produce(self.topic, "tkey", payload, timeout=5)
consumed = kc.consume_one(self.topic, 0, offset)
latency = time.time() - start
self.logger.info(
f"_ping_pong produced '{payload}' consumed '{consumed}' in {(latency)*1000.0:.2f} ms"
)
if consumed['payload'] != payload:
raise RuntimeError(f"expected '{payload}' got '{consumed}'")
def test_produce_topic(self):
"""
Create a topic and verify that pandaproxy can produce to it.
"""
name = create_topic_names(1)[0]
data = '''
{
"records": [
{"value": "dmVjdG9yaXplZA==", "partition": 0},
{"value": "cGFuZGFwcm94eQ==", "partition": 1},
{"value": "bXVsdGlicm9rZXI=", "partition": 2}
]
}'''
self.logger.info(f"Producing to non-existant topic: {name}")
produce_result = self._produce_topic(name, data)
for o in produce_result["offsets"]:
assert o["error_code"] == 3
assert o["offset"] == -1
self.logger.info(f"Creating test topic: {name}")
self._create_topics([name], partitions=3)
self.logger.debug("Waiting for leaders to settle")
self._wait_for_topic(name)
self.logger.info(f"Producing to topic: {name}")
produce_result = self._produce_topic(name, data)
for o in produce_result["offsets"]:
assert o["offset"] == 1, f'error_code {o["error_code"]}'
self.logger.info(f"Consuming from topic: {name}")
kc = KafkaCat(self.redpanda)
assert kc.consume_one(name, 0, 1)["payload"] == "vectorized"
assert kc.consume_one(name, 1, 1)["payload"] == "pandaproxy"
assert kc.consume_one(name, 2, 1)["payload"] == "multibroker"
