def test_produce_topic(self): """ Create a topic and verify that pandaproxy can produce to it. """ name = create_topic_names(1)[0] data = ''' { "records": [ {"value": "dmVjdG9yaXplZA==", "partition": 0}, {"value": "cGFuZGFwcm94eQ==", "partition": 1}, {"value": "bXVsdGlicm9rZXI=", "partition": 2} ] }''' self.logger.info(f"Producing to non-existant topic: {name}") produce_result_raw = self._produce_topic(name, data) assert produce_result_raw.status_code == requests.codes.ok produce_result = produce_result_raw.json() for o in produce_result["offsets"]: assert o["error_code"] == 3 assert o["offset"] == -1 self.logger.info(f"Creating test topic: {name}") self._create_topics([name], partitions=3) self.logger.info(f"Producing to topic: {name}") produce_result_raw = self._produce_topic(name, data) assert produce_result_raw.status_code == requests.codes.ok assert produce_result_raw.headers[ "Content-Type"] == "application/vnd.kafka.v2+json" produce_result = produce_result_raw.json() for o in produce_result["offsets"]: assert o["offset"] == 0, f'error_code {o["error_code"]}' self.logger.info(f"Consuming from topic: {name}") kc = KafkaCat(self.redpanda) assert kc.consume_one(name, 0, 0)["payload"] == "vectorized" assert kc.consume_one(name, 1, 0)["payload"] == "pandaproxy" assert kc.consume_one(name, 2, 0)["payload"] == "multibroker" self.logger.info(f"Producing to topic without partition: {name}") produce_result_raw = self._produce_topic( name, ''' { "records": [ {"value": "dmVjdG9yaXplZA=="}, {"value": "cGFuZGFwcm94eQ=="}, {"value": "bXVsdGlicm9rZXI="} ] }''') assert produce_result_raw.status_code == requests.codes.ok produce_result = produce_result_raw.json() for o in produce_result["offsets"]: assert o["offset"] == 1, f'error_code {o["error_code"]}'
def test_produce_topic(self): """ Create a topic and verify that pandaproxy can produce to it. """ name = "pandaproxy-topic-{}".format(uuid.uuid4()) self.logger.debug("Topic name %s", name) prev = set(self._get_topics()) self.logger.debug("Existing topics %s", prev) assert prev.isdisjoint(name) data = '{"records": [{"value": "dmVjdG9yaXplZA==", "partition": 0},{"value": "cGFuZGFwcm94eQ==", "partition": 1},{"value": "bXVsdGlicm9rZXI=", "partition": 2}]}' self.logger.debug("Producing to non-existant topic") produce_result = self._produce_topic(name, data) for o in produce_result["offsets"]: assert o["error_code"] == 3 assert o["offset"] == -1 kc = KafkaCat(self.redpanda) self.logger.debug("Creating test topic") kafka_tools = KafkaCliTools(self.redpanda) kafka_tools.create_topic( TopicSpec(name=name, replication_factor=1, partition_count=3)) self.logger.debug("Waiting for leaders to settle") has_leaders = False while not has_leaders: topics = kc.metadata()["topics"] maybe_leaders = True for t in topics: if t["topic"] == name: for p in t["partitions"]: if p["leader"] == -1: maybe_leaders = False has_leaders = maybe_leaders # TODO: # Despite the above test, Pandaproxy can still get back no leaders # Query Pandaproxy metadata to see when leaders have settled # The retry logic for produce should have sufficient time for this # additional settle time. self.logger.debug("Producing to topic") produce_result = self._produce_topic(name, data) self.logger.debug("Producing to topic: %s", produce_result) for o in produce_result["offsets"]: assert o["offset"] == 1, f'error_code {o["error_code"]}' self.logger.debug(f"Consuming topic: {name}") assert kc.consume_one(name, 0, 1)["payload"] == "vectorized" assert kc.consume_one(name, 1, 1)["payload"] == "pandaproxy" assert kc.consume_one(name, 2, 1)["payload"] == "multibroker"
def test_produce_topic(self): """ Create a topic and verify that pandaproxy can produce to it. """ name = create_topic_names(1)[0] data = ''' { "records": [ {"value": "dmVjdG9yaXplZA==", "partition": 0}, {"value": "cGFuZGFwcm94eQ==", "partition": 1}, {"value": "bXVsdGlicm9rZXI=", "partition": 2} ] }''' self.logger.info(f"Producing to non-existant topic: {name}") produce_result = self._produce_topic(name, data) for o in produce_result["offsets"]: assert o["error_code"] == 3 assert o["offset"] == -1 kc = KafkaCat(self.redpanda) self.logger.info(f"Creating test topic: {name}") self._create_topics([name], partitions=3) self.logger.debug("Waiting for leaders to settle") has_leaders = False while not has_leaders: topics = kc.metadata()["topics"] maybe_leaders = True for t in topics: if t["topic"] == name: for p in t["partitions"]: if p["leader"] == -1: maybe_leaders = False has_leaders = maybe_leaders # TODO: # Despite the above test, Pandaproxy can still get back no leaders # Query Pandaproxy metadata to see when leaders have settled # The retry logic for produce should have sufficient time for this # additional settle time. self.logger.info(f"Producing to topic: {name}") produce_result = self._produce_topic(name, data) for o in produce_result["offsets"]: assert o["offset"] == 1, f'error_code {o["error_code"]}' self.logger.info(f"Consuming from topic: {name}") assert kc.consume_one(name, 0, 1)["payload"] == "vectorized" assert kc.consume_one(name, 1, 1)["payload"] == "pandaproxy" assert kc.consume_one(name, 2, 1)["payload"] == "multibroker"
def _ping_pong(self): kc = KafkaCat(self.redpanda) rpk = RpkTool(self.redpanda) payload = str(random.randint(0, 1000)) start = time.time() offset = rpk.produce(self.topic, "tkey", payload, timeout=5) consumed = kc.consume_one(self.topic, 0, offset) latency = time.time() - start self.logger.info( f"_ping_pong produced '{payload}' consumed '{consumed}' in {(latency)*1000.0:.2f} ms" ) if consumed['payload'] != payload: raise RuntimeError(f"expected '{payload}' got '{consumed}'")
def test_produce_topic(self): """ Create a topic and verify that pandaproxy can produce to it. """ name = create_topic_names(1)[0] data = ''' { "records": [ {"value": "dmVjdG9yaXplZA==", "partition": 0}, {"value": "cGFuZGFwcm94eQ==", "partition": 1}, {"value": "bXVsdGlicm9rZXI=", "partition": 2} ] }''' self.logger.info(f"Producing to non-existant topic: {name}") produce_result = self._produce_topic(name, data) for o in produce_result["offsets"]: assert o["error_code"] == 3 assert o["offset"] == -1 self.logger.info(f"Creating test topic: {name}") self._create_topics([name], partitions=3) self.logger.debug("Waiting for leaders to settle") self._wait_for_topic(name) self.logger.info(f"Producing to topic: {name}") produce_result = self._produce_topic(name, data) for o in produce_result["offsets"]: assert o["offset"] == 1, f'error_code {o["error_code"]}' self.logger.info(f"Consuming from topic: {name}") kc = KafkaCat(self.redpanda) assert kc.consume_one(name, 0, 1)["payload"] == "vectorized" assert kc.consume_one(name, 1, 1)["payload"] == "pandaproxy" assert kc.consume_one(name, 2, 1)["payload"] == "multibroker"