def _get_partition_leaders(self): kcat = KafkaCat(self.redpanda) m = kcat.metadata() self.logger.info(f"kcat.metadata() == {m}") brokers = {} for b in m['brokers']: id = b['id'] ip = b['name'] ip = ip[:ip.index(':')] for n in self.redpanda.nodes: n_ip = n.account.hostname self.logger.debug(f"matching {n_ip} over {ip}") if n_ip == ip: brokers[id] = n break self.logger.debug(f"found brokers {brokers}") assert len(brokers) == 3 leaders = {} for topic in m['topics']: if topic['topic'] == ArchivalTest.s3_topic_name: for part in topic['partitions']: leader_id = part['leader'] partition_id = part['partition'] leader = brokers[leader_id] leaders[partition_id] = leader return leaders
def controller(self): kc = KafkaCat(self) cid = kc.metadata()["controllerid"] self.logger.debug("Controller reported with id: {}".format(cid)) if cid != -1: node = self.get_node(cid) self.logger.debug("Controller node found: {}".format(node)) return node
def registered(self, node): idx = self.idx(node) self.logger.debug("Checking if broker %d/%s is registered", idx, node) kc = KafkaCat(self) brokers = kc.metadata()["brokers"] brokers = {b["id"]: b for b in brokers} broker = brokers.get(idx, None) self.logger.debug("Found broker info: %s", broker) return broker is not None
def _registered(self, service, node): idx = service.idx(node) service.logger.debug("Checking if broker %d/%s is registered", idx, node) kc = KafkaCat(RedpandaMuServiceServiceProxy(service, self)) brokers = kc.metadata()["brokers"] brokers = {b["id"]: b for b in brokers} broker = brokers.get(idx, None) service.logger.debug("Found broker info: %s", broker) return broker is not None
def registered(self, node): idx = self.idx(node) self.logger.debug( f"Checking if broker {idx} ({node.name} is registered") kc = KafkaCat(self) brokers = kc.metadata()["brokers"] brokers = {b["id"]: b for b in brokers} broker = brokers.get(idx, None) self.logger.debug(f"Found broker info: {broker}") return broker is not None
def test_produce_topic(self): """ Create a topic and verify that pandaproxy can produce to it. """ name = "pandaproxy-topic-{}".format(uuid.uuid4()) self.logger.debug("Topic name %s", name) prev = set(self._get_topics()) self.logger.debug("Existing topics %s", prev) assert prev.isdisjoint(name) data = '{"records": [{"value": "dmVjdG9yaXplZA==", "partition": 0},{"value": "cGFuZGFwcm94eQ==", "partition": 1},{"value": "bXVsdGlicm9rZXI=", "partition": 2}]}' self.logger.debug("Producing to non-existant topic") produce_result = self._produce_topic(name, data) for o in produce_result["offsets"]: assert o["error_code"] == 3 assert o["offset"] == -1 kc = KafkaCat(self.redpanda) self.logger.debug("Creating test topic") kafka_tools = KafkaCliTools(self.redpanda) kafka_tools.create_topic( TopicSpec(name=name, replication_factor=1, partition_count=3)) self.logger.debug("Waiting for leaders to settle") has_leaders = False while not has_leaders: topics = kc.metadata()["topics"] maybe_leaders = True for t in topics: if t["topic"] == name: for p in t["partitions"]: if p["leader"] == -1: maybe_leaders = False has_leaders = maybe_leaders # TODO: # Despite the above test, Pandaproxy can still get back no leaders # Query Pandaproxy metadata to see when leaders have settled # The retry logic for produce should have sufficient time for this # additional settle time. self.logger.debug("Producing to topic") produce_result = self._produce_topic(name, data) self.logger.debug("Producing to topic: %s", produce_result) for o in produce_result["offsets"]: assert o["offset"] == 1, f'error_code {o["error_code"]}' self.logger.debug(f"Consuming topic: {name}") assert kc.consume_one(name, 0, 1)["payload"] == "vectorized" assert kc.consume_one(name, 1, 1)["payload"] == "pandaproxy" assert kc.consume_one(name, 2, 1)["payload"] == "multibroker"
def _wait_for_topic(self, name): kc = KafkaCat(self.redpanda) has_leaders = False while not has_leaders: topics = kc.metadata()["topics"] maybe_leaders = True for t in topics: if t["topic"] == name: for p in t["partitions"]: if p["leader"] == -1: maybe_leaders = False has_leaders = maybe_leaders
def test_produce_topic(self): """ Create a topic and verify that pandaproxy can produce to it. """ name = create_topic_names(1)[0] data = ''' { "records": [ {"value": "dmVjdG9yaXplZA==", "partition": 0}, {"value": "cGFuZGFwcm94eQ==", "partition": 1}, {"value": "bXVsdGlicm9rZXI=", "partition": 2} ] }''' self.logger.info(f"Producing to non-existant topic: {name}") produce_result = self._produce_topic(name, data) for o in produce_result["offsets"]: assert o["error_code"] == 3 assert o["offset"] == -1 kc = KafkaCat(self.redpanda) self.logger.info(f"Creating test topic: {name}") self._create_topics([name], partitions=3) self.logger.debug("Waiting for leaders to settle") has_leaders = False while not has_leaders: topics = kc.metadata()["topics"] maybe_leaders = True for t in topics: if t["topic"] == name: for p in t["partitions"]: if p["leader"] == -1: maybe_leaders = False has_leaders = maybe_leaders # TODO: # Despite the above test, Pandaproxy can still get back no leaders # Query Pandaproxy metadata to see when leaders have settled # The retry logic for produce should have sufficient time for this # additional settle time. self.logger.info(f"Producing to topic: {name}") produce_result = self._produce_topic(name, data) for o in produce_result["offsets"]: assert o["offset"] == 1, f'error_code {o["error_code"]}' self.logger.info(f"Consuming from topic: {name}") assert kc.consume_one(name, 0, 1)["payload"] == "vectorized" assert kc.consume_one(name, 1, 1)["payload"] == "pandaproxy" assert kc.consume_one(name, 2, 1)["payload"] == "multibroker"
def partitions(self, topic): """ Return partition metadata for the topic. """ kc = KafkaCat(self) md = kc.metadata() topic = next(filter(lambda t: t["topic"] == topic, md["topics"])) def make_partition(p): index = p["partition"] leader_id = p["leader"] leader = None if leader_id == -1 else self.get_node(leader_id) replicas = [self.get_node(r["id"]) for r in p["replicas"]] return Partition(index, leader, replicas) return [make_partition(p) for p in topic["partitions"]]
def test_controller_recovery(self): kc = KafkaCat(self.redpanda) # choose a partition and a target node partition = self._get_partition(kc) target_node_id = next( filter(lambda r: r["id"] != partition["leader"], partition["replicas"]))["id"] self.logger.debug( f"Transfering leader from {partition['leader']} to {target_node_id}" ) # build the transfer url meta = kc.metadata() brokers = meta["brokers"] source_broker = next( filter(lambda b: b["id"] == partition["leader"], brokers)) target_broker = next( filter(lambda b: b["id"] == target_node_id, brokers)) self.logger.debug(f"Source broker {source_broker}") self.logger.debug(f"Target broker {target_broker}") host = source_broker["name"] host = host.split(":")[0] partition_id = partition["partition"] url = "http://{}:9644/v1/kafka/{}/{}/transfer_leadership?target={}".format( host, self.topic, partition["partition"], target_node_id) def try_transfer(): self.logger.debug(url) res = requests.post(url) self.logger.debug(res.text) for _ in range(3): # just give it a moment time.sleep(1) meta = kc.metadata() partition = next( filter(lambda p: p["partition"] == partition_id, meta["topics"][0]["partitions"])) if partition["leader"] == target_node_id: return True return False wait_until(lambda: try_transfer(), timeout_sec=30, backoff_sec=5, err_msg="Transfer did not complete")
def test_controller_recovery(self): kc = KafkaCat(self.redpanda) # choose a partition and a target node partition = self._get_partition(kc) target_node_id = next( filter(lambda r: r["id"] != partition["leader"], partition["replicas"]))["id"] self.logger.debug( f"Transfering leader from {partition['leader']} to {target_node_id}" ) # build the transfer url meta = kc.metadata() brokers = meta["brokers"] source_broker = next( filter(lambda b: b["id"] == partition["leader"], brokers)) target_broker = next( filter(lambda b: b["id"] == target_node_id, brokers)) self.logger.debug(f"Source broker {source_broker}") self.logger.debug(f"Target broker {target_broker}") # Send the request to any host, they should redirect to # the leader of the partition. partition_id = partition['partition'] admin = Admin(self.redpanda) admin.partition_transfer_leadership("kafka", self.topic, partition_id, target_node_id) def transfer_complete(): for _ in range(3): # just give it a moment time.sleep(1) meta = kc.metadata() partition = next( filter(lambda p: p["partition"] == partition_id, meta["topics"][0]["partitions"])) if partition["leader"] == target_node_id: return True return False wait_until(lambda: transfer_complete(), timeout_sec=30, backoff_sec=5, err_msg="Transfer did not complete")
def _get_leaders_by_node(self): kc = KafkaCat(self.redpanda) md = kc.metadata() topic = next(filter(lambda t: t["topic"] == self.topic, md["topics"])) leaders = (p["leader"] for p in topic["partitions"]) return collections.Counter(leaders)