def metadata_updated(): client = PythonLibrdkafka(self.redpanda) brokers = client.brokers() self.redpanda.logger.debug(f"brokers metadata: {brokers}") ports = [b.port for _, b in brokers.items()] ports.sort() return ports == [10091, 10092, 10093]
def registered(self, node): idx = self.idx(node) self.logger.debug( f"Checking if broker {idx} ({node.name} is registered") client = PythonLibrdkafka(self) brokers = client.brokers() broker = brokers.get(idx, None) self.logger.debug(f"Found broker info: {broker}") return broker is not None
def metadata_updated(): client = PythonLibrdkafka(self.redpanda) brokers = client.brokers() self.logger.debug(f"brokers metadata: {brokers}") if brokers[1].port != 10091: return False if brokers[2].port != 10092: return False if brokers[3].port != 10093: return False return True
def registered(self, node): """ Check if a newly added node is fully registered with the cluster, such that a kafka metadata request to any node in the cluster will include it. We first check the admin API to do a kafka-independent check, and then verify that kafka clients see the same thing. """ idx = self.idx(node) self.logger.debug( f"registered: checking if broker {idx} ({node.name} is registered..." ) # Query all nodes' admin APIs, so that we don't advance during setup until # the node is stored in raft0 AND has been replayed on all nodes. Otherwise # a kafka metadata request to the last node to join could return incomplete # metadata and cause strange issues within a test. admin = Admin(self) for peer in self._started: try: admin_brokers = admin.get_brokers(node=peer) except requests.exceptions.RequestException as e: # We run during startup, when admin API may not even be listening yet: tolerate # API errors but presume that if some APIs are not up yet, then node registration # is also not complete. self.logger.debug( f"registered: peer {peer.name} admin API not yet available ({e})" ) return False found = idx in [b['node_id'] for b in admin_brokers] if not found: self.logger.info( f"registered: node {node.name} not yet found in peer {peer.name}'s broker list ({admin_brokers})" ) return False else: self.logger.debug( f"registered: node {node.name} now visible in peer {peer.name}'s broker list ({admin_brokers})" ) client = PythonLibrdkafka(self) brokers = client.brokers() broker = brokers.get(idx, None) if broker is None: # This should never happen, because we already checked via the admin API # that the node of interest had become visible to all peers. self.logger.error( f"registered: node {node.name} not found in kafka metadata!") assert broker is not None self.logger.debug(f"registered: found broker info: {broker}") return True
def brokers(self): """ Note for implementers: this method is expected to return the set of brokers reported by the cluster, rather than the nodes allocated for use in the self._redpanda service. """ client = PythonLibrdkafka(self._redpanda) brokers = client.brokers() return { b.id: BrokerDescription(id=b.id, host=b.host, port=b.port) for b in brokers.values() }
def test_enable_sasl_live(self): """ Verify that when enable_sasl is set to true at runtime, subsequent unauthenticated kafka clients are rejected. """ unauthenticated_client = PythonLibrdkafka(self.redpanda) topic = TopicSpec(replication_factor=1) unauthenticated_client.create_topic(topic) assert len(unauthenticated_client.topics()) == 1 # Switch on authentication admin = Admin(self.redpanda) admin.patch_cluster_config(upsert={'enable_sasl': True}) # An unauthenticated client should be rejected try: unauthenticated_client.topics() except Exception as e: self.logger.exception(f"Unauthenticated: {e}") else: self.logger.error( "Unauthenticated client should have been rejected") assert False # Switch off authentication admin.patch_cluster_config(upsert={'enable_sasl': False}) # An unauthenticated client should be accepted again assert len(unauthenticated_client.topics()) == 1
def test_custom_assignment_validation(self): client = PythonLibrdkafka(self.redpanda).get_client() def expect_failed_create_topic(name, custom_assignment, expected_error): topics = [ NewTopic(name, num_partitions=len(custom_assignment), replica_assignment=custom_assignment) ] res = client.create_topics(topics, request_timeout=10) assert len(res) == 1 fut = res[name] try: fut.result() assert False except KafkaException as e: kafka_error = e.args[0] self.redpanda.logger.debug( f"topic {name} creation failed: {kafka_error}, expected error: {expected_error}" ) assert kafka_error.code() == expected_error # not unique replicas expect_failed_create_topic("invalid-1", [[1, 1, 2]], KafkaError.INVALID_REQUEST) # not existing broker expect_failed_create_topic("invalid-1", [[1, 10, 2]], KafkaError.BROKER_NOT_AVAILABLE) # different replication factors expect_failed_create_topic("invalid-1", [[1, 2, 3], [4]], KafkaError.INVALID_REPLICATION_FACTOR)
def make_superuser_client(self, password_override=None): username, password, algorithm = self.redpanda.SUPERUSER_CREDENTIALS password = password_override or password return PythonLibrdkafka(self.redpanda, username=username, password=password, algorithm=algorithm)
def broker_configuration_updated(): client = PythonLibrdkafka(self.redpanda) brokers = client.brokers() self.logger.debug(f"brokers metadata: {brokers}") return brokers[2].port == 10091