def check_version(self, node_id=None, timeout=2, strict=False): """Attempt to guess the version of a Kafka broker. Note: It is possible that this method blocks longer than the specified timeout. This can happen if the entire cluster is down and the client enters a bootstrap backoff sleep. This is only possible if node_id is None. Returns: version tuple, i.e. (0, 10), (0, 9), (0, 8, 2), ... Raises: NodeNotReadyError (if node_id is provided) NoBrokersAvailable (if node_id is None) UnrecognizedBrokerVersion: please file bug if seen! AssertionError (if strict=True): please file bug if seen! """ self._lock.acquire() end = time.time() + timeout while time.time() < end: # It is possible that least_loaded_node falls back to bootstrap, # which can block for an increasing backoff period try_node = node_id or self.least_loaded_node() if try_node is None: self._lock.release() raise Errors.NoBrokersAvailable() self._maybe_connect(try_node) conn = self._conns[try_node] # We will intentionally cause socket failures # These should not trigger metadata refresh self._refresh_on_disconnects = False try: remaining = end - time.time() version = conn.check_version( timeout=remaining, strict=strict, topics=list(self.config["bootstrap_topics_filter"]), ) if version >= (0, 10, 0): # cache the api versions map if it's available (starting # in 0.10 cluster version) self._api_versions = conn.get_api_versions() self._lock.release() return version except Errors.NodeNotReadyError: # Only raise to user if this is a node-specific request if node_id is not None: self._lock.release() raise finally: self._refresh_on_disconnects = True # Timeout else: self._lock.release() raise Errors.NoBrokersAvailable()
def _send_group_coordinator_request(self): """Discover the current coordinator for the group. Returns: Future: resolves to the node id of the coordinator """ node_id = self._client.least_loaded_node() if node_id is None: return Future().failure(Errors.NoBrokersAvailable()) elif not self._client.ready(node_id, metadata_priority=False): e = Errors.NodeNotReadyError(node_id) return Future().failure(e) log.debug( "Sending group coordinator request for group %s to broker %s", self.group_id, node_id, ) request = GroupCoordinatorRequest[0](self.group_id) future = Future() _f = self._client.send(node_id, request) _f.add_callback(self._handle_group_coordinator_response, future) _f.add_errback(self._failed_request, node_id, request, future) return future
def test_kafka_connection(): try: consumer = KafkaConsumer( KAFKA_TOPIC, auto_offset_reset="earliest", bootstrap_servers=KAFKA_SERVER, client_id="demo-client-1", group_id="demo-group", security_protocol="SSL", ssl_cafile=FILE_SSL_CAFILE, ssl_certfile=FILE_SSL_CERTFILE, ssl_keyfile=FILE_SSL_KEYFILE, ) r = consumer.topics() assert r == set(['metrics']) except AssertionError as e: raise AssertionError("Please create 'metrics' topic") from e except errors.NoBrokersAvailable as e: raise errors.NoBrokersAvailable("Kafka connection error") from e
def update_metadata(self, metadata): """Update cluster state given a MetadataResponse. Arguments: metadata (MetadataResponse): broker response to a metadata request Returns: None """ # In the common case where we ask for a single topic and get back an # error, we should fail the future if len(metadata.topics) == 1 and metadata.topics[0][0] != 0: error_code, topic = metadata.topics[0][:2] error = Errors.for_code(error_code)(topic) return self.failed_update(error) if not metadata.brokers: log.warning("No broker metadata found in MetadataResponse") return Errors.NoBrokersAvailable() _new_brokers = {} for broker in metadata.brokers: if metadata.API_VERSION == 0: node_id, host, port = broker rack = None else: node_id, host, port, rack = broker _new_brokers.update({ node_id: BrokerMetadata(node_id, host, port, rack) }) if metadata.API_VERSION == 0: _new_controller = None else: _new_controller = _new_brokers.get(metadata.controller_id) _new_partitions = {} _new_broker_partitions = collections.defaultdict(set) _new_unauthorized_topics = set() _new_internal_topics = set() for topic_data in metadata.topics: if metadata.API_VERSION == 0: error_code, topic, partitions = topic_data is_internal = False else: error_code, topic, is_internal, partitions = topic_data if is_internal: _new_internal_topics.add(topic) error_type = Errors.for_code(error_code) if error_type is Errors.NoError: _new_partitions[topic] = {} for p_error, partition, leader, replicas, isr in partitions: _new_partitions[topic][partition] = PartitionMetadata( topic=topic, partition=partition, leader=leader, replicas=replicas, isr=isr, error=p_error) if leader != -1: _new_broker_partitions[leader].add( TopicPartition(topic, partition)) elif error_type is Errors.LeaderNotAvailableError: log.warning("Topic %s is not available during auto-create" " initialization", topic) elif error_type is Errors.UnknownTopicOrPartitionError: log.error("Topic %s not found in cluster metadata", topic) elif error_type is Errors.TopicAuthorizationFailedError: log.error("Topic %s is not authorized for this client", topic) _new_unauthorized_topics.add(topic) elif error_type is Errors.InvalidTopicError: log.error("'%s' is not a valid topic name", topic) else: log.error("Error fetching metadata for topic %s: %s", topic, error_type) with self._lock: self._brokers = _new_brokers self.controller = _new_controller self._partitions = _new_partitions self._broker_partitions = _new_broker_partitions self.unauthorized_topics = _new_unauthorized_topics self.internal_topics = _new_internal_topics f = None if self._future: f = self._future self._future = None self._need_update = False now = time.time() * 1000 self._last_refresh_ms = now self._last_successful_refresh_ms = now if f: f.success(self) log.debug("Updated cluster metadata to %s", self) for listener in self._listeners: listener(self) if self.need_all_topic_metadata: # the listener may change the interested topics, # which could cause another metadata refresh. # If we have already fetched all topics, however, # another fetch should be unnecessary. self._need_update = False