def _send_broker_unaware_request(self, payloads, encoder_fn, decoder_fn): """ Attempt to send a broker-agnostic request to one of the available brokers. Keep trying until you succeed. """ hosts = set() for broker in self.brokers.values(): host, port, afi = get_ip_port_afi(broker.host) hosts.add((host, broker.port, afi)) hosts.update(self.hosts) hosts = list(hosts) random.shuffle(hosts) for (host, port, afi) in hosts: try: conn = self._get_conn(host, port, afi) except ConnectionError: log.warning("Skipping unconnected connection: %s:%s (AFI %s)", host, port, afi) continue request = encoder_fn(payloads=payloads) future = conn.send(request) # Block while not future.is_done: conn.recv() if future.failed(): log.error("Request failed: %s", future.exception) continue return decoder_fn(future.value) raise KafkaUnavailableError('All servers failed to process request: %s' % hosts)
def _maybe_connect(self, node_id): """Idempotent non-blocking connection attempt to the given node id.""" with self._lock: broker = self.cluster.broker_metadata(node_id) conn = self._conns.get(node_id) if conn is None: assert broker, 'Broker id %s not in current metadata' % node_id log.debug("Initiating connection to node %s at %s:%s", node_id, broker.host, broker.port) host, port, afi = get_ip_port_afi(broker.host) cb = functools.partial(WeakMethod(self._conn_state_change), node_id) conn = BrokerConnection(host, broker.port, afi, state_change_callback=cb, node_id=node_id, **self.config) self._conns[node_id] = conn # Check if existing connection should be recreated because host/port changed elif conn.disconnected() and broker is not None: host, _, __ = get_ip_port_afi(broker.host) if conn.host != host or conn.port != broker.port: log.info("Broker metadata change detected for node %s" " from %s:%s to %s:%s", node_id, conn.host, conn.port, broker.host, broker.port) # Drop old connection object. # It will be recreated on next _maybe_connect self._conns.pop(node_id) return False elif conn.connected(): return True conn.connect() return conn.connected()
def _send_broker_unaware_request(self, payloads, encoder_fn, decoder_fn): """ Attempt to send a broker-agnostic request to one of the available brokers. Keep trying until you succeed. """ hosts = set() for broker in self.brokers.values(): host, port, afi = get_ip_port_afi(broker.host) hosts.add((host, broker.port, afi)) hosts.update(self.hosts) hosts = list(hosts) random.shuffle(hosts) for (host, port, afi) in hosts: try: conn = self._get_conn(host, port, afi) except ConnectionError: log.warning("Skipping unconnected connection: %s:%s (AFI %s)", host, port, afi) continue request = encoder_fn(payloads=payloads) future = conn.send(request) # Block while not future.is_done: conn.recv() if future.failed(): log.error("Request failed: %s", future.exception) continue return decoder_fn(future.value) raise KafkaUnavailableError( 'All servers failed to process request: %s' % hosts)
def _send_consumer_aware_request(self, group, payloads, encoder_fn, decoder_fn): """ Send a list of requests to the consumer coordinator for the group specified using the supplied encode/decode functions. As the payloads that use consumer-aware requests do not contain the group (e.g. OffsetFetchRequest), all payloads must be for a single group. Arguments: group: the name of the consumer group (str) the payloads are for payloads: list of object-like entities with topic (str) and partition (int) attributes; payloads with duplicate topic+partition are not supported. encode_fn: a method to encode the list of payloads to a request body, must accept client_id, correlation_id, and payloads as keyword arguments decode_fn: a method to decode a response body into response objects. The response objects must be object-like and have topic and partition attributes Returns: List of response objects in the same order as the supplied payloads """ # encoders / decoders do not maintain ordering currently # so we need to keep this so we can rebuild order before returning original_ordering = [(p.topic, p.partition) for p in payloads] broker = self._get_coordinator_for_group(group) # Send the list of request payloads and collect the responses and # errors responses = {} request_id = self._next_id() log.debug('Request %s to %s: %s', request_id, broker, payloads) request = encoder_fn(client_id=self.client_id, correlation_id=request_id, payloads=payloads) # Send the request, recv the response try: host, port, afi = get_ip_port_afi(broker.host) conn = self._get_conn(host, broker.port, afi) conn.send(request_id, request) except ConnectionError as e: log.warning( 'ConnectionError attempting to send request %s ' 'to server %s: %s', request_id, broker, e) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = FailedPayloadsError(payload) # No exception, try to get response else: # decoder_fn=None signal that the server is expected to not # send a response. This probably only applies to # ProduceRequest w/ acks = 0 if decoder_fn is None: log.debug( 'Request %s does not expect a response ' '(skipping conn.recv)', request_id) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = None return [] try: response = conn.recv(request_id) except ConnectionError as e: log.warning( 'ConnectionError attempting to receive a ' 'response to request %s from server %s: %s', request_id, broker, e) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = FailedPayloadsError(payload) else: _resps = [] for payload_response in decoder_fn(response): topic_partition = (payload_response.topic, payload_response.partition) responses[topic_partition] = payload_response _resps.append(payload_response) log.debug('Response %s: %s', request_id, _resps) # Return responses in the same order as provided return [responses[tp] for tp in original_ordering]
def _send_broker_aware_request(self, payloads, encoder_fn, decoder_fn): """ Group a list of request payloads by topic+partition and send them to the leader broker for that partition using the supplied encode/decode functions Arguments: payloads: list of object-like entities with a topic (str) and partition (int) attribute; payloads with duplicate topic-partitions are not supported. encode_fn: a method to encode the list of payloads to a request body, must accept client_id, correlation_id, and payloads as keyword arguments decode_fn: a method to decode a response body into response objects. The response objects must be object-like and have topic and partition attributes Returns: List of response objects in the same order as the supplied payloads """ # encoders / decoders do not maintain ordering currently # so we need to keep this so we can rebuild order before returning original_ordering = [(p.topic, p.partition) for p in payloads] # Connection errors generally mean stale metadata # although sometimes it means incorrect api request # Unfortunately there is no good way to tell the difference # so we'll just reset metadata on all errors to be safe refresh_metadata = False # For each broker, send the list of request payloads # and collect the responses and errors payloads_by_broker = self._payloads_by_broker(payloads) responses = {} def failed_payloads(payloads): for payload in payloads: topic_partition = (str(payload.topic), payload.partition) responses[(topic_partition)] = FailedPayloadsError(payload) # For each BrokerConnection keep the real socket so that we can use # a select to perform unblocking I/O connections_by_future = {} for broker, broker_payloads in six.iteritems(payloads_by_broker): if broker is None: failed_payloads(broker_payloads) continue host, port, afi = get_ip_port_afi(broker.host) try: conn = self._get_conn(host, broker.port, afi) except ConnectionError: refresh_metadata = True failed_payloads(broker_payloads) continue request = encoder_fn(payloads=broker_payloads) future = conn.send(request) if future.failed(): refresh_metadata = True failed_payloads(broker_payloads) continue if not request.expect_response(): for payload in broker_payloads: topic_partition = (str(payload.topic), payload.partition) responses[topic_partition] = None continue connections_by_future[future] = (conn, broker) conn = None while connections_by_future: futures = list(connections_by_future.keys()) # block until a socket is ready to be read sockets = [ conn._sock for future, (conn, _) in six.iteritems(connections_by_future) if not future.is_done and conn._sock is not None ] if sockets: read_socks, _, _ = select.select(sockets, [], []) for future in futures: if not future.is_done: conn, _ = connections_by_future[future] conn.recv() continue _, broker = connections_by_future.pop(future) if future.failed(): refresh_metadata = True failed_payloads(payloads_by_broker[broker]) else: for payload_response in decoder_fn(future.value): topic_partition = (str(payload_response.topic), payload_response.partition) responses[topic_partition] = payload_response if refresh_metadata: self.reset_all_metadata() # Return responses in the same order as provided return [responses[tp] for tp in original_ordering]
def _send_consumer_aware_request(self, group, payloads, encoder_fn, decoder_fn): """ Send a list of requests to the consumer coordinator for the group specified using the supplied encode/decode functions. As the payloads that use consumer-aware requests do not contain the group (e.g. OffsetFetchRequest), all payloads must be for a single group. Arguments: group: the name of the consumer group (str) the payloads are for payloads: list of object-like entities with topic (str) and partition (int) attributes; payloads with duplicate topic+partition are not supported. encode_fn: a method to encode the list of payloads to a request body, must accept client_id, correlation_id, and payloads as keyword arguments decode_fn: a method to decode a response body into response objects. The response objects must be object-like and have topic and partition attributes Returns: List of response objects in the same order as the supplied payloads """ # encoders / decoders do not maintain ordering currently # so we need to keep this so we can rebuild order before returning original_ordering = [(p.topic, p.partition) for p in payloads] retries = 0 broker = None while not broker: try: broker = self._get_coordinator_for_group(group) except (GroupCoordinatorNotAvailableError, GroupLoadInProgressError) as e: if retries == CONSUMER_OFFSET_TOPIC_CREATION_RETRIES: raise e time.sleep(CONSUMER_OFFSET_RETRY_INTERVAL_SEC) retries += 1 # Send the list of request payloads and collect the responses and # errors responses = {} def failed_payloads(payloads): for payload in payloads: topic_partition = (str(payload.topic), payload.partition) responses[topic_partition] = FailedPayloadsError(payload) host, port, afi = get_ip_port_afi(broker.host) try: conn = self._get_conn(host, broker.port, afi) except ConnectionError: failed_payloads(payloads) else: request = encoder_fn(payloads=payloads) # decoder_fn=None signal that the server is expected to not # send a response. This probably only applies to # ProduceRequest w/ acks = 0 future = conn.send(request) while not future.is_done: for r, f in conn.recv(): f.success(r) if future.failed(): failed_payloads(payloads) elif not request.expect_response(): failed_payloads(payloads) else: for payload_response in decoder_fn(future.value): topic_partition = (str(payload_response.topic), payload_response.partition) responses[topic_partition] = payload_response # Return responses in the same order as provided return [responses[tp] for tp in original_ordering]
def _send_consumer_aware_request(self, group, payloads, encoder_fn, decoder_fn): """ Send a list of requests to the consumer coordinator for the group specified using the supplied encode/decode functions. As the payloads that use consumer-aware requests do not contain the group (e.g. OffsetFetchRequest), all payloads must be for a single group. Arguments: group: the name of the consumer group (str) the payloads are for payloads: list of object-like entities with topic (str) and partition (int) attributes; payloads with duplicate topic+partition are not supported. encode_fn: a method to encode the list of payloads to a request body, must accept client_id, correlation_id, and payloads as keyword arguments decode_fn: a method to decode a response body into response objects. The response objects must be object-like and have topic and partition attributes Returns: List of response objects in the same order as the supplied payloads """ # encoders / decoders do not maintain ordering currently # so we need to keep this so we can rebuild order before returning original_ordering = [(p.topic, p.partition) for p in payloads] broker = self._get_coordinator_for_group(group) # Send the list of request payloads and collect the responses and # errors responses = {} request_id = self._next_id() log.debug('Request %s to %s: %s', request_id, broker, payloads) request = encoder_fn(client_id=self.client_id, correlation_id=request_id, payloads=payloads) # Send the request, recv the response try: host, port, afi = get_ip_port_afi(broker.host) conn = self._get_conn(host, broker.port, afi) conn.send(request_id, request) except ConnectionError as e: log.warning('ConnectionError attempting to send request %s ' 'to server %s: %s', request_id, broker, e) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = FailedPayloadsError(payload) # No exception, try to get response else: # decoder_fn=None signal that the server is expected to not # send a response. This probably only applies to # ProduceRequest w/ acks = 0 if decoder_fn is None: log.debug('Request %s does not expect a response ' '(skipping conn.recv)', request_id) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = None return [] try: response = conn.recv(request_id) except ConnectionError as e: log.warning('ConnectionError attempting to receive a ' 'response to request %s from server %s: %s', request_id, broker, e) for payload in payloads: topic_partition = (payload.topic, payload.partition) responses[topic_partition] = FailedPayloadsError(payload) else: _resps = [] for payload_response in decoder_fn(response): topic_partition = (payload_response.topic, payload_response.partition) responses[topic_partition] = payload_response _resps.append(payload_response) log.debug('Response %s: %s', request_id, _resps) # Return responses in the same order as provided return [responses[tp] for tp in original_ordering]
def _send_broker_aware_request(self, payloads, encoder_fn, decoder_fn): """ Group a list of request payloads by topic+partition and send them to the leader broker for that partition using the supplied encode/decode functions Arguments: payloads: list of object-like entities with a topic (str) and partition (int) attribute; payloads with duplicate topic-partitions are not supported. encode_fn: a method to encode the list of payloads to a request body, must accept client_id, correlation_id, and payloads as keyword arguments decode_fn: a method to decode a response body into response objects. The response objects must be object-like and have topic and partition attributes Returns: List of response objects in the same order as the supplied payloads """ # encoders / decoders do not maintain ordering currently # so we need to keep this so we can rebuild order before returning original_ordering = [(p.topic, p.partition) for p in payloads] # Connection errors generally mean stale metadata # although sometimes it means incorrect api request # Unfortunately there is no good way to tell the difference # so we'll just reset metadata on all errors to be safe refresh_metadata = False # For each broker, send the list of request payloads # and collect the responses and errors payloads_by_broker = self._payloads_by_broker(payloads) responses = {} def failed_payloads(payloads): for payload in payloads: topic_partition = (str(payload.topic), payload.partition) responses[(topic_partition)] = FailedPayloadsError(payload) # For each BrokerConnection keep the real socket so that we can use # a select to perform unblocking I/O connections_by_future = {} for broker, broker_payloads in six.iteritems(payloads_by_broker): if broker is None: failed_payloads(broker_payloads) continue host, port, afi = get_ip_port_afi(broker.host) try: conn = self._get_conn(host, broker.port, afi) except ConnectionError: refresh_metadata = True failed_payloads(broker_payloads) continue request = encoder_fn(payloads=broker_payloads) future = conn.send(request) if future.failed(): refresh_metadata = True failed_payloads(broker_payloads) continue if not request.expect_response(): for payload in broker_payloads: topic_partition = (str(payload.topic), payload.partition) responses[topic_partition] = None continue connections_by_future[future] = (conn, broker) conn = None while connections_by_future: futures = list(connections_by_future.keys()) # block until a socket is ready to be read sockets = [ conn._sock for future, (conn, _) in six.iteritems(connections_by_future) if not future.is_done and conn._sock is not None] if sockets: read_socks, _, _ = select.select(sockets, [], []) for future in futures: if not future.is_done: conn, _ = connections_by_future[future] conn.recv() continue _, broker = connections_by_future.pop(future) if future.failed(): refresh_metadata = True failed_payloads(payloads_by_broker[broker]) else: for payload_response in decoder_fn(future.value): topic_partition = (str(payload_response.topic), payload_response.partition) responses[topic_partition] = payload_response if refresh_metadata: self.reset_all_metadata() # Return responses in the same order as provided return [responses[tp] for tp in original_ordering]
def _send_consumer_aware_request(self, group, payloads, encoder_fn, decoder_fn): """ Send a list of requests to the consumer coordinator for the group specified using the supplied encode/decode functions. As the payloads that use consumer-aware requests do not contain the group (e.g. OffsetFetchRequest), all payloads must be for a single group. Arguments: group: the name of the consumer group (str) the payloads are for payloads: list of object-like entities with topic (str) and partition (int) attributes; payloads with duplicate topic+partition are not supported. encode_fn: a method to encode the list of payloads to a request body, must accept client_id, correlation_id, and payloads as keyword arguments decode_fn: a method to decode a response body into response objects. The response objects must be object-like and have topic and partition attributes Returns: List of response objects in the same order as the supplied payloads """ # encoders / decoders do not maintain ordering currently # so we need to keep this so we can rebuild order before returning original_ordering = [(p.topic, p.partition) for p in payloads] retries = 0 broker = None while not broker: try: broker = self._get_coordinator_for_group(group) except (GroupCoordinatorNotAvailableError, GroupLoadInProgressError) as e: if retries == CONSUMER_OFFSET_TOPIC_CREATION_RETRIES: raise e time.sleep(CONSUMER_OFFSET_RETRY_INTERVAL_SEC) retries += 1 # Send the list of request payloads and collect the responses and # errors responses = {} def failed_payloads(payloads): for payload in payloads: topic_partition = (str(payload.topic), payload.partition) responses[topic_partition] = FailedPayloadsError(payload) host, port, afi = get_ip_port_afi(broker.host) try: conn = self._get_conn(host, broker.port, afi, broker.nodeId) except ConnectionError: failed_payloads(payloads) else: request = encoder_fn(payloads=payloads) # decoder_fn=None signal that the server is expected to not # send a response. This probably only applies to # ProduceRequest w/ acks = 0 expect_response = (decoder_fn is not None) future = conn.send(request, expect_response=expect_response) while not future.is_done: conn.recv() if future.failed(): failed_payloads(payloads) elif not expect_response: failed_payloads(payloads) else: for payload_response in decoder_fn(future.value): topic_partition = (str(payload_response.topic), payload_response.partition) responses[topic_partition] = payload_response # Return responses in the same order as provided return [responses[tp] for tp in original_ordering]
def _send_broker_aware_request(self, payloads, encoder_fn, decoder_fn): """ Group a list of request payloads by topic+partition and send them to the leader broker for that partition using the supplied encode/decode functions Arguments: payloads: list of object-like entities with a topic (str) and partition (int) attribute; payloads with duplicate topic-partitions are not supported. encode_fn: a method to encode the list of payloads to a request body, must accept client_id, correlation_id, and payloads as keyword arguments decode_fn: a method to decode a response body into response objects. The response objects must be object-like and have topic and partition attributes Returns: List of response objects in the same order as the supplied payloads """ # encoders / decoders do not maintain ordering currently # so we need to keep this so we can rebuild order before returning original_ordering = [(p.topic, p.partition) for p in payloads] # Connection errors generally mean stale metadata # although sometimes it means incorrect api request # Unfortunately there is no good way to tell the difference # so we'll just reset metadata on all errors to be safe refresh_metadata = False # For each broker, send the list of request payloads # and collect the responses and errors payloads_by_broker = self._payloads_by_broker(payloads) responses = {} def failed_payloads(payloads): for payload in payloads: topic_partition = (str(payload.topic), payload.partition) responses[topic_partition] = FailedPayloadsError(payload) futures_by_connection = {} selector = selectors.DefaultSelector() for broker, broker_payloads in six.iteritems(payloads_by_broker): if broker is None: failed_payloads(broker_payloads) continue host, port, afi = get_ip_port_afi(broker.host) try: conn = self._get_conn(host, broker.port, afi, broker.nodeId) except ConnectionError: refresh_metadata = True failed_payloads(broker_payloads) continue request = encoder_fn(payloads=broker_payloads) # decoder_fn=None signal that the server is expected to not # send a response. This probably only applies to # ProduceRequest w/ acks = 0 expect_response = (decoder_fn is not None) if expect_response: selector.register(conn._sock, selectors.EVENT_READ, conn) future = conn.send(request, expect_response=expect_response) if future.failed(): log.error("Request failed: %s", future.exception) selector.unregister(conn._sock) refresh_metadata = True failed_payloads(broker_payloads) continue if not expect_response: for payload in broker_payloads: topic_partition = (str(payload.topic), payload.partition) responses[topic_partition] = None continue futures_by_connection[conn] = (future, broker) timeout = self.timeout while futures_by_connection: start_time = time.time() ready = selector.select(timeout) for key, _ in ready: conn = key.data future, _ = futures_by_connection[conn] while not future.is_done: conn.recv() _, broker = futures_by_connection.pop(conn) if future.failed(): log.error("Request failed: %s", future.exception) refresh_metadata = True failed_payloads(payloads_by_broker[broker]) else: for payload_response in decoder_fn(future.value): topic_partition = (str(payload_response.topic), payload_response.partition) responses[topic_partition] = payload_response timeout -= time.time() - start_time if timeout < 0: log.error("%s requests timed out.", len(futures_by_connection)) for _, broker in six.itervalues(futures_by_connection): failed_payloads(payloads_by_broker[broker]) refresh_metadata = True break if refresh_metadata: self.reset_all_metadata() selector.close() # Return responses in the same order as provided return [responses[tp] for tp in original_ordering]
def load_metadata_for_topics(self, *topics, **kwargs): """Fetch broker and topic-partition metadata from the server. Updates internal data: broker list, topic/partition list, and topic/parition -> broker map. This method should be called after receiving any error. Note: Exceptions *will not* be raised in a full refresh (i.e. no topic list). In this case, error codes will be logged as errors. Partition-level errors will also not be raised here (a single partition w/o a leader, for example). Arguments: *topics (optional): If a list of topics is provided, the metadata refresh will be limited to the specified topics only. ignore_leadernotavailable (bool): suppress LeaderNotAvailableError so that metadata is loaded correctly during auto-create. Default: False. Raises: UnknownTopicOrPartitionError: Raised for topics that do not exist, unless the broker is configured to auto-create topics. LeaderNotAvailableError: Raised for topics that do not exist yet, when the broker is configured to auto-create topics. Retry after a short backoff (topics/partitions are initializing). """ if 'ignore_leadernotavailable' in kwargs: ignore_leadernotavailable = kwargs['ignore_leadernotavailable'] else: ignore_leadernotavailable = False if topics: self.reset_topic_metadata(*topics) else: self.reset_all_metadata() resp = self.send_metadata_request(topics) log.debug('Updating broker metadata: %s', resp.brokers) log.debug('Updating topic metadata: %s', [topic for _, topic, _ in resp.topics]) if not self._specific_broker: self.brokers = dict([(nodeId, BrokerMetadata(nodeId, host, port, None)) for nodeId, host, port in resp.brokers]) else: for nodeId, host, port in resp.brokers: if str(host) + ':' + str(port) == self._specific_broker: self.brokers = { nodeId: BrokerMetadata(nodeId, host, port, None) } break if not self.brokers: h, p, a = get_ip_port_afi(self._specific_broker) self.brokers = {1010: BrokerMetadata(1010, h, p, None)} # raise ReplicaNotAvailableError("%s" % self._specific_broker) for error, topic, partitions in resp.topics: # Errors expected for new topics if error: error_type = kafka.errors.kafka_errors.get(error, UnknownError) if error_type in (UnknownTopicOrPartitionError, LeaderNotAvailableError): log.error('Error loading topic metadata for %s: %s (%s)', topic, error_type, error) if topic not in topics: continue elif (error_type is LeaderNotAvailableError): # elif (error_type is LeaderNotAvailableError and # ignore_leadernotavailable): continue raise error_type(topic) self.topic_partitions[topic] = {} for error, partition, leader, _, _ in partitions: self.topic_partitions[topic][partition] = leader # Populate topics_to_brokers dict topic_part = TopicPartition(topic, partition) # Check for partition errors if error: error_type = kafka.errors.kafka_errors.get( error, UnknownError) # If No Leader, topics_to_brokers topic_partition -> None if error_type is LeaderNotAvailableError: log.error('No leader for topic %s partition %d', topic, partition) self.topics_to_brokers[topic_part] = None continue # If one of the replicas is unavailable -- ignore # this error code is provided for admin purposes only # we never talk to replicas, only the leader elif error_type is ReplicaNotAvailableError: log.debug( 'Some (non-leader) replicas not available for topic %s partition %d', topic, partition) else: raise error_type(topic_part) # If Known Broker, topic_partition -> BrokerMetadata if leader in self.brokers: self.topics_to_brokers[topic_part] = self.brokers[leader] # If Unknown Broker, fake BrokerMetadata so we don't lose the id # (not sure how this could happen. server could be in bad state) else: self.topics_to_brokers[topic_part] = BrokerMetadata( leader, None, None, None)