def _handle_metadata_update(self, cluster): # if we encounter any unauthorized topics, raise an exception if cluster.unauthorized_topics: raise Errors.TopicAuthorizationFailedError( cluster.unauthorized_topics) if self._subscription.subscribed_pattern: topics = [] for topic in cluster.topics( self.config["exclude_internal_topics"]): if self._subscription.subscribed_pattern.match(topic): topics.append(topic) if set(topics) != self._subscription.subscription: self._subscription.change_subscription(topics) self._client.set_topics( self._subscription.group_subscription()) # check if there are any changes to the metadata which should trigger # a rebalance if self._subscription.partitions_auto_assigned(): metadata_snapshot = self._build_metadata_snapshot( self._subscription, cluster) if self._metadata_snapshot != metadata_snapshot: self._metadata_snapshot = metadata_snapshot # If we haven't got group coordinator support, # just assign all partitions locally if self._auto_assign_all_partitions(): self._subscription.assign_from_subscribed([ TopicPartition(topic, partition) for topic in self._subscription.subscription for partition in self._metadata_snapshot[topic] ])
def _raise_if_unauthorized_topics(self): """Check FetchResponses for topic authorization failures. Raises: TopicAuthorizationFailedError """ if self._unauthorized_topics: topics = set(self._unauthorized_topics) self._unauthorized_topics.clear() raise Errors.TopicAuthorizationFailedError(topics)
def _wait_on_metadata(self, topic, max_wait): """ Wait for cluster metadata including partitions for the given topic to be available. Arguments: topic (str): topic we want metadata for max_wait (float): maximum time in secs for waiting on the metadata Returns: set: partition ids for the topic Raises: KafkaTimeoutError: if partitions for topic were not obtained before specified max_wait timeout """ # add topic to metadata topic list if it is not there already. self._sender.add_topic(topic) begin = time.time() elapsed = 0.0 metadata_event = None while True: partitions = self._metadata.partitions_for_topic(topic) if partitions is not None: return partitions if not metadata_event: metadata_event = threading.Event() log.debug("Requesting metadata update for topic %s", topic) metadata_event.clear() future = self._metadata.request_update() future.add_both(lambda e, *args: e.set(), metadata_event) self._sender.wakeup() metadata_event.wait(max_wait - elapsed) elapsed = time.time() - begin if not metadata_event.is_set(): raise Errors.KafkaTimeoutError( "Failed to update metadata after %.1f secs." % (max_wait, )) elif topic in self._metadata.unauthorized_topics: raise Errors.TopicAuthorizationFailedError(topic) else: log.debug("_wait_on_metadata woke after %s secs.", elapsed)
def _parse_fetched_data(self, completed_fetch): tp = completed_fetch.topic_partition fetch_offset = completed_fetch.fetched_offset num_bytes = 0 records_count = 0 parsed_records = None error_code, highwater = completed_fetch.partition_data[:2] error_type = Errors.for_code(error_code) try: if not self._subscriptions.is_fetchable(tp): # this can happen when a rebalance happened or a partition # consumption paused while fetch is still in-flight log.debug( "Ignoring fetched records for partition %s" " since it is no longer fetchable", tp) elif error_type is Errors.NoError: self._subscriptions.assignment[tp].highwater = highwater # we are interested in this fetch only if the beginning # offset (of the *request*) matches the current consumed position # Note that the *response* may return a messageset that starts # earlier (e.g., compressed messages) or later (e.g., compacted topic) position = self._subscriptions.assignment[tp].position if position is None or position != fetch_offset: log.debug( "Discarding fetch response for partition %s" " since its offset %d does not match the" " expected offset %d", tp, fetch_offset, position) return None records = MemoryRecords(completed_fetch.partition_data[-1]) if records.has_next(): log.debug( "Adding fetched record for partition %s with" " offset %d to buffered record list", tp, position) unpacked = list(self._unpack_message_set(tp, records)) parsed_records = self.PartitionRecords( fetch_offset, tp, unpacked) if unpacked: last_offset = unpacked[-1].offset self._sensors.records_fetch_lag.record(highwater - last_offset) num_bytes = records.valid_bytes() records_count = len(unpacked) elif records.size_in_bytes() > 0: # we did not read a single message from a non-empty # buffer because that message's size is larger than # fetch size, in this case record this exception record_too_large_partitions = {tp: fetch_offset} raise RecordTooLargeError( "There are some messages at [Partition=Offset]: %s " " whose size is larger than the fetch size %s" " and hence cannot be ever returned." " Increase the fetch size, or decrease the maximum message" " size the broker will allow." % (record_too_large_partitions, self.config['max_partition_fetch_bytes']), record_too_large_partitions) self._sensors.record_topic_fetch_metrics( tp.topic, num_bytes, records_count) elif error_type in (Errors.NotLeaderForPartitionError, Errors.UnknownTopicOrPartitionError): self._client.cluster.request_update() elif error_type is Errors.OffsetOutOfRangeError: position = self._subscriptions.assignment[tp].position if position is None or position != fetch_offset: log.debug( "Discarding stale fetch response for partition %s" " since the fetched offset %d does not match the" " current offset %d", tp, fetch_offset, position) elif self._subscriptions.has_default_offset_reset_policy(): log.info( "Fetch offset %s is out of range for topic-partition %s", fetch_offset, tp) self._subscriptions.need_offset_reset(tp) else: raise Errors.OffsetOutOfRangeError({tp: fetch_offset}) elif error_type is Errors.TopicAuthorizationFailedError: log.warning("Not authorized to read from topic %s.", tp.topic) raise Errors.TopicAuthorizationFailedError(set(tp.topic)) elif error_type is Errors.UnknownError: log.warning( "Unknown error fetching data for topic-partition %s", tp) else: raise error_type('Unexpected error while fetching data') finally: completed_fetch.metric_aggregator.record(tp, num_bytes, records_count) return parsed_records
def _handle_offset_commit_response(self, offsets, future, send_time, response): # TODO look at adding request_latency_ms to response (like java kafka) self.consumer_sensors.commit_latency.record( (time.time() - send_time) * 1000) unauthorized_topics = set() for topic, partitions in response.topics: for partition, error_code in partitions: tp = TopicPartition(topic, partition) offset = offsets[tp] error_type = Errors.for_code(error_code) if error_type is Errors.NoError: log.debug( "Group %s committed offset %s for partition %s", self.group_id, offset, tp, ) if self._subscription.is_assigned(tp): self._subscription.assignment[tp].committed = offset elif error_type is Errors.GroupAuthorizationFailedError: log.error("Not authorized to commit offsets for group %s", self.group_id) future.failure(error_type(self.group_id)) return elif error_type is Errors.TopicAuthorizationFailedError: unauthorized_topics.add(topic) elif error_type in ( Errors.OffsetMetadataTooLargeError, Errors.InvalidCommitOffsetSizeError, ): # raise the error to the user log.debug( "OffsetCommit for group %s failed on partition %s" " %s", self.group_id, tp, error_type.__name__, ) future.failure(error_type()) return elif error_type is Errors.GroupLoadInProgressError: # just retry log.debug( "OffsetCommit for group %s failed: %s", self.group_id, error_type.__name__, ) future.failure(error_type(self.group_id)) return elif error_type in ( Errors.GroupCoordinatorNotAvailableError, Errors.NotCoordinatorForGroupError, Errors.RequestTimedOutError, ): log.debug( "OffsetCommit for group %s failed: %s", self.group_id, error_type.__name__, ) self.coordinator_dead(error_type()) future.failure(error_type(self.group_id)) return elif error_type in ( Errors.UnknownMemberIdError, Errors.IllegalGenerationError, Errors.RebalanceInProgressError, ): # need to re-join group error = error_type(self.group_id) log.debug("OffsetCommit for group %s failed: %s", self.group_id, error) self.reset_generation() future.failure(Errors.CommitFailedError()) return else: log.error( "Group %s failed to commit partition %s at offset" " %s: %s", self.group_id, tp, offset, error_type.__name__, ) future.failure(error_type()) return if unauthorized_topics: log.error( "Not authorized to commit to topics %s for group %s", unauthorized_topics, self.group_id, ) future.failure( Errors.TopicAuthorizationFailedError(unauthorized_topics)) else: future.success(None)
def _handle_offset_commit_response(self, offsets, future, send_time, response): # TODO look at adding request_latency_ms to response (like java kafka) self.consumer_sensors.commit_latency.record( (time.time() - send_time) * 1000) unauthorized_topics = set() for topic, partitions in response.topics: for partition, error_code in partitions: tp = TopicPartition(topic, partition) offset = offsets[tp] error_type = Errors.for_code(error_code) if error_type is Errors.NoError: log.debug("Group %s committed offset %s for partition %s", self.group_id, offset, tp) if self._subscription.is_assigned(tp): self._subscription.assignment[ tp].committed = offset.offset elif error_type is Errors.GroupAuthorizationFailedError: log.error("Not authorized to commit offsets for group %s", self.group_id) future.failure(error_type(self.group_id)) return elif error_type is Errors.TopicAuthorizationFailedError: unauthorized_topics.add(topic) elif error_type in (Errors.OffsetMetadataTooLargeError, Errors.InvalidCommitOffsetSizeError): # raise the error to the user log.debug( "OffsetCommit for group %s failed on partition %s" " %s", self.group_id, tp, error_type.__name__) future.failure(error_type()) return elif error_type is Errors.GroupLoadInProgressError: # just retry log.debug("OffsetCommit for group %s failed: %s", self.group_id, error_type.__name__) future.failure(error_type(self.group_id)) return elif error_type in (Errors.GroupCoordinatorNotAvailableError, Errors.NotCoordinatorForGroupError, Errors.RequestTimedOutError): log.debug("OffsetCommit for group %s failed: %s", self.group_id, error_type.__name__) self.coordinator_dead(error_type()) future.failure(error_type(self.group_id)) return elif error_type in (Errors.UnknownMemberIdError, Errors.IllegalGenerationError, Errors.RebalanceInProgressError): # need to re-join group error = error_type(self.group_id) log.debug("OffsetCommit for group %s failed: %s", self.group_id, error) self.reset_generation() future.failure( Errors.CommitFailedError( "Commit cannot be completed since the group has" " already rebalanced and assigned the partitions to" " another member. This means that the time between" " subsequent calls to poll() was longer than the" " configured session_timeout_ms, which typically" " implies that the poll loop is spending too much time" " message processing. You can address this either by" " increasing the session timeout or by reducing the" " maximum size of batches returned in poll() with" " max_poll_records.")) return else: log.error( "Group %s failed to commit partition %s at offset" " %s: %s", self.group_id, tp, offset, error_type.__name__) future.failure(error_type()) return if unauthorized_topics: log.error("Not authorized to commit to topics %s for group %s", unauthorized_topics, self.group_id) future.failure( Errors.TopicAuthorizationFailedError(unauthorized_topics)) else: future.success(None)