async def _wait_on_metadata(self, topic): """ Wait for cluster metadata including partitions for the given topic to be available. Arguments: topic (str): topic we want metadata for Returns: set: partition ids for the topic Raises: UnknownTopicOrPartitionError: if no topic or partitions found in cluster metadata """ if topic in self.cluster.topics(): return self.cluster.partitions_for_topic(topic) # add topic to metadata topic list if it is not there already. self.add_topic(topic) t0 = self._loop.time() while True: await self.force_metadata_update() if topic in self.cluster.topics(): break if (self._loop.time() - t0) > (self._request_timeout_ms / 1000): raise UnknownTopicOrPartitionError() if topic in self.cluster.unauthorized_topics: raise Errors.TopicAuthorizationFailedError(topic) await asyncio.sleep(self._retry_backoff, loop=self._loop) return self.cluster.partitions_for_topic(topic)
def _proc_fetch_request(self, assignment, node_id, request): needs_wakeup = False try: response = yield from self._client.send(node_id, request) except Errors.KafkaError as err: log.error("Failed fetch messages from %s: %s", node_id, err) return False except asyncio.CancelledError: # Either `close()` or partition unassigned. Either way the result # is no longer of interest. return False if not assignment.active: log.debug( "Discarding fetch response since the assignment changed during" " fetch") return False fetch_offsets = {} for topic, partitions in request.topics: for partition, offset, _ in partitions: fetch_offsets[TopicPartition(topic, partition)] = offset for topic, partitions in response.topics: for partition, error_code, highwater, *part_data in partitions: tp = TopicPartition(topic, partition) error_type = Errors.for_code(error_code) fetch_offset = fetch_offsets[tp] tp_state = assignment.state_value(tp) if not tp_state.has_valid_position or \ tp_state.position != fetch_offset: log.debug( "Discarding fetch response for partition %s " "since its offset %s does not match the current " "position", tp, fetch_offset) continue if error_type is Errors.NoError: tp_state.highwater = highwater # part_data also contains lso, aborted_transactions. # message_set is last records = MemoryRecords(part_data[-1]) if records.has_next(): log.debug( "Adding fetched record for partition %s with" " offset %d to buffered record list", tp, fetch_offset) message_iterator = self._unpack_records(tp, records) self._records[tp] = FetchResult( tp, message_iterator=message_iterator, assignment=assignment, backoff=self._prefetch_backoff, fetch_offset=fetch_offset, loop=self._loop) # We added at least 1 successful record needs_wakeup = True elif records.size_in_bytes() > 0: # we did not read a single message from a non-empty # buffer because that message's size is larger than # fetch size, in this case record this exception err = RecordTooLargeError( "There are some messages at [Partition=Offset]: " "%s=%s whose size is larger than the fetch size %s" " and hence cannot be ever returned. " "Increase the fetch size, or decrease the maximum " "message size the broker will allow.", tp, fetch_offset, self._max_partition_fetch_bytes) self._set_error(tp, err) tp_state.consumed_to(tp_state.position + 1) needs_wakeup = True elif error_type in (Errors.NotLeaderForPartitionError, Errors.UnknownTopicOrPartitionError): self._client.force_metadata_update() elif error_type is Errors.OffsetOutOfRangeError: if self._default_reset_strategy != \ OffsetResetStrategy.NONE: tp_state.await_reset(self._default_reset_strategy) else: err = Errors.OffsetOutOfRangeError({tp: fetch_offset}) self._set_error(tp, err) needs_wakeup = True log.info( "Fetch offset %s is out of range for partition %s," " resetting offset", fetch_offset, tp) elif error_type is Errors.TopicAuthorizationFailedError: log.warning("Not authorized to read from topic %s.", tp.topic) err = Errors.TopicAuthorizationFailedError(tp.topic) self._set_error(tp, err) needs_wakeup = True else: log.warning('Unexpected error while fetching data: %s', error_type.__name__) return needs_wakeup
def commit_offsets(self, offsets): """Commit specific offsets asynchronously. Arguments: offsets (dict {TopicPartition: OffsetAndMetadata}): what to commit Raises error on failure """ self._subscription.needs_fetch_committed_offsets = True if not offsets: log.debug('No offsets to commit') return True if (yield from self.coordinator_unknown()): raise Errors.GroupCoordinatorNotAvailableError() node_id = self.coordinator_id # create the offset commit request offset_data = collections.defaultdict(list) for tp, offset in offsets.items(): offset_data[tp.topic].append( (tp.partition, offset.offset, offset.metadata)) request = OffsetCommitRequest( self.group_id, self.generation, self.member_id, OffsetCommitRequest.DEFAULT_RETENTION_TIME, [(topic, tp_offsets) for topic, tp_offsets in offset_data.items()]) log.debug("Sending offset-commit request with %s for group %s to %s", offsets, self.group_id, node_id) response = yield from self._send_req( node_id, request, group=ConnectionGroup.COORDINATION) unauthorized_topics = set() for topic, partitions in response.topics: for partition, error_code in partitions: tp = TopicPartition(topic, partition) offset = offsets[tp] error_type = Errors.for_code(error_code) if error_type is Errors.NoError: log.debug("Committed offset %s for partition %s", offset, tp) if self._subscription.is_assigned(tp): partition = self._subscription.assignment[tp] partition.committed = offset.offset elif error_type is Errors.GroupAuthorizationFailedError: log.error("OffsetCommit failed for group %s - %s", self.group_id, error_type.__name__) raise error_type() elif error_type is Errors.TopicAuthorizationFailedError: unauthorized_topics.add(topic) elif error_type in (Errors.OffsetMetadataTooLargeError, Errors.InvalidCommitOffsetSizeError): # raise the error to the user log.info( "OffsetCommit failed for group %s on partition %s" " due to %s, will retry", self.group_id, tp, error_type.__name__) raise error_type() elif error_type is Errors.GroupLoadInProgressError: # just retry log.info( "OffsetCommit failed for group %s because group is" " initializing (%s), will retry", self.group_id, error_type.__name__) raise error_type() elif error_type in (Errors.GroupCoordinatorNotAvailableError, Errors.NotCoordinatorForGroupError, Errors.RequestTimedOutError): log.info( "OffsetCommit failed for group %s due to a" " coordinator error (%s), will find new coordinator" " and retry", self.group_id, error_type.__name__) self.coordinator_dead() raise error_type() elif error_type in (Errors.UnknownMemberIdError, Errors.IllegalGenerationError, Errors.RebalanceInProgressError): # need to re-join group error = error_type(self.group_id) log.error( "OffsetCommit failed for group %s due to group" " error (%s), will rejoin", self.group_id, error) self._subscription.mark_for_reassignment() raise error else: log.error( "OffsetCommit failed for group %s on partition %s" " with offset %s: %s", self.group_id, tp, offset, error_type.__name__) raise error_type() if unauthorized_topics: log.error("OffsetCommit failed for unauthorized topics %s", unauthorized_topics) raise Errors.TopicAuthorizationFailedError(unauthorized_topics)
def _proc_fetch_request(self, node_id, request): needs_wakeup = False needs_position_update = [] try: response = yield from self._client.send(node_id, request) except Errors.KafkaError as err: log.error("Failed fetch messages from %s: %s", node_id, err) return False finally: self._in_flight.remove(node_id) fetch_offsets = {} for topic, partitions in request.topics: for partition, offset, _ in partitions: fetch_offsets[TopicPartition(topic, partition)] = offset for topic, partitions in response.topics: for partition, error_code, highwater, raw_batch in partitions: tp = TopicPartition(topic, partition) error_type = Errors.for_code(error_code) if not self._subscriptions.is_fetchable(tp): # this can happen when a rebalance happened log.debug( "Ignoring fetched records for partition %s" " since it is no longer fetchable", tp) elif error_type is Errors.NoError: tp_assignment = self._subscriptions.assignment[tp] tp_assignment.highwater = highwater # `drop_pending_message_set` is set after a seek to another # position. If we request the *new* position we have to # drop this flag, so we catch future seek's. fetch_offset = fetch_offsets[tp] if fetch_offset == tp_assignment.position: tp_assignment.drop_pending_message_set = False records = MemoryRecords(raw_batch) if records.has_next(): log.debug( "Adding fetched record for partition %s with" " offset %d to buffered record list", tp, fetch_offset) message_iterator = self._unpack_records(tp, records) self._records[tp] = FetchResult( tp, records=message_iterator, subscriptions=self._subscriptions, backoff=self._prefetch_backoff, loop=self._loop) # We added at least 1 successful record needs_wakeup = True elif records.size_in_bytes() > 0: # we did not read a single message from a non-empty # buffer because that message's size is larger than # fetch size, in this case record this exception err = RecordTooLargeError( "There are some messages at [Partition=Offset]: " "%s=%s whose size is larger than the fetch size %s" " and hence cannot be ever returned. " "Increase the fetch size, or decrease the maximum " "message size the broker will allow.", tp, fetch_offset, self._max_partition_fetch_bytes) self._set_error(tp, err) needs_wakeup = True self._subscriptions.assignment[tp].position += 1 elif error_type in (Errors.NotLeaderForPartitionError, Errors.UnknownTopicOrPartitionError): self._client.force_metadata_update() elif error_type is Errors.OffsetOutOfRangeError: fetch_offset = fetch_offsets[tp] if self._subscriptions.has_default_offset_reset_policy(): self._subscriptions.need_offset_reset(tp) needs_position_update.append(tp) else: err = Errors.OffsetOutOfRangeError({tp: fetch_offset}) self._set_error(tp, err) needs_wakeup = True log.info( "Fetch offset %s is out of range for partition %s," " resetting offset", fetch_offset, tp) elif error_type is Errors.TopicAuthorizationFailedError: log.warn("Not authorized to read from topic %s.", tp.topic) err = Errors.TopicAuthorizationFailedError(tp.topic) self._set_error(tp, err) needs_wakeup = True else: log.warn('Unexpected error while fetching data: %s', error_type.__name__) if needs_position_update: try: yield from self.update_fetch_positions(needs_position_update) except Exception: # pragma: no cover log.error("Unexpected error updating fetch positions", exc_info=True) return needs_wakeup