def _raise_if_offset_out_of_range(self):
"""Check FetchResponses for offset out of range.
Raises:
OffsetOutOfRangeError: if any partition from previous FetchResponse
contains OffsetOutOfRangeError and the default_reset_policy is
None
"""
if not self._offset_out_of_range_partitions:
return
current_out_of_range_partitions = {}
# filter only the fetchable partitions
for partition, offset in self._offset_out_of_range_partitions:
if not self._subscriptions.is_fetchable(partition):
log.debug(
"Ignoring fetched records for %s since it is no"
" longer fetchable", partition)
continue
position = self._subscriptions.assignment[partition].position
# ignore partition if the current position != offset in FetchResponse
# e.g. after seek()
if position is not None and offset == position:
current_out_of_range_partitions[partition] = position
self._offset_out_of_range_partitions.clear()
if current_out_of_range_partitions:
raise Errors.OffsetOutOfRangeError(current_out_of_range_partitions)
def _parse_fetched_data(self, completed_fetch):
tp = completed_fetch.topic_partition
fetch_offset = completed_fetch.fetched_offset
num_bytes = 0
records_count = 0
parsed_records = None
error_code, highwater = completed_fetch.partition_data[:2]
error_type = Errors.for_code(error_code)
try:
if not self._subscriptions.is_fetchable(tp):
# this can happen when a rebalance happened or a partition
# consumption paused while fetch is still in-flight
log.debug(
"Ignoring fetched records for partition %s"
" since it is no longer fetchable", tp)
elif error_type is Errors.NoError:
self._subscriptions.assignment[tp].highwater = highwater
# we are interested in this fetch only if the beginning
# offset (of the *request*) matches the current consumed position
# Note that the *response* may return a messageset that starts
# earlier (e.g., compressed messages) or later (e.g., compacted topic)
position = self._subscriptions.assignment[tp].position
if position is None or position != fetch_offset:
log.debug(
"Discarding fetch response for partition %s"
" since its offset %d does not match the"
" expected offset %d", tp, fetch_offset, position)
return None
records = MemoryRecords(completed_fetch.partition_data[-1])
if records.has_next():
log.debug(
"Adding fetched record for partition %s with"
" offset %d to buffered record list", tp, position)
unpacked = list(self._unpack_message_set(tp, records))
parsed_records = self.PartitionRecords(
fetch_offset, tp, unpacked)
if unpacked:
last_offset = unpacked[-1].offset
self._sensors.records_fetch_lag.record(highwater -
last_offset)
num_bytes = records.valid_bytes()
records_count = len(unpacked)
elif records.size_in_bytes() > 0:
# we did not read a single message from a non-empty
# buffer because that message's size is larger than
# fetch size, in this case record this exception
record_too_large_partitions = {tp: fetch_offset}
raise RecordTooLargeError(
"There are some messages at [Partition=Offset]: %s "
" whose size is larger than the fetch size %s"
" and hence cannot be ever returned."
" Increase the fetch size, or decrease the maximum message"
" size the broker will allow." %
(record_too_large_partitions,
self.config['max_partition_fetch_bytes']),
record_too_large_partitions)
self._sensors.record_topic_fetch_metrics(
tp.topic, num_bytes, records_count)
elif error_type in (Errors.NotLeaderForPartitionError,
Errors.UnknownTopicOrPartitionError):
self._client.cluster.request_update()
elif error_type is Errors.OffsetOutOfRangeError:
position = self._subscriptions.assignment[tp].position
if position is None or position != fetch_offset:
log.debug(
"Discarding stale fetch response for partition %s"
" since the fetched offset %d does not match the"
" current offset %d", tp, fetch_offset, position)
elif self._subscriptions.has_default_offset_reset_policy():
log.info(
"Fetch offset %s is out of range for topic-partition %s",
fetch_offset, tp)
self._subscriptions.need_offset_reset(tp)
else:
raise Errors.OffsetOutOfRangeError({tp: fetch_offset})
elif error_type is Errors.TopicAuthorizationFailedError:
log.warning("Not authorized to read from topic %s.", tp.topic)
raise Errors.TopicAuthorizationFailedError(set(tp.topic))
elif error_type is Errors.UnknownError:
log.warning(
"Unknown error fetching data for topic-partition %s", tp)
else:
raise error_type('Unexpected error while fetching data')
finally:
completed_fetch.metric_aggregator.record(tp, num_bytes,
records_count)
return parsed_records
