def _handle_metadata_update(self, cluster):
# if we encounter any unauthorized topics, raise an exception
if cluster.unauthorized_topics:
raise Errors.TopicAuthorizationFailedError(
cluster.unauthorized_topics)
if self._subscription.subscribed_pattern:
topics = []
for topic in cluster.topics(
self.config["exclude_internal_topics"]):
if self._subscription.subscribed_pattern.match(topic):
topics.append(topic)
if set(topics) != self._subscription.subscription:
self._subscription.change_subscription(topics)
self._client.set_topics(
self._subscription.group_subscription())
# check if there are any changes to the metadata which should trigger
# a rebalance
if self._subscription.partitions_auto_assigned():
metadata_snapshot = self._build_metadata_snapshot(
self._subscription, cluster)
if self._metadata_snapshot != metadata_snapshot:
self._metadata_snapshot = metadata_snapshot
# If we haven't got group coordinator support,
# just assign all partitions locally
if self._auto_assign_all_partitions():
self._subscription.assign_from_subscribed([
TopicPartition(topic, partition)
for topic in self._subscription.subscription
for partition in self._metadata_snapshot[topic]
])
def _raise_if_unauthorized_topics(self):
"""Check FetchResponses for topic authorization failures.
Raises:
TopicAuthorizationFailedError
"""
if self._unauthorized_topics:
topics = set(self._unauthorized_topics)
self._unauthorized_topics.clear()
raise Errors.TopicAuthorizationFailedError(topics)
def _wait_on_metadata(self, topic, max_wait):
"""
Wait for cluster metadata including partitions for the given topic to
be available.
Arguments:
topic (str): topic we want metadata for
max_wait (float): maximum time in secs for waiting on the metadata
Returns:
set: partition ids for the topic
Raises:
KafkaTimeoutError: if partitions for topic were not obtained before
specified max_wait timeout
"""
# add topic to metadata topic list if it is not there already.
self._sender.add_topic(topic)
begin = time.time()
elapsed = 0.0
metadata_event = None
while True:
partitions = self._metadata.partitions_for_topic(topic)
if partitions is not None:
return partitions
if not metadata_event:
metadata_event = threading.Event()
log.debug("Requesting metadata update for topic %s", topic)
metadata_event.clear()
future = self._metadata.request_update()
future.add_both(lambda e, *args: e.set(), metadata_event)
self._sender.wakeup()
metadata_event.wait(max_wait - elapsed)
elapsed = time.time() - begin
if not metadata_event.is_set():
raise Errors.KafkaTimeoutError(
"Failed to update metadata after %.1f secs." %
(max_wait, ))
elif topic in self._metadata.unauthorized_topics:
raise Errors.TopicAuthorizationFailedError(topic)
else:
log.debug("_wait_on_metadata woke after %s secs.", elapsed)
def _parse_fetched_data(self, completed_fetch):
tp = completed_fetch.topic_partition
fetch_offset = completed_fetch.fetched_offset
num_bytes = 0
records_count = 0
parsed_records = None
error_code, highwater = completed_fetch.partition_data[:2]
error_type = Errors.for_code(error_code)
try:
if not self._subscriptions.is_fetchable(tp):
# this can happen when a rebalance happened or a partition
# consumption paused while fetch is still in-flight
log.debug(
"Ignoring fetched records for partition %s"
" since it is no longer fetchable", tp)
elif error_type is Errors.NoError:
self._subscriptions.assignment[tp].highwater = highwater
# we are interested in this fetch only if the beginning
# offset (of the *request*) matches the current consumed position
# Note that the *response* may return a messageset that starts
# earlier (e.g., compressed messages) or later (e.g., compacted topic)
position = self._subscriptions.assignment[tp].position
if position is None or position != fetch_offset:
log.debug(
"Discarding fetch response for partition %s"
" since its offset %d does not match the"
" expected offset %d", tp, fetch_offset, position)
return None
records = MemoryRecords(completed_fetch.partition_data[-1])
if records.has_next():
log.debug(
"Adding fetched record for partition %s with"
" offset %d to buffered record list", tp, position)
unpacked = list(self._unpack_message_set(tp, records))
parsed_records = self.PartitionRecords(
fetch_offset, tp, unpacked)
if unpacked:
last_offset = unpacked[-1].offset
self._sensors.records_fetch_lag.record(highwater -
last_offset)
num_bytes = records.valid_bytes()
records_count = len(unpacked)
elif records.size_in_bytes() > 0:
# we did not read a single message from a non-empty
# buffer because that message's size is larger than
# fetch size, in this case record this exception
record_too_large_partitions = {tp: fetch_offset}
raise RecordTooLargeError(
"There are some messages at [Partition=Offset]: %s "
" whose size is larger than the fetch size %s"
" and hence cannot be ever returned."
" Increase the fetch size, or decrease the maximum message"
" size the broker will allow." %
(record_too_large_partitions,
self.config['max_partition_fetch_bytes']),
record_too_large_partitions)
self._sensors.record_topic_fetch_metrics(
tp.topic, num_bytes, records_count)
elif error_type in (Errors.NotLeaderForPartitionError,
Errors.UnknownTopicOrPartitionError):
self._client.cluster.request_update()
elif error_type is Errors.OffsetOutOfRangeError:
position = self._subscriptions.assignment[tp].position
if position is None or position != fetch_offset:
log.debug(
"Discarding stale fetch response for partition %s"
" since the fetched offset %d does not match the"
" current offset %d", tp, fetch_offset, position)
elif self._subscriptions.has_default_offset_reset_policy():
log.info(
"Fetch offset %s is out of range for topic-partition %s",
fetch_offset, tp)
self._subscriptions.need_offset_reset(tp)
else:
raise Errors.OffsetOutOfRangeError({tp: fetch_offset})
elif error_type is Errors.TopicAuthorizationFailedError:
log.warning("Not authorized to read from topic %s.", tp.topic)
raise Errors.TopicAuthorizationFailedError(set(tp.topic))
elif error_type is Errors.UnknownError:
log.warning(
"Unknown error fetching data for topic-partition %s", tp)
else:
raise error_type('Unexpected error while fetching data')
finally:
completed_fetch.metric_aggregator.record(tp, num_bytes,
records_count)
return parsed_records
def _handle_offset_commit_response(self, offsets, future, send_time,
response):
# TODO look at adding request_latency_ms to response (like java kafka)
self.consumer_sensors.commit_latency.record(
(time.time() - send_time) * 1000)
unauthorized_topics = set()
for topic, partitions in response.topics:
for partition, error_code in partitions:
tp = TopicPartition(topic, partition)
offset = offsets[tp]
error_type = Errors.for_code(error_code)
if error_type is Errors.NoError:
log.debug(
"Group %s committed offset %s for partition %s",
self.group_id,
offset,
tp,
)
if self._subscription.is_assigned(tp):
self._subscription.assignment[tp].committed = offset
elif error_type is Errors.GroupAuthorizationFailedError:
log.error("Not authorized to commit offsets for group %s",
self.group_id)
future.failure(error_type(self.group_id))
return
elif error_type is Errors.TopicAuthorizationFailedError:
unauthorized_topics.add(topic)
elif error_type in (
Errors.OffsetMetadataTooLargeError,
Errors.InvalidCommitOffsetSizeError,
):
# raise the error to the user
log.debug(
"OffsetCommit for group %s failed on partition %s"
" %s",
self.group_id,
tp,
error_type.__name__,
)
future.failure(error_type())
return
elif error_type is Errors.GroupLoadInProgressError:
# just retry
log.debug(
"OffsetCommit for group %s failed: %s",
self.group_id,
error_type.__name__,
)
future.failure(error_type(self.group_id))
return
elif error_type in (
Errors.GroupCoordinatorNotAvailableError,
Errors.NotCoordinatorForGroupError,
Errors.RequestTimedOutError,
):
log.debug(
"OffsetCommit for group %s failed: %s",
self.group_id,
error_type.__name__,
)
self.coordinator_dead(error_type())
future.failure(error_type(self.group_id))
return
elif error_type in (
Errors.UnknownMemberIdError,
Errors.IllegalGenerationError,
Errors.RebalanceInProgressError,
):
# need to re-join group
error = error_type(self.group_id)
log.debug("OffsetCommit for group %s failed: %s",
self.group_id, error)
self.reset_generation()
future.failure(Errors.CommitFailedError())
return
else:
log.error(
"Group %s failed to commit partition %s at offset"
" %s: %s",
self.group_id,
tp,
offset,
error_type.__name__,
)
future.failure(error_type())
return
if unauthorized_topics:
log.error(
"Not authorized to commit to topics %s for group %s",
unauthorized_topics,
self.group_id,
)
future.failure(
Errors.TopicAuthorizationFailedError(unauthorized_topics))
else:
future.success(None)
def _handle_offset_commit_response(self, offsets, future, send_time,
response):
# TODO look at adding request_latency_ms to response (like java kafka)
self.consumer_sensors.commit_latency.record(
(time.time() - send_time) * 1000)
unauthorized_topics = set()
for topic, partitions in response.topics:
for partition, error_code in partitions:
tp = TopicPartition(topic, partition)
offset = offsets[tp]
error_type = Errors.for_code(error_code)
if error_type is Errors.NoError:
log.debug("Group %s committed offset %s for partition %s",
self.group_id, offset, tp)
if self._subscription.is_assigned(tp):
self._subscription.assignment[
tp].committed = offset.offset
elif error_type is Errors.GroupAuthorizationFailedError:
log.error("Not authorized to commit offsets for group %s",
self.group_id)
future.failure(error_type(self.group_id))
return
elif error_type is Errors.TopicAuthorizationFailedError:
unauthorized_topics.add(topic)
elif error_type in (Errors.OffsetMetadataTooLargeError,
Errors.InvalidCommitOffsetSizeError):
# raise the error to the user
log.debug(
"OffsetCommit for group %s failed on partition %s"
" %s", self.group_id, tp, error_type.__name__)
future.failure(error_type())
return
elif error_type is Errors.GroupLoadInProgressError:
# just retry
log.debug("OffsetCommit for group %s failed: %s",
self.group_id, error_type.__name__)
future.failure(error_type(self.group_id))
return
elif error_type in (Errors.GroupCoordinatorNotAvailableError,
Errors.NotCoordinatorForGroupError,
Errors.RequestTimedOutError):
log.debug("OffsetCommit for group %s failed: %s",
self.group_id, error_type.__name__)
self.coordinator_dead(error_type())
future.failure(error_type(self.group_id))
return
elif error_type in (Errors.UnknownMemberIdError,
Errors.IllegalGenerationError,
Errors.RebalanceInProgressError):
# need to re-join group
error = error_type(self.group_id)
log.debug("OffsetCommit for group %s failed: %s",
self.group_id, error)
self.reset_generation()
future.failure(
Errors.CommitFailedError(
"Commit cannot be completed since the group has"
" already rebalanced and assigned the partitions to"
" another member. This means that the time between"
" subsequent calls to poll() was longer than the"
" configured session_timeout_ms, which typically"
" implies that the poll loop is spending too much time"
" message processing. You can address this either by"
" increasing the session timeout or by reducing the"
" maximum size of batches returned in poll() with"
" max_poll_records."))
return
else:
log.error(
"Group %s failed to commit partition %s at offset"
" %s: %s", self.group_id, tp, offset,
error_type.__name__)
future.failure(error_type())
return
if unauthorized_topics:
log.error("Not authorized to commit to topics %s for group %s",
unauthorized_topics, self.group_id)
future.failure(
Errors.TopicAuthorizationFailedError(unauthorized_topics))
else:
future.success(None)