def _raise_if_unauthorized_topics(self):
"""Check FetchResponses for topic authorization failures.
Raises:
TopicAuthorizationFailedError
"""
if self._unauthorized_topics:
topics = set(self._unauthorized_topics)
self._unauthorized_topics.clear()
raise Errors.TopicAuthorizationFailedError(topics)
def _wait_on_metadata(self, topic, max_wait):
"""
Wait for cluster metadata including partitions for the given topic to
be available.
Arguments:
topic (str): topic we want metadata for
max_wait (float): maximum time in secs for waiting on the metadata
Returns:
set: partition ids for the topic
Raises:
TimeoutException: if partitions for topic were not obtained before
specified max_wait timeout
"""
# add topic to metadata topic list if it is not there already.
self._sender.add_topic(topic)
begin = time.time()
elapsed = 0.0
metadata_event = None
while True:
partitions = self._metadata.partitions_for_topic(topic)
if partitions is not None:
return partitions
if not metadata_event:
metadata_event = threading.Event()
log.debug("Requesting metadata update for topic %s", topic)
metadata_event.clear()
future = self._metadata.request_update()
future.add_both(lambda e, *args: e.set(), metadata_event)
self._sender.wakeup()
metadata_event.wait(max_wait - elapsed)
elapsed = time.time() - begin
if not metadata_event.is_set():
raise Errors.KafkaTimeoutError(
"Failed to update metadata after %s secs.", max_wait)
elif topic in self._metadata.unauthorized_topics:
raise Errors.TopicAuthorizationFailedError(topic)
else:
log.debug("_wait_on_metadata woke after %s secs.", elapsed)
def commit_offsets(self, offsets):
"""Commit specific offsets asynchronously.
Arguments:
offsets (dict {TopicPartition: OffsetAndMetadata}): what to commit
Raises error on failure
"""
self._subscription.needs_fetch_committed_offsets = True
if not offsets:
log.debug('No offsets to commit')
return True
if (yield from self.coordinator_unknown()):
raise Errors.GroupCoordinatorNotAvailableError()
node_id = self.coordinator_id
# create the offset commit request
offset_data = collections.defaultdict(list)
for tp, offset in offsets.items():
offset_data[tp.topic].append(
(tp.partition, offset.offset, offset.metadata))
request = OffsetCommitRequest(
self.group_id, self.generation, self.member_id,
OffsetCommitRequest.DEFAULT_RETENTION_TIME,
[(topic, tp_offsets) for topic, tp_offsets in offset_data.items()])
log.debug("Sending offset-commit request with %s to %s", offsets,
node_id)
response = yield from self._send_req(node_id, request)
unauthorized_topics = set()
for topic, partitions in response.topics:
for partition, error_code in partitions:
tp = TopicPartition(topic, partition)
offset = offsets[tp]
error_type = Errors.for_code(error_code)
if error_type is Errors.NoError:
log.debug("Committed offset %s for partition %s", offset,
tp)
if self._subscription.is_assigned(tp):
partition = self._subscription.assignment[tp]
partition.committed = offset.offset
elif error_type is Errors.GroupAuthorizationFailedError:
log.error("OffsetCommit failed for group %s - %s",
self.group_id, error_type.__name__)
raise error_type()
elif error_type is Errors.TopicAuthorizationFailedError:
unauthorized_topics.add(topic)
elif error_type in (Errors.OffsetMetadataTooLargeError,
Errors.InvalidCommitOffsetSizeError):
# raise the error to the user
log.info(
"OffsetCommit failed for group %s on partition %s"
" due to %s, will retry", self.group_id, tp,
error_type.__name__)
raise error_type()
elif error_type is Errors.GroupLoadInProgressError:
# just retry
log.info(
"OffsetCommit failed for group %s because group is"
" initializing (%s), will retry", self.group_id,
error_type.__name__)
raise error_type()
elif error_type in (Errors.GroupCoordinatorNotAvailableError,
Errors.NotCoordinatorForGroupError,
Errors.RequestTimedOutError):
log.info(
"OffsetCommit failed for group %s due to a"
" coordinator error (%s), will find new coordinator"
" and retry", self.group_id, error_type.__name__)
self.coordinator_dead()
raise error_type()
elif error_type in (Errors.UnknownMemberIdError,
Errors.IllegalGenerationError,
Errors.RebalanceInProgressError):
# need to re-join group
error = error_type(self.group_id)
log.error(
"OffsetCommit failed for group %s due to group"
" error (%s), will rejoin", self.group_id, error)
self._subscription.mark_for_reassignment()
raise error
else:
log.error(
"OffsetCommit failed for group %s on partition %s"
" with offset %s: %s", self.group_id, tp, offset,
error_type.__name__)
raise error_type()
if unauthorized_topics:
log.error("OffsetCommit failed for unauthorized topics %s",
unauthorized_topics)
raise Errors.TopicAuthorizationFailedError(unauthorized_topics)
def _proc_fetch_request(self, node_id, request):
needs_wakeup = False
try:
response = yield from self._client.send(node_id, request)
except Errors.KafkaError as err:
log.error("Failed fetch messages from %s: %s", node_id, err)
return False
finally:
self._in_flight.remove(node_id)
fetch_offsets = {}
for topic, partitions in request.topics:
for partition, offset, _ in partitions:
fetch_offsets[TopicPartition(topic, partition)] = offset
for topic, partitions in response.topics:
for partition, error_code, highwater, messages in partitions:
tp = TopicPartition(topic, partition)
error_type = Errors.for_code(error_code)
if not self._subscriptions.is_fetchable(tp):
# this can happen when a rebalance happened
log.debug(
"Ignoring fetched records for partition %s"
" since it is no longer fetchable", tp)
elif error_type is Errors.NoError:
self._subscriptions.assignment[tp].highwater = highwater
# we are interested in this fetch only if the beginning
# offset matches the current consumed position
fetch_offset = fetch_offsets[tp]
partial = None
if messages and \
isinstance(messages[-1][-1], PartialMessage):
partial = messages.pop()
if messages:
log.debug(
"Adding fetched record for partition %s with"
" offset %d to buffered record list", tp,
fetch_offset)
try:
messages = collections.deque(
self._unpack_message_set(tp, messages))
except Errors.InvalidMessageError as err:
self._set_error(tp, err)
continue
self._records[tp] = FetchResult(
tp,
messages=messages,
subscriptions=self._subscriptions,
backoff=self._prefetch_backoff,
loop=self._loop)
# We added at least 1 successful record
needs_wakeup = True
elif partial:
# we did not read a single message from a non-empty
# buffer because that message's size is larger than
# fetch size, in this case record this exception
err = RecordTooLargeError(
"There are some messages at [Partition=Offset]: "
"%s=%s whose size is larger than the fetch size %s"
" and hence cannot be ever returned. "
"Increase the fetch size, or decrease the maximum "
"message size the broker will allow.", tp,
fetch_offset, self._max_partition_fetch_bytes)
self._set_error(tp, err)
needs_wakeup = True
self._subscriptions.assignment[tp].position += 1
elif error_type in (Errors.NotLeaderForPartitionError,
Errors.UnknownTopicOrPartitionError):
self._client.force_metadata_update()
elif error_type is Errors.OffsetOutOfRangeError:
fetch_offset = fetch_offsets[tp]
if self._subscriptions.has_default_offset_reset_policy():
self._subscriptions.need_offset_reset(tp)
else:
err = Errors.OffsetOutOfRangeError({tp: fetch_offset})
self._set_error(tp, err)
needs_wakeup = True
log.info(
"Fetch offset %s is out of range, resetting offset",
fetch_offset)
elif error_type is Errors.TopicAuthorizationFailedError:
log.warn("Not authorized to read from topic %s.", tp.topic)
err = Errors.TopicAuthorizationFailedError(tp.topic)
self._set_error(tp, err)
needs_wakeup = True
else:
log.warn('Unexpected error while fetching data: %s',
error_type.__name__)
return needs_wakeup
def _handle_offset_commit_response(self, offsets, future, response):
#self.sensors.commit_latency.record(response.requestLatencyMs())
unauthorized_topics = set()
for topic, partitions in response.topics:
for partition, error_code in partitions:
tp = TopicPartition(topic, partition)
offset = offsets[tp]
error_type = Errors.for_code(error_code)
if error_type is Errors.NoError:
log.debug("Committed offset %s for partition %s", offset, tp)
if self._subscription.is_assigned(tp):
self._subscription.assignment[tp].committed = offset.offset
elif error_type is Errors.GroupAuthorizationFailedError:
log.error("OffsetCommit failed for group %s - %s",
self.group_id, error_type.__name__)
future.failure(error_type(self.group_id))
return
elif error_type is Errors.TopicAuthorizationFailedError:
unauthorized_topics.add(topic)
elif error_type in (Errors.OffsetMetadataTooLargeError,
Errors.InvalidCommitOffsetSizeError):
# raise the error to the user
log.info("OffsetCommit failed for group %s on partition %s"
" due to %s, will retry", self.group_id, tp,
error_type.__name__)
future.failure(error_type())
return
elif error_type is Errors.GroupLoadInProgressError:
# just retry
log.info("OffsetCommit failed for group %s because group is"
" initializing (%s), will retry", self.group_id,
error_type.__name__)
future.failure(error_type(self.group_id))
return
elif error_type in (Errors.GroupCoordinatorNotAvailableError,
Errors.NotCoordinatorForGroupError,
Errors.RequestTimedOutError):
log.info("OffsetCommit failed for group %s due to a"
" coordinator error (%s), will find new coordinator"
" and retry", self.group_id, error_type.__name__)
self.coordinator_dead()
future.failure(error_type(self.group_id))
return
elif error_type in (Errors.UnknownMemberIdError,
Errors.IllegalGenerationError,
Errors.RebalanceInProgressError):
# need to re-join group
error = error_type(self.group_id)
log.error("OffsetCommit failed for group %s due to group"
" error (%s), will rejoin", self.group_id, error)
self._subscription.mark_for_reassignment()
# Errors.CommitFailedError("Commit cannot be completed due to group rebalance"))
future.failure(error)
return
else:
log.error("OffsetCommit failed for group % on partition %s"
" with offset %s: %s", self.group_id, tp, offset,
error_type.__name__)
future.failure(error_type())
return
if unauthorized_topics:
log.error("OffsetCommit failed for unauthorized topics %s",
unauthorized_topics)
future.failure(Errors.TopicAuthorizationFailedError(unauthorized_topics))
else:
future.success(True)