def _send_offset_commit_request(self, offsets):
"""Commit offsets for the specified list of topics and partitions.
This is a non-blocking call which returns a request future that can be
polled in the case of a synchronous commit or ignored in the
asynchronous case.
Arguments:
offsets (dict of {TopicPartition: OffsetAndMetadata}): what should
be committed
Returns:
Future: indicating whether the commit was successful or not
"""
assert self.config["api_version"] >= (0, 8,
1), "Unsupported Broker API"
assert all(map(lambda k: isinstance(k, TopicPartition), offsets))
assert all(
map(lambda v: isinstance(v, OffsetAndMetadata), offsets.values()))
if not offsets:
log.debug("No offsets to commit")
return Future().success(None)
node_id = self.coordinator()
if node_id is None:
return Future().failure(Errors.GroupCoordinatorNotAvailableError)
# create the offset commit request
offset_data = collections.defaultdict(dict)
for tp, offset in six.iteritems(offsets):
offset_data[tp.topic][tp.partition] = offset
if self._subscription.partitions_auto_assigned():
generation = self.generation()
else:
generation = Generation.NO_GENERATION
# if the generation is None, we are not part of an active group
# (and we expect to be). The only thing we can do is fail the commit
# and let the user rejoin the group in poll()
if self.config["api_version"] >= (0, 9) and generation is None:
return Future().failure(Errors.CommitFailedError())
if self.config["api_version"] >= (0, 9):
request = OffsetCommitRequest[2](
self.group_id,
generation.generation_id,
generation.member_id,
OffsetCommitRequest[2].DEFAULT_RETENTION_TIME,
[(
topic,
[(partition, offset.offset, offset.metadata)
for partition, offset in six.iteritems(partitions)],
) for topic, partitions in six.iteritems(offset_data)],
)
elif self.config["api_version"] >= (0, 8, 2):
request = OffsetCommitRequest[1](
self.group_id,
-1,
"",
[(
topic,
[(partition, offset.offset, -1, offset.metadata)
for partition, offset in six.iteritems(partitions)],
) for topic, partitions in six.iteritems(offset_data)],
)
elif self.config["api_version"] >= (0, 8, 1):
request = OffsetCommitRequest[0](
self.group_id,
[(
topic,
[(partition, offset.offset, offset.metadata)
for partition, offset in six.iteritems(partitions)],
) for topic, partitions in six.iteritems(offset_data)],
)
log.debug(
"Sending offset-commit request with %s for group %s to %s",
offsets,
self.group_id,
node_id,
)
future = Future()
_f = self._client.send(node_id, request)
_f.add_callback(self._handle_offset_commit_response, offsets, future,
time.time())
_f.add_errback(self._failed_request, node_id, request, future)
return future
def _handle_offset_commit_response(self, offsets, future, send_time,
response):
# TODO look at adding request_latency_ms to response (like java kafka)
self.consumer_sensors.commit_latency.record(
(time.time() - send_time) * 1000)
unauthorized_topics = set()
for topic, partitions in response.topics:
for partition, error_code in partitions:
tp = TopicPartition(topic, partition)
offset = offsets[tp]
error_type = Errors.for_code(error_code)
if error_type is Errors.NoError:
log.debug(
"Group %s committed offset %s for partition %s",
self.group_id,
offset,
tp,
)
if self._subscription.is_assigned(tp):
self._subscription.assignment[tp].committed = offset
elif error_type is Errors.GroupAuthorizationFailedError:
log.error("Not authorized to commit offsets for group %s",
self.group_id)
future.failure(error_type(self.group_id))
return
elif error_type is Errors.TopicAuthorizationFailedError:
unauthorized_topics.add(topic)
elif error_type in (
Errors.OffsetMetadataTooLargeError,
Errors.InvalidCommitOffsetSizeError,
):
# raise the error to the user
log.debug(
"OffsetCommit for group %s failed on partition %s"
" %s",
self.group_id,
tp,
error_type.__name__,
)
future.failure(error_type())
return
elif error_type is Errors.GroupLoadInProgressError:
# just retry
log.debug(
"OffsetCommit for group %s failed: %s",
self.group_id,
error_type.__name__,
)
future.failure(error_type(self.group_id))
return
elif error_type in (
Errors.GroupCoordinatorNotAvailableError,
Errors.NotCoordinatorForGroupError,
Errors.RequestTimedOutError,
):
log.debug(
"OffsetCommit for group %s failed: %s",
self.group_id,
error_type.__name__,
)
self.coordinator_dead(error_type())
future.failure(error_type(self.group_id))
return
elif error_type in (
Errors.UnknownMemberIdError,
Errors.IllegalGenerationError,
Errors.RebalanceInProgressError,
):
# need to re-join group
error = error_type(self.group_id)
log.debug("OffsetCommit for group %s failed: %s",
self.group_id, error)
self.reset_generation()
future.failure(Errors.CommitFailedError())
return
else:
log.error(
"Group %s failed to commit partition %s at offset"
" %s: %s",
self.group_id,
tp,
offset,
error_type.__name__,
)
future.failure(error_type())
return
if unauthorized_topics:
log.error(
"Not authorized to commit to topics %s for group %s",
unauthorized_topics,
self.group_id,
)
future.failure(
Errors.TopicAuthorizationFailedError(unauthorized_topics))
else:
future.success(None)
def _handle_offset_commit_response(self, offsets, future, send_time,
response):
# TODO look at adding request_latency_ms to response (like java kafka)
self.consumer_sensors.commit_latency.record(
(time.time() - send_time) * 1000)
unauthorized_topics = set()
for topic, partitions in response.topics:
for partition, error_code in partitions:
tp = TopicPartition(topic, partition)
offset = offsets[tp]
error_type = Errors.for_code(error_code)
if error_type is Errors.NoError:
log.debug("Group %s committed offset %s for partition %s",
self.group_id, offset, tp)
if self._subscription.is_assigned(tp):
self._subscription.assignment[
tp].committed = offset.offset
elif error_type is Errors.GroupAuthorizationFailedError:
log.error("Not authorized to commit offsets for group %s",
self.group_id)
future.failure(error_type(self.group_id))
return
elif error_type is Errors.TopicAuthorizationFailedError:
unauthorized_topics.add(topic)
elif error_type in (Errors.OffsetMetadataTooLargeError,
Errors.InvalidCommitOffsetSizeError):
# raise the error to the user
log.debug(
"OffsetCommit for group %s failed on partition %s"
" %s", self.group_id, tp, error_type.__name__)
future.failure(error_type())
return
elif error_type is Errors.GroupLoadInProgressError:
# just retry
log.debug("OffsetCommit for group %s failed: %s",
self.group_id, error_type.__name__)
future.failure(error_type(self.group_id))
return
elif error_type in (Errors.GroupCoordinatorNotAvailableError,
Errors.NotCoordinatorForGroupError,
Errors.RequestTimedOutError):
log.debug("OffsetCommit for group %s failed: %s",
self.group_id, error_type.__name__)
self.coordinator_dead(error_type())
future.failure(error_type(self.group_id))
return
elif error_type in (Errors.UnknownMemberIdError,
Errors.IllegalGenerationError,
Errors.RebalanceInProgressError):
# need to re-join group
error = error_type(self.group_id)
log.debug("OffsetCommit for group %s failed: %s",
self.group_id, error)
self.reset_generation()
future.failure(
Errors.CommitFailedError(
"Commit cannot be completed since the group has"
" already rebalanced and assigned the partitions to"
" another member. This means that the time between"
" subsequent calls to poll() was longer than the"
" configured session_timeout_ms, which typically"
" implies that the poll loop is spending too much time"
" message processing. You can address this either by"
" increasing the session timeout or by reducing the"
" maximum size of batches returned in poll() with"
" max_poll_records."))
return
else:
log.error(
"Group %s failed to commit partition %s at offset"
" %s: %s", self.group_id, tp, offset,
error_type.__name__)
future.failure(error_type())
return
if unauthorized_topics:
log.error("Not authorized to commit to topics %s for group %s",
unauthorized_topics, self.group_id)
future.failure(
Errors.TopicAuthorizationFailedError(unauthorized_topics))
else:
future.success(None)