def _send_offset_commit_request(self, offsets): """Commit offsets for the specified list of topics and partitions. This is a non-blocking call which returns a request future that can be polled in the case of a synchronous commit or ignored in the asynchronous case. Arguments: offsets (dict of {TopicPartition: OffsetAndMetadata}): what should be committed Returns: Future: indicating whether the commit was successful or not """ assert self.config["api_version"] >= (0, 8, 1), "Unsupported Broker API" assert all(map(lambda k: isinstance(k, TopicPartition), offsets)) assert all( map(lambda v: isinstance(v, OffsetAndMetadata), offsets.values())) if not offsets: log.debug("No offsets to commit") return Future().success(None) node_id = self.coordinator() if node_id is None: return Future().failure(Errors.GroupCoordinatorNotAvailableError) # create the offset commit request offset_data = collections.defaultdict(dict) for tp, offset in six.iteritems(offsets): offset_data[tp.topic][tp.partition] = offset if self._subscription.partitions_auto_assigned(): generation = self.generation() else: generation = Generation.NO_GENERATION # if the generation is None, we are not part of an active group # (and we expect to be). The only thing we can do is fail the commit # and let the user rejoin the group in poll() if self.config["api_version"] >= (0, 9) and generation is None: return Future().failure(Errors.CommitFailedError()) if self.config["api_version"] >= (0, 9): request = OffsetCommitRequest[2]( self.group_id, generation.generation_id, generation.member_id, OffsetCommitRequest[2].DEFAULT_RETENTION_TIME, [( topic, [(partition, offset.offset, offset.metadata) for partition, offset in six.iteritems(partitions)], ) for topic, partitions in six.iteritems(offset_data)], ) elif self.config["api_version"] >= (0, 8, 2): request = OffsetCommitRequest[1]( self.group_id, -1, "", [( topic, [(partition, offset.offset, -1, offset.metadata) for partition, offset in six.iteritems(partitions)], ) for topic, partitions in six.iteritems(offset_data)], ) elif self.config["api_version"] >= (0, 8, 1): request = OffsetCommitRequest[0]( self.group_id, [( topic, [(partition, offset.offset, offset.metadata) for partition, offset in six.iteritems(partitions)], ) for topic, partitions in six.iteritems(offset_data)], ) log.debug( "Sending offset-commit request with %s for group %s to %s", offsets, self.group_id, node_id, ) future = Future() _f = self._client.send(node_id, request) _f.add_callback(self._handle_offset_commit_response, offsets, future, time.time()) _f.add_errback(self._failed_request, node_id, request, future) return future
def _handle_offset_commit_response(self, offsets, future, send_time, response): # TODO look at adding request_latency_ms to response (like java kafka) self.consumer_sensors.commit_latency.record( (time.time() - send_time) * 1000) unauthorized_topics = set() for topic, partitions in response.topics: for partition, error_code in partitions: tp = TopicPartition(topic, partition) offset = offsets[tp] error_type = Errors.for_code(error_code) if error_type is Errors.NoError: log.debug( "Group %s committed offset %s for partition %s", self.group_id, offset, tp, ) if self._subscription.is_assigned(tp): self._subscription.assignment[tp].committed = offset elif error_type is Errors.GroupAuthorizationFailedError: log.error("Not authorized to commit offsets for group %s", self.group_id) future.failure(error_type(self.group_id)) return elif error_type is Errors.TopicAuthorizationFailedError: unauthorized_topics.add(topic) elif error_type in ( Errors.OffsetMetadataTooLargeError, Errors.InvalidCommitOffsetSizeError, ): # raise the error to the user log.debug( "OffsetCommit for group %s failed on partition %s" " %s", self.group_id, tp, error_type.__name__, ) future.failure(error_type()) return elif error_type is Errors.GroupLoadInProgressError: # just retry log.debug( "OffsetCommit for group %s failed: %s", self.group_id, error_type.__name__, ) future.failure(error_type(self.group_id)) return elif error_type in ( Errors.GroupCoordinatorNotAvailableError, Errors.NotCoordinatorForGroupError, Errors.RequestTimedOutError, ): log.debug( "OffsetCommit for group %s failed: %s", self.group_id, error_type.__name__, ) self.coordinator_dead(error_type()) future.failure(error_type(self.group_id)) return elif error_type in ( Errors.UnknownMemberIdError, Errors.IllegalGenerationError, Errors.RebalanceInProgressError, ): # need to re-join group error = error_type(self.group_id) log.debug("OffsetCommit for group %s failed: %s", self.group_id, error) self.reset_generation() future.failure(Errors.CommitFailedError()) return else: log.error( "Group %s failed to commit partition %s at offset" " %s: %s", self.group_id, tp, offset, error_type.__name__, ) future.failure(error_type()) return if unauthorized_topics: log.error( "Not authorized to commit to topics %s for group %s", unauthorized_topics, self.group_id, ) future.failure( Errors.TopicAuthorizationFailedError(unauthorized_topics)) else: future.success(None)
def _handle_offset_commit_response(self, offsets, future, send_time, response): # TODO look at adding request_latency_ms to response (like java kafka) self.consumer_sensors.commit_latency.record( (time.time() - send_time) * 1000) unauthorized_topics = set() for topic, partitions in response.topics: for partition, error_code in partitions: tp = TopicPartition(topic, partition) offset = offsets[tp] error_type = Errors.for_code(error_code) if error_type is Errors.NoError: log.debug("Group %s committed offset %s for partition %s", self.group_id, offset, tp) if self._subscription.is_assigned(tp): self._subscription.assignment[ tp].committed = offset.offset elif error_type is Errors.GroupAuthorizationFailedError: log.error("Not authorized to commit offsets for group %s", self.group_id) future.failure(error_type(self.group_id)) return elif error_type is Errors.TopicAuthorizationFailedError: unauthorized_topics.add(topic) elif error_type in (Errors.OffsetMetadataTooLargeError, Errors.InvalidCommitOffsetSizeError): # raise the error to the user log.debug( "OffsetCommit for group %s failed on partition %s" " %s", self.group_id, tp, error_type.__name__) future.failure(error_type()) return elif error_type is Errors.GroupLoadInProgressError: # just retry log.debug("OffsetCommit for group %s failed: %s", self.group_id, error_type.__name__) future.failure(error_type(self.group_id)) return elif error_type in (Errors.GroupCoordinatorNotAvailableError, Errors.NotCoordinatorForGroupError, Errors.RequestTimedOutError): log.debug("OffsetCommit for group %s failed: %s", self.group_id, error_type.__name__) self.coordinator_dead(error_type()) future.failure(error_type(self.group_id)) return elif error_type in (Errors.UnknownMemberIdError, Errors.IllegalGenerationError, Errors.RebalanceInProgressError): # need to re-join group error = error_type(self.group_id) log.debug("OffsetCommit for group %s failed: %s", self.group_id, error) self.reset_generation() future.failure( Errors.CommitFailedError( "Commit cannot be completed since the group has" " already rebalanced and assigned the partitions to" " another member. This means that the time between" " subsequent calls to poll() was longer than the" " configured session_timeout_ms, which typically" " implies that the poll loop is spending too much time" " message processing. You can address this either by" " increasing the session timeout or by reducing the" " maximum size of batches returned in poll() with" " max_poll_records.")) return else: log.error( "Group %s failed to commit partition %s at offset" " %s: %s", self.group_id, tp, offset, error_type.__name__) future.failure(error_type()) return if unauthorized_topics: log.error("Not authorized to commit to topics %s for group %s", unauthorized_topics, self.group_id) future.failure( Errors.TopicAuthorizationFailedError(unauthorized_topics)) else: future.success(None)