def _send_offset_request(self, partition, timestamp): """Fetch a single offset before the given timestamp for the partition. Arguments: partition (TopicPartition): partition that needs fetching offset timestamp (int): timestamp for fetching offset Returns: Future: resolves to the corresponding offset """ node_id = self._client.cluster.leader_for_partition(partition) if node_id is None: log.debug( "Partition %s is unknown for fetching offset," " wait for metadata refresh", partition) return Future().failure(Errors.StaleMetadata(partition)) elif node_id == -1: log.debug( "Leader for partition %s unavailable for fetching offset," " wait for metadata refresh", partition) return Future().failure(Errors.LeaderNotAvailableError(partition)) request = OffsetRequest( -1, [(partition.topic, [(partition.partition, timestamp, 1)])]) # Client returns a future that only fails on network issues # so create a separate future and attach a callback to update it # based on response error codes future = Future() if not self._client.ready(node_id): return future.failure(Errors.NodeNotReadyError(node_id)) _f = self._client.send(node_id, request) _f.add_callback(self._handle_offset_response, partition, future) _f.add_errback(lambda e: future.failure(e)) return future
def _proc_offset_request(self, partition, timestamp): """Fetch a single offset before the given timestamp for the partition. Arguments: partition (TopicPartition): partition that needs fetching offset timestamp (int): timestamp for fetching offset Returns: Future: resolves to the corresponding offset """ node_id = self._client.cluster.leader_for_partition(partition) if node_id is None: log.debug( "Partition %s is unknown for fetching offset," " wait for metadata refresh", partition) raise Errors.StaleMetadata(partition) elif node_id == -1: log.debug( "Leader for partition %s unavailable for fetching offset," " wait for metadata refresh", partition) raise Errors.LeaderNotAvailableError(partition) request = OffsetRequest( -1, [(partition.topic, [(partition.partition, timestamp, 1)])]) if not (yield from self._client.ready(node_id)): raise Errors.NodeNotReadyError(node_id) response = yield from self._client.send(node_id, request) topic, partition_info = response.topics[0] assert len(response.topics) == 1 and len(partition_info) == 1, ( 'OffsetResponse should only be for a single topic-partition') part, error_code, offsets = partition_info[0] assert topic == partition.topic and part == partition.partition, ( 'OffsetResponse partition does not match OffsetRequest partition') error_type = Errors.for_code(error_code) if error_type is Errors.NoError: if not offsets: return -1 assert len(offsets) == 1, 'Expected OffsetResponse with one offset' offset = offsets[0] log.debug("Fetched offset %d for partition %s", offset, partition) return offset elif error_type in (Errors.NotLeaderForPartitionError, Errors.UnknownTopicOrPartitionError): log.warning( "Attempt to fetch offsets for partition %s failed due" " to obsolete leadership information, retrying.", partition) raise error_type(partition) else: log.error( "Attempt to fetch offsets for partition %s failed due to:" " %s", partition, error_type) raise error_type(partition)