def _send_offset_request(self, partition, timestamp): """Fetch a single offset before the given timestamp for the partition. Arguments: partition (TopicPartition): partition that needs fetching offset timestamp (int): timestamp for fetching offset Returns: Future: resolves to the corresponding offset """ node_id = self._client.cluster.leader_for_partition(partition) if node_id is None: log.debug( "Partition %s is unknown for fetching offset," " wait for metadata refresh", partition) return Future().failure(Errors.StaleMetadata(partition)) elif node_id == -1: log.debug( "Leader for partition %s unavailable for fetching offset," " wait for metadata refresh", partition) return Future().failure(Errors.LeaderNotAvailableError(partition)) request = OffsetRequest[0]( -1, [(partition.topic, [(partition.partition, timestamp, 1)])]) # Client returns a future that only fails on network issues # so create a separate future and attach a callback to update it # based on response error codes future = Future() _f = self._client.send(node_id, request) _f.add_callback(self._handle_offset_response, partition, future) _f.add_errback(lambda e: future.failure(e)) return future
def _send_offset_requests(self, timestamps): """Fetch offsets for each partition in timestamps dict. This may send request to multiple nodes, based on who is Leader for partition. Arguments: timestamps (dict): {TopicPartition: int} mapping of fetching timestamps. Returns: Future: resolves to a mapping of retrieved offsets """ timestamps_by_node = collections.defaultdict(dict) for partition, timestamp in six.iteritems(timestamps): node_id = self._client.cluster.leader_for_partition(partition) if node_id is None: self._client.add_topic(partition.topic) log.debug( "Partition %s is unknown for fetching offset," " wait for metadata refresh", partition) return Future().failure(Errors.StaleMetadata(partition)) elif node_id == -1: log.debug( "Leader for partition %s unavailable for fetching " "offset, wait for metadata refresh", partition) return Future().failure( Errors.LeaderNotAvailableError(partition)) else: timestamps_by_node[node_id][partition] = timestamp # Aggregate results until we have all list_offsets_future = Future() responses = [] node_count = len(timestamps_by_node) def on_success(value): responses.append(value) if len(responses) == node_count: offsets = {} for r in responses: offsets.update(r) list_offsets_future.success(offsets) def on_fail(err): if not list_offsets_future.is_done: list_offsets_future.failure(err) for node_id, timestamps in six.iteritems(timestamps_by_node): _f = self._send_offset_request(node_id, timestamps) _f.add_callback(on_success) _f.add_errback(on_fail) return list_offsets_future
def _send_offset_request(self, partitions, timestamp): """Fetch a single offset before the given timestamp for the partition. Arguments: partitions iterable of TopicPartition: partitions that needs fetching offset timestamp (int): timestamp for fetching offset Returns: list of Future: resolves to the corresponding offset """ topic = partitions[0].topic nodes_per_partitions = {} for partition in partitions: node_id = self._client.cluster.leader_for_partition(partition) if node_id is None: log.debug( "Partition %s is unknown for fetching offset," " wait for metadata refresh", partition) return [Future().failure(Errors.StaleMetadata(partition))] elif node_id == -1: log.debug( "Leader for partition %s unavailable for fetching offset," " wait for metadata refresh", partition) return [ Future().failure(Errors.LeaderNotAvailableError(partition)) ] nodes_per_partitions.setdefault(node_id, []).append(partition) # Client returns a future that only fails on network issues # so create a separate future and attach a callback to update it # based on response error codes futures = [] for node_id, partitions in six.iteritems(nodes_per_partitions): request = OffsetRequest[0]( -1, [(topic, [(partition.partition, timestamp, 1) for partition in partitions])]) future_request = Future() _f = self._client.send(node_id, request) _f.add_callback(self._handle_offset_response, partitions, future_request) def errback(e): log.error("Offset request errback error %s", e) future_request.failure(e) _f.add_errback(errback) futures.append(future_request) return futures