def _send_offset_request(self, partition, timestamp):
"""Fetch a single offset before the given timestamp for the partition.
Arguments:
partition (TopicPartition): partition that needs fetching offset
timestamp (int): timestamp for fetching offset
Returns:
Future: resolves to the corresponding offset
"""
node_id = self._client.cluster.leader_for_partition(partition)
if node_id is None:
log.debug(
"Partition %s is unknown for fetching offset,"
" wait for metadata refresh", partition)
return Future().failure(Errors.StaleMetadata(partition))
elif node_id == -1:
log.debug(
"Leader for partition %s unavailable for fetching offset,"
" wait for metadata refresh", partition)
return Future().failure(Errors.LeaderNotAvailableError(partition))
request = OffsetRequest[0](
-1, [(partition.topic, [(partition.partition, timestamp, 1)])])
# Client returns a future that only fails on network issues
# so create a separate future and attach a callback to update it
# based on response error codes
future = Future()
_f = self._client.send(node_id, request)
_f.add_callback(self._handle_offset_response, partition, future)
_f.add_errback(lambda e: future.failure(e))
return future
def _send_offset_requests(self, timestamps):
"""Fetch offsets for each partition in timestamps dict. This may send
request to multiple nodes, based on who is Leader for partition.
Arguments:
timestamps (dict): {TopicPartition: int} mapping of fetching
timestamps.
Returns:
Future: resolves to a mapping of retrieved offsets
"""
timestamps_by_node = collections.defaultdict(dict)
for partition, timestamp in six.iteritems(timestamps):
node_id = self._client.cluster.leader_for_partition(partition)
if node_id is None:
self._client.add_topic(partition.topic)
log.debug(
"Partition %s is unknown for fetching offset,"
" wait for metadata refresh", partition)
return Future().failure(Errors.StaleMetadata(partition))
elif node_id == -1:
log.debug(
"Leader for partition %s unavailable for fetching "
"offset, wait for metadata refresh", partition)
return Future().failure(
Errors.LeaderNotAvailableError(partition))
else:
timestamps_by_node[node_id][partition] = timestamp
# Aggregate results until we have all
list_offsets_future = Future()
responses = []
node_count = len(timestamps_by_node)
def on_success(value):
responses.append(value)
if len(responses) == node_count:
offsets = {}
for r in responses:
offsets.update(r)
list_offsets_future.success(offsets)
def on_fail(err):
if not list_offsets_future.is_done:
list_offsets_future.failure(err)
for node_id, timestamps in six.iteritems(timestamps_by_node):
_f = self._send_offset_request(node_id, timestamps)
_f.add_callback(on_success)
_f.add_errback(on_fail)
return list_offsets_future
def _send_offset_request(self, partitions, timestamp):
"""Fetch a single offset before the given timestamp for the partition.
Arguments:
partitions iterable of TopicPartition: partitions that needs fetching offset
timestamp (int): timestamp for fetching offset
Returns:
list of Future: resolves to the corresponding offset
"""
topic = partitions[0].topic
nodes_per_partitions = {}
for partition in partitions:
node_id = self._client.cluster.leader_for_partition(partition)
if node_id is None:
log.debug(
"Partition %s is unknown for fetching offset,"
" wait for metadata refresh", partition)
return [Future().failure(Errors.StaleMetadata(partition))]
elif node_id == -1:
log.debug(
"Leader for partition %s unavailable for fetching offset,"
" wait for metadata refresh", partition)
return [
Future().failure(Errors.LeaderNotAvailableError(partition))
]
nodes_per_partitions.setdefault(node_id, []).append(partition)
# Client returns a future that only fails on network issues
# so create a separate future and attach a callback to update it
# based on response error codes
futures = []
for node_id, partitions in six.iteritems(nodes_per_partitions):
request = OffsetRequest[0](
-1, [(topic, [(partition.partition, timestamp, 1)
for partition in partitions])])
future_request = Future()
_f = self._client.send(node_id, request)
_f.add_callback(self._handle_offset_response, partitions,
future_request)
def errback(e):
log.error("Offset request errback error %s", e)
future_request.failure(e)
_f.add_errback(errback)
futures.append(future_request)
return futures
