def send(self, node_id, request):
"""Send a request to a specific node.
Arguments:
node_id (int): destination node
request (Struct): request object (not-encoded)
Raises:
NodeNotReadyError: if node_id is not ready
Returns:
Future: resolves to Response struct
"""
if not self._can_send_request(node_id):
raise Errors.NodeNotReadyError(
"Attempt to send a request to node"
" which is not ready (node id %s)." % node_id)
# Every request gets a response, except one special case:
expect_response = True
if isinstance(request, ProduceRequest) and request.required_acks == 0:
expect_response = False
return self._conns[node_id].send(request,
expect_response=expect_response)
def _send_offset_request(self, partition, timestamp):
"""Fetch a single offset before the given timestamp for the partition.
Arguments:
partition (TopicPartition): partition that needs fetching offset
timestamp (int): timestamp for fetching offset
Returns:
Future: resolves to the corresponding offset
"""
node_id = self._client.cluster.leader_for_partition(partition)
if node_id is None:
log.debug(
"Partition %s is unknown for fetching offset,"
" wait for metadata refresh", partition)
return Future().failure(Errors.StaleMetadata(partition))
elif node_id == -1:
log.debug(
"Leader for partition %s unavailable for fetching offset,"
" wait for metadata refresh", partition)
return Future().failure(Errors.LeaderNotAvailableError(partition))
request = OffsetRequest(
-1, [(partition.topic, [(partition.partition, timestamp, 1)])])
# Client returns a future that only fails on network issues
# so create a separate future and attach a callback to update it
# based on response error codes
future = Future()
if not self._client.ready(node_id):
return future.failure(Errors.NodeNotReadyError(node_id))
_f = self._client.send(node_id, request)
_f.add_callback(self._handle_offset_response, partition, future)
_f.add_errback(lambda e: future.failure(e))
return future
def connect():
timeout = time.time() + 10
# brokers < 0.9 do not return any broker metadata if there are no topics
# so we're left with a single bootstrap connection
while not self.ready(node_id):
if time.time() >= timeout:
raise Errors.NodeNotReadyError(node_id)
time.sleep(0.025)
def _proc_offset_request(self, partition, timestamp):
"""Fetch a single offset before the given timestamp for the partition.
Arguments:
partition (TopicPartition): partition that needs fetching offset
timestamp (int): timestamp for fetching offset
Returns:
Future: resolves to the corresponding offset
"""
node_id = self._client.cluster.leader_for_partition(partition)
if node_id is None:
log.debug(
"Partition %s is unknown for fetching offset,"
" wait for metadata refresh", partition)
raise Errors.StaleMetadata(partition)
elif node_id == -1:
log.debug(
"Leader for partition %s unavailable for fetching offset,"
" wait for metadata refresh", partition)
raise Errors.LeaderNotAvailableError(partition)
request = OffsetRequest(
-1, [(partition.topic, [(partition.partition, timestamp, 1)])])
if not (yield from self._client.ready(node_id)):
raise Errors.NodeNotReadyError(node_id)
response = yield from self._client.send(node_id, request)
topic, partition_info = response.topics[0]
assert len(response.topics) == 1 and len(partition_info) == 1, (
'OffsetResponse should only be for a single topic-partition')
part, error_code, offsets = partition_info[0]
assert topic == partition.topic and part == partition.partition, (
'OffsetResponse partition does not match OffsetRequest partition')
error_type = Errors.for_code(error_code)
if error_type is Errors.NoError:
if not offsets:
return -1
assert len(offsets) == 1, 'Expected OffsetResponse with one offset'
offset = offsets[0]
log.debug("Fetched offset %d for partition %s", offset, partition)
return offset
elif error_type in (Errors.NotLeaderForPartitionError,
Errors.UnknownTopicOrPartitionError):
log.warning(
"Attempt to fetch offsets for partition %s failed due"
" to obsolete leadership information, retrying.", partition)
raise error_type(partition)
else:
log.error(
"Attempt to fetch offsets for partition %s failed due to:"
" %s", partition, error_type)
raise error_type(partition)
def connect(node_id):
timeout_at = time.time() + timeout
# brokers < 0.9 do not return any broker metadata if there are no topics
# so we're left with a single bootstrap connection
while not self.ready(node_id):
if time.time() >= timeout_at:
raise Errors.NodeNotReadyError(node_id)
time.sleep(0.025)
# Monkeypatch the connection request timeout
# Generally this timeout should not get triggered
# but in case it does, we want it to be reasonably short
self._conns[node_id].config['request_timeout_ms'] = timeout * 1000