def send(self, request, expect_response=True):
if self._writer is None:
raise Errors.ConnectionError(
"No connection to broker at {0}:{1}"
.format(self._host, self._port))
correlation_id = self._next_correlation_id()
header = RequestHeader(request,
correlation_id=correlation_id,
client_id=self._client_id)
message = header.encode() + request.encode()
size = self.HEADER.pack(len(message))
try:
self._writer.write(size + message)
except OSError as err:
self.close()
raise Errors.ConnectionError(
"Connection at {0}:{1} broken: {2}".format(
self._host, self._port, err))
if not expect_response:
return self._writer.drain()
fut = asyncio.Future(loop=self._loop)
self._requests.append((correlation_id, request.RESPONSE_TYPE, fut))
return asyncio.wait_for(fut, self._request_timeout, loop=self._loop)
def _raise_connection_error(self):
# Cleanup socket if we have one
if self._sock:
self.close()
# And then raise
raise Errors.ConnectionError("Kafka @ {0}:{1} went away".format(self.host, self.port))
def send(self, request, expect_response=True):
"""send request, return Future()
Can block on network if request is larger than send_buffer_bytes
"""
future = Future()
if not self.connected():
return future.failure(Errors.ConnectionError())
if not self.can_send_more():
return future.failure(Errors.TooManyInFlightRequests())
correlation_id = self._next_correlation_id()
header = RequestHeader(request,
correlation_id=correlation_id,
client_id=self.config['client_id'])
message = b''.join([header.encode(), request.encode()])
size = Int32.encode(len(message))
try:
# In the future we might manage an internal write buffer
# and send bytes asynchronously. For now, just block
# sending each request payload
self._sock.setblocking(True)
sent_bytes = self._sock.send(size)
assert sent_bytes == len(size)
sent_bytes = self._sock.send(message)
assert sent_bytes == len(message)
self._sock.setblocking(False)
except (AssertionError, ConnectionError) as e:
log.exception("Error sending %s to %s", request, self)
error = Errors.ConnectionError(e)
self.close(error=error)
return future.failure(error)
log.debug('%s Request %d: %s', self, correlation_id, request)
if expect_response:
ifr = InFlightRequest(request=request,
correlation_id=correlation_id,
response_type=request.RESPONSE_TYPE,
future=future,
timestamp=time.time())
self.in_flight_requests.append(ifr)
else:
future.success(None)
return future
def _read(self):
try:
while True:
resp = yield from self._reader.readexactly(4)
size, = self.HEADER.unpack(resp)
resp = yield from self._reader.readexactly(size)
recv_correlation_id, = self.HEADER.unpack(resp[:4])
correlation_id, resp_type, fut = self._requests.pop(0)
if (self._api_version == (0, 8, 2)
and resp_type is GroupCoordinatorResponse
and correlation_id != 0 and recv_correlation_id == 0):
self.log.warning(
'Kafka 0.8.2 quirk -- GroupCoordinatorResponse'
' coorelation id does not match request. This'
' should go away once at least one topic has been'
' initialized on the broker')
elif correlation_id != recv_correlation_id:
error = Errors.CorrelationIdError(
'Correlation ids do not match: sent {}, recv {}'.
format(correlation_id, recv_correlation_id))
if not fut.done():
fut.set_exception(error)
self.close()
break
if not fut.done():
response = resp_type.decode(resp[4:])
self.log.debug('%s Response %d: %s', self, correlation_id,
response)
fut.set_result(response)
# Update idle timer.
self._last_action = self._loop.time()
except (OSError, EOFError, ConnectionError) as exc:
conn_exc = Errors.ConnectionError(
"Connection at {0}:{1} broken".format(self._host, self._port))
conn_exc.__cause__ = exc
conn_exc.__context__ = exc
for _, _, fut in self._requests:
fut.set_exception(conn_exc)
self.close()
except asyncio.CancelledError:
pass
def close(self):
if self._reader is not None:
self._writer.close()
self._writer = self._reader = None
self._read_task.cancel()
self._read_task = None
error = Errors.ConnectionError(
"Connection at {0}:{1} closed".format(self._host, self._port))
for _, _, fut in self._requests:
if not fut.done():
fut.set_exception(error)
self._requests = []
if self._idle_handle is not None:
self._idle_handle.cancel()
# transport.close() will close socket, but not right ahead. Return
# a future in case we need to wait on it.
return self._closed_fut
def close(self, error=None):
"""Close socket and fail all in-flight-requests.
Arguments:
error (Exception, optional): pending in-flight-requests
will be failed with this exception.
Default: kafka.common.ConnectionError.
"""
if self._sock:
self._sock.close()
self._sock = None
self.state = ConnectionStates.DISCONNECTED
self._receiving = False
self._next_payload_bytes = 0
self._rbuffer.seek(0)
self._rbuffer.truncate()
if error is None:
error = Errors.ConnectionError()
while self.in_flight_requests:
ifr = self.in_flight_requests.popleft()
ifr.future.failure(error)
def recv(self, timeout=0):
"""Non-blocking network receive.
Return response if available
"""
assert not self._processing, 'Recursion not supported'
if not self.connected():
log.warning('%s cannot recv: socket not connected', self)
# If requests are pending, we should close the socket and
# fail all the pending request futures
if self.in_flight_requests:
self.close()
return None
elif not self.in_flight_requests:
log.warning('%s: No in-flight-requests to recv', self)
return None
elif self._requests_timed_out():
log.warning('%s timed out after %s ms. Closing connection.',
self, self.config['request_timeout_ms'])
self.close(error=Errors.RequestTimedOutError(
'Request timed out after %s ms' %
self.config['request_timeout_ms']))
return None
readable, _, _ = select([self._sock], [], [], timeout)
if not readable:
return None
# Not receiving is the state of reading the payload header
if not self._receiving:
try:
# An extremely small, but non-zero, probability that there are
# more than 0 but not yet 4 bytes available to read
self._rbuffer.write(self._sock.recv(4 - self._rbuffer.tell()))
except ConnectionError as e:
if six.PY2 and e.errno == errno.EWOULDBLOCK:
# This shouldn't happen after selecting above
# but just in case
return None
log.exception('%s: Error receiving 4-byte payload header -'
' closing socket', self)
self.close(error=Errors.ConnectionError(e))
return None
except BlockingIOError:
if six.PY3:
return None
raise
if self._rbuffer.tell() == 4:
self._rbuffer.seek(0)
self._next_payload_bytes = Int32.decode(self._rbuffer)
# reset buffer and switch state to receiving payload bytes
self._rbuffer.seek(0)
self._rbuffer.truncate()
self._receiving = True
elif self._rbuffer.tell() > 4:
raise Errors.KafkaError('this should not happen - are you threading?')
if self._receiving:
staged_bytes = self._rbuffer.tell()
try:
self._rbuffer.write(self._sock.recv(self._next_payload_bytes - staged_bytes))
except ConnectionError as e:
# Extremely small chance that we have exactly 4 bytes for a
# header, but nothing to read in the body yet
if six.PY2 and e.errno == errno.EWOULDBLOCK:
return None
log.exception('%s: Error in recv', self)
self.close(error=Errors.ConnectionError(e))
return None
except BlockingIOError:
if six.PY3:
return None
raise
staged_bytes = self._rbuffer.tell()
if staged_bytes > self._next_payload_bytes:
self.close(error=Errors.KafkaError('Receive buffer has more bytes than expected?'))
if staged_bytes != self._next_payload_bytes:
return None
self._receiving = False
self._next_payload_bytes = 0
self._rbuffer.seek(0)
response = self._process_response(self._rbuffer)
self._rbuffer.seek(0)
self._rbuffer.truncate()
return response
