def send(self, request, expect_response=True): if self._writer is None: raise Errors.ConnectionError( "No connection to broker at {0}:{1}" .format(self._host, self._port)) correlation_id = self._next_correlation_id() header = RequestHeader(request, correlation_id=correlation_id, client_id=self._client_id) message = header.encode() + request.encode() size = self.HEADER.pack(len(message)) try: self._writer.write(size + message) except OSError as err: self.close() raise Errors.ConnectionError( "Connection at {0}:{1} broken: {2}".format( self._host, self._port, err)) if not expect_response: return self._writer.drain() fut = asyncio.Future(loop=self._loop) self._requests.append((correlation_id, request.RESPONSE_TYPE, fut)) return asyncio.wait_for(fut, self._request_timeout, loop=self._loop)
def _raise_connection_error(self): # Cleanup socket if we have one if self._sock: self.close() # And then raise raise Errors.ConnectionError("Kafka @ {0}:{1} went away".format(self.host, self.port))
def send(self, request, expect_response=True): """send request, return Future() Can block on network if request is larger than send_buffer_bytes """ future = Future() if not self.connected(): return future.failure(Errors.ConnectionError()) if not self.can_send_more(): return future.failure(Errors.TooManyInFlightRequests()) correlation_id = self._next_correlation_id() header = RequestHeader(request, correlation_id=correlation_id, client_id=self.config['client_id']) message = b''.join([header.encode(), request.encode()]) size = Int32.encode(len(message)) try: # In the future we might manage an internal write buffer # and send bytes asynchronously. For now, just block # sending each request payload self._sock.setblocking(True) sent_bytes = self._sock.send(size) assert sent_bytes == len(size) sent_bytes = self._sock.send(message) assert sent_bytes == len(message) self._sock.setblocking(False) except (AssertionError, ConnectionError) as e: log.exception("Error sending %s to %s", request, self) error = Errors.ConnectionError(e) self.close(error=error) return future.failure(error) log.debug('%s Request %d: %s', self, correlation_id, request) if expect_response: ifr = InFlightRequest(request=request, correlation_id=correlation_id, response_type=request.RESPONSE_TYPE, future=future, timestamp=time.time()) self.in_flight_requests.append(ifr) else: future.success(None) return future
def _read(self): try: while True: resp = yield from self._reader.readexactly(4) size, = self.HEADER.unpack(resp) resp = yield from self._reader.readexactly(size) recv_correlation_id, = self.HEADER.unpack(resp[:4]) correlation_id, resp_type, fut = self._requests.pop(0) if (self._api_version == (0, 8, 2) and resp_type is GroupCoordinatorResponse and correlation_id != 0 and recv_correlation_id == 0): self.log.warning( 'Kafka 0.8.2 quirk -- GroupCoordinatorResponse' ' coorelation id does not match request. This' ' should go away once at least one topic has been' ' initialized on the broker') elif correlation_id != recv_correlation_id: error = Errors.CorrelationIdError( 'Correlation ids do not match: sent {}, recv {}'. format(correlation_id, recv_correlation_id)) if not fut.done(): fut.set_exception(error) self.close() break if not fut.done(): response = resp_type.decode(resp[4:]) self.log.debug('%s Response %d: %s', self, correlation_id, response) fut.set_result(response) # Update idle timer. self._last_action = self._loop.time() except (OSError, EOFError, ConnectionError) as exc: conn_exc = Errors.ConnectionError( "Connection at {0}:{1} broken".format(self._host, self._port)) conn_exc.__cause__ = exc conn_exc.__context__ = exc for _, _, fut in self._requests: fut.set_exception(conn_exc) self.close() except asyncio.CancelledError: pass
def close(self): if self._reader is not None: self._writer.close() self._writer = self._reader = None self._read_task.cancel() self._read_task = None error = Errors.ConnectionError( "Connection at {0}:{1} closed".format(self._host, self._port)) for _, _, fut in self._requests: if not fut.done(): fut.set_exception(error) self._requests = [] if self._idle_handle is not None: self._idle_handle.cancel() # transport.close() will close socket, but not right ahead. Return # a future in case we need to wait on it. return self._closed_fut
def close(self, error=None): """Close socket and fail all in-flight-requests. Arguments: error (Exception, optional): pending in-flight-requests will be failed with this exception. Default: kafka.common.ConnectionError. """ if self._sock: self._sock.close() self._sock = None self.state = ConnectionStates.DISCONNECTED self._receiving = False self._next_payload_bytes = 0 self._rbuffer.seek(0) self._rbuffer.truncate() if error is None: error = Errors.ConnectionError() while self.in_flight_requests: ifr = self.in_flight_requests.popleft() ifr.future.failure(error)
def recv(self, timeout=0): """Non-blocking network receive. Return response if available """ assert not self._processing, 'Recursion not supported' if not self.connected(): log.warning('%s cannot recv: socket not connected', self) # If requests are pending, we should close the socket and # fail all the pending request futures if self.in_flight_requests: self.close() return None elif not self.in_flight_requests: log.warning('%s: No in-flight-requests to recv', self) return None elif self._requests_timed_out(): log.warning('%s timed out after %s ms. Closing connection.', self, self.config['request_timeout_ms']) self.close(error=Errors.RequestTimedOutError( 'Request timed out after %s ms' % self.config['request_timeout_ms'])) return None readable, _, _ = select([self._sock], [], [], timeout) if not readable: return None # Not receiving is the state of reading the payload header if not self._receiving: try: # An extremely small, but non-zero, probability that there are # more than 0 but not yet 4 bytes available to read self._rbuffer.write(self._sock.recv(4 - self._rbuffer.tell())) except ConnectionError as e: if six.PY2 and e.errno == errno.EWOULDBLOCK: # This shouldn't happen after selecting above # but just in case return None log.exception('%s: Error receiving 4-byte payload header -' ' closing socket', self) self.close(error=Errors.ConnectionError(e)) return None except BlockingIOError: if six.PY3: return None raise if self._rbuffer.tell() == 4: self._rbuffer.seek(0) self._next_payload_bytes = Int32.decode(self._rbuffer) # reset buffer and switch state to receiving payload bytes self._rbuffer.seek(0) self._rbuffer.truncate() self._receiving = True elif self._rbuffer.tell() > 4: raise Errors.KafkaError('this should not happen - are you threading?') if self._receiving: staged_bytes = self._rbuffer.tell() try: self._rbuffer.write(self._sock.recv(self._next_payload_bytes - staged_bytes)) except ConnectionError as e: # Extremely small chance that we have exactly 4 bytes for a # header, but nothing to read in the body yet if six.PY2 and e.errno == errno.EWOULDBLOCK: return None log.exception('%s: Error in recv', self) self.close(error=Errors.ConnectionError(e)) return None except BlockingIOError: if six.PY3: return None raise staged_bytes = self._rbuffer.tell() if staged_bytes > self._next_payload_bytes: self.close(error=Errors.KafkaError('Receive buffer has more bytes than expected?')) if staged_bytes != self._next_payload_bytes: return None self._receiving = False self._next_payload_bytes = 0 self._rbuffer.seek(0) response = self._process_response(self._rbuffer) self._rbuffer.seek(0) self._rbuffer.truncate() return response