def send(self, request, expect_response=True): if self._writer is None: raise Errors.ConnectionError( "No connection to broker at {0}:{1}" .format(self._host, self._port)) correlation_id = self._next_correlation_id() header = RequestHeader(request, correlation_id=correlation_id, client_id=self._client_id) message = header.encode() + request.encode() size = self.HEADER.pack(len(message)) try: self._writer.write(size + message) except OSError as err: self.close() raise Errors.ConnectionError( "Connection at {0}:{1} broken: {2}".format( self._host, self._port, err)) if not expect_response: return self._writer.drain() fut = asyncio.Future(loop=self._loop) self._requests.append((correlation_id, request.RESPONSE_TYPE, fut)) return asyncio.wait_for(fut, self._request_timeout, loop=self._loop)
def send(self, request, expect_response=True): if self._writer is None: raise Errors.KafkaConnectionError( "No connection to broker at {0}:{1}".format( self._host, self._port)) correlation_id = self._next_correlation_id() header = RequestHeader(request, correlation_id=correlation_id, client_id=self._client_id) message = header.encode() + request.encode() size = struct.pack(">i", len(message)) try: self._writer.write(size + message) except OSError as err: self.close(reason=CloseReason.CONNECTION_BROKEN) raise Errors.KafkaConnectionError( "Connection at {0}:{1} broken: {2}".format( self._host, self._port, err)) self.log.debug('%s Request %d: %s', self, correlation_id, request) if not expect_response: return self._writer.drain() fut = self._loop.create_future() self._requests.append((correlation_id, request.RESPONSE_TYPE, fut)) return asyncio.wait_for(fut, self._request_timeout)
def send_request(self, request, correlation_id=None): """Encode and queue a kafka api request for sending. Arguments: request (object): An un-encoded kafka request. correlation_id (int, optional): Optionally specify an ID to correlate requests with responses. If not provided, an ID will be generated automatically. Returns: correlation_id """ log.debug("Sending request %s", request) if correlation_id is None: correlation_id = self._next_correlation_id() header = RequestHeader(request, correlation_id=correlation_id, client_id=self._client_id) message = b"".join([header.encode(), request.encode()]) size = Int32.encode(len(message)) data = size + message self.bytes_to_send.append(data) if request.expect_response(): ifr = (correlation_id, request) self.in_flight_requests.append(ifr) return correlation_id
def send_request(self, request, correlation_id=None): """Encode and queue a kafka api request for sending. Arguments: request (object): An un-encoded kafka request. correlation_id (int, optional): Optionally specify an ID to correlate requests with responses. If not provided, an ID will be generated automatically. Returns: correlation_id """ log.debug('Sending request %s', request) if correlation_id is None: correlation_id = self._next_correlation_id() header = RequestHeader(request, correlation_id=correlation_id, client_id=self._client_id) message = b''.join([header.encode(), request.encode()]) size = Int32.encode(len(message)) data = size + message self.bytes_to_send.append(data) if request.expect_response(): ifr = (correlation_id, request) self.in_flight_requests.append(ifr) return correlation_id
def send(self, request, expect_response=True): if self._writer is None: raise Errors.ConnectionError( "No connection to broker at {0}:{1}" .format(self._host, self._port)) correlation_id = self._next_correlation_id() header = RequestHeader(request, correlation_id=correlation_id, client_id=self._client_id) message = header.encode() + request.encode() size = struct.pack(">i", len(message)) try: self._writer.write(size + message) except OSError as err: self.close(reason=CloseReason.CONNECTION_BROKEN) raise Errors.ConnectionError( "Connection at {0}:{1} broken: {2}".format( self._host, self._port, err)) self.log.debug( '%s Request %d: %s', self, correlation_id, request) if not expect_response: return self._writer.drain() fut = create_future(loop=self._loop) self._requests.append((correlation_id, request.RESPONSE_TYPE, fut)) return asyncio.wait_for(fut, self._request_timeout, loop=self._loop)
def test_encode_message_header(): expect = b''.join([ struct.pack('>h', 10), # API Key struct.pack('>h', 0), # API Version struct.pack('>i', 4), # Correlation Id struct.pack('>h', len('client3')), # Length of clientId b'client3', # ClientId ]) req = GroupCoordinatorRequest[0]('foo') header = RequestHeader(req, correlation_id=4, client_id='client3') assert header.encode() == expect
def test_send_response(_socket, conn): conn.connect() assert conn.state is ConnectionStates.CONNECTED req = MetadataRequest[0]([]) header = RequestHeader(req, client_id=conn.config['client_id']) payload_bytes = len(header.encode()) + len(req.encode()) third = payload_bytes // 3 remainder = payload_bytes % 3 _socket.send.side_effect = [4, third, third, third, remainder] assert len(conn.in_flight_requests) == 0 f = conn.send(req) assert f.is_done is False assert len(conn.in_flight_requests) == 1
def test_send_no_response(_socket, conn): conn.connect() assert conn.state is ConnectionStates.CONNECTED req = ProduceRequest[0](required_acks=0, timeout=0, topics=()) header = RequestHeader(req, client_id=conn.config['client_id']) payload_bytes = len(header.encode()) + len(req.encode()) third = payload_bytes // 3 remainder = payload_bytes % 3 _socket.send.side_effect = [4, third, third, third, remainder] assert len(conn.in_flight_requests) == 0 f = conn.send(req) assert f.succeeded() is True assert f.value is None assert len(conn.in_flight_requests) == 0
def test_send_no_response(socket, conn): conn.connect() assert conn.state is ConnectionStates.CONNECTED req = MetadataRequest[0]([]) header = RequestHeader(req, client_id=conn.config['client_id']) payload_bytes = len(header.encode()) + len(req.encode()) third = payload_bytes // 3 remainder = payload_bytes % 3 socket.send.side_effect = [4, third, third, third, remainder] assert len(conn.in_flight_requests) == 0 f = conn.send(req, expect_response=False) assert f.succeeded() is True assert f.value is None assert len(conn.in_flight_requests) == 0
def send(self, request, expect_response=True): """send request, return Future() Can block on network if request is larger than send_buffer_bytes """ future = Future() if self.connecting(): return future.failure(Errors.NodeNotReadyError(str(self))) elif not self.connected(): return future.failure(Errors.ConnectionError(str(self))) elif not self.can_send_more(): return future.failure(Errors.TooManyInFlightRequests(str(self))) correlation_id = self._next_correlation_id() header = RequestHeader(request, correlation_id=correlation_id, client_id=self.config['client_id']) message = b''.join([header.encode(), request.encode()]) size = Int32.encode(len(message)) try: # In the future we might manage an internal write buffer # and send bytes asynchronously. For now, just block # sending each request payload self._sock.setblocking(True) for data in (size, message): total_sent = 0 while total_sent < len(data): sent_bytes = self._sock.send(data[total_sent:]) total_sent += sent_bytes assert total_sent == len(data) self._sock.setblocking(False) except (AssertionError, ConnectionError) as e: log.exception("Error sending %s to %s", request, self) error = Errors.ConnectionError("%s: %s" % (str(self), e)) self.close(error=error) return future.failure(error) log.debug('%s Request %d: %s', self, correlation_id, request) if expect_response: ifr = InFlightRequest(request=request, correlation_id=correlation_id, response_type=request.RESPONSE_TYPE, future=future, timestamp=time.time()) self.in_flight_requests.append(ifr) else: future.success(None) return future
def test_recv_disconnected_too(_socket, conn): conn.connect() assert conn.connected() req = MetadataRequest[0]([]) header = RequestHeader(req, client_id=conn.config['client_id']) payload_bytes = len(header.encode()) + len(req.encode()) _socket.send.side_effect = [4, payload_bytes] conn.send(req) # Empty data on recv means the socket is disconnected _socket.recv.return_value = b'' # Attempt to receive should mark connection as disconnected assert conn.connected() conn.recv() assert conn.disconnected()
def _send(self, request): assert self.state in (ConnectionStates.AUTHENTICATING, ConnectionStates.CONNECTED) future = Future() correlation_id = self._next_correlation_id() header = RequestHeader(request, correlation_id=correlation_id, client_id=self.config['client_id']) message = b''.join([header.encode(), request.encode()]) size = Int32.encode(len(message)) data = size + message try: # In the future we might manage an internal write buffer # and send bytes asynchronously. For now, just block # sending each request payload self._sock.setblocking(True) total_sent = 0 while total_sent < len(data): sent_bytes = self._sock.send(data[total_sent:]) total_sent += sent_bytes assert total_sent == len(data) if self._sensors: self._sensors.bytes_sent.record(total_sent) self._sock.setblocking(False) except (AssertionError, ConnectionError) as e: log.exception("Error sending %s to %s", request, self) error = Errors.ConnectionError("%s: %s" % (self, e)) self.close(error=error) return future.failure(error) log.debug('%s Request %d: %s', self, correlation_id, request) if request.expect_response(): ifr = InFlightRequest(request=request, correlation_id=correlation_id, response_type=request.RESPONSE_TYPE, future=future, timestamp=time.time()) self.in_flight_requests.append(ifr) else: future.success(None) return future
def _send(self, request, expect_response=True): future = Future() correlation_id = self._next_correlation_id() header = RequestHeader(request, correlation_id=correlation_id, client_id=self.config['client_id']) message = b''.join([header.encode(), request.encode()]) size = Int32.encode(len(message)) data = size + message try: # In the future we might manage an internal write buffer # and send bytes asynchronously. For now, just block # sending each request payload self._sock.setblocking(True) total_sent = 0 while total_sent < len(data): sent_bytes = self._sock.send(data[total_sent:]) total_sent += sent_bytes assert total_sent == len(data) if self._sensors: self._sensors.bytes_sent.record(total_sent) self._sock.setblocking(False) except (AssertionError, ConnectionError) as e: log.exception("Error sending %s to %s", request, self) error = Errors.ConnectionError("%s: %s" % (str(self), e)) self.close(error=error) return future.failure(error) log.debug('%s Request %d: %s', self, correlation_id, request) if expect_response: ifr = InFlightRequest(request=request, correlation_id=correlation_id, response_type=request.RESPONSE_TYPE, future=future, timestamp=time.time()) self.in_flight_requests.append(ifr) else: future.success(None) return future
def send(request, wait_response=True): global correlation_id with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect(('127.0.0.1', PORT)) header = RequestHeader( request=request, correlation_id=correlation_id, client_id='test' ) correlation_id += 1 message = b''.join([header.encode(), request.encode()]) size = Int32.encode(len(message)) s.send(size) s.send(message) if wait_response: size = s.recv(4, socket.MSG_WAITALL) size = Int32.decode(BytesIO(size)) s.recv(size, socket.MSG_WAITALL)