def receive_bytes(self, data): """Process bytes received from the network. Arguments: data (bytes): any length bytes received from a network connection to a kafka broker. Returns: responses (list of (correlation_id, response)): any/all completed responses, decoded from bytes to python objects. Raises: KafkaProtocolError: if the bytes received could not be decoded. CorrelationIdError: if the response does not match the request correlation id. """ i = 0 n = len(data) responses = [] while i < n: # Not receiving is the state of reading the payload header if not self._receiving: bytes_to_read = min(4 - self._header.tell(), n - i) self._header.write(data[i:i + bytes_to_read]) i += bytes_to_read if self._header.tell() == 4: self._header.seek(0) nbytes = Int32.decode(self._header) # reset buffer and switch state to receiving payload bytes self._rbuffer = KafkaBytes(nbytes) self._receiving = True elif self._header.tell() > 4: raise Errors.KafkaError( "this should not happen - are you threading?") if self._receiving: total_bytes = len(self._rbuffer) staged_bytes = self._rbuffer.tell() bytes_to_read = min(total_bytes - staged_bytes, n - i) self._rbuffer.write(data[i:i + bytes_to_read]) i += bytes_to_read staged_bytes = self._rbuffer.tell() if staged_bytes > total_bytes: raise Errors.KafkaError( "Receive buffer has more bytes than expected?") if staged_bytes != total_bytes: break self._receiving = False self._rbuffer.seek(0) resp = self._process_response(self._rbuffer) responses.append(resp) self._reset_buffer() return responses
def _send_join_group_request(self): """Join the group and return the assignment for the next generation. This function handles both JoinGroup and SyncGroup, delegating to :meth:`._perform_assignment` if elected leader by the coordinator. Returns: Future: resolves to the encoded-bytes assignment returned from the group leader """ if self.coordinator_unknown(): e = Errors.GroupCoordinatorNotAvailableError(self.coordinator_id) return Future().failure(e) elif not self._client.ready(self.coordinator_id, metadata_priority=False): e = Errors.NodeNotReadyError(self.coordinator_id) return Future().failure(e) # send a join group request to the coordinator log.info("(Re-)joining group %s", self.group_id) member_metadata = [ (protocol, metadata if isinstance(metadata, bytes) else metadata.encode()) for protocol, metadata in self.group_protocols() ] if self.config['api_version'] < (0, 9): raise Errors.KafkaError( 'JoinGroupRequest api requires 0.9+ brokers') elif (0, 9) <= self.config['api_version'] < (0, 10, 1): request = JoinGroupRequest[0](self.group_id, self.config['session_timeout_ms'], self._generation.member_id, self.protocol_type(), member_metadata) elif (0, 10, 1) <= self.config['api_version'] < (0, 11, 0): request = JoinGroupRequest[1](self.group_id, self.config['session_timeout_ms'], self.config['max_poll_interval_ms'], self._generation.member_id, self.protocol_type(), member_metadata) else: request = JoinGroupRequest[2](self.group_id, self.config['session_timeout_ms'], self.config['max_poll_interval_ms'], self._generation.member_id, self.protocol_type(), member_metadata) # create the request for the coordinator log.debug("Sending JoinGroup (%s) to coordinator %s", request, self.coordinator_id) future = Future() _f = self._client.send(self.coordinator_id, request) _f.add_callback(self._handle_join_group_response, future, time.time()) _f.add_errback(self._failed_request, self.coordinator_id, request, future) return future
def _recv(self): # Not receiving is the state of reading the payload header if not self._receiving: try: bytes_to_read = 4 - self._rbuffer.tell() data = self._sock.recv(bytes_to_read) # We expect socket.recv to raise an exception if there is not # enough data to read the full bytes_to_read # but if the socket is disconnected, we will get empty data # without an exception raised if not data: log.error('%s: socket disconnected', self) self.close(error=Errors.ConnectionError('socket disconnected')) return None self._rbuffer.write(data) except ssl.SSLWantReadError: return None except ConnectionError as e: if six.PY2 and e.errno == errno.EWOULDBLOCK: return None log.exception('%s: Error receiving 4-byte payload header -' ' closing socket', self) self.close(error=Errors.ConnectionError(e)) return None except BlockingIOError: if six.PY3: return None raise if self._rbuffer.tell() == 4: self._rbuffer.seek(0) self._next_payload_bytes = Int32.decode(self._rbuffer) # reset buffer and switch state to receiving payload bytes self._rbuffer.seek(0) self._rbuffer.truncate() self._receiving = True elif self._rbuffer.tell() > 4: raise Errors.KafkaError('this should not happen - are you threading?') if self._receiving: staged_bytes = self._rbuffer.tell() try: bytes_to_read = self._next_payload_bytes - staged_bytes data = self._sock.recv(bytes_to_read) # We expect socket.recv to raise an exception if there is not # enough data to read the full bytes_to_read # but if the socket is disconnected, we will get empty data # without an exception raised if bytes_to_read and not data: log.error('%s: socket disconnected', self) self.close(error=Errors.ConnectionError('socket disconnected')) return None self._rbuffer.write(data) except ssl.SSLWantReadError: return None except ConnectionError as e: # Extremely small chance that we have exactly 4 bytes for a # header, but nothing to read in the body yet if six.PY2 and e.errno == errno.EWOULDBLOCK: return None log.exception('%s: Error in recv', self) self.close(error=Errors.ConnectionError(e)) return None except BlockingIOError: if six.PY3: return None raise staged_bytes = self._rbuffer.tell() if staged_bytes > self._next_payload_bytes: self.close(error=Errors.KafkaError('Receive buffer has more bytes than expected?')) if staged_bytes != self._next_payload_bytes: return None self._receiving = False self._next_payload_bytes = 0 if self._sensors: self._sensors.bytes_received.record(4 + self._rbuffer.tell()) self._rbuffer.seek(0) response = self._process_response(self._rbuffer) self._rbuffer.seek(0) self._rbuffer.truncate() return response
def recv(self): """Non-blocking network receive. Return response if available """ assert not self._processing, 'Recursion not supported' if not self.connected(): log.warning('%s cannot recv: socket not connected', self) # If requests are pending, we should close the socket and # fail all the pending request futures if self.in_flight_requests: self.close() return None elif not self.in_flight_requests: log.warning('%s: No in-flight-requests to recv', self) return None elif self._requests_timed_out(): log.warning('%s timed out after %s ms. Closing connection.', self, self.config['request_timeout_ms']) self.close(error=Errors.RequestTimedOutError( 'Request timed out after %s ms' % self.config['request_timeout_ms'])) return None # Not receiving is the state of reading the payload header if not self._receiving: try: bytes_to_read = 4 - self._rbuffer.tell() data = self._sock.recv(bytes_to_read) # We expect socket.recv to raise an exception if there is not # enough data to read the full bytes_to_read # but if the socket is disconnected, we will get empty data # without an exception raised if not data: log.error('%s: socket disconnected', self) self.close(error=Errors.ConnectionError('socket disconnected')) return None self._rbuffer.write(data) except ssl.SSLWantReadError: return None except ConnectionError as e: if six.PY2 and e.errno == errno.EWOULDBLOCK: return None log.exception('%s: Error receiving 4-byte payload header -' ' closing socket', self) self.close(error=Errors.ConnectionError(e)) return None except BlockingIOError: if six.PY3: return None raise if self._rbuffer.tell() == 4: self._rbuffer.seek(0) self._next_payload_bytes = Int32.decode(self._rbuffer) # reset buffer and switch state to receiving payload bytes self._rbuffer.seek(0) self._rbuffer.truncate() self._receiving = True elif self._rbuffer.tell() > 4: raise Errors.KafkaError('this should not happen - are you threading?') if self._receiving: staged_bytes = self._rbuffer.tell() try: bytes_to_read = self._next_payload_bytes - staged_bytes data = self._sock.recv(bytes_to_read) # We expect socket.recv to raise an exception if there is not # enough data to read the full bytes_to_read # but if the socket is disconnected, we will get empty data # without an exception raised if not data: log.error('%s: socket disconnected', self) self.close(error=Errors.ConnectionError('socket disconnected')) return None self._rbuffer.write(data) except ssl.SSLWantReadError: return None except ConnectionError as e: # Extremely small chance that we have exactly 4 bytes for a # header, but nothing to read in the body yet if six.PY2 and e.errno == errno.EWOULDBLOCK: return None log.exception('%s: Error in recv', self) self.close(error=Errors.ConnectionError(e)) return None except BlockingIOError: if six.PY3: return None raise staged_bytes = self._rbuffer.tell() if staged_bytes > self._next_payload_bytes: self.close(error=Errors.KafkaError('Receive buffer has more bytes than expected?')) if staged_bytes != self._next_payload_bytes: return None self._receiving = False self._next_payload_bytes = 0 self._rbuffer.seek(0) response = self._process_response(self._rbuffer) self._rbuffer.seek(0) self._rbuffer.truncate() return response