def _poll(self, timeout): """Returns list of (response, future) tuples""" processed = set() start_select = time.time() ready = self._selector.select(timeout) end_select = time.time() if self._sensors: self._sensors.select_time.record( (end_select - start_select) * 1000000000) for key, events in ready: if key.fileobj is self._wake_r: self._clear_wake_fd() continue elif not (events & selectors.EVENT_READ): continue conn = key.data processed.add(conn) if not conn.in_flight_requests: # if we got an EVENT_READ but there were no in-flight requests, one of # two things has happened: # # 1. The remote end closed the connection (because it died, or because # a firewall timed out, or whatever) # 2. The protocol is out of sync. # # either way, we can no longer safely use this connection # # Do a 1-byte read to check protocol didnt get out of sync, and then close the conn try: unexpected_data = key.fileobj.recv(1) if unexpected_data: # anything other than a 0-byte read means protocol issues log.warning('Protocol out of sync on %r, closing', conn) except socket.error: pass conn.close( Errors.KafkaConnectionError( 'Socket EVENT_READ without in-flight-requests')) continue self._idle_expiry_manager.update(conn.node_id) self._pending_completion.extend(conn.recv()) # Check for additional pending SSL bytes if self.config['security_protocol'] in ('SSL', 'SASL_SSL'): # TODO: optimize for conn in self._conns.values(): if conn not in processed and conn.connected( ) and conn._sock.pending(): self._pending_completion.extend(conn.recv()) for conn in six.itervalues(self._conns): if conn.requests_timed_out(): log.warning('%s timed out after %s ms. Closing connection.', conn, conn.config['request_timeout_ms']) conn.close(error=Errors.RequestTimedOutError( 'Request timed out after %s ms' % conn.config['request_timeout_ms'])) if self._sensors: self._sensors.io_time.record( (time.time() - end_select) * 1000000000) self._maybe_close_oldest_connection()
def _poll(self, timeout): # This needs to be locked, but since it is only called from within the # locked section of poll(), there is no additional lock acquisition here processed = set() # Send pending requests first, before polling for responses self._register_send_sockets() start_select = time.time() ready = self._selector.select(timeout) end_select = time.time() if self._sensors: self._sensors.select_time.record( (end_select - start_select) * 1000000000) for key, events in ready: if key.fileobj is self._wake_r: self._clear_wake_fd() continue # Send pending requests if socket is ready to write if events & selectors.EVENT_WRITE: conn = key.data if conn.connecting(): conn.connect() else: if conn.send_pending_requests_v2(): # If send is complete, we dont need to track write readiness # for this socket anymore if key.events ^ selectors.EVENT_WRITE: self._selector.modify( key.fileobj, key.events ^ selectors.EVENT_WRITE, key.data) else: self._selector.unregister(key.fileobj) if not (events & selectors.EVENT_READ): continue conn = key.data processed.add(conn) if not conn.in_flight_requests: # if we got an EVENT_READ but there were no in-flight requests, one of # two things has happened: # # 1. The remote end closed the connection (because it died, or because # a firewall timed out, or whatever) # 2. The protocol is out of sync. # # either way, we can no longer safely use this connection # # Do a 1-byte read to check protocol didnt get out of sync, and then close the conn try: unexpected_data = key.fileobj.recv(1) if unexpected_data: # anything other than a 0-byte read means protocol issues log.warning('Protocol out of sync on %r, closing', conn) except socket.error: pass conn.close( Errors.KafkaConnectionError( 'Socket EVENT_READ without in-flight-requests')) continue self._idle_expiry_manager.update(conn.node_id) self._pending_completion.extend(conn.recv()) # Check for additional pending SSL bytes if self.config['security_protocol'] in ('SSL', 'SASL_SSL'): # TODO: optimize for conn in self._conns.values(): if conn not in processed and conn.connected( ) and conn._sock.pending(): self._pending_completion.extend(conn.recv()) for conn in six.itervalues(self._conns): if conn.requests_timed_out(): log.warning('%s timed out after %s ms. Closing connection.', conn, conn.config['request_timeout_ms']) conn.close(error=Errors.RequestTimedOutError( 'Request timed out after %s ms' % conn.config['request_timeout_ms'])) if self._sensors: self._sensors.io_time.record( (time.time() - end_select) * 1000000000) self._maybe_close_oldest_connection()