def _raise_connection_error(self): # Cleanup socket if we have one if self._sock: self.close() # And then raise raise ConnectionError("Kafka @ {0}:{1} went away".format(self.host, self.port))
def _read(self): try: while True: resp = yield from self._reader.readexactly(4) size, = self.HEADER.unpack(resp) resp = yield from self._reader.readexactly(size) fut = self._requests.pop(0) if not fut.cancelled(): fut.set_result(resp) except OSError as exc: conn_exc = ConnectionError("Kafka at {0}:{1} went away".format( self._host, self._port)) conn_exc.__cause__ = exc conn_exc.__context__ = exc fut = self._requests.pop(0) fut.set_exception(conn_exc) self.close()
def bootstrap(self): """Try to to bootstrap initial cluster metadata""" # using request v0 for bootstap (bcs api version is not detected yet) metadata_request = MetadataRequest[0]([]) for host, port, _ in self.hosts: log.debug("Attempting to bootstrap via node at %s:%s", host, port) try: bootstrap_conn = yield from create_conn( host, port, loop=self._loop, client_id=self._client_id, request_timeout_ms=self._request_timeout_ms, ssl_context=self._ssl_context, security_protocol=self._security_protocol, max_idle_ms=self._connections_max_idle_ms) except (OSError, asyncio.TimeoutError) as err: log.error('Unable connect to "%s:%s": %s', host, port, err) continue try: metadata = yield from bootstrap_conn.send(metadata_request) except KafkaError as err: log.warning('Unable to request metadata from "%s:%s": %s', host, port, err) bootstrap_conn.close() continue self.cluster.update_metadata(metadata) # A cluster with no topics can return no broker metadata # in that case, we should keep the bootstrap connection if not len(self.cluster.brokers()): self._conns['bootstrap'] = bootstrap_conn else: bootstrap_conn.close() log.debug('Received cluster metadata: %s', self.cluster) break else: raise ConnectionError('Unable to bootstrap from {}'.format( self.hosts)) # detect api version if need if self._api_version == 'auto': self._api_version = yield from self.check_version() if type(self._api_version) is not tuple: self._api_version = tuple(map(int, self._api_version.split('.'))) if self._sync_task is None: # starting metadata synchronizer task self._sync_task = ensure_future(self._md_synchronizer(), loop=self._loop)
def check_version(self, node_id=None): """Attempt to guess the broker version""" if node_id is None: if self._conns: node_id = list(self._conns.keys())[0] else: assert self.cluster.brokers(), 'no brokers in metadata' node_id = list(self.cluster.brokers())[0].nodeId from kafka.protocol.admin import (ListGroupsRequest_v0, ApiVersionRequest_v0) from kafka.protocol.commit import (OffsetFetchRequest_v0, GroupCoordinatorRequest_v0) from kafka.protocol.metadata import MetadataRequest_v0 test_cases = [ ('0.10', ApiVersionRequest_v0()), ('0.9', ListGroupsRequest_v0()), ('0.8.2', GroupCoordinatorRequest_v0('aiokafka-default-group')), ('0.8.1', OffsetFetchRequest_v0('aiokafka-default-group', [])), ('0.8.0', MetadataRequest_v0([])), ] # kafka kills the connection when it doesnt recognize an API request # so we can send a test request and then follow immediately with a # vanilla MetadataRequest. If the server did not recognize the first # request, both will be failed with a ConnectionError that wraps # socket.error (32, 54, or 104) conn = yield from self._get_conn(node_id) if conn is None: raise ConnectionError( "No connection to node with id {}".format(node_id)) for version, request in test_cases: try: if not conn.connected(): yield from conn.connect() assert conn, 'no connection to node with id {}'.format(node_id) # request can be ignored by Kafka broker, # so we send metadata request and wait response task = self._loop.create_task(conn.send(request)) yield from asyncio.wait([task], timeout=0.1, loop=self._loop) try: yield from conn.send(MetadataRequest_v0([])) except KafkaError: # metadata request can be cancelled in case # of invalid correlationIds order pass yield from task except KafkaError: continue else: return version raise UnrecognizedBrokerVersion()
def bootstrap(self): """Try to to bootstrap initial cluster metadata""" metadata_request = MetadataRequest([]) for host, port, _ in self.hosts: log.debug("Attempting to bootstrap via node at %s:%s", host, port) try: bootstrap_conn = yield from create_conn( host, port, loop=self._loop, client_id=self._client_id, request_timeout_ms=self._request_timeout_ms) except (OSError, asyncio.TimeoutError) as err: log.error('Unable connect to "%s:%s": %s', host, port, err) continue try: metadata = yield from bootstrap_conn.send(metadata_request) except KafkaError as err: log.warning('Unable to request metadata from "%s:%s": %s', host, port, err) bootstrap_conn.close() continue self.cluster.update_metadata(metadata) # A cluster with no topics can return no broker metadata # in that case, we should keep the bootstrap connection if not len(self.cluster.brokers()): self._conns['bootstrap'] = bootstrap_conn else: bootstrap_conn.close() log.debug('Received cluster metadata: %s', self.cluster) break else: raise ConnectionError('Unable to bootstrap from {}'.format( self.hosts)) if self._sync_task is None: # starting metadata synchronizer task self._sync_task = ensure_future(self._md_synchronizer(), loop=self._loop)
def check_version(self, node_id=None): """Attempt to guess the broker version""" if node_id is None: if self._conns: node_id = list(self._conns.keys())[0] else: assert self.cluster.brokers(), 'no brokers in metadata' node_id = list(self.cluster.brokers())[0].nodeId from kafka.protocol.admin import ListGroupsRequest_v0 from kafka.protocol.commit import (OffsetFetchRequest_v0, GroupCoordinatorRequest_v0) from kafka.protocol.metadata import MetadataRequest_v0 test_cases = [ ('0.9', ListGroupsRequest_v0()), ('0.8.2', GroupCoordinatorRequest_v0('aiokafka-default-group')), ('0.8.1', OffsetFetchRequest_v0('aiokafka-default-group', [])), ('0.8.0', MetadataRequest_v0([])), ] # kafka kills the connection when it doesnt recognize an API request # so we can send a test request and then follow immediately with a # vanilla MetadataRequest. If the server did not recognize the first # request, both will be failed with a ConnectionError that wraps # socket.error (32, 54, or 104) conn = yield from self._get_conn(node_id) if conn is None: raise ConnectionError( "No connection to node with id {}".format(node_id)) for version, request in test_cases: try: if not conn.connected(): yield from conn.connect() assert conn, 'no connection to node with id {}'.format(node_id) yield from conn.send(request) except KafkaError: continue else: return version raise UnrecognizedBrokerVersion()
def _raise_connection_error(self): self._dirty = True raise ConnectionError("Kafka @ {0}:{1} went away".format( self.host, self.port))