def check_version(self, node_id=None): """Attempt to guess the broker version""" if node_id is None: if self._conns: node_id = list(self._conns.keys())[0] else: assert self.cluster.brokers(), 'no brokers in metadata' node_id = list(self.cluster.brokers())[0].nodeId from kafka.protocol.admin import (ListGroupsRequest_v0, ApiVersionRequest_v0) from kafka.protocol.commit import (OffsetFetchRequest_v0, GroupCoordinatorRequest_v0) from kafka.protocol.metadata import MetadataRequest_v0 test_cases = [ ('0.10', ApiVersionRequest_v0()), ('0.9', ListGroupsRequest_v0()), ('0.8.2', GroupCoordinatorRequest_v0('aiokafka-default-group')), ('0.8.1', OffsetFetchRequest_v0('aiokafka-default-group', [])), ('0.8.0', MetadataRequest_v0([])), ] # kafka kills the connection when it doesnt recognize an API request # so we can send a test request and then follow immediately with a # vanilla MetadataRequest. If the server did not recognize the first # request, both will be failed with a ConnectionError that wraps # socket.error (32, 54, or 104) conn = yield from self._get_conn(node_id) if conn is None: raise ConnectionError( "No connection to node with id {}".format(node_id)) for version, request in test_cases: try: if not conn.connected(): yield from conn.connect() assert conn, 'no connection to node with id {}'.format(node_id) # request can be ignored by Kafka broker, # so we send metadata request and wait response task = self._loop.create_task(conn.send(request)) yield from asyncio.wait([task], timeout=0.1, loop=self._loop) try: yield from conn.send(MetadataRequest_v0([])) except KafkaError: # metadata request can be cancelled in case # of invalid correlationIds order pass yield from task except KafkaError: continue else: # To avoid having a connection in undefined state if node_id != "bootstrap" and conn.connected(): conn.close() return version raise UnrecognizedBrokerVersion()
def check_version(self, node_id=None): """Attempt to guess the broker version""" if node_id is None: if self._conns: node_id = list(self._conns.keys())[0] else: assert self.cluster.brokers(), 'no brokers in metadata' node_id = list(self.cluster.brokers())[0].nodeId from kafka.protocol.admin import ListGroupsRequest_v0 from kafka.protocol.commit import (OffsetFetchRequest_v0, GroupCoordinatorRequest_v0) from kafka.protocol.metadata import MetadataRequest_v0 test_cases = [ ('0.9', ListGroupsRequest_v0()), ('0.8.2', GroupCoordinatorRequest_v0('aiokafka-default-group')), ('0.8.1', OffsetFetchRequest_v0('aiokafka-default-group', [])), ('0.8.0', MetadataRequest_v0([])), ] # kafka kills the connection when it doesnt recognize an API request # so we can send a test request and then follow immediately with a # vanilla MetadataRequest. If the server did not recognize the first # request, both will be failed with a ConnectionError that wraps # socket.error (32, 54, or 104) conn = yield from self._get_conn(node_id) if conn is None: raise ConnectionError( "No connection to node with id {}".format(node_id)) for version, request in test_cases: try: if not conn.connected(): yield from conn.connect() assert conn, 'no connection to node with id {}'.format(node_id) yield from conn.send(request) except KafkaError: continue else: return version raise UnrecognizedBrokerVersion()
async def wait_no_lag(self): print("Ensuring no lag") consumer_group = "f-simple" client = self.client await self.client.bootstrap() source = self.source source_builder = self._source_builder source_highwaters = source_builder._highwaters() source_tps = source_builder._assignment protocol_tps = [(source, [tp.partition for tp in source_tps])] node_id = next(broker.nodeId for broker in client.cluster.brokers()) coordinator_request = GroupCoordinatorRequest_v0(consumer_group) coordinator_response = await client.send(node_id, coordinator_request) coordinator_id = coordinator_response.coordinator_id while True: consumer_offsets_req = OffsetFetchRequest_v1( consumer_group, protocol_tps) consumer_offsets_resp = await client.send(coordinator_id, consumer_offsets_req) topics = consumer_offsets_resp.topics assert len(topics) == 1, f"{topics!r}" topic, partition_resps = topics[0] assert topic == source, f"{source}" assert len(partition_resps) == len(source_tps) # + 1 is to account for the difference in how faust commits positions = { TopicPartition(topic=source, partition=partition): offset + 1 for partition, offset, _, _ in partition_resps } if positions != source_highwaters: print("There is lag. Waiting!") await asyncio.sleep(2.0) else: return
async def check_version(self, node_id=None): """Attempt to guess the broker version""" if node_id is None: default_group_conns = [ n_id for (n_id, group) in self._conns.keys() if group == ConnectionGroup.DEFAULT ] if default_group_conns: node_id = default_group_conns[0] else: assert self.cluster.brokers(), 'no brokers in metadata' node_id = list(self.cluster.brokers())[0].nodeId from kafka.protocol.admin import (ListGroupsRequest_v0, ApiVersionRequest_v0) from kafka.protocol.commit import (OffsetFetchRequest_v0, GroupCoordinatorRequest_v0) from kafka.protocol.metadata import MetadataRequest_v0 test_cases = [ ((0, 10), ApiVersionRequest_v0()), ((0, 9), ListGroupsRequest_v0()), ((0, 8, 2), GroupCoordinatorRequest_v0('aiokafka-default-group')), ((0, 8, 1), OffsetFetchRequest_v0('aiokafka-default-group', [])), ((0, 8, 0), MetadataRequest_v0([])), ] # kafka kills the connection when it doesnt recognize an API request # so we can send a test request and then follow immediately with a # vanilla MetadataRequest. If the server did not recognize the first # request, both will be failed with a ConnectionError that wraps # socket.error (32, 54, or 104) conn = await self._get_conn(node_id, no_hint=True) if conn is None: raise ConnectionError( "No connection to node with id {}".format(node_id)) for version, request in test_cases: try: if not conn.connected(): await conn.connect() assert conn, 'no connection to node with id {}'.format(node_id) # request can be ignored by Kafka broker, # so we send metadata request and wait response task = self._loop.create_task(conn.send(request)) await asyncio.wait([task], timeout=0.1, loop=self._loop) try: await conn.send(MetadataRequest_v0([])) except KafkaError: # metadata request can be cancelled in case # of invalid correlationIds order pass response = await task except KafkaError: continue else: # To avoid having a connection in undefined state if node_id != "bootstrap" and conn.connected(): conn.close() if isinstance(request, ApiVersionRequest_v0): # Starting from 0.10 kafka broker we determine version # by looking at ApiVersionResponse version = self._check_api_version_response(response) return version raise UnrecognizedBrokerVersion()