def __init__(self, source, changelog, loop): self.source = source self.loop = loop self.client = AIOKafkaClient(bootstrap_servers=bootstrap_servers, loop=self.loop) self._source_builder = SourceTableBuilder(source, loop) self.changelog = changelog self._changelog_builder = ChangelogTableBuilder(changelog, loop)
async def send_one(): client = AIOKafkaClient(loop=loop, bootstrap_servers='192.168.13.163:9092') await producer.start() try: # Produce message await producer.send_and_wait("report_apm", str("test").encode()) finally: # Wait for all pending messages to be delivered or expire. await producer.stop()
async def _get_controller_node(self, owner: Service, client: aiokafka.AIOKafkaClient, timeout: int = 30000) -> Optional[int]: nodes = [broker.nodeId for broker in client.cluster.brokers()] for node_id in nodes: if node_id is None: raise RuntimeError('Not connected to Kafka Broker') request = MetadataRequest_v1([]) wait_result = await owner.wait( client.send(node_id, request), timeout=timeout, ) if wait_result.stopped: owner.log.info(f'Shutting down - skipping creation.') return None response = wait_result.result return response.controller_id raise Exception(f'Controller node not found')
async def _really_create_topic( self, owner: Service, client: aiokafka.AIOKafkaClient, topic: str, partitions: int, replication: int, *, config: Mapping[str, Any] = None, timeout: int = 30000, retention: int = None, compacting: bool = None, deleting: bool = None, ensure_created: bool = False) -> None: # pragma: no cover owner.log.info('Creating topic %r', topic) if topic in client.cluster.topics(): owner.log.debug('Topic %r exists, skipping creation.', topic) return protocol_version = 1 extra_configs = config or {} config = self._topic_config(retention, compacting, deleting) config.update(extra_configs) controller_node = await self._get_controller_node(owner, client, timeout=timeout) owner.log.debug('Found controller: %r', controller_node) if controller_node is None: if owner.should_stop: owner.log.info('Shutting down hence controller not found') return else: raise Exception('Controller node is None') request = CreateTopicsRequest[protocol_version]( [(topic, partitions, replication, [], list(config.items()))], timeout, False, ) wait_result = await owner.wait( client.send(controller_node, request), timeout=timeout, ) if wait_result.stopped: owner.log.debug(f'Shutting down - skipping creation.') return response = wait_result.result assert len(response.topic_error_codes), 'single topic' _, code, reason = response.topic_error_codes[0] if code != 0: if not ensure_created and code == TopicExistsError.errno: owner.log.debug( 'Topic %r exists, skipping creation.', topic) return elif code == NotControllerError.errno: raise RuntimeError(f'Invalid controller: {controller_node}') else: raise for_code(code)( f'Cannot create topic: {topic} ({code}): {reason}') else: owner.log.info('Topic %r created.', topic) return
async def check_if_kafka_is_alive(my_client: AIOKafkaClient): while 1: conns: Dict[str, AIOKafkaConnection] = my_client.__getattribute__('_conns') print(my_client._bootstrap_servers) print('Host = ', my_client.hosts) if not is_connected(conns): print('RENEW CONNECTION') try: my_client.__setattr__( 'cluster', ClusterMetadata(metadata_max_age_ms=300000)) my_client.__setattr__('_topics', set()) my_client.__setattr__('_conns', {}) my_client.__setattr__('_sync_task', None) loop = asyncio.get_event_loop() my_client.__setattr__('_md_update_fut', None) my_client.__setattr__('_md_update_waiter', loop.create_future()) my_client.__setattr__('_get_conn_lock', asyncio.Lock(loop=loop)) await my_client.bootstrap() except ConnectionError: pass else: for conn in conns.items(): print(conn) print(conn[1].connected()) try: if not conn[1].connected(): print('TRY RE CONNECTION') await conn[1].connect() if not conn[1].connected(): print('RENEW CONNECTION') await my_client.bootstrap() except Exception as err: print(err) await asyncio.sleep(1)
async def _really_create_topic(self, owner: Service, client: aiokafka.AIOKafkaClient, topic: str, partitions: int, replication: int, *, config: Mapping[str, Any] = None, timeout: int = 30000, retention: int = None, compacting: bool = None, deleting: bool = None, ensure_created: bool = False) -> None: owner.log.info(f'Creating topic {topic}') protocol_version = 1 extra_configs = config or {} config = self._topic_config(retention, compacting, deleting) config.update(extra_configs) # Create topic request needs to be sent to the kafka cluster controller # Since aiokafka client doesn't currently support MetadataRequest # version 1, client.controller will always be None. Hence we cycle # through all brokers if we get Error 41 (not controller) until we # hit the controller nodes = [broker.nodeId for broker in client.cluster.brokers()] owner.log.info(f'Nodes: {nodes}') for node_id in nodes: if node_id is None: raise RuntimeError('Not connected to Kafka broker') request = CreateTopicsRequest[protocol_version]( [(topic, partitions, replication, [], list(config.items()))], timeout, False, ) wait_result = await owner.wait( client.send(node_id, request), timeout=timeout, ) if wait_result.stopped: owner.log.info(f'Shutting down - skipping creation.') return response = wait_result.result assert len(response.topic_error_codes), 'single topic' _, code, reason = response.topic_error_codes[0] if code != 0: if not ensure_created and code == TopicExistsError.errno: owner.log.debug( f'Topic {topic} exists, skipping creation.') return elif code == NotControllerError.errno: owner.log.debug(f'Broker: {node_id} is not controller.') continue else: raise for_code(code)( f'Cannot create topic: {topic} ({code}): {reason}') else: owner.log.info(f'Topic {topic} created.') return raise Exception(f'No controller found among brokers: {nodes}')