def __init__(self, source, changelog, loop):
self.source = source
self.loop = loop
self.client = AIOKafkaClient(bootstrap_servers=bootstrap_servers,
loop=self.loop)
self._source_builder = SourceTableBuilder(source, loop)
self.changelog = changelog
self._changelog_builder = ChangelogTableBuilder(changelog, loop)
async def send_one():
client = AIOKafkaClient(loop=loop, bootstrap_servers='192.168.13.163:9092')
await producer.start()
try:
# Produce message
await producer.send_and_wait("report_apm", str("test").encode())
finally:
# Wait for all pending messages to be delivered or expire.
await producer.stop()
async def _get_controller_node(self,
owner: Service,
client: aiokafka.AIOKafkaClient,
timeout: int = 30000) -> Optional[int]:
nodes = [broker.nodeId for broker in client.cluster.brokers()]
for node_id in nodes:
if node_id is None:
raise RuntimeError('Not connected to Kafka Broker')
request = MetadataRequest_v1([])
wait_result = await owner.wait(
client.send(node_id, request),
timeout=timeout,
)
if wait_result.stopped:
owner.log.info(f'Shutting down - skipping creation.')
return None
response = wait_result.result
return response.controller_id
raise Exception(f'Controller node not found')
async def _really_create_topic(
self,
owner: Service,
client: aiokafka.AIOKafkaClient,
topic: str,
partitions: int,
replication: int,
*,
config: Mapping[str, Any] = None,
timeout: int = 30000,
retention: int = None,
compacting: bool = None,
deleting: bool = None,
ensure_created: bool = False) -> None: # pragma: no cover
owner.log.info('Creating topic %r', topic)
if topic in client.cluster.topics():
owner.log.debug('Topic %r exists, skipping creation.', topic)
return
protocol_version = 1
extra_configs = config or {}
config = self._topic_config(retention, compacting, deleting)
config.update(extra_configs)
controller_node = await self._get_controller_node(owner, client,
timeout=timeout)
owner.log.debug('Found controller: %r', controller_node)
if controller_node is None:
if owner.should_stop:
owner.log.info('Shutting down hence controller not found')
return
else:
raise Exception('Controller node is None')
request = CreateTopicsRequest[protocol_version](
[(topic, partitions, replication, [], list(config.items()))],
timeout,
False,
)
wait_result = await owner.wait(
client.send(controller_node, request),
timeout=timeout,
)
if wait_result.stopped:
owner.log.debug(f'Shutting down - skipping creation.')
return
response = wait_result.result
assert len(response.topic_error_codes), 'single topic'
_, code, reason = response.topic_error_codes[0]
if code != 0:
if not ensure_created and code == TopicExistsError.errno:
owner.log.debug(
'Topic %r exists, skipping creation.', topic)
return
elif code == NotControllerError.errno:
raise RuntimeError(f'Invalid controller: {controller_node}')
else:
raise for_code(code)(
f'Cannot create topic: {topic} ({code}): {reason}')
else:
owner.log.info('Topic %r created.', topic)
return
async def check_if_kafka_is_alive(my_client: AIOKafkaClient):
while 1:
conns: Dict[str,
AIOKafkaConnection] = my_client.__getattribute__('_conns')
print(my_client._bootstrap_servers)
print('Host = ', my_client.hosts)
if not is_connected(conns):
print('RENEW CONNECTION')
try:
my_client.__setattr__(
'cluster', ClusterMetadata(metadata_max_age_ms=300000))
my_client.__setattr__('_topics', set())
my_client.__setattr__('_conns', {})
my_client.__setattr__('_sync_task', None)
loop = asyncio.get_event_loop()
my_client.__setattr__('_md_update_fut', None)
my_client.__setattr__('_md_update_waiter',
loop.create_future())
my_client.__setattr__('_get_conn_lock',
asyncio.Lock(loop=loop))
await my_client.bootstrap()
except ConnectionError:
pass
else:
for conn in conns.items():
print(conn)
print(conn[1].connected())
try:
if not conn[1].connected():
print('TRY RE CONNECTION')
await conn[1].connect()
if not conn[1].connected():
print('RENEW CONNECTION')
await my_client.bootstrap()
except Exception as err:
print(err)
await asyncio.sleep(1)
async def _really_create_topic(self,
owner: Service,
client: aiokafka.AIOKafkaClient,
topic: str,
partitions: int,
replication: int,
*,
config: Mapping[str, Any] = None,
timeout: int = 30000,
retention: int = None,
compacting: bool = None,
deleting: bool = None,
ensure_created: bool = False) -> None:
owner.log.info(f'Creating topic {topic}')
protocol_version = 1
extra_configs = config or {}
config = self._topic_config(retention, compacting, deleting)
config.update(extra_configs)
# Create topic request needs to be sent to the kafka cluster controller
# Since aiokafka client doesn't currently support MetadataRequest
# version 1, client.controller will always be None. Hence we cycle
# through all brokers if we get Error 41 (not controller) until we
# hit the controller
nodes = [broker.nodeId for broker in client.cluster.brokers()]
owner.log.info(f'Nodes: {nodes}')
for node_id in nodes:
if node_id is None:
raise RuntimeError('Not connected to Kafka broker')
request = CreateTopicsRequest[protocol_version](
[(topic, partitions, replication, [], list(config.items()))],
timeout,
False,
)
wait_result = await owner.wait(
client.send(node_id, request),
timeout=timeout,
)
if wait_result.stopped:
owner.log.info(f'Shutting down - skipping creation.')
return
response = wait_result.result
assert len(response.topic_error_codes), 'single topic'
_, code, reason = response.topic_error_codes[0]
if code != 0:
if not ensure_created and code == TopicExistsError.errno:
owner.log.debug(
f'Topic {topic} exists, skipping creation.')
return
elif code == NotControllerError.errno:
owner.log.debug(f'Broker: {node_id} is not controller.')
continue
else:
raise for_code(code)(
f'Cannot create topic: {topic} ({code}): {reason}')
else:
owner.log.info(f'Topic {topic} created.')
return
raise Exception(f'No controller found among brokers: {nodes}')
