async def send_confirmation(request_id, status): # pragma: no cover
"""
Send kafka validation message to Insights Upload service.
When a new file lands for topic 'hccm' we must validate it
so that it will be made perminenantly available to other
apps listening on the 'platform.upload.available' topic.
Args:
request_id (String): Request ID for file being confirmed.
status (String): Either 'success' or 'failure'
Returns:
None
"""
producer = AIOKafkaProducer(
loop=EVENT_LOOP, bootstrap_servers=Config.INSIGHTS_KAFKA_ADDRESS)
try:
await producer.start()
except (KafkaError, TimeoutError) as err:
await producer.stop()
LOG.exception(str(err))
KAFKA_CONNECTION_ERRORS_COUNTER.inc()
raise KafkaMsgHandlerError(
'Unable to connect to kafka server. Closing producer.')
try:
validation = {'request_id': request_id, 'validation': status}
msg = bytes(json.dumps(validation), 'utf-8')
LOG.info('Validating message: %s', str(msg))
await producer.send_and_wait(VALIDATION_TOPIC, msg)
LOG.info('Validating message complete.')
finally:
await producer.stop()
async def listen_for_messages(consumer): # pragma: no cover
"""
Listen for messages on the available and hccm topics.
Once a message from one of these topics arrives, we add
them to the MSG_PENDING_QUEUE.
Args:
None
Returns:
None
"""
try:
await consumer.start()
except KafkaError as err:
await consumer.stop()
LOG.exception(str(err))
KAFKA_CONNECTION_ERRORS_COUNTER.inc()
raise KafkaMsgHandlerError('Unable to connect to kafka server.')
LOG.info('Listener started. Waiting for messages...')
try:
# Consume messages
async for msg in consumer:
await MSG_PENDING_QUEUE.put(msg)
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
async def listen_for_messages(consumer):
"""
Listen for messages on the hccm topic.
Once a message from one of these topics arrives, we add
them extract the payload and line item process the report files.
Once all files from the manifest are complete a celery job is
dispatched to the worker to complete summary processing for the manifest.
Several exceptions can occur while listening for messages:
Database Errors - Re-processing attempts will be made until successful.
Internal Errors - Re-processing attempts will be made until successful.
Report Processing Errors - Kafka message will be committed with an error.
Errors of this type would require a report processor
fix and we do not want to block the message queue.
Upon successful processing the kafka message is manually committed. Manual
commits are used so we can use the message queue to store unprocessed messages
to make the service more tolerant of SIGTERM events.
Args:
consumer - (AIOKafkaConsumer): kafka consumer for HCCM ingress topic.
Returns:
None
"""
LOG.info("Kafka consumer starting...")
await consumer.start()
LOG.info("Listener started. Waiting for messages...")
try:
async for msg in consumer:
process_complete = False
try:
process_complete = await process_messages(msg)
if process_complete:
await consumer.commit()
except (InterfaceError, OperationalError, ReportProcessorDBError) as err:
connection.close()
LOG.error(f"[listen_for_messages] database error. Seeking to committed. Error: {str(err)}")
await asyncio.sleep(Config.RETRY_SECONDS)
await consumer.seek_to_committed()
except KafkaMsgHandlerError as error:
LOG.error(f"[listen_for_messages] internal error. Seeking to committed. Error: {str(error)}")
await asyncio.sleep(Config.RETRY_SECONDS)
await consumer.seek_to_committed()
except ReportProcessorError as error:
LOG.error(f"Report processing error: {str(error)}")
await consumer.commit()
except KafkaError as error:
LOG.error(f"[listen_for_messages] Kafka error encountered: {type(error).__name__}: {error}", exc_info=True)
KAFKA_CONNECTION_ERRORS_COUNTER.inc()
except Exception as error:
traceback.print_exc()
LOG.error(f"[listen_for_messages] UNKNOWN error encountered: {type(error).__name__}: {error}", exc_info=True)
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
def is_kafka_connected(host, port):
"""Wait for Kafka to become available."""
count = 0
result = False
while not result:
result = check_kafka_connection(host, port)
if result:
LOG.info("Test connection to Kafka was successful.")
else:
LOG.error(f"Unable to connect to Kafka server: {host}:{port}")
KAFKA_CONNECTION_ERRORS_COUNTER.inc()
backoff(count)
count += 1
return result
def listen_for_messages_loop():
"""Wrap listen_for_messages in while true."""
consumer = get_consumer()
LOG.info("Consumer is listening for messages...")
for _ in itertools.count(): # equivalent to while True, but mockable
msg = consumer.poll(timeout=1.0)
if msg is None:
continue
if msg.error():
KAFKA_CONNECTION_ERRORS_COUNTER.inc()
LOG.error(f"[listen_for_messages_loop] consumer.poll message: {msg}. Error: {msg.error()}")
continue
listen_for_messages(msg, consumer)
def check_kafka_connection(): # pragma: no cover
"""
Check connectability to Kafka messenger.
This method runs when asyncio_sources_thread is initialized. It
creates a temporary thread and consumer. The consumer is started
to check our connection to Kafka. If the consumer starts successfully,
then Kafka is running. The consumer is stopped and the function
returns. If there is no Kafka connection, the consumer.start() will
fail, raising an exception. The function will retry to start the
consumer, and will continue until a connection is possible.
This method will block sources integration initialization until
Kafka is connected.
"""
async def test_consumer(consumer, method):
started = None
if method == "start":
await consumer.start()
started = True
else:
await consumer.stop()
return started
count = 0
result = False
temp_loop = asyncio.new_event_loop()
consumer = AIOKafkaConsumer(loop=temp_loop,
bootstrap_servers=Config.SOURCES_KAFKA_ADDRESS,
group_id=None)
while not result:
try:
result = temp_loop.run_until_complete(
test_consumer(consumer, "start"))
LOG.info(f"Test consumer connection to Kafka was successful.")
break
except KafkaError as err:
LOG.error(f"Unable to connect to Kafka server. Error: {err}")
KAFKA_CONNECTION_ERRORS_COUNTER.inc()
backoff(count)
count += 1
finally:
temp_loop.run_until_complete(test_consumer(
consumer, "stop")) # stop any consumers started
temp_loop.stop() # loop must be stopped before calling .close()
temp_loop.close() # eliminate the temporary loop
return result
def is_kafka_connected(): # pragma: no cover
"""
Check connectability to Kafka messenger.
This method will block sources integration initialization until
Kafka is connected.
"""
count = 0
result = False
while not result:
result = check_kafka_connection()
if result:
LOG.info("Test connection to Kafka was successful.")
else:
LOG.error("Unable to connect to Kafka server.")
KAFKA_CONNECTION_ERRORS_COUNTER.inc()
backoff(count)
count += 1
return result