async def configure_internal_integrations() -> None:
"""
Configure internal integrations to support:
- Kafka
- NATS Messaging/Jetstream
"""
get_kafka_producer()
await get_nats_client()
await get_jetstream_context()
await create_nats_subscribers()
create_kafka_listeners()
async def do_retransmit(message: dict, queue_pos: int):
"""
Process messages from NATS or the nats_retransmit_queue.
:param message: the LFH message containing the data to retransmit
:param queue_pos: the position of the message in the queue. queue_pos will be -1 if
the message has not yet been queued or 0 <= queue_pos < len(nats_retransmit_queue).
"""
global nats_retransmit_queue
settings = get_settings()
max_retries = settings.nats_retransmit_max_retries
kafka_producer = get_kafka_producer()
resource = decode_to_dict(message["data"])
if "retransmit_count" not in message:
message["retransmit_count"] = 0
message["retransmit_count"] += 1
target_endpoint_url = message["target_endpoint_urls"][0]
try:
# attempt to retransmit the message
logger.trace(
f"do_retransmit #{message['retransmit_count']}: retransmitting to: {target_endpoint_url}"
)
async with AsyncClient(verify=settings.certificate_verify) as client:
if message["operation"] == "POST":
await client.post(target_endpoint_url, json=resource)
elif message["operation"] == "PUT":
await client.put(target_endpoint_url, json=resource)
elif message["operation"] == "PATCH":
await client.patch(target_endpoint_url, json=resource)
# if the message came from the retransmit queue, remove it
if not queue_pos == -1:
nats_retransmit_queue.pop(queue_pos)
message["status"] = "SUCCESS"
logger.trace(
f"do_retransmit: successfully retransmitted message with id {message['uuid']} "
+ f"after {message['retransmit_count']} retries")
except Exception as ex:
logger.trace(f"do_retransmit: exception {type(ex)}")
if queue_pos == -1:
nats_retransmit_queue.append(message)
logger.trace(f"do_retransmit: queued message for retransmitter()")
if not max_retries == -1 and message["retransmit_count"] >= max_retries:
nats_retransmit_queue.pop(queue_pos)
message["status"] = "FAILED"
logger.trace(
f"do_retransmit: failed retransmit of message with id {message['uuid']} "
+ f"after {message['retransmit_count']} retries")
# send outcome to kafka
if message["status"] == "SUCCESS" or message["status"] == "FAILED":
transmit_delta = datetime.now() - datetime.strptime(
message["transmit_start"], "%Y-%m-%dT%H:%M:%S.%f")
message["elapsed_transmit_time"] = transmit_delta.total_seconds()
message["elapsed_total_time"] += transmit_delta.total_seconds()
await kafka_producer.produce("RETRANSMIT",
json.dumps(message, cls=ConnectEncoder))
logger.trace(f"do_retransmit: sent message to kafka topic RETRANSMIT")
async def close_internal_clients() -> None:
"""
Closes internal Connect client connections:
- Kafka
- NATS
"""
logger.info("Shutting down internal clients")
try:
kafka_producer = get_kafka_producer()
kafka_producer.close()
stop_kafka_listeners()
await stop_nats_clients()
except CancelledError:
pass
async def nats_sync_event_handler(msg: Msg):
"""
Callback for NATS 'nats_sync_subject' messages
"""
subject = msg.subject
reply = msg.reply
data = msg.data.decode()
logger.debug(
f'nats_sync_event_handler: received a message on {subject} {reply}: {data}'
)
# if the message is from our local LFH, don't store in kafka
message = json.loads(data)
if (get_settings().lfh_id == message['lfh_id']):
logger.debug(
'nats_sync_event_handler: detected local LFH message, not storing in kafka'
)
return
# store the message in kafka
kafka_producer = get_kafka_producer()
kafka_cb = KafkaCallback()
await kafka_producer.produce_with_callback(
kafka_sync_topic, data, on_delivery=kafka_cb.get_kafka_result)
logger.debug(
f'nats_sync_event_handler: stored msg in kafka topic {kafka_sync_topic} at {kafka_cb.kafka_result}'
)
# process the message into the local store
settings = get_settings()
msg_data = decode_to_dict(message['data'])
workflow = core.CoreWorkflow(
message=msg_data,
origin_url=message['consuming_endpoint_url'],
certificate_verify=settings.certificate_verify,
lfh_id=message['lfh_id'],
data_format=message['data_format'],
transmit_server=None,
do_sync=False)
result = await workflow.run(None)
location = result['data_record_location']
logger.debug(
f'nats_sync_event_handler: replayed nats sync message, data record location = {location}'
)
async def error(self, error) -> str:
"""
On error, store the error message and the current message in
Kafka for persistence and further error handling.
Input:
self.message: The python dict for the current message being processed
:param error: The error message tp be stored in kafka
:return: The json string for the error message stored in Kafka
"""
logger.debug(
f'{self.__class__.__name__} error: incoming error = {error}')
data_str = json.dumps(self.message, cls=ConnectEncoder)
data = json.loads(data_str)
message = {
'uuid': uuid.uuid4(),
'error_date': datetime.utcnow().replace(microsecond=0),
'error_msg': str(error),
'data': data
}
error = LFHError(**message)
kafka_producer = get_kafka_producer()
kafka_cb = KafkaCallback()
await kafka_producer.produce_with_callback(
self.lfh_exception_topic,
error.json(),
on_delivery=kafka_cb.get_kafka_result)
logger.debug(
f'{self.__class__.__name__} error: stored resource location = {kafka_cb.kafka_result}'
)
message['data_record_location'] = kafka_cb.kafka_result
error = LFHError(**message).json()
logger.debug(
f'{self.__class__.__name__} error: outgoing message = {error}')
return error
async def nats_sync_event_handler(msg: Msg):
"""
Callback for NATS 'nats_sync_subject' messages
:param msg: a message delivered from the NATS server
"""
subject = msg.subject
reply = msg.reply
data = msg.data.decode()
message = json.loads(data)
logger.trace(
f"nats_sync_event_handler: received a message with id={message['uuid']} on {subject} {reply}"
)
response = await msg.ack_sync()
logger.trace(f"nats_sync_event_handler: ack response={response}")
# Emit an app_sync message so LFH clients that are listening only for
# messages from this LFH node will be able to get all sync'd messages
# from all LFH nodes.
js = await get_jetstream_context()
await js.publish(nats_app_sync_subject, msg.data)
# if the message is from our local LFH, don't store in kafka
if get_settings().connect_lfh_id == message["lfh_id"]:
logger.trace(
"nats_sync_event_handler: detected local LFH message, not storing in kafka",
)
return
# store the message in kafka
kafka_producer = get_kafka_producer()
kafka_cb = KafkaCallback()
await kafka_producer.produce_with_callback(
kafka_sync_topic, data, on_delivery=kafka_cb.get_kafka_result)
logger.trace(
f"nats_sync_event_handler: stored msg in kafka topic {kafka_sync_topic} at {kafka_cb.kafka_result}",
)
# set up transmit servers, if defined
transmit_servers = []
settings = get_settings()
if message["data_format"].startswith("FHIR-R4_"):
for s in settings.connect_external_fhir_servers:
if settings.connect_generate_fhir_server_url:
origin_url_elements = message["consuming_endpoint_url"].split(
"/")
resource_type = origin_url_elements[len(origin_url_elements) -
1]
transmit_servers.append(f"{s}/{resource_type}")
else:
transmit_servers.append(s)
# perform message type-specific decoding
if message["data_format"].startswith("X12_"):
msg_data = decode_to_str(message["data"])
else:
msg_data = decode_to_dict(message["data"])
# process the message into the local store
workflow = core.CoreWorkflow(
message=msg_data,
origin_url=message["consuming_endpoint_url"],
certificate_verify=settings.certificate_verify,
data_format=message["data_format"],
lfh_id=message["lfh_id"],
transmit_servers=transmit_servers,
do_sync=False,
operation=message["operation"],
do_retransmit=settings.nats_enable_retransmit,
)
result = await workflow.run()
logger.trace(
f"nats_sync_event_handler: successfully replayed nats sync message with id={message['uuid']}"
)
async def persist(self):
"""
Store the message in Kafka for persistence after converting it to the LinuxForHealth
message format.
Input:
self.message: The object to be stored in Kafka
self.origin_url: The originating endpoint url
self.data_format: The data_format of the data being stored
self.start_time: The transaction start time
Output:
self.message: The python dict for LinuxForHealthDataRecordResponse instance with
the original object instance in the data field as a byte string
"""
logger.debug(
f'{self.__class__.__name__}: incoming message = {self.message}')
logger.debug(
f'{self.__class__.__name__}: incoming message type = {type(self.message)}'
)
if hasattr(self.message, 'dict'):
encoded_data = encode_from_dict(self.message.dict())
elif isinstance(self.message, dict):
encoded_data = encode_from_dict(self.message)
else:
encoded_data = encode_from_str(self.message)
message = {
'uuid': str(uuid.uuid4()),
'lfh_id': self.lfh_id,
'creation_date':
str(datetime.utcnow().replace(microsecond=0)) + 'Z',
'store_date': str(datetime.utcnow().replace(microsecond=0)) + 'Z',
'consuming_endpoint_url': self.origin_url,
'data_format': self.data_format,
'data': encoded_data,
'target_endpoint_url': self.transmit_server
}
response = LinuxForHealthDataRecordResponse(**message)
kafka_producer = get_kafka_producer()
kafka_cb = KafkaCallback()
storage_start = datetime.now()
await kafka_producer.produce_with_callback(
self.data_format,
response.json(),
on_delivery=kafka_cb.get_kafka_result)
storage_delta = datetime.now() - storage_start
logger.debug(
f' {self.__class__.__name__} persist: stored resource location = {kafka_cb.kafka_result}'
)
total_time = datetime.utcnow() - self.start_time
message['elapsed_storage_time'] = storage_delta.total_seconds()
message['elapsed_total_time'] = total_time.total_seconds()
message['data_record_location'] = kafka_cb.kafka_result
message['status'] = kafka_cb.kafka_status
response = LinuxForHealthDataRecordResponse(**message).dict()
logger.debug(
f'{self.__class__.__name__} persist: outgoing message = {response}'
)
self.message = response