async def configure_internal_integrations() -> None: """ Configure internal integrations to support: - Kafka - NATS Messaging/Jetstream """ get_kafka_producer() await get_nats_client() await get_jetstream_context() await create_nats_subscribers() create_kafka_listeners()
async def do_retransmit(message: dict, queue_pos: int): """ Process messages from NATS or the nats_retransmit_queue. :param message: the LFH message containing the data to retransmit :param queue_pos: the position of the message in the queue. queue_pos will be -1 if the message has not yet been queued or 0 <= queue_pos < len(nats_retransmit_queue). """ global nats_retransmit_queue settings = get_settings() max_retries = settings.nats_retransmit_max_retries kafka_producer = get_kafka_producer() resource = decode_to_dict(message["data"]) if "retransmit_count" not in message: message["retransmit_count"] = 0 message["retransmit_count"] += 1 target_endpoint_url = message["target_endpoint_urls"][0] try: # attempt to retransmit the message logger.trace( f"do_retransmit #{message['retransmit_count']}: retransmitting to: {target_endpoint_url}" ) async with AsyncClient(verify=settings.certificate_verify) as client: if message["operation"] == "POST": await client.post(target_endpoint_url, json=resource) elif message["operation"] == "PUT": await client.put(target_endpoint_url, json=resource) elif message["operation"] == "PATCH": await client.patch(target_endpoint_url, json=resource) # if the message came from the retransmit queue, remove it if not queue_pos == -1: nats_retransmit_queue.pop(queue_pos) message["status"] = "SUCCESS" logger.trace( f"do_retransmit: successfully retransmitted message with id {message['uuid']} " + f"after {message['retransmit_count']} retries") except Exception as ex: logger.trace(f"do_retransmit: exception {type(ex)}") if queue_pos == -1: nats_retransmit_queue.append(message) logger.trace(f"do_retransmit: queued message for retransmitter()") if not max_retries == -1 and message["retransmit_count"] >= max_retries: nats_retransmit_queue.pop(queue_pos) message["status"] = "FAILED" logger.trace( f"do_retransmit: failed retransmit of message with id {message['uuid']} " + f"after {message['retransmit_count']} retries") # send outcome to kafka if message["status"] == "SUCCESS" or message["status"] == "FAILED": transmit_delta = datetime.now() - datetime.strptime( message["transmit_start"], "%Y-%m-%dT%H:%M:%S.%f") message["elapsed_transmit_time"] = transmit_delta.total_seconds() message["elapsed_total_time"] += transmit_delta.total_seconds() await kafka_producer.produce("RETRANSMIT", json.dumps(message, cls=ConnectEncoder)) logger.trace(f"do_retransmit: sent message to kafka topic RETRANSMIT")
async def close_internal_clients() -> None: """ Closes internal Connect client connections: - Kafka - NATS """ logger.info("Shutting down internal clients") try: kafka_producer = get_kafka_producer() kafka_producer.close() stop_kafka_listeners() await stop_nats_clients() except CancelledError: pass
async def nats_sync_event_handler(msg: Msg): """ Callback for NATS 'nats_sync_subject' messages """ subject = msg.subject reply = msg.reply data = msg.data.decode() logger.debug( f'nats_sync_event_handler: received a message on {subject} {reply}: {data}' ) # if the message is from our local LFH, don't store in kafka message = json.loads(data) if (get_settings().lfh_id == message['lfh_id']): logger.debug( 'nats_sync_event_handler: detected local LFH message, not storing in kafka' ) return # store the message in kafka kafka_producer = get_kafka_producer() kafka_cb = KafkaCallback() await kafka_producer.produce_with_callback( kafka_sync_topic, data, on_delivery=kafka_cb.get_kafka_result) logger.debug( f'nats_sync_event_handler: stored msg in kafka topic {kafka_sync_topic} at {kafka_cb.kafka_result}' ) # process the message into the local store settings = get_settings() msg_data = decode_to_dict(message['data']) workflow = core.CoreWorkflow( message=msg_data, origin_url=message['consuming_endpoint_url'], certificate_verify=settings.certificate_verify, lfh_id=message['lfh_id'], data_format=message['data_format'], transmit_server=None, do_sync=False) result = await workflow.run(None) location = result['data_record_location'] logger.debug( f'nats_sync_event_handler: replayed nats sync message, data record location = {location}' )
async def error(self, error) -> str: """ On error, store the error message and the current message in Kafka for persistence and further error handling. Input: self.message: The python dict for the current message being processed :param error: The error message tp be stored in kafka :return: The json string for the error message stored in Kafka """ logger.debug( f'{self.__class__.__name__} error: incoming error = {error}') data_str = json.dumps(self.message, cls=ConnectEncoder) data = json.loads(data_str) message = { 'uuid': uuid.uuid4(), 'error_date': datetime.utcnow().replace(microsecond=0), 'error_msg': str(error), 'data': data } error = LFHError(**message) kafka_producer = get_kafka_producer() kafka_cb = KafkaCallback() await kafka_producer.produce_with_callback( self.lfh_exception_topic, error.json(), on_delivery=kafka_cb.get_kafka_result) logger.debug( f'{self.__class__.__name__} error: stored resource location = {kafka_cb.kafka_result}' ) message['data_record_location'] = kafka_cb.kafka_result error = LFHError(**message).json() logger.debug( f'{self.__class__.__name__} error: outgoing message = {error}') return error
async def nats_sync_event_handler(msg: Msg): """ Callback for NATS 'nats_sync_subject' messages :param msg: a message delivered from the NATS server """ subject = msg.subject reply = msg.reply data = msg.data.decode() message = json.loads(data) logger.trace( f"nats_sync_event_handler: received a message with id={message['uuid']} on {subject} {reply}" ) response = await msg.ack_sync() logger.trace(f"nats_sync_event_handler: ack response={response}") # Emit an app_sync message so LFH clients that are listening only for # messages from this LFH node will be able to get all sync'd messages # from all LFH nodes. js = await get_jetstream_context() await js.publish(nats_app_sync_subject, msg.data) # if the message is from our local LFH, don't store in kafka if get_settings().connect_lfh_id == message["lfh_id"]: logger.trace( "nats_sync_event_handler: detected local LFH message, not storing in kafka", ) return # store the message in kafka kafka_producer = get_kafka_producer() kafka_cb = KafkaCallback() await kafka_producer.produce_with_callback( kafka_sync_topic, data, on_delivery=kafka_cb.get_kafka_result) logger.trace( f"nats_sync_event_handler: stored msg in kafka topic {kafka_sync_topic} at {kafka_cb.kafka_result}", ) # set up transmit servers, if defined transmit_servers = [] settings = get_settings() if message["data_format"].startswith("FHIR-R4_"): for s in settings.connect_external_fhir_servers: if settings.connect_generate_fhir_server_url: origin_url_elements = message["consuming_endpoint_url"].split( "/") resource_type = origin_url_elements[len(origin_url_elements) - 1] transmit_servers.append(f"{s}/{resource_type}") else: transmit_servers.append(s) # perform message type-specific decoding if message["data_format"].startswith("X12_"): msg_data = decode_to_str(message["data"]) else: msg_data = decode_to_dict(message["data"]) # process the message into the local store workflow = core.CoreWorkflow( message=msg_data, origin_url=message["consuming_endpoint_url"], certificate_verify=settings.certificate_verify, data_format=message["data_format"], lfh_id=message["lfh_id"], transmit_servers=transmit_servers, do_sync=False, operation=message["operation"], do_retransmit=settings.nats_enable_retransmit, ) result = await workflow.run() logger.trace( f"nats_sync_event_handler: successfully replayed nats sync message with id={message['uuid']}" )
async def persist(self): """ Store the message in Kafka for persistence after converting it to the LinuxForHealth message format. Input: self.message: The object to be stored in Kafka self.origin_url: The originating endpoint url self.data_format: The data_format of the data being stored self.start_time: The transaction start time Output: self.message: The python dict for LinuxForHealthDataRecordResponse instance with the original object instance in the data field as a byte string """ logger.debug( f'{self.__class__.__name__}: incoming message = {self.message}') logger.debug( f'{self.__class__.__name__}: incoming message type = {type(self.message)}' ) if hasattr(self.message, 'dict'): encoded_data = encode_from_dict(self.message.dict()) elif isinstance(self.message, dict): encoded_data = encode_from_dict(self.message) else: encoded_data = encode_from_str(self.message) message = { 'uuid': str(uuid.uuid4()), 'lfh_id': self.lfh_id, 'creation_date': str(datetime.utcnow().replace(microsecond=0)) + 'Z', 'store_date': str(datetime.utcnow().replace(microsecond=0)) + 'Z', 'consuming_endpoint_url': self.origin_url, 'data_format': self.data_format, 'data': encoded_data, 'target_endpoint_url': self.transmit_server } response = LinuxForHealthDataRecordResponse(**message) kafka_producer = get_kafka_producer() kafka_cb = KafkaCallback() storage_start = datetime.now() await kafka_producer.produce_with_callback( self.data_format, response.json(), on_delivery=kafka_cb.get_kafka_result) storage_delta = datetime.now() - storage_start logger.debug( f' {self.__class__.__name__} persist: stored resource location = {kafka_cb.kafka_result}' ) total_time = datetime.utcnow() - self.start_time message['elapsed_storage_time'] = storage_delta.total_seconds() message['elapsed_total_time'] = total_time.total_seconds() message['data_record_location'] = kafka_cb.kafka_result message['status'] = kafka_cb.kafka_status response = LinuxForHealthDataRecordResponse(**message).dict() logger.debug( f'{self.__class__.__name__} persist: outgoing message = {response}' ) self.message = response