def state_event(self):
state_event = {
"flow_run_id": f"{self._flow_id}#{self._run_id}",
"state_id": uuid.uuid4().hex,
"state_type": self._state_type,
"state_timestamp": self._timestamp,
"state_properties": [
{
"property_name": "task_name",
"property_value": self._task_name
},
{
"property_name": "task_kwargs",
"property_value": ajson.dumps(self._task_kwargs)
}
]
}
if self._state_type == 'EventCompleted':
state_event['state_properties'].append({
"property_name": "task_results",
"property_value": ajson.dumps(self._results)
})
if self._state_type == 'EventFailed':
state_event['state_properties'].append({
"property_name": "fail_details",
"property_value": ajson.dumps(self._fail_details)
})
return state_event
def leech_integration_event(request):
params = request.param
event = _read_test_event(params[0])
event_string = ajson.dumps(event)
message_object = {'Message': event_string}
body_object = {'body': ajson.dumps(message_object)}
return {'Records': [body_object]}
def mark_event_failed(self, flow_id, run_id, task_name, task_kwargs,
failure_details):
state_properties = [{
'property_name': 'task_name',
'property_value': task_name
}, {
'property_name': 'task_kwargs',
'property_value': ajson.dumps(task_kwargs)
}, {
'property_name': 'failure_details',
'property_value': ajson.dumps(failure_details)
}]
state_timestamp = datetime.utcnow().timestamp()
self.add_state_event(flow_id, run_id, 'EventFailed', state_timestamp,
state_properties)
def potential_connections_unit_event(request):
params = request.param
schema_entry = _read_schema(params[1], params[2])
schema = _read_schema(params[1])
test_event = _read_test_event(params[0])
task_kwargs = ajson.loads(ajson.dumps(test_event))
task_kwargs.update({'schema': schema, 'schema_entry': schema_entry})
event = {
'task_name': 'derive_potential_connections',
'task_kwargs': task_kwargs
}
event_string = ajson.dumps(event)
message_object = {'Message': event_string}
body_object = {'body': ajson.dumps(message_object)}
return {'Records': [body_object]}
def mark_event_completed(self, flow_id, run_id, task_name, task_kwargs,
results):
state_properties = [{
'property_name': 'task_name',
'property_value': task_name
}, {
'property_name': 'task_kwargs',
'property_value': ajson.dumps(task_kwargs)
}, {
'property_name': 'task_results',
'property_value': ajson.dumps(results)
}]
state_timestamp = datetime.utcnow().timestamp()
self.add_state_event(flow_id, run_id, 'EventCompleted',
state_timestamp, state_properties)
def handler(event, context):
logging.info(
f'received a call to filter with the oyster, event: {event}, context: {context}'
)
_check_payload(event)
sapper = Sapper(event.get('state_gql_endpoint'))
flow_id, run_id = event['flow_id'], event['run_id']
history = sapper.retrieve_state_history(flow_id, run_id)
if history.is_completed:
logging.info(
f'completed a call to the oyster, event: {event}, task is completed, not fired'
)
return
if history.is_open:
if not history.is_timed_out:
logging.info(
f'completed a call to the oyster, event: {event}, task is open and not timed out, not fired'
)
return
if history.is_timed_out:
logging.info(f'task: {event}, has timed out, fire again')
sapper.mark_event_failed(failure_details={'cause': 'timed_out'},
**event)
if history.is_failed:
logging.info(f'task: {event} has failed, fire again')
task_manager = TaskManager(sapper=sapper, **event)
results = task_manager.send_task()
logging.info(
f'completed a call to filter with the oyster, event: {event}, context: {context}, results: {results}'
)
return ajson.dumps(results)
def test_add_vertex_response(self, db_add_vertex_response):
vertex = rapidjson.loads(rapidjson.dumps(db_add_vertex_response),
object_hook=TridentDecoder.object_hook)
vertex_string = ajson.dumps(vertex)
gql = rapidjson.loads(vertex_string,
object_hook=GqlDecoder.object_hook)
assert gql
def handler(event, context):
logging.info(
f'received a call to run a graph_object command: event/context: {event}/{context}'
)
_load_config(ENVIRON_VARIABLES)
if isinstance(event, list):
results = []
for entry in event:
result = handler(entry, context)
results.append(result)
return results
gql_context = event['context']
field_name = event['field_name']
type_name = event['type_name']
args = gql_context['arguments']
source = gql_context['source']
result = gql_context['result']
request = gql_context['request']
identity = gql_context['identity']
results = _decision_tree(type_name, field_name, args, source, result,
request, identity)
logging.debug(f'results after the decision tree: {results}')
strung_results = ajson.dumps(results)
logging.debug(
f'results after first round of json encoding: {strung_results}')
encoded_results = rapidjson.loads(strung_results,
object_hook=GqlDecoder.object_hook)
logging.info(f'results after GQL encoding: {encoded_results}')
return encoded_results
def send_task(self):
self._sapper.mark_event_started(**self.payload)
try:
response = self._fire_lambda()
response_payload = response['Payload'].read()
status_code = response['StatusCode']
if status_code != 200:
fn_error = response['FunctionError']
log_results = response['LogResult']
failure_details = {
'cause': fn_error,
'failure_logs': log_results,
'status_code': status_code,
'fn_response': response_payload
}
self._sapper.mark_event_failed(failure_details=failure_details,
**self.payload)
return
result = ajson.loads(response_payload)
self._sapper.mark_event_completed(results={'results': result},
**self.payload)
return result
except Exception as e:
failure_details = ajson.dumps({'cause': e.args})
self._sapper.mark_event_failed(failure_details=failure_details,
**self.payload)
def distill_documentation(encounter_vertex):
if isinstance(encounter_vertex, str):
encounter_vertex = rapidjson.loads(encounter_vertex)
vertex_properties = encounter_vertex['vertex_properties']
documentation = _retrieve_documentation(vertex_properties)
id_source = _find_vertex_property('id_source', 'local_properties',
vertex_properties)
encounter_id = int(encounter_vertex['id_value']['property_value'])
distilled_text = _distill_strings(documentation)
extracted_data = {
'object_type': 'DocumentationText',
'identifier': f'#{id_source}#Encounter#',
'id_value': encounter_id,
'extracted_data': {
'source': {
'id_source': id_source,
'encounter_id': encounter_id,
'utc_generated_datetime': datetime.utcnow().isoformat(),
'documentation_text': distilled_text
}
}
}
stored_data = StoredData.from_object(uuid.uuid4(),
extracted_data,
full_unpack=True)
return ajson.dumps(stored_data)
def test(self):
session = boto3.Session(profile_name='dev')
sns = session.client('sns')
listener_arn = 'arn:aws:sns:us-east-1:726075243133:leech-dev-Listener-1EV4D8VOW7L37'
bull_horn = Bullhorn(sns)
schema = Schema.retrieve()
schema_entry = schema['ExternalId']
message = ajson.dumps({
'task_name': 'generate_source_vertex',
'task_args': None,
'task_kwargs': {
'schema': schema,
'schema_entry': schema_entry,
'extracted_data': {
'source': {
'id_source': 'Algernon',
'id_type': 'Employees',
'id_name': 'emp_id',
'id_value': 1001
}
}
}
})
message_id = bull_horn.publish('new_event', listener_arn, message)
assert message_id
def publish_to_leech(bullhorn, flow_id, id_source, patient_id, provider_id,
encounter_id, encounter_text, encounter_datetime_in,
encounter_datetime_out, encounter_type, patient_last_name,
patient_first_name, patient_dob):
encounter_data = {
'source': {
'patient_id': patient_id,
'provider_id': provider_id,
'id_source': id_source,
'encounter_id': encounter_id,
'documentation': encounter_text,
'encounter_datetime_in': encounter_datetime_in,
'encounter_datetime_out': encounter_datetime_out,
'encounter_type': encounter_type
},
'patient_data': [{
'last_name': patient_last_name,
'first_name': patient_first_name,
'dob': patient_dob
}]
}
message = {
'task_name': 'leech',
'task_kwargs': {
'object_type': 'Encounter',
'extracted_data': encounter_data
},
'flow_id': f'{flow_id}#leech'
}
listener_arn = bullhorn.find_task_arn('leech')
strung_event = ajson.dumps(message)
return bullhorn.publish('new_event', listener_arn, strung_event)
def publish_to_incredible(bullhorn, flow_id, id_source, patient_id,
provider_id, encounter_id, encounter_datetime_in,
encounter_datetime_out, encounter_type,
patient_last_name, patient_first_name, patient_dob):
next_task_name = 'get_encounter'
encounter_data = {
'encounter_id': encounter_id,
'encounter_datetime_in': encounter_datetime_in,
'encounter_datetime_out': encounter_datetime_out,
'patient_id': patient_id,
'provider_id': provider_id,
'encounter_type': encounter_type,
'id_source': id_source,
'patient_last_name': patient_last_name,
'patient_first_name': patient_first_name,
'patient_dob': patient_dob
}
message = {
'task_name': next_task_name,
'task_kwargs': encounter_data,
'flow_id': f'{flow_id}#{patient_id}#{next_task_name}-{encounter_id}'
}
listener_arn = bullhorn.find_task_arn('credible_tasks')
strung_event = ajson.dumps(message)
return bullhorn.publish('new_event', listener_arn, strung_event)
def test_credential_to_json(self):
id_source = 'ICFS'
credentials = CredibleLoginCredentials.retrieve(id_source)
json_string = ajson.dumps(credentials)
assert json_string
rebuilt_credentials = ajson.loads(json_string)
assert isinstance(rebuilt_credentials, CredibleLoginCredentials)
assert rebuilt_credentials.validate()
def source_vertex_task_integration_event(request):
params = request.param
schema_entry = _read_schema(params[1], params[2])
schema = _read_schema(params[1])
extracted_data = _read_test_event(params[0])
event = {
'task_name': 'generate_source_vertex',
'task_kwargs': {
'schema': schema,
'schema_entry': schema_entry,
'extracted_data': extracted_data
}
}
event_string = ajson.dumps(event)
message_object = {'Message': event_string}
body_object = {'body': ajson.dumps(message_object)}
return {'Records': [body_object]}
def package(self) -> str:
unencrypted_text = ajson.dumps({
'pagination_id': self._pagination_id,
'inclusive_start': self._inclusive_start,
'page_size': self._page_size
})
return SneakyKipper('pagination').encrypt(unencrypted_text,
{'username': self._username})
def test_vertex_vertex_properties(self,
db_vertex_vertex_properties_response):
vertex_properties = rapidjson.loads(
rapidjson.dumps(db_vertex_vertex_properties_response),
object_hook=TridentDecoder.object_hook)
vertex_properties_string = ajson.dumps(vertex_properties)
gql = rapidjson.loads(vertex_properties_string,
object_hook=GqlDecoder.object_hook)
assert gql
def store(cls, object_data, **kwargs):
data_type = kwargs['data_type']
bucket_name = kwargs['bucket_name']
object_key = kwargs['object_key']
bucket = boto3.resource('s3').Bucket(bucket_name)
if not _check_for_object(bucket_name, object_key):
bucket.put_object(Key=object_key, Body=ajson.dumps(object_data))
storage_uri = f's3://{bucket_name}/{object_key}'
return cls(data_type, 's3', storage_uri)
def store_engine_data(bucket_name, file_key, engine_data):
s3 = boto3.resource('s3')
stored_object = s3.Object(bucket_name, file_key)
try:
stored_object.load()
raise RuntimeError(f'can not overwrite file at key: {file_key}')
except ClientError as e:
if e.response['Error']['Code'] != '404':
raise e
stored_object.put(Body=ajson.dumps(engine_data))
def handler(event, context):
logging.info(
f'received a call to run an auditor task, event/context: {event}/{context}'
)
task_name = event['task_name']
task_kwargs = event['task_kwargs']
flow_id = event['flow_id']
task_kwargs['flow_id'] = flow_id
task_fn = _find_task(task_name)
results = task_fn(**task_kwargs)
logging.info(f'completed an auditor task, results: {results}')
return ajson.dumps(results)
def generate_edge_integration_event(request):
params = request.param
schema_entry = _read_schema(params[1], params[2])
test_event = _read_test_event(params[0])
edge_type = params[3]
rule_entry = _generate_linking_rule(schema_entry, edge_type)
edge_schema_entry = _read_schema(params[1], edge_type)
task_kwargs = ajson.loads(ajson.dumps(test_event))
task_kwargs.update({
'schema_entry': edge_schema_entry,
'rule_entry': rule_entry
})
event = {
'task_name': 'generate_potential_edge',
'task_kwargs': task_kwargs,
'flow_id': 'some_flow_id'
}
event_string = ajson.dumps(event)
message_object = {'Message': event_string}
body_object = {'body': ajson.dumps(message_object)}
return {'Records': [body_object]}
def parse_batch_encounters(event, context):
_load_config(ENVIRON_VARIABLES)
migration_table_name = os.environ['PROGRESS_TABLE_NAME']
if 'Records' in event:
return _process_queued_parse_task(event, context)
event = rebuild_event(event)
parsed_events = _parse_batch_event(event)
_fire_hose_parsed_events(parsed_events)
for parsed_event in parsed_events:
_mark_parsing_complete(parsed_event, migration_table_name)
return ajson.dumps(
StoredData.from_object(uuid.uuid4(), parsed_events, full_unpack=True))
def find_existing_vertexes(request):
params = request.param
schema_entry = _read_schema(params[1], params[2])
schema = _read_schema(params[1])
test_event = _read_test_event(params[0])
task_kwargs = ajson.loads(ajson.dumps(test_event))
edge_type = params[3]
rule_entry = _generate_linking_rule(schema_entry, edge_type)
task_kwargs.update({
'schema': schema,
'schema_entry': schema_entry,
'rule_entry': rule_entry
})
event = {
'task_name': 'check_for_existing_vertexes',
'task_kwargs': task_kwargs
}
event_string = ajson.dumps(event)
message_object = {'Message': event_string}
body_object = {'body': ajson.dumps(message_object)}
return {'Records': [body_object]}
def handler(event, context):
event = rebuild_event(event)
logging.info(
f'received a call to run a query_s3_csv command: event/context: {event}/{context}'
)
function_payload = {
'bucket_name': event['bucket_name'],
'file_key': event['file_key'],
'expression': event['expression']
}
results = query_s3_csv(**function_payload)
logging.info(
f'completed a call to run a query_s3_csv command: results: {results}')
return ajson.dumps(results)
def source_vertex_task_deployment_event(request):
params = request.param
schema_entry = _read_schema(params[1], params[2])
schema = _read_schema(params[1])
extracted_data = _read_test_event(params[0])
event = {
'task_name': 'generate_source_vertex',
'task_kwargs': {
'schema': schema,
'schema_entry': schema_entry,
'extracted_data': extracted_data
}
}
event_string = ajson.dumps(event)
return event_string
def _mark_parsing_complete(parsed_event, table_name):
resource = boto3.resource('dynamodb')
table = resource.Table(table_name)
table.update_item(Key={
'identifier': parsed_event['identifier'],
'id_value': int(parsed_event['id_value'])
},
UpdateExpression='SET #x=:x',
ExpressionAttributeNames={'#x': 'extraction'},
ExpressionAttributeValues={
':x': {
'completed_at': datetime.now().isoformat(),
'stage_results': ajson.dumps(parsed_event)
}
})
def _generate_task_manager(**kwargs):
payload_mock = MagicMock(name='payload')
payload_mock.read.return_value = ajson.dumps(kwargs.get('payload', {'some_result': 'here'}))
response = {
'Payload': payload_mock,
'StatusCode': kwargs.get('status_code', 200)
}
if kwargs.get('function_error'):
response['FunctionError'] = kwargs.get('function_error')
if kwargs.get('log_result'):
response['LogResult'] = kwargs.get('log_result')
retrieve_patch_obj = patch('toll_booth.obj.shucker.TaskManager._fire_lambda')
patches.append(retrieve_patch_obj)
mock_obj = retrieve_patch_obj.start()
mock_obj.return_value = response
return mock_obj
def generate_extraction_config(
id_source: str,
source_name: str,
extraction_type: str,
extraction_params: Mapping = None
):
source_config, extraction_config = config_tasks.build_configs(
id_source=id_source,
source_name=source_name,
extraction_type=extraction_type,
extraction_params=extraction_params
)
results = {
'source_config': source_config,
'extraction_config': extraction_config
}
return rapidjson.loads(ajson.dumps(results))
def get_encounter_text(encounter_id: str,
driver: CredibleFrontEndDriver,
next_task_name: str = None,
**kwargs):
encounter_text = driver.retrieve_client_encounter(encounter_id)
if next_task_name:
bullhorn = kwargs['bullhorn']
topic_arn = bullhorn.find_task_arn(next_task_name)
msg = {
'task_name': next_task_name,
'task_kwargs': {
'id_source': kwargs.get('id_source'),
'encounter_id': encounter_id,
'documentation': encounter_text
}
}
bullhorn.publish('new_event', topic_arn, ajson.dumps(msg))
return encounter_text
def _publish_documentation_field_node(encounter_id, id_source, parser_id, field_name, field_documentation, bullhorn):
task_name = 'leech'
leech_arn = bullhorn.find_task_arn(task_name)
documentation_field_node = {
'encounter_id': encounter_id,
'id_source': id_source,
'parser_id': parser_id,
'field_name': field_name,
'field_documentation': field_documentation
}
message = {
'task_name': task_name,
'task_kwargs': {
'object_type': 'DocumentationField',
'extracted_data': {'source': documentation_field_node}
}
}
strung_event = ajson.dumps(message)
bullhorn.publish('new_event', leech_arn, strung_event)
