def state_event(self): state_event = { "flow_run_id": f"{self._flow_id}#{self._run_id}", "state_id": uuid.uuid4().hex, "state_type": self._state_type, "state_timestamp": self._timestamp, "state_properties": [ { "property_name": "task_name", "property_value": self._task_name }, { "property_name": "task_kwargs", "property_value": ajson.dumps(self._task_kwargs) } ] } if self._state_type == 'EventCompleted': state_event['state_properties'].append({ "property_name": "task_results", "property_value": ajson.dumps(self._results) }) if self._state_type == 'EventFailed': state_event['state_properties'].append({ "property_name": "fail_details", "property_value": ajson.dumps(self._fail_details) }) return state_event
def leech_integration_event(request): params = request.param event = _read_test_event(params[0]) event_string = ajson.dumps(event) message_object = {'Message': event_string} body_object = {'body': ajson.dumps(message_object)} return {'Records': [body_object]}
def mark_event_failed(self, flow_id, run_id, task_name, task_kwargs, failure_details): state_properties = [{ 'property_name': 'task_name', 'property_value': task_name }, { 'property_name': 'task_kwargs', 'property_value': ajson.dumps(task_kwargs) }, { 'property_name': 'failure_details', 'property_value': ajson.dumps(failure_details) }] state_timestamp = datetime.utcnow().timestamp() self.add_state_event(flow_id, run_id, 'EventFailed', state_timestamp, state_properties)
def potential_connections_unit_event(request): params = request.param schema_entry = _read_schema(params[1], params[2]) schema = _read_schema(params[1]) test_event = _read_test_event(params[0]) task_kwargs = ajson.loads(ajson.dumps(test_event)) task_kwargs.update({'schema': schema, 'schema_entry': schema_entry}) event = { 'task_name': 'derive_potential_connections', 'task_kwargs': task_kwargs } event_string = ajson.dumps(event) message_object = {'Message': event_string} body_object = {'body': ajson.dumps(message_object)} return {'Records': [body_object]}
def mark_event_completed(self, flow_id, run_id, task_name, task_kwargs, results): state_properties = [{ 'property_name': 'task_name', 'property_value': task_name }, { 'property_name': 'task_kwargs', 'property_value': ajson.dumps(task_kwargs) }, { 'property_name': 'task_results', 'property_value': ajson.dumps(results) }] state_timestamp = datetime.utcnow().timestamp() self.add_state_event(flow_id, run_id, 'EventCompleted', state_timestamp, state_properties)
def handler(event, context): logging.info( f'received a call to filter with the oyster, event: {event}, context: {context}' ) _check_payload(event) sapper = Sapper(event.get('state_gql_endpoint')) flow_id, run_id = event['flow_id'], event['run_id'] history = sapper.retrieve_state_history(flow_id, run_id) if history.is_completed: logging.info( f'completed a call to the oyster, event: {event}, task is completed, not fired' ) return if history.is_open: if not history.is_timed_out: logging.info( f'completed a call to the oyster, event: {event}, task is open and not timed out, not fired' ) return if history.is_timed_out: logging.info(f'task: {event}, has timed out, fire again') sapper.mark_event_failed(failure_details={'cause': 'timed_out'}, **event) if history.is_failed: logging.info(f'task: {event} has failed, fire again') task_manager = TaskManager(sapper=sapper, **event) results = task_manager.send_task() logging.info( f'completed a call to filter with the oyster, event: {event}, context: {context}, results: {results}' ) return ajson.dumps(results)
def test_add_vertex_response(self, db_add_vertex_response): vertex = rapidjson.loads(rapidjson.dumps(db_add_vertex_response), object_hook=TridentDecoder.object_hook) vertex_string = ajson.dumps(vertex) gql = rapidjson.loads(vertex_string, object_hook=GqlDecoder.object_hook) assert gql
def handler(event, context): logging.info( f'received a call to run a graph_object command: event/context: {event}/{context}' ) _load_config(ENVIRON_VARIABLES) if isinstance(event, list): results = [] for entry in event: result = handler(entry, context) results.append(result) return results gql_context = event['context'] field_name = event['field_name'] type_name = event['type_name'] args = gql_context['arguments'] source = gql_context['source'] result = gql_context['result'] request = gql_context['request'] identity = gql_context['identity'] results = _decision_tree(type_name, field_name, args, source, result, request, identity) logging.debug(f'results after the decision tree: {results}') strung_results = ajson.dumps(results) logging.debug( f'results after first round of json encoding: {strung_results}') encoded_results = rapidjson.loads(strung_results, object_hook=GqlDecoder.object_hook) logging.info(f'results after GQL encoding: {encoded_results}') return encoded_results
def send_task(self): self._sapper.mark_event_started(**self.payload) try: response = self._fire_lambda() response_payload = response['Payload'].read() status_code = response['StatusCode'] if status_code != 200: fn_error = response['FunctionError'] log_results = response['LogResult'] failure_details = { 'cause': fn_error, 'failure_logs': log_results, 'status_code': status_code, 'fn_response': response_payload } self._sapper.mark_event_failed(failure_details=failure_details, **self.payload) return result = ajson.loads(response_payload) self._sapper.mark_event_completed(results={'results': result}, **self.payload) return result except Exception as e: failure_details = ajson.dumps({'cause': e.args}) self._sapper.mark_event_failed(failure_details=failure_details, **self.payload)
def distill_documentation(encounter_vertex): if isinstance(encounter_vertex, str): encounter_vertex = rapidjson.loads(encounter_vertex) vertex_properties = encounter_vertex['vertex_properties'] documentation = _retrieve_documentation(vertex_properties) id_source = _find_vertex_property('id_source', 'local_properties', vertex_properties) encounter_id = int(encounter_vertex['id_value']['property_value']) distilled_text = _distill_strings(documentation) extracted_data = { 'object_type': 'DocumentationText', 'identifier': f'#{id_source}#Encounter#', 'id_value': encounter_id, 'extracted_data': { 'source': { 'id_source': id_source, 'encounter_id': encounter_id, 'utc_generated_datetime': datetime.utcnow().isoformat(), 'documentation_text': distilled_text } } } stored_data = StoredData.from_object(uuid.uuid4(), extracted_data, full_unpack=True) return ajson.dumps(stored_data)
def test(self): session = boto3.Session(profile_name='dev') sns = session.client('sns') listener_arn = 'arn:aws:sns:us-east-1:726075243133:leech-dev-Listener-1EV4D8VOW7L37' bull_horn = Bullhorn(sns) schema = Schema.retrieve() schema_entry = schema['ExternalId'] message = ajson.dumps({ 'task_name': 'generate_source_vertex', 'task_args': None, 'task_kwargs': { 'schema': schema, 'schema_entry': schema_entry, 'extracted_data': { 'source': { 'id_source': 'Algernon', 'id_type': 'Employees', 'id_name': 'emp_id', 'id_value': 1001 } } } }) message_id = bull_horn.publish('new_event', listener_arn, message) assert message_id
def publish_to_leech(bullhorn, flow_id, id_source, patient_id, provider_id, encounter_id, encounter_text, encounter_datetime_in, encounter_datetime_out, encounter_type, patient_last_name, patient_first_name, patient_dob): encounter_data = { 'source': { 'patient_id': patient_id, 'provider_id': provider_id, 'id_source': id_source, 'encounter_id': encounter_id, 'documentation': encounter_text, 'encounter_datetime_in': encounter_datetime_in, 'encounter_datetime_out': encounter_datetime_out, 'encounter_type': encounter_type }, 'patient_data': [{ 'last_name': patient_last_name, 'first_name': patient_first_name, 'dob': patient_dob }] } message = { 'task_name': 'leech', 'task_kwargs': { 'object_type': 'Encounter', 'extracted_data': encounter_data }, 'flow_id': f'{flow_id}#leech' } listener_arn = bullhorn.find_task_arn('leech') strung_event = ajson.dumps(message) return bullhorn.publish('new_event', listener_arn, strung_event)
def publish_to_incredible(bullhorn, flow_id, id_source, patient_id, provider_id, encounter_id, encounter_datetime_in, encounter_datetime_out, encounter_type, patient_last_name, patient_first_name, patient_dob): next_task_name = 'get_encounter' encounter_data = { 'encounter_id': encounter_id, 'encounter_datetime_in': encounter_datetime_in, 'encounter_datetime_out': encounter_datetime_out, 'patient_id': patient_id, 'provider_id': provider_id, 'encounter_type': encounter_type, 'id_source': id_source, 'patient_last_name': patient_last_name, 'patient_first_name': patient_first_name, 'patient_dob': patient_dob } message = { 'task_name': next_task_name, 'task_kwargs': encounter_data, 'flow_id': f'{flow_id}#{patient_id}#{next_task_name}-{encounter_id}' } listener_arn = bullhorn.find_task_arn('credible_tasks') strung_event = ajson.dumps(message) return bullhorn.publish('new_event', listener_arn, strung_event)
def test_credential_to_json(self): id_source = 'ICFS' credentials = CredibleLoginCredentials.retrieve(id_source) json_string = ajson.dumps(credentials) assert json_string rebuilt_credentials = ajson.loads(json_string) assert isinstance(rebuilt_credentials, CredibleLoginCredentials) assert rebuilt_credentials.validate()
def source_vertex_task_integration_event(request): params = request.param schema_entry = _read_schema(params[1], params[2]) schema = _read_schema(params[1]) extracted_data = _read_test_event(params[0]) event = { 'task_name': 'generate_source_vertex', 'task_kwargs': { 'schema': schema, 'schema_entry': schema_entry, 'extracted_data': extracted_data } } event_string = ajson.dumps(event) message_object = {'Message': event_string} body_object = {'body': ajson.dumps(message_object)} return {'Records': [body_object]}
def package(self) -> str: unencrypted_text = ajson.dumps({ 'pagination_id': self._pagination_id, 'inclusive_start': self._inclusive_start, 'page_size': self._page_size }) return SneakyKipper('pagination').encrypt(unencrypted_text, {'username': self._username})
def test_vertex_vertex_properties(self, db_vertex_vertex_properties_response): vertex_properties = rapidjson.loads( rapidjson.dumps(db_vertex_vertex_properties_response), object_hook=TridentDecoder.object_hook) vertex_properties_string = ajson.dumps(vertex_properties) gql = rapidjson.loads(vertex_properties_string, object_hook=GqlDecoder.object_hook) assert gql
def store(cls, object_data, **kwargs): data_type = kwargs['data_type'] bucket_name = kwargs['bucket_name'] object_key = kwargs['object_key'] bucket = boto3.resource('s3').Bucket(bucket_name) if not _check_for_object(bucket_name, object_key): bucket.put_object(Key=object_key, Body=ajson.dumps(object_data)) storage_uri = f's3://{bucket_name}/{object_key}' return cls(data_type, 's3', storage_uri)
def store_engine_data(bucket_name, file_key, engine_data): s3 = boto3.resource('s3') stored_object = s3.Object(bucket_name, file_key) try: stored_object.load() raise RuntimeError(f'can not overwrite file at key: {file_key}') except ClientError as e: if e.response['Error']['Code'] != '404': raise e stored_object.put(Body=ajson.dumps(engine_data))
def handler(event, context): logging.info( f'received a call to run an auditor task, event/context: {event}/{context}' ) task_name = event['task_name'] task_kwargs = event['task_kwargs'] flow_id = event['flow_id'] task_kwargs['flow_id'] = flow_id task_fn = _find_task(task_name) results = task_fn(**task_kwargs) logging.info(f'completed an auditor task, results: {results}') return ajson.dumps(results)
def generate_edge_integration_event(request): params = request.param schema_entry = _read_schema(params[1], params[2]) test_event = _read_test_event(params[0]) edge_type = params[3] rule_entry = _generate_linking_rule(schema_entry, edge_type) edge_schema_entry = _read_schema(params[1], edge_type) task_kwargs = ajson.loads(ajson.dumps(test_event)) task_kwargs.update({ 'schema_entry': edge_schema_entry, 'rule_entry': rule_entry }) event = { 'task_name': 'generate_potential_edge', 'task_kwargs': task_kwargs, 'flow_id': 'some_flow_id' } event_string = ajson.dumps(event) message_object = {'Message': event_string} body_object = {'body': ajson.dumps(message_object)} return {'Records': [body_object]}
def parse_batch_encounters(event, context): _load_config(ENVIRON_VARIABLES) migration_table_name = os.environ['PROGRESS_TABLE_NAME'] if 'Records' in event: return _process_queued_parse_task(event, context) event = rebuild_event(event) parsed_events = _parse_batch_event(event) _fire_hose_parsed_events(parsed_events) for parsed_event in parsed_events: _mark_parsing_complete(parsed_event, migration_table_name) return ajson.dumps( StoredData.from_object(uuid.uuid4(), parsed_events, full_unpack=True))
def find_existing_vertexes(request): params = request.param schema_entry = _read_schema(params[1], params[2]) schema = _read_schema(params[1]) test_event = _read_test_event(params[0]) task_kwargs = ajson.loads(ajson.dumps(test_event)) edge_type = params[3] rule_entry = _generate_linking_rule(schema_entry, edge_type) task_kwargs.update({ 'schema': schema, 'schema_entry': schema_entry, 'rule_entry': rule_entry }) event = { 'task_name': 'check_for_existing_vertexes', 'task_kwargs': task_kwargs } event_string = ajson.dumps(event) message_object = {'Message': event_string} body_object = {'body': ajson.dumps(message_object)} return {'Records': [body_object]}
def handler(event, context): event = rebuild_event(event) logging.info( f'received a call to run a query_s3_csv command: event/context: {event}/{context}' ) function_payload = { 'bucket_name': event['bucket_name'], 'file_key': event['file_key'], 'expression': event['expression'] } results = query_s3_csv(**function_payload) logging.info( f'completed a call to run a query_s3_csv command: results: {results}') return ajson.dumps(results)
def source_vertex_task_deployment_event(request): params = request.param schema_entry = _read_schema(params[1], params[2]) schema = _read_schema(params[1]) extracted_data = _read_test_event(params[0]) event = { 'task_name': 'generate_source_vertex', 'task_kwargs': { 'schema': schema, 'schema_entry': schema_entry, 'extracted_data': extracted_data } } event_string = ajson.dumps(event) return event_string
def _mark_parsing_complete(parsed_event, table_name): resource = boto3.resource('dynamodb') table = resource.Table(table_name) table.update_item(Key={ 'identifier': parsed_event['identifier'], 'id_value': int(parsed_event['id_value']) }, UpdateExpression='SET #x=:x', ExpressionAttributeNames={'#x': 'extraction'}, ExpressionAttributeValues={ ':x': { 'completed_at': datetime.now().isoformat(), 'stage_results': ajson.dumps(parsed_event) } })
def _generate_task_manager(**kwargs): payload_mock = MagicMock(name='payload') payload_mock.read.return_value = ajson.dumps(kwargs.get('payload', {'some_result': 'here'})) response = { 'Payload': payload_mock, 'StatusCode': kwargs.get('status_code', 200) } if kwargs.get('function_error'): response['FunctionError'] = kwargs.get('function_error') if kwargs.get('log_result'): response['LogResult'] = kwargs.get('log_result') retrieve_patch_obj = patch('toll_booth.obj.shucker.TaskManager._fire_lambda') patches.append(retrieve_patch_obj) mock_obj = retrieve_patch_obj.start() mock_obj.return_value = response return mock_obj
def generate_extraction_config( id_source: str, source_name: str, extraction_type: str, extraction_params: Mapping = None ): source_config, extraction_config = config_tasks.build_configs( id_source=id_source, source_name=source_name, extraction_type=extraction_type, extraction_params=extraction_params ) results = { 'source_config': source_config, 'extraction_config': extraction_config } return rapidjson.loads(ajson.dumps(results))
def get_encounter_text(encounter_id: str, driver: CredibleFrontEndDriver, next_task_name: str = None, **kwargs): encounter_text = driver.retrieve_client_encounter(encounter_id) if next_task_name: bullhorn = kwargs['bullhorn'] topic_arn = bullhorn.find_task_arn(next_task_name) msg = { 'task_name': next_task_name, 'task_kwargs': { 'id_source': kwargs.get('id_source'), 'encounter_id': encounter_id, 'documentation': encounter_text } } bullhorn.publish('new_event', topic_arn, ajson.dumps(msg)) return encounter_text
def _publish_documentation_field_node(encounter_id, id_source, parser_id, field_name, field_documentation, bullhorn): task_name = 'leech' leech_arn = bullhorn.find_task_arn(task_name) documentation_field_node = { 'encounter_id': encounter_id, 'id_source': id_source, 'parser_id': parser_id, 'field_name': field_name, 'field_documentation': field_documentation } message = { 'task_name': task_name, 'task_kwargs': { 'object_type': 'DocumentationField', 'extracted_data': {'source': documentation_field_node} } } strung_event = ajson.dumps(message) bullhorn.publish('new_event', leech_arn, strung_event)