def resolve_field_value_template(tree, data, data_index): object_paths = as_list(resolve_path(tree, ['start', 'object_path'])) for object_path in object_paths: ids = as_list(resolve_path(data, get_field_path(object_path))) if not ids: return None data = remove_none(list(map(lambda id: data_index.get(id), ids))) value_paths = as_list(resolve_path(tree, ['start', 'value_path'])) if len(value_paths) == 1: return resolve_path(data, get_field_path(value_paths[0])) else: new_value = [] for data in as_list(data): field_values = remove_empty( map( lambda value_path: as_list( resolve_path(data, get_field_path(value_path))) or None, value_paths)) product = itertools.product(*field_values) joined = map( lambda field_value: reduce( lambda acc, s: s if s.startswith(acc) else acc + " " + s, field_value, ""), product) if joined: new_value.extend(joined) return list(distinct(remove_empty(new_value)))
def test_resolve_simple_path(self): input_path = "a" input_value = {"a": 1} expected = 1 actual = resolve_path(input_value, input_path) self.assertEqual(expected, actual)
def test_resolve_invalid_simple_path(self): input_path = ["a", "z"] input_value = {"a": {"b": {"c": 1}}} expected = None actual = resolve_path(input_value, input_path) self.assertEqual(expected, actual)
def test_resolve_heterogeneous_nested_multi_value_path(self): input_path = ["a", "b", "c"] input_value = [{ "a": { "b": { "c": 1 } } }, { "a": [{ "b": { "c": 3 } }] }, { "a": { "b": [{ "c": 4 }, { "c": 5 }] } }] expected = [1, 3, 4, 5] actual = resolve_path(input_value, input_path) self.assertEqual(expected, actual)
def test_resolve_nested_multi_value_path(self): input_path = ["a", "b", "c"] input_value = [{ "a": { "b": { "c": 1 } } }, { "a": { "b": { "c": 2 } } }, { "a": { "b": { "c": 3 } } }] expected = [1, 2, 3] actual = resolve_path(input_value, input_path) self.assertEqual(expected, actual)
def test_resolve_nested_path(self): input_path = ["a", "b", "c"] input_value = {"a": {"b": {"c": 1}}} expected = 1 actual = resolve_path(input_value, input_path) self.assertEqual(expected, actual)
def test_resolve_simple_multi_value_path(self): input_path = "a" input_value = [{"a": 1}, {"a": 2}, {"a": 3}] expected = [1, 2, 3] actual = resolve_path(input_value, input_path) self.assertEqual(expected, actual)
def fetch_details(options): """ Fetch details call for a BrAPI object (ex: /brapi/v1/studies/{id}) """ source, logger, entity, object_id = options if 'detail' not in entity: return detail_call_group = entity['detail'] in_store = object_id in entity['store'] skip_if_in_store = detail_call_group.get('skip-if-in-store') already_detailed = resolve_path(entity['store'], [object_id, 'etl:detailed']) if in_store and (skip_if_in_store or already_detailed): return entity_name = entity['name'] entity_id = entity_name + 'DbId' detail_call = get_implemented_call(source, detail_call_group, {entity_id: object_id}) if not detail_call: return details = BreedingAPIIterator.fetch_all(source['brapi:endpointUrl'], detail_call, logger).__next__() details['etl:detailed'] = True return entity_name, [details]
def remove_internal_objects(entities): """ Remove objects referenced inside others (example: trial.studies or study.location) """ for (entity_name, entity) in entities.items(): for link in (entity.get('links') or []): if link['type'] != 'internal-object': continue for (_, data) in entity['store'].items(): link_path = link['json-path'] link_path_list = remove_empty(link_path.split('.')) context_path, last = link_path_list[:-1], link_path_list[-1] link_context = resolve_path(data, context_path) if link_context and last in link_context: del link_context[last]
def test_resolve_invalid_multi_path(self): input_path = ["a", "c", "d"] input_value = [{ "a": { "b": { "c": 1 } } }, { "a": { "b": [{ "c": 4 }, { "c": 5 }] } }] expected = None actual = resolve_path(input_value, input_path) self.assertEqual(expected, actual)
def fetch_all_links(source, logger, entities): """ Link objects across entities. - Internal: link an object (ex: study) to another using an identifier inside the JSON object (ex: link a location via study.locationDbId) - Internal object: link an object (ex: study) to another contained inside the first (ex: link a location via study.location.locationDbId) - External object: link an object (ex: study) to another using a dedicated call (ex: link to observation variables via /brapi/v1/studies/{id}/observationvariables) """ for (entity_name, entity) in entities.items(): if 'links' not in entity: continue for link in entity['links']: for (object_id, object) in entity['store'].items(): linked_entity_name = link['entity'] linked_entity = entities[linked_entity_name] linked_objects_by_id = {} if link['type'].startswith('internal'): link_path = link['json-path'] link_path_list = remove_empty(link_path.split('.')) link_values = remove_none( as_list(resolve_path(object, link_path_list))) if not link_values: if link.get('required'): raise BrokenLink( "Could not find required field '{}' in {} object id '{}'" .format(link_path, entity_name, object_id)) continue if link['type'] == 'internal-object': for link_value in link_values: link_id = get_identifier(linked_entity_name, link_value) linked_objects_by_id[link_id] = link_value elif link['type'] == 'internal': link_id_field = linked_entity['name'] + 'DbId' link_name_field = linked_entity['name'] + 'Name' for link_value in link_values: link_id = link_value.get(link_id_field) link_name = link_value.get(link_name_field) if link_id: linked_objects_by_id[link_id] = { link_id_field: link_id, link_name_field: link_name } elif link['type'] == 'external-object': call = get_implemented_call(source, link, context=object) if not call: continue link_values = list( BreedingAPIIterator.fetch_all( source['brapi:endpointUrl'], call, logger)) for link_value in link_values: link_id = get_identifier(linked_entity_name, link_value) linked_objects_by_id[link_id] = link_value link_objects(entity, object, linked_entity, linked_objects_by_id)
def get_field_path(tree): return resolve_path(tree, ['field_path', 'FIELD'])