def resolve_field_value_template(tree, data, data_index):
object_paths = as_list(resolve_path(tree, ['start', 'object_path']))
for object_path in object_paths:
ids = as_list(resolve_path(data, get_field_path(object_path)))
if not ids:
return None
data = remove_none(list(map(lambda id: data_index.get(id), ids)))
value_paths = as_list(resolve_path(tree, ['start', 'value_path']))
if len(value_paths) == 1:
return resolve_path(data, get_field_path(value_paths[0]))
else:
new_value = []
for data in as_list(data):
field_values = remove_empty(
map(
lambda value_path: as_list(
resolve_path(data, get_field_path(value_path))) or
None, value_paths))
product = itertools.product(*field_values)
joined = map(
lambda field_value: reduce(
lambda acc, s: s
if s.startswith(acc) else acc + " " + s, field_value, ""),
product)
if joined:
new_value.extend(joined)
return list(distinct(remove_empty(new_value)))
def test_resolve_simple_path(self):
input_path = "a"
input_value = {"a": 1}
expected = 1
actual = resolve_path(input_value, input_path)
self.assertEqual(expected, actual)
def test_resolve_invalid_simple_path(self):
input_path = ["a", "z"]
input_value = {"a": {"b": {"c": 1}}}
expected = None
actual = resolve_path(input_value, input_path)
self.assertEqual(expected, actual)
def test_resolve_heterogeneous_nested_multi_value_path(self):
input_path = ["a", "b", "c"]
input_value = [{
"a": {
"b": {
"c": 1
}
}
}, {
"a": [{
"b": {
"c": 3
}
}]
}, {
"a": {
"b": [{
"c": 4
}, {
"c": 5
}]
}
}]
expected = [1, 3, 4, 5]
actual = resolve_path(input_value, input_path)
self.assertEqual(expected, actual)
def test_resolve_nested_multi_value_path(self):
input_path = ["a", "b", "c"]
input_value = [{
"a": {
"b": {
"c": 1
}
}
}, {
"a": {
"b": {
"c": 2
}
}
}, {
"a": {
"b": {
"c": 3
}
}
}]
expected = [1, 2, 3]
actual = resolve_path(input_value, input_path)
self.assertEqual(expected, actual)
def test_resolve_nested_path(self):
input_path = ["a", "b", "c"]
input_value = {"a": {"b": {"c": 1}}}
expected = 1
actual = resolve_path(input_value, input_path)
self.assertEqual(expected, actual)
def test_resolve_simple_multi_value_path(self):
input_path = "a"
input_value = [{"a": 1}, {"a": 2}, {"a": 3}]
expected = [1, 2, 3]
actual = resolve_path(input_value, input_path)
self.assertEqual(expected, actual)
def fetch_details(options):
"""
Fetch details call for a BrAPI object (ex: /brapi/v1/studies/{id})
"""
source, logger, entity, object_id = options
if 'detail' not in entity:
return
detail_call_group = entity['detail']
in_store = object_id in entity['store']
skip_if_in_store = detail_call_group.get('skip-if-in-store')
already_detailed = resolve_path(entity['store'],
[object_id, 'etl:detailed'])
if in_store and (skip_if_in_store or already_detailed):
return
entity_name = entity['name']
entity_id = entity_name + 'DbId'
detail_call = get_implemented_call(source, detail_call_group,
{entity_id: object_id})
if not detail_call:
return
details = BreedingAPIIterator.fetch_all(source['brapi:endpointUrl'],
detail_call, logger).__next__()
details['etl:detailed'] = True
return entity_name, [details]
def remove_internal_objects(entities):
"""
Remove objects referenced inside others (example: trial.studies or study.location)
"""
for (entity_name, entity) in entities.items():
for link in (entity.get('links') or []):
if link['type'] != 'internal-object':
continue
for (_, data) in entity['store'].items():
link_path = link['json-path']
link_path_list = remove_empty(link_path.split('.'))
context_path, last = link_path_list[:-1], link_path_list[-1]
link_context = resolve_path(data, context_path)
if link_context and last in link_context:
del link_context[last]
def test_resolve_invalid_multi_path(self):
input_path = ["a", "c", "d"]
input_value = [{
"a": {
"b": {
"c": 1
}
}
}, {
"a": {
"b": [{
"c": 4
}, {
"c": 5
}]
}
}]
expected = None
actual = resolve_path(input_value, input_path)
self.assertEqual(expected, actual)
def fetch_all_links(source, logger, entities):
"""
Link objects across entities.
- Internal: link an object (ex: study) to another using an identifier inside the JSON object
(ex: link a location via study.locationDbId)
- Internal object: link an object (ex: study) to another contained inside the first
(ex: link a location via study.location.locationDbId)
- External object: link an object (ex: study) to another using a dedicated call
(ex: link to observation variables via /brapi/v1/studies/{id}/observationvariables)
"""
for (entity_name, entity) in entities.items():
if 'links' not in entity:
continue
for link in entity['links']:
for (object_id, object) in entity['store'].items():
linked_entity_name = link['entity']
linked_entity = entities[linked_entity_name]
linked_objects_by_id = {}
if link['type'].startswith('internal'):
link_path = link['json-path']
link_path_list = remove_empty(link_path.split('.'))
link_values = remove_none(
as_list(resolve_path(object, link_path_list)))
if not link_values:
if link.get('required'):
raise BrokenLink(
"Could not find required field '{}' in {} object id '{}'"
.format(link_path, entity_name, object_id))
continue
if link['type'] == 'internal-object':
for link_value in link_values:
link_id = get_identifier(linked_entity_name,
link_value)
linked_objects_by_id[link_id] = link_value
elif link['type'] == 'internal':
link_id_field = linked_entity['name'] + 'DbId'
link_name_field = linked_entity['name'] + 'Name'
for link_value in link_values:
link_id = link_value.get(link_id_field)
link_name = link_value.get(link_name_field)
if link_id:
linked_objects_by_id[link_id] = {
link_id_field: link_id,
link_name_field: link_name
}
elif link['type'] == 'external-object':
call = get_implemented_call(source, link, context=object)
if not call:
continue
link_values = list(
BreedingAPIIterator.fetch_all(
source['brapi:endpointUrl'], call, logger))
for link_value in link_values:
link_id = get_identifier(linked_entity_name,
link_value)
linked_objects_by_id[link_id] = link_value
link_objects(entity, object, linked_entity,
linked_objects_by_id)
def get_field_path(tree):
return resolve_path(tree, ['field_path', 'FIELD'])
