def resolve_field_value_template(tree, data, data_index):
object_paths = as_list(resolve_path(tree, ['start', 'object_path']))
for object_path in object_paths:
ids = as_list(resolve_path(data, get_field_path(object_path)))
if not ids:
return None
data = remove_none(list(map(lambda id: data_index.get(id), ids)))
value_paths = as_list(resolve_path(tree, ['start', 'value_path']))
if len(value_paths) == 1:
return resolve_path(data, get_field_path(value_paths[0]))
else:
new_value = []
for data in as_list(data):
field_values = remove_empty(
map(
lambda value_path: as_list(
resolve_path(data, get_field_path(value_path))) or
None, value_paths))
product = itertools.product(*field_values)
joined = map(
lambda field_value: reduce(
lambda acc, s: s
if s.startswith(acc) else acc + " " + s, field_value, ""),
product)
if joined:
new_value.extend(joined)
return list(distinct(remove_empty(new_value)))
def tree_to_dict(tree):
if isinstance(tree, lark.tree.Tree):
new_children = remove_none(list(map(tree_to_dict, tree.children)))
return {'{lark}': 'TREE', tree.data: new_children}
if isinstance(tree, lark.lexer.Token):
return {'{lark}': 'TOKEN', tree.type: tree.value}
return tree
def parse_template(template):
if isinstance(template, str):
return parse_string_template(template)
elif isinstance(template, list):
return list(remove_none(map(parse_template, template)))
elif isinstance(template, dict):
template_parsers = {
'{if}': parse_if_template,
'{or}': parse_or_template,
'{join}': parse_join_template,
'{flatten_distinct}': parse_flatten_template,
'{list}': parse_list_template,
'{map}': parse_map_template,
'{merge}': parse_merge_template,
}
for key, parser in template_parsers.items():
if key in template:
return parser(template)
new_dict = dict()
for (key, value) in template.items():
new_value = parse_template(value)
if new_value:
new_dict[key] = new_value
return new_dict
return template
def extract_links(id_field):
def extract_link(entry):
key, value = entry
match = re.search("^(\w+)"+id_field+"(s?)$", key)
if match and value:
entity_name, plural = match.groups()
return [entity_name, key, plural, value]
return remove_none(map(extract_link, data.items()))
def get_links(path, data):
if is_collection(data):
if isinstance(data, dict):
entries = data.items()
else:
entries = enumerate(data)
return itertools.chain.from_iterable(
remove_none(map(partial(get_entry_link, path), entries)))
def fetch_all_links(source, logger, entities):
"""
Link objects across entities.
- Internal: link an object (ex: study) to another using an identifier inside the JSON object
(ex: link a location via study.locationDbId)
- Internal object: link an object (ex: study) to another contained inside the first
(ex: link a location via study.location.locationDbId)
- External object: link an object (ex: study) to another using a dedicated call
(ex: link to observation variables via /brapi/v1/studies/{id}/observationVariables)
"""
for (entity_name, entity) in entities.items():
if 'links' not in entity:
continue
for link in entity['links']:
for (object_id, object) in entity['store'].items():
linked_entity_name = link['entity']
linked_entity = entities[linked_entity_name]
linked_objects_by_id = {}
if link['type'].startswith('internal'):
link_path = link['json-path']
link_path_list = remove_empty(link_path.split('.'))
link_values = remove_none(as_list(get_in(object, link_path_list)))
if not link_values:
if link.get('required'):
raise BrokenLink("Could not find required field '{}' in {} object id '{}'"
.format(link_path, entity_name, object_id))
continue
if link['type'] == 'internal-object':
for link_value in link_values:
link_id = get_identifier(linked_entity_name, link_value)
linked_objects_by_id[link_id] = link_value
elif link['type'] == 'internal':
link_id_field = linked_entity['name'] + 'DbId'
link_name_field = linked_entity['name'] + 'Name'
for link_value in link_values:
link_id = link_value.get(link_id_field)
link_name = link_value.get(link_name_field)
if link_id:
linked_objects_by_id[link_id] = {link_id_field: link_id, link_name_field: link_name}
elif link['type'] == 'external-object':
call = get_implemented_call(source, link, context=object)
if not call:
continue
link_values = list(BreedingAPIIterator.fetch_all(source['brapi:endpointUrl'], call, logger))
for link_value in link_values:
link_id = get_identifier(linked_entity_name, link_value)
linked_objects_by_id[link_id] = link_value
link_objects(entity, object, linked_entity, linked_objects_by_id)
def resolve_join_template(template, data, data_index):
accept_none = False if template.get('{accept_none}') == False else True
separator = template.get('{separator}') or ''
elements = template.get('{join}')
flattened_elements = flatten(resolve(elements, data, data_index))
if not accept_none:
for elem in as_list(flattened_elements):
if not elem:
return None
filtered_elements = remove_none(flattened_elements)
return coll_as_str(filtered_elements, separator)
def remove_white_space_token(tree):
if isinstance(tree, lark.tree.Tree):
def not_ws_token(child):
if not (isinstance(child, lark.lexer.Token)
and child.type == "WS"):
return child
without_ws = filter(not_ws_token, tree.children)
new_children = remove_none(map(remove_white_space_token, without_ws))
return lark.tree.Tree(tree.data, new_children)
return tree
def resolve_field_value_template(tree, data, data_index):
object_paths = as_list(get_in(tree, ['start', 'object_path']))
for object_path in object_paths:
field_path = get_field_path(object_path)
ids = as_list(get_in(data, field_path))
if not ids:
return None
field = field_path[-1]
entity = re.sub(r"(\w+)URIs?", "\\1", field)
entity_index = data_index[entity]
try:
dataList = []
for id in ids:
dataList.append(json.loads(entity_index[id].decode()))
data = remove_none(dataList)
except AttributeError:
data = remove_none(list(map(lambda id: entity_index[id], ids)))
if getattr(entity_index, 'close', False):
entity_index.close()
value_paths = as_list(get_in(tree, ['start', 'value_path']))
if len(value_paths) == 1:
return get_in(data, get_field_path(value_paths[0]))
else:
new_value = []
for data in as_list(data):
field_values = remove_empty(
map(
lambda value_path: as_list(
get_in(data, get_field_path(value_path))) or None,
value_paths))
product = itertools.product(*field_values)
joined = map(
lambda field_value: reduce(
lambda acc, s: s
if s.startswith(acc) else acc + " " + s, field_value, ""),
product)
if joined:
new_value.extend(joined)
return list(distinct(remove_empty(new_value)))