def _validate_otu_key_if_present(self, nd_id_nd_iter, otu_by_id, vc):
erred = False
for nd_id, nd in nd_id_nd_iter:
o = nd.get('@otu')
if o is not None:
if o not in otu_by_id:
erred = True
self._error_event(_NEXEL.NODE,
obj=nd,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=nd_id,
key_list=[o])
if erred:
errorReturn('Unknown "@otu" id')
return True
def _validate_otu_key_if_present(self, nd_id_nd_iter, otu_by_id, vc):
erred = False
for nd_id, nd in nd_id_nd_iter:
o = nd.get('@otu')
if o is not None:
if o not in otu_by_id:
erred = True
self._error_event(_NEXEL.NODE,
obj=nd,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=nd_id,
key_list=[o])
if erred:
errorReturn('Unknown "@otu" id')
return True
def _post_key_check_validate_otus_obj(self, og_nex_id, otus_group, vc):
otu_dict = {}
otu_list = otus_group.get('otu', [])
if isinstance(otu_list, dict):
otu_list = [otu_list]
if not otu_list:
return
vc.push_context(_NEXEL.OTU, (otus_group, og_nex_id))
try:
without_id = []
otu_tuple_list = []
for otu in otu_list:
oid = otu.get('@id')
if oid is None:
without_id.append(otu)
else:
otu_tuple_list.append((oid, otu))
otu_dict[oid] = otu
if without_id:
self._error_event(_NEXEL.NEXML,
obj=without_id,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=None,
key_list=['@id'])
return errorReturn('lack of "@id" in "otu"')
r = self._validate_otu_list(otu_tuple_list, vc)
if r:
self._otu_by_otug[og_nex_id] = otu_dict
return r
except:
vc.pop_context()
def _post_key_check_validate_otus_obj(self, og_nex_id, otus_group, vc):
otu_dict = {}
otu_list = otus_group.get('otu', [])
if isinstance(otu_list, dict):
otu_list = [otu_list]
if not otu_list:
return
vc.push_context(_NEXEL.OTU, (otus_group, og_nex_id))
try:
without_id = []
otu_tuple_list = []
for otu in otu_list:
oid = otu.get('@id')
if oid is None:
without_id.append(otu)
else:
otu_tuple_list.append((oid, otu))
otu_dict[oid] = otu
if without_id:
self._error_event(_NEXEL.NEXML,
obj=without_id,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=None,
key_list=['@id'])
return errorReturn('lack of "@id" in "otu"')
r = self._validate_otu_list(otu_tuple_list, vc)
if r:
self._otu_by_otug[og_nex_id] = otu_dict
return r
except:
vc.pop_context()
def _post_key_check_validate_tree_group(self, tg_nex_id, trees_group, vc):
otus_el = trees_group.get('@otus')
if otus_el not in self._otu_group_by_id:
kl = ['@otus="{oe}"'.format(oe=otus_el)]
self._error_event(_NEXEL.TREES,
obj=trees_group,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tg_nex_id,
key_list=kl)
return errorReturn('bad "@otus" in trees group')
tree_list = trees_group.get('tree', [])
if isinstance(tree_list, dict):
tree_list = [tree_list]
elif not isinstance(tree_list, list):
self._error_event(_NEXEL.TREES,
obj=trees_group,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=tg_nex_id,
key_list=['tree'])
return errorReturn('lack of "tree" in trees group')
for tree_obj in tree_list:
t_nex_id = tree_obj.get('@id')
vc.push_context(_NEXEL.TREE, (tree_obj, t_nex_id))
try:
if t_nex_id is None:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tg_nex_id,
key_list=['@id'])
return errorReturn('no "@id" in tree')
if not self._validate_tree(t_nex_id,
tree_obj,
vc,
otus_group_id=otus_el):
return False
finally:
vc.pop_context()
return True
def _post_key_check_validate_tree_group(self, tg_nex_id, trees_group, vc):
otus_el = trees_group.get('@otus')
if otus_el not in self._otu_group_by_id:
kl = ['@otus="{oe}"'.format(oe=otus_el)]
self._error_event(_NEXEL.TREES,
obj=trees_group,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tg_nex_id,
key_list=kl)
return errorReturn('bad "@otus" in trees group')
tree_list = trees_group.get('tree', [])
if isinstance(tree_list, dict):
tree_list = [tree_list]
elif not isinstance(tree_list, list):
self._error_event(_NEXEL.TREES,
obj=trees_group,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=tg_nex_id,
key_list=['tree'])
return errorReturn('lack of "tree" in trees group')
for tree_obj in tree_list:
t_nex_id = tree_obj.get('@id')
vc.push_context(_NEXEL.TREE, (trees_group, tg_nex_id))
try:
if t_nex_id is None:
self._error_event(
_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tg_nex_id,
key_list=['@id'])
return errorReturn('no "@id" in tree')
if not self._validate_tree(
t_nex_id, tree_obj, vc, otus_group_id=otus_el):
return False
finally:
vc.pop_context()
return True
def _post_key_check_validate_tree_group(self, tg_nex_id, tree_group_obj,
vc):
tree_id_order = tree_group_obj.get('^ot:treeElementOrder', [])
tree_by_id = tree_group_obj.get('treeById', {})
if (not isinstance(tree_by_id, dict)) \
or (not isinstance(tree_id_order, list)):
self._error_event(_NEXEL.TREES,
obj=tree_group_obj,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=tg_nex_id,
key_list=['treeById', '^ot:treeElementOrder'])
return errorReturn('no "treeById" in trees group')
if ((not tree_by_id) and tree_id_order) or ((not tree_id_order)
and tree_by_id):
self._error_event(_NEXEL.TREES,
obj=tree_group_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tg_nex_id,
key_list=['treeById', '^ot:treeElementOrder'])
return errorReturn('no "treeById" in trees group')
otus_el = tree_group_obj.get('@otus')
if otus_el not in self._otu_group_by_id:
self._error_event(_NEXEL.TREES,
obj=tree_group_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tg_nex_id,
key_list=[otus_el])
return errorReturn('no "@otus" in trees group')
for t_nex_id, tree_obj in tree_by_id.items():
vc.push_context(_NEXEL.TREE, (tree_group_obj, tg_nex_id))
try:
if not self._validate_tree(
t_nex_id, tree_obj, vc, otus_group_id=otus_el):
return False
finally:
vc.pop_context()
return True
def _post_key_check_validate_otus_obj(self, og_nex_id, otus_group, vc):
otu_obj = otus_group.get('otuById', {})
if not isinstance(otu_obj, dict):
self._error_event(_NEXEL.OTUS,
obj=otus_group,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=og_nex_id,
key_list=['otuById'])
return errorReturn('no "otuById" in otus group')
self._otu_group_by_id[og_nex_id] = otus_group
vc.push_context(_NEXEL.OTU, (otus_group, og_nex_id))
try:
not_dict_otu = []
ndo_id_list = []
otu_id_obj_list = []
for id_obj_pair in otu_obj.items():
if not isinstance(id_obj_pair[1], dict):
r = check_raw_dict(id_obj_pair[1], otus_group, None, vc)
assert r[0] is False
t = r[1]
not_dict_otu.append(t)
ndo_id_list.append(id_obj_pair[0])
else:
otu_id_obj_list.append(id_obj_pair)
if not_dict_otu:
self._error_event(_NEXEL.OTU,
obj=otu_obj,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=ndo_id_list,
key_val_type_list=[not_dict_otu])
return errorReturn('otu is wrong type')
r = self._validate_otu_list(otu_id_obj_list, vc)
if r:
self._otu_by_otug[og_nex_id] = otu_obj
return r
finally:
vc.pop_context()
def _post_key_check_validate_otus_obj(self, og_nex_id, otus_group, vc):
otu_obj = otus_group.get('otuById', {})
if not isinstance(otu_obj, dict):
self._error_event(_NEXEL.OTUS,
obj=otus_group,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=og_nex_id,
key_list=['otuById'])
return errorReturn('no "otuById" in otus group')
self._otu_group_by_id[og_nex_id] = otus_group
vc.push_context(_NEXEL.OTU, (otus_group, og_nex_id))
try:
not_dict_otu = []
ndo_id_list = []
otu_id_obj_list = []
for id_obj_pair in otu_obj.items():
if not isinstance(id_obj_pair[1], dict):
r = check_raw_dict(id_obj_pair[1], otus_group, None, vc)
assert r[0] is False
t = r[1]
not_dict_otu.append(t)
ndo_id_list.append(id_obj_pair[0])
else:
otu_id_obj_list.append(id_obj_pair)
if not_dict_otu:
self._error_event(_NEXEL.OTU,
obj=otu_obj,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=ndo_id_list,
key_val_type_list=[not_dict_otu])
return errorReturn('otu is wrong type')
r = self._validate_otu_list(otu_id_obj_list, vc)
if r:
self._otu_by_otug[og_nex_id] = otu_obj
return r
finally:
vc.pop_context()
def _post_key_check_validate_tree_group(self, tg_nex_id, tree_group_obj, vc):
tree_id_order = tree_group_obj.get('^ot:treeElementOrder', [])
tree_by_id = tree_group_obj.get('treeById', {})
if (not isinstance(tree_by_id, dict)) \
or (not isinstance(tree_id_order, list)):
self._error_event(_NEXEL.TREES,
obj=tree_group_obj,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=tg_nex_id,
key_list=['treeById', '^ot:treeElementOrder'])
return errorReturn('no "treeById" in trees group')
if ((not tree_by_id) and tree_id_order) or ((not tree_id_order) and tree_by_id):
self._error_event(_NEXEL.TREES,
obj=tree_group_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tg_nex_id,
key_list=['treeById', '^ot:treeElementOrder'])
return errorReturn('no "treeById" in trees group')
otus_el = tree_group_obj.get('@otus')
if otus_el not in self._otu_group_by_id:
self._error_event(_NEXEL.TREES,
obj=tree_group_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tg_nex_id,
key_list=[otus_el])
return errorReturn('no "@otus" in trees group')
for t_nex_id, tree_obj in tree_by_id.items():
vc.push_context(_NEXEL.TREE, (tree_group_obj, tg_nex_id))
try:
if not self._validate_tree(t_nex_id, tree_obj, vc, otus_group_id=otus_el):
return False
finally:
vc.pop_context()
return True
def _post_key_check_validate_tree(self,
tree_nex_id,
tree_obj,
vc,
otus_group_id=None):
node_list = tree_obj.get('node')
if isinstance(node_list, dict):
node_list = [node_list]
elif (not node_list) or (not isinstance(node_list, list)):
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=['node',])
return errorReturn('no "node" in "trees"')
edge_list = tree_obj.get('edge')
if isinstance(edge_list, dict):
edge_list = [edge_list]
elif (not edge_list) or (not isinstance(edge_list, list)):
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=['edge',])
return errorReturn('no "edge" in tree')
edge_id_list = [(i.get('@id'), i) for i in edge_list]
valid = self._validate_edge_list(edge_id_list, vc)
if not valid:
return False
node_id_obj_list = [(i.get('@id'), i) for i in node_list]
valid = self._validate_node_list(node_id_obj_list, vc)
if not valid:
return False
node_dict = {}
for i in node_id_obj_list:
nid, nd = i
node_dict[nid] = nd
missing_src = []
missing_target = []
for el in edge_id_list:
e = el[1]
sid = e.get('@source')
tid = e.get('@target')
if sid not in node_dict:
missing_src.append(sid)
if tid not in node_dict:
missing_target.append(tid)
if missing_src:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=missing_src)
return errorReturn('no "@source" in edge')
if missing_target:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=missing_target)
return errorReturn('no "@target" in edge')
if otus_group_id is None:
tree_group = vc.anc_list[-1][1]
otus_group_id = tree_group.get('@otus')
lowest_nodeid_set = set()
encountered_nodes = set()
edge_by_target = {}
edge_by_source = {}
multi_parent_node = []
for e in edge_list:
t = e.get('@target')
if t in edge_by_target:
multi_parent_node.append(t)
else:
edge_by_target[t] = e
#_LOG.debug('e=' + str(e))
sid = e['@source']
edge_by_source.setdefault(sid, []).append(e)
if multi_parent_node:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_NodeWithMultipleParents,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=multi_parent_node)
otuid2leaf = {}
unflagged_leaves = []
nonleaves_with_leaf_flags = []
with_at_root_prop = {}
first_lowest_node = None
for nd in node_list:
nid = nd.get('@id')
cycle_node, path_to_root = construct_path_to_root(nd, encountered_nodes, edge_by_target)
if cycle_node:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_TreeCycleWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
cycle_node=cycle_node)
return errorReturn('"@id" in node')
if path_to_root:
lowest_nodeid_set.add(path_to_root[-1])
if first_lowest_node is None:
first_lowest_node = path_to_root[-1]
is_flagged_as_leaf = self._find_first_literal_meta(nd, 'ot:isLeaf')
ch_list = edge_by_source.get(nid)
if ch_list is None:
otu_id = nd.get('@otu')
if otu_id is None:
vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
try:
self._error_event(_NEXEL.NODE,
obj=nd,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=nid,
key_list=['@otu',])
return errorReturn('"@otu" in leaf')
finally:
vc.pop_context()
else:
if otu_id in otuid2leaf:
vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
try:
self._error_event(_NEXEL.NODE,
obj=nd,
err_type=gen_RepeatedOTUWarning,
anc=vc.anc_list,
obj_nex_id=nid,
key_list=[otu_id])
return errorReturn('repeated "@otu" in leaves')
finally:
vc.pop_context()
otuid2leaf[otu_id] = nd
if not is_flagged_as_leaf:
if not self._logger.retain_deprecated:
add_literal_meta(nd, 'ot:isLeaf', True, self._syntax_version)
else:
unflagged_leaves.append(nid)
elif is_flagged_as_leaf:
if not self._logger.retain_deprecated:
delete_first_literal_meta(nd, 'ot:isLeaf', self._syntax_version)
else:
nonleaves_with_leaf_flags.append(nid)
if nd.get('@root'):
with_at_root_prop[nid] = nd
if unflagged_leaves:
vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
try:
#_LOG.debug('unflagged_leaves="{f}"'.format(f=unflagged_leaves))
self._error_event(_NEXEL.NODE,
obj=tree_obj,
err_type=gen_MissingMandatoryKeyWarning,
anc=vc.anc_list,
obj_nex_id=unflagged_leaves,
key_list=['ot:isLeaf'])
finally:
vc.pop_context()
if nonleaves_with_leaf_flags:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_InvalidKeyWarning,
anc=vc.anc_list,
obj_nex_id=nonleaves_with_leaf_flags,
key_list=['ot:isLeaf'])
return errorReturn('"ot:isLeaf" for internal')
self._detect_multilabelled_tree(otus_group_id,
tree_nex_id,
otuid2leaf)
if len(lowest_nodeid_set) > 1:
lowest_nodeid_set = list(lowest_nodeid_set)
lowest_nodeid_set.sort()
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MultipleRootsWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=lowest_nodeid_set)
return errorReturn('multiple roots in a tree')
root_node_id = first_lowest_node
if root_node_id not in with_at_root_prop:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MultipleRootsWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=list(with_at_root_prop.keys()) + [root_node_id])
return errorReturn('root without "@root"')
elif len(with_at_root_prop) > 1:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MultipleRootsWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=list(with_at_root_prop.keys()))
return errorReturn('Multiple nodes with "@root"')
elif len(with_at_root_prop) == 0:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_NoRootWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id)
return errorReturn('no node with "@root"')
og = set([i['@id'] for i in self._otu_group_by_id[otus_group_id]['otu']])
nli = [(i['@id'], i) for i in node_list]
return self._validate_otu_key_if_present(nli, og, vc)
def _post_key_check_validate_nexml_obj(self, nex_obj, obj_nex_id, vc):
otus_group_list = nex_obj.get('otus', [])
if otus_group_list and isinstance(otus_group_list, dict):
otus_group_list = [otus_group_list]
if not isinstance(otus_group_list, list):
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=['otus'])
return errorReturn('no "otus" in nexml')
vc.push_context(_NEXEL.OTUS, (nex_obj, obj_nex_id))
try:
without_id = []
og_tuple_list = []
for og in otus_group_list:
ogid = og.get('@id')
if ogid is None:
without_id.append(None)
else:
og_tuple_list.append((ogid, og))
if without_id:
self._error_event(_NEXEL.OTUS,
obj=nex_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=None,
key_list=['otus[*]/@id'])
return errorReturn('otu without "@id"')
if not self._validate_otus_group_list(og_tuple_list, vc):
return False
finally:
vc.pop_context()
# and now the trees...
trees_group_list = nex_obj.get('trees', [])
if trees_group_list and isinstance(trees_group_list, dict):
trees_group_list = [trees_group_list]
if not isinstance(trees_group_list, list):
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=['trees'])
return errorReturn('No "trees" in nexml')
vc.push_context(_NEXEL.TREES, (nex_obj, obj_nex_id))
try:
without_id = []
tg_tuple_list = []
for tg in trees_group_list:
tgid = tg.get('@id')
if tgid is None:
without_id.append(tg)
else:
tg_tuple_list.append((tgid, tg))
if without_id:
self._error_event(_NEXEL.TREES,
obj=without_id,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=None,
key_list=['@id'])
return errorReturn('No "@id" in trees group"')
if not self._validate_trees_group_list(tg_tuple_list, vc):
return False
finally:
vc.pop_context()
ogid2og = {}
for og in otus_group_list:
ogid = og.get('@id')
ogid2og[ogid] = og
if not self._find_first_literal_meta(nex_obj,
'ot:notIntendedForSynthesis'):
cs = self._find_first_literal_meta(nex_obj,
'ot:candidateTreeForSynthesis')
if cs:
if not isinstance(cs, list):
tree_list = [cs]
else:
tree_list = cs
else:
tree_list = []
for tg in trees_group_list:
stree_list = tg.get('tree')
if not isinstance(stree_list, list):
stree_list = [stree_list]
tree_list.extend([i.get('@id') for i in stree_list])
self._generate_ott_warnings(ogid2og, tree_list,
(nex_obj, obj_nex_id), vc)
return True
def _post_key_check_validate_tree(self,
tree_nex_id,
tree_obj,
vc,
otus_group_id=None):
# pylint: disable=R0914
node_by_id = tree_obj.get('nodeById')
edge_by_source = tree_obj.get('edgeBySourceId')
root_node_id = tree_obj.get('^ot:rootNodeId')
if (not node_by_id) or (not isinstance(node_by_id, dict)):
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=[
'nodeById',
])
return errorReturn('no "nodeById" in tree')
if (not edge_by_source) or (not isinstance(edge_by_source, dict)):
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=[
'edgeBySourceId',
])
return errorReturn('no "edgeBySourceId" in tree')
if not is_str_type(root_node_id):
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=[
'^ot:rootNodeId',
])
return errorReturn('no "^ot:rootNodeId" in tree')
edge_dict = {}
edge_by_target = {}
internal_nodes = []
if otus_group_id is None:
tree_group = vc.anc_list[-1][1]
otus_group_id = tree_group.get('@otus')
bad_node_ref = []
repeated_edge_id = []
missing_target = []
repeated_target = []
reachable_nodes = set()
for par_node_id, edge_by_id in edge_by_source.items():
if par_node_id not in node_by_id:
bad_node_ref.append(par_node_id)
else:
reachable_nodes.add(par_node_id)
for edge_id, edge in edge_by_id.items():
if edge_id in edge_dict:
repeated_edge_id.append(edge_id)
else:
edge_dict[edge_id] = edge
try:
t = edge.get('@target')
except:
t = None
if t is None:
missing_target.append(edge_id)
elif t in edge_by_target:
repeated_target.append(t)
else:
edge_by_target[t] = edge
reachable_nodes.add(t)
node_set = set(node_by_id.keys())
if node_set != reachable_nodes:
unreachable = list(node_set - reachable_nodes)
unreachable.sort()
not_in_node_by_id = list(reachable_nodes - node_set)
not_in_node_by_id.sort()
if unreachable:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_UnreachableNodeWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=unreachable)
return errorReturn('unreachable node in tree tree')
if not_in_node_by_id:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=not_in_node_by_id)
return errorReturn(
'referenced node id not in "nodeById" in tree')
if bad_node_ref:
bad_node_ref.sort()
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=bad_node_ref)
return errorReturn(
'referenced parent node not in "nodeById" in tree')
if missing_target:
missing_target.sort()
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=missing_target)
return errorReturn('no "@target" in edge')
if repeated_target:
repeated_target.sort()
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_NodeWithMultipleParents,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=repeated_target)
return errorReturn(
'same node used as "@target" for different edges')
if repeated_edge_id:
repeated_edge_id.sort()
self._error_event(_NEXEL.EDGE,
obj=tree_obj,
err_type=gen_RepeatedIDWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=repeated_edge_id)
return errorReturn('edge "@id" repeated')
node_set = set(edge_by_target.keys())
internal_node_set = set(edge_by_source.keys())
leaf_set = node_set - internal_node_set
leaves = [(i, node_by_id[i]) for i in leaf_set]
vc.push_context(_NEXEL.LEAF_NODE, (tree_obj, tree_nex_id))
try:
if not self._validate_leaf_list(leaves, vc):
return False
finally:
vc.pop_context()
internal_nodes = [(i, node_by_id[i]) for i in internal_node_set]
with_at_root_prop = {}
for nid, n_obj in internal_nodes:
if n_obj.get('@root'):
with_at_root_prop[nid] = n_obj
if len(with_at_root_prop) > 1:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MultipleRootsWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=list(with_at_root_prop.keys()))
return errorReturn('multiple "@root" nodes')
if len(with_at_root_prop) == 0:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_NoRootWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id)
return errorReturn('No node labelled as "@root"')
if root_node_id not in with_at_root_prop:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MultipleRootsWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=list(with_at_root_prop.keys()) +
[root_node_id])
return errorReturn('root node not labelled as root')
vc.push_context(_NEXEL.INTERNAL_NODE, (tree_obj, tree_nex_id))
try:
if not self._validate_internal_node_list(internal_nodes, vc):
return False
finally:
vc.pop_context()
edges = [i for i in edge_dict.items()]
vc.push_context(_NEXEL.EDGE, (tree_obj, tree_nex_id))
try:
if not self._validate_edge_list(edges, vc):
return False
finally:
vc.pop_context()
otuid2leaf = {}
for nd_id, nd in leaves:
otuid = nd['@otu']
if otuid in otuid2leaf:
vc.push_context(_NEXEL.LEAF_NODE, (tree_obj, tree_nex_id))
try:
self._error_event(_NEXEL.LEAF_NODE,
obj=nd,
err_type=gen_RepeatedOTUWarning,
anc=vc.anc_list,
obj_nex_id=nd_id,
key_list=[otuid])
return errorReturn('Repeated "@otu" id')
finally:
vc.pop_context()
otuid2leaf[otuid] = nd_id
self._detect_multilabelled_tree(otus_group_id=otus_group_id,
tree_id=tree_nex_id,
otuid2leaf=otuid2leaf)
og = self._otu_group_by_id[otus_group_id]['otuById']
return self._validate_otu_key_if_present(node_by_id.items(), og, vc)
def _post_key_check_validate_tree(self,
tree_nex_id,
tree_obj,
vc,
otus_group_id=None):
node_list = tree_obj.get('node')
if isinstance(node_list, dict):
node_list = [node_list]
elif (not node_list) or (not isinstance(node_list, list)):
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=[
'node',
])
return errorReturn('no "node" in "trees"')
edge_list = tree_obj.get('edge')
if isinstance(edge_list, dict):
edge_list = [edge_list]
elif (not edge_list) or (not isinstance(edge_list, list)):
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=[
'edge',
])
return errorReturn('no "edge" in tree')
edge_id_list = [(i.get('@id'), i) for i in edge_list]
vc.push_context(_NEXEL.EDGE, (tree_obj, tree_nex_id))
try:
valid = self._validate_edge_list(edge_id_list, vc)
if not valid:
return False
finally:
vc.pop_context()
node_id_obj_list = [(i.get('@id'), i) for i in node_list]
vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
try:
valid = self._validate_node_list(node_id_obj_list, vc)
if not valid:
return False
finally:
vc.pop_context()
node_dict = {}
for i in node_id_obj_list:
nid, nd = i
node_dict[nid] = nd
missing_src = []
missing_target = []
for el in edge_id_list:
e = el[1]
sid = e.get('@source')
tid = e.get('@target')
if sid not in node_dict:
missing_src.append(sid)
if tid not in node_dict:
missing_target.append(tid)
if missing_src:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=missing_src)
return errorReturn('no "@source" in edge')
if missing_target:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=missing_target)
return errorReturn('no "@target" in edge')
if otus_group_id is None:
tree_group = vc.anc_list[-1][1]
otus_group_id = tree_group.get('@otus')
lowest_nodeid_set = set()
encountered_nodes = set()
edge_by_target = {}
edge_by_source = {}
multi_parent_node = []
for e in edge_list:
t = e.get('@target')
if t in edge_by_target:
multi_parent_node.append(t)
else:
edge_by_target[t] = e
#_LOG.debug('e=' + str(e))
sid = e['@source']
edge_by_source.setdefault(sid, []).append(e)
if multi_parent_node:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_NodeWithMultipleParents,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=multi_parent_node)
otuid2leaf = {}
unflagged_leaves = []
nonleaves_with_leaf_flags = []
with_at_root_prop = {}
first_lowest_node = None
for nd in node_list:
nid = nd.get('@id')
cycle_node, path_to_root = construct_path_to_root(
nd, encountered_nodes, edge_by_target)
if cycle_node:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_TreeCycleWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
cycle_node=cycle_node)
return errorReturn('"@id" in node')
if path_to_root:
lowest_nodeid_set.add(path_to_root[-1])
if first_lowest_node is None:
first_lowest_node = path_to_root[-1]
is_flagged_as_leaf = self._find_first_literal_meta(nd, 'ot:isLeaf')
ch_list = edge_by_source.get(nid)
if ch_list is None:
otu_id = nd.get('@otu')
if otu_id is None:
vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
try:
self._error_event(
_NEXEL.NODE,
obj=nd,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=nid,
key_list=[
'@otu',
])
return errorReturn('"@otu" in leaf')
finally:
vc.pop_context()
else:
if otu_id in otuid2leaf:
vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
try:
self._error_event(_NEXEL.NODE,
obj=nd,
err_type=gen_RepeatedOTUWarning,
anc=vc.anc_list,
obj_nex_id=nid,
key_list=[otu_id])
return errorReturn('repeated "@otu" in leaves')
finally:
vc.pop_context()
otuid2leaf[otu_id] = nd
if not is_flagged_as_leaf:
if not self._logger.retain_deprecated:
add_literal_meta(nd, 'ot:isLeaf', True,
self._syntax_version)
else:
unflagged_leaves.append(nid)
elif is_flagged_as_leaf:
if not self._logger.retain_deprecated:
delete_first_literal_meta(nd, 'ot:isLeaf',
self._syntax_version)
else:
nonleaves_with_leaf_flags.append(nid)
if nd.get('@root'):
with_at_root_prop[nid] = nd
if unflagged_leaves:
vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
try:
#_LOG.debug('unflagged_leaves="{f}"'.format(f=unflagged_leaves))
self._error_event(_NEXEL.NODE,
obj=tree_obj,
err_type=gen_MissingMandatoryKeyWarning,
anc=vc.anc_list,
obj_nex_id=unflagged_leaves,
key_list=['ot:isLeaf'])
finally:
vc.pop_context()
if nonleaves_with_leaf_flags:
vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
try:
self._error_event(_NEXEL.NODE,
obj=tree_obj,
err_type=gen_InvalidKeyWarning,
anc=vc.anc_list,
obj_nex_id=nonleaves_with_leaf_flags,
key_list=['ot:isLeaf'])
return errorReturn('"ot:isLeaf" for internal')
finally:
vc.pop_context()
self._detect_multilabelled_tree(otus_group_id, tree_nex_id, otuid2leaf)
if len(lowest_nodeid_set) > 1:
lowest_nodeid_set = list(lowest_nodeid_set)
lowest_nodeid_set.sort()
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MultipleRootsWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=lowest_nodeid_set)
return errorReturn('multiple roots in a tree')
root_node_id = first_lowest_node
if root_node_id not in with_at_root_prop:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MultipleRootsWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=list(with_at_root_prop.keys()) +
[root_node_id])
return errorReturn('root without "@root"')
elif len(with_at_root_prop) > 1:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MultipleRootsWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=list(with_at_root_prop.keys()))
return errorReturn('Multiple nodes with "@root"')
elif len(with_at_root_prop) == 0:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_NoRootWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id)
return errorReturn('no node with "@root"')
og = set(
[i['@id'] for i in self._otu_group_by_id[otus_group_id]['otu']])
nli = [(i['@id'], i) for i in node_list]
return self._validate_otu_key_if_present(nli, og, vc)
def _post_key_check_validate_nexml_obj(self, nex_obj, obj_nex_id, vc):
otus = nex_obj.get('otusById', {})
otus_order_list = nex_obj.get('^ot:otusElementOrder', [])
if not isinstance(otus, dict) or (otus_order_list and (not otus)):
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=['otusById'])
return errorReturn('Missing "otusById"')
if (not isinstance(otus_order_list, list)) or ((not otus_order_list)
and otus):
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=['^ot:otusElementOrder'])
return errorReturn('Missing "^ot:otusElementOrder"')
otus_group_list = []
missing_ogid = []
not_dict_og = []
for ogid in otus_order_list:
og = otus.get(ogid)
if og is None:
missing_ogid.append(ogid)
elif not isinstance(og, dict):
r = check_raw_dict(og, otus, ogid, vc)
assert r[0] is False
t = r[1]
not_dict_og.append(t)
else:
otus_group_list.append((ogid, og))
if missing_ogid:
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=missing_ogid)
return errorReturn('Missing "@id" for otus group')
vc.push_context(_NEXEL.OTUS, (nex_obj, obj_nex_id))
try:
if not_dict_og:
self._error_event(_NEXEL.OTUS,
obj=otus,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=None,
key_val_type_list=[not_dict_og])
return errorReturn('Otus objects of the wrong type of object')
if not self._validate_otus_group_list(otus_group_list, vc):
return False
finally:
vc.pop_context()
# and now the trees...
trees = nex_obj.get('treesById')
trees_order_list = nex_obj.get('^ot:treesElementOrder')
if (not isinstance(trees, dict)) or ((not trees) and trees_order_list):
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=['treesById'])
return errorReturn('Missing "treesById"')
if (not isinstance(trees_order_list, list)) or ((not trees_order_list)
and trees):
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=['^ot:treesElementOrder'])
return errorReturn('Missing "^ot:treesElementOrder"')
trees_group_list = []
missing_tgid = []
not_dict_tg = []
for tgid in trees_order_list:
tg = trees.get(tgid)
if tg is None:
missing_tgid.append(tgid)
elif not isinstance(tg, dict):
r = check_raw_dict(tg, trees, tgid, vc)
assert r[0] is False
t = r[1]
not_dict_tg.append(t)
else:
trees_group_list.append((tgid, tg))
if missing_tgid:
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=missing_tgid)
return errorReturn('Missing trees group id')
vc.push_context(_NEXEL.TREES, (nex_obj, obj_nex_id))
try:
if not_dict_tg:
self._error_event(_NEXEL.TREES,
obj=trees,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=None,
key_val_type_list=[not_dict_tg])
return errorReturn('Trees element of the wrong type')
if not self._validate_trees_group_list(trees_group_list, vc):
return False
finally:
vc.pop_context()
if not nex_obj.get('ot:notIntendedForSynthesis'):
cs = nex_obj.get('ot:candidateTreeForSynthesis')
if cs:
if not isinstance(cs, list):
tree_list = [cs]
else:
tree_list = cs
else:
tree_list = []
for tg in trees.values():
tbi = tg.get('treeById', {})
tree_list.extend(tbi.keys())
self._generate_ott_warnings(otus, tree_list, (nex_obj, obj_nex_id),
vc)
return True
def _post_key_check_validate_nexml_obj(self, nex_obj, obj_nex_id, vc):
otus_group_list = nex_obj.get('otus', [])
if otus_group_list and isinstance(otus_group_list, dict):
otus_group_list = [otus_group_list]
if not isinstance(otus_group_list, list):
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=['otus'])
return errorReturn('no "otus" in nexml')
vc.push_context(_NEXEL.OTUS, (nex_obj, obj_nex_id))
try:
without_id = []
og_tuple_list = []
for og in otus_group_list:
ogid = og.get('@id')
if ogid is None:
without_id.append(None)
else:
og_tuple_list.append((ogid, og))
if without_id:
self._error_event(_NEXEL.OTUS,
obj=nex_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=None,
key_list=['otus[*]/@id'])
return errorReturn('otu without "@id"')
if not self._validate_otus_group_list(og_tuple_list, vc):
return False
finally:
vc.pop_context()
# and now the trees...
trees_group_list = nex_obj.get('trees', [])
if trees_group_list and isinstance(trees_group_list, dict):
trees_group_list = [trees_group_list]
if not isinstance(trees_group_list, list):
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=['trees'])
return errorReturn('No "trees" in nexml')
vc.push_context(_NEXEL.TREES, (nex_obj, obj_nex_id))
try:
without_id = []
tg_tuple_list = []
for tg in trees_group_list:
tgid = tg.get('@id')
if tgid is None:
without_id.append(tg)
else:
tg_tuple_list.append((tgid, tg))
if without_id:
self._error_event(_NEXEL.TREES,
obj=without_id,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=None,
key_list=['@id'])
return errorReturn('No "@id" in trees group"')
if not self._validate_trees_group_list(tg_tuple_list, vc):
return False
finally:
vc.pop_context()
ogid2og = {}
for og in otus_group_list:
ogid = og.get('@id')
ogid2og[ogid] = og
if not self._find_first_literal_meta(nex_obj, 'ot:notIntendedForSynthesis'):
cs = self._find_first_literal_meta(nex_obj, 'ot:candidateTreeForSynthesis')
if cs:
if not isinstance(cs, list):
tree_list = [cs]
else:
tree_list = cs
else:
tree_list = []
for tg in trees_group_list:
stree_list = tg.get('tree')
if not isinstance(stree_list, list):
stree_list = [stree_list]
tree_list.extend([i.get('@id') for i in stree_list])
self._generate_ott_warnings(ogid2og, tree_list, (nex_obj, obj_nex_id), vc)
return True
def _post_key_check_validate_nexml_obj(self, nex_obj, obj_nex_id, vc):
otus = nex_obj.get('otusById', {})
otus_order_list = nex_obj.get('^ot:otusElementOrder', [])
if not isinstance(otus, dict) or (otus_order_list and (not otus)):
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=['otusById'])
return errorReturn('Missing "otusById"')
if (not isinstance(otus_order_list, list)) or ((not otus_order_list) and otus):
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=['^ot:otusElementOrder'])
return errorReturn('Missing "^ot:otusElementOrder"')
otus_group_list = []
missing_ogid = []
not_dict_og = []
for ogid in otus_order_list:
og = otus.get(ogid)
if og is None:
missing_ogid.append(ogid)
elif not isinstance(og, dict):
r = check_raw_dict(og, otus, ogid, vc)
assert r[0] is False
t = r[1]
not_dict_og.append(t)
else:
otus_group_list.append((ogid, og))
if missing_ogid:
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=missing_ogid)
return errorReturn('Missing "@id" for otus group')
vc.push_context(_NEXEL.OTUS, (nex_obj, obj_nex_id))
try:
if not_dict_og:
self._error_event(_NEXEL.OTUS,
obj=otus,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=None,
key_val_type_list=[not_dict_og])
return errorReturn('Otus objects of the wrong type of object')
if not self._validate_otus_group_list(otus_group_list, vc):
return False
finally:
vc.pop_context()
# and now the trees...
trees = nex_obj.get('treesById')
trees_order_list = nex_obj.get('^ot:treesElementOrder')
if (not isinstance(trees, dict)) or ((not trees) and trees_order_list):
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=['treesById'])
return errorReturn('Missing "treesById"')
if (not isinstance(trees_order_list, list)) or ((not trees_order_list) and trees):
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=['^ot:treesElementOrder'])
return errorReturn('Missing "^ot:treesElementOrder"')
trees_group_list = []
missing_tgid = []
not_dict_tg = []
for tgid in trees_order_list:
tg = trees.get(tgid)
if tg is None:
missing_tgid.append(tgid)
elif not isinstance(tg, dict):
r = check_raw_dict(tg, trees, tgid, vc)
assert r[0] is False
t = r[1]
not_dict_tg.append(t)
else:
trees_group_list.append((tgid, tg))
if missing_tgid:
self._error_event(_NEXEL.NEXML,
obj=nex_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=obj_nex_id,
key_list=missing_tgid)
return errorReturn('Missing trees group id')
vc.push_context(_NEXEL.TREES, (nex_obj, obj_nex_id))
try:
if not_dict_tg:
self._error_event(_NEXEL.TREES,
obj=trees,
err_type=gen_WrongValueTypeWarning,
anc=vc.anc_list,
obj_nex_id=None,
key_val_type_list=[not_dict_tg])
return errorReturn('Trees element of the wrong type')
if not self._validate_trees_group_list(trees_group_list, vc):
return False
finally:
vc.pop_context()
if not nex_obj.get('ot:notIntendedForSynthesis'):
cs = nex_obj.get('ot:candidateTreeForSynthesis')
if cs:
if not isinstance(cs, list):
tree_list = [cs]
else:
tree_list = cs
else:
tree_list = []
for tg in trees.values():
tbi = tg.get('treeById', {})
tree_list.extend(tbi.keys())
self._generate_ott_warnings(otus, tree_list, (nex_obj, obj_nex_id), vc)
return True
def _post_key_check_validate_tree(self,
tree_nex_id,
tree_obj,
vc,
otus_group_id=None):
#pylint: disable=R0914
node_by_id = tree_obj.get('nodeById')
edge_by_source = tree_obj.get('edgeBySourceId')
root_node_id = tree_obj.get('^ot:rootNodeId')
if (not node_by_id) or (not isinstance(node_by_id, dict)):
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=['nodeById',])
return errorReturn('no "nodeById" in tree')
if (not edge_by_source) or (not isinstance(edge_by_source, dict)):
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=['edgeBySourceId',])
return errorReturn('no "edgeBySourceId" in tree')
if not is_str_type(root_node_id):
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MissingCrucialContentWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=['^ot:rootNodeId',])
return errorReturn('no "^ot:rootNodeId" in tree')
edge_dict = {}
edge_by_target = {}
internal_nodes = []
if otus_group_id is None:
tree_group = vc.anc_list[-1][1]
otus_group_id = tree_group.get('@otus')
bad_node_ref = []
repeated_edge_id = []
missing_target = []
repeated_target = []
reachable_nodes = set()
for par_node_id, edge_by_id in edge_by_source.items():
if par_node_id not in node_by_id:
bad_node_ref.append(par_node_id)
else:
reachable_nodes.add(par_node_id)
for edge_id, edge in edge_by_id.items():
if edge_id in edge_dict:
repeated_edge_id.append(edge_id)
else:
edge_dict[edge_id] = edge
try:
t = edge.get('@target')
except:
t = None
if t is None:
missing_target.append(edge_id)
elif t in edge_by_target:
repeated_target.append(t)
else:
edge_by_target[t] = edge
reachable_nodes.add(t)
node_set = set(node_by_id.keys())
if node_set != reachable_nodes:
unreachable = list(node_set - reachable_nodes)
unreachable.sort()
not_in_node_by_id = list(reachable_nodes - node_set)
not_in_node_by_id.sort()
if unreachable:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_UnreachableNodeWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=unreachable)
return errorReturn('unreachable node in tree tree')
if not_in_node_by_id:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=not_in_node_by_id)
return errorReturn('referenced node id not in "nodeById" in tree')
if bad_node_ref:
bad_node_ref.sort()
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=bad_node_ref)
return errorReturn('referenced parent node not in "nodeById" in tree')
if missing_target:
missing_target.sort()
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_ReferencedIDNotFoundWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=missing_target)
return errorReturn('no "@target" in edge')
if repeated_target:
repeated_target.sort()
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_NodeWithMultipleParents,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=repeated_target)
return errorReturn('same node used as "@target" for different edges')
if repeated_edge_id:
repeated_edge_id.sort()
self._error_event(_NEXEL.EDGE,
obj=tree_obj,
err_type=gen_RepeatedIDWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
key_list=repeated_edge_id)
return errorReturn('edge "@id" repeated')
node_set = set(edge_by_target.keys())
internal_node_set = set(edge_by_source.keys())
leaf_set = node_set - internal_node_set
leaves = [(i, node_by_id[i]) for i in leaf_set]
vc.push_context(_NEXEL.LEAF_NODE, (tree_obj, tree_nex_id))
try:
if not self._validate_leaf_list(leaves, vc):
return False
finally:
vc.pop_context()
internal_nodes = [(i, node_by_id[i]) for i in internal_node_set]
with_at_root_prop = {}
for nid, n_obj in internal_nodes:
if n_obj.get('@root'):
with_at_root_prop[nid] = n_obj
if len(with_at_root_prop) > 1:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MultipleRootsWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=list(with_at_root_prop.keys()))
return errorReturn('multiple "@root" nodes')
if len(with_at_root_prop) == 0:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_NoRootWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id)
return errorReturn('No node labelled as "@root"')
if root_node_id not in with_at_root_prop:
self._error_event(_NEXEL.TREE,
obj=tree_obj,
err_type=gen_MultipleRootsWarning,
anc=vc.anc_list,
obj_nex_id=tree_nex_id,
node_id_list=list(with_at_root_prop.keys()) + [root_node_id])
return errorReturn('root node not labelled as root')
vc.push_context(_NEXEL.INTERNAL_NODE, (tree_obj, tree_nex_id))
try:
if not self._validate_internal_node_list(internal_nodes, vc):
return False
finally:
vc.pop_context()
edges = [i for i in edge_dict.items()]
vc.push_context(_NEXEL.EDGE, (tree_obj, tree_nex_id))
try:
if not self._validate_edge_list(edges, vc):
return False
finally:
vc.pop_context()
otuid2leaf = {}
for nd_id, nd in leaves:
otuid = nd['@otu']
if otuid in otuid2leaf:
vc.push_context(_NEXEL.LEAF_NODE, (tree_obj, tree_nex_id))
try:
self._error_event(_NEXEL.LEAF_NODE,
obj=nd,
err_type=gen_RepeatedOTUWarning,
anc=vc.anc_list,
obj_nex_id=nd_id,
key_list=[otuid])
return errorReturn('Repeated "@otu" id')
finally:
vc.pop_context()
otuid2leaf[otuid] = nd_id
self._detect_multilabelled_tree(otus_group_id=otus_group_id,
tree_id=tree_nex_id,
otuid2leaf=otuid2leaf)
og = self._otu_group_by_id[otus_group_id]['otuById']
return self._validate_otu_key_if_present(node_by_id.items(), og, vc)
def _validate_id_obj_list_by_schema(self,
id_obj_list,
vc,
group_by_warning=False):
element_type = vc.curr_element_type
assert element_type is not None
schema = vc.schema
anc_list = vc.anc_list
#_LOG.debug('using schema type = ' + vc.schema_name())
using_hbf_meta = vc._using_hbf_meta #pylint: disable=W0212
_by_warn_type = {}
for obj_nex_id, obj in id_obj_list:
wrong_type = []
unrec_meta_keys = []
unrec_non_meta_keys = []
if using_hbf_meta:
for k, v in obj.items():
is_meta = k[0] == '^'
if k not in schema.ALLOWED_KEY_SET:
if is_meta:
unrec_meta_keys.append(k)
else:
unrec_non_meta_keys.append(k)
else:
correct_type, info = schema.K2VT[k](v, obj, k, vc)
if not correct_type:
wrong_type.append((k, v, info))
else:
for k, v in obj.items():
if k not in schema.ALLOWED_KEY_SET:
unrec_non_meta_keys.append(k)
else:
correct_type, info = schema.K2VT[k](v, obj, k, vc)
if not correct_type:
wrong_type.append((k, v, info))
m = self._get_list_key(obj, 'meta', vc)
if m:
# might want a flag of meta?
md = self._bf_meta_list_to_dict(m, obj, vc)
#_LOG.debug('md = ' + str(md))
mrmk = [
i for i in schema.REQUIRED_META_KEY_SET if i not in md
]
memk = [
i for i in schema.EXPECTED_META_KEY_SET if i not in md
]
if memk:
if group_by_warning:
memk.sort()
foks = frozenset(memk)
t = _by_warn_type.setdefault(foks, [[], []])
t[0].append(obj)
t[1].append(obj_nex_id)
else:
self._warn_event(
element_type,
obj=obj,
err_type=gen_MissingOptionalKeyWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_list=memk)
if mrmk:
self._error_event(
element_type,
obj=obj,
err_type=gen_MissingMandatoryKeyWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_list=mrmk)
msgf = 'missing mandatory meta key(s) according to {s} schema'
msg = msgf.format(s=vc.schema_name())
return errorReturn(msg)
for k, v in md.items():
if k not in schema.ALLOWED_META_KEY_SET:
unrec_meta_keys.append(k)
else:
#_LOG.debug('{k} --> "{v}"'.format(k=k, v=repr(v)))
correct_type, info = schema.K2VT[k](v, obj, k, vc)
if not correct_type:
v = extract_meta(v)
wrong_type.append((k, v, info))
if wrong_type:
self._error_event(element_type,
obj=obj,
err_type=gen_WrongValueTypeWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_val_type_list=wrong_type)
return errorReturn(
'wrong value type according to {s} schema'.format(
s=vc.schema_name()))
if unrec_non_meta_keys:
self._warn_event(element_type,
obj=obj,
err_type=gen_UnrecognizedKeyWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_list=unrec_non_meta_keys)
if unrec_meta_keys:
# might want a flag of meta?
self._warn_event(element_type,
obj=obj,
err_type=gen_UnrecognizedKeyWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_list=unrec_meta_keys)
off_key = [k for k in schema.EXPECETED_KEY_SET if k not in obj]
if off_key:
if group_by_warning:
off_key.sort()
foks = frozenset(off_key)
t = _by_warn_type.setdefault(foks, [[], []])
t[0].append(obj)
t[1].append(obj_nex_id)
else:
self._warn_event(element_type,
obj=obj,
err_type=gen_MissingOptionalKeyWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_list=off_key)
off_key = [k for k in schema.REQUIRED_KEY_SET if k not in obj]
if off_key:
self._error_event(element_type,
obj=obj,
err_type=gen_MissingMandatoryKeyWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_list=off_key)
return errorReturn(
'missing key(s) according to {s} schema'.format(
s=vc.schema_name()))
if _by_warn_type:
for ks, obj_lists in _by_warn_type.items():
mlist = list(ks)
mlist.sort()
id_arg = obj_lists[1]
if len(id_arg) == 1:
id_arg = id_arg[0]
obj_arg = obj_lists[0]
if len(obj_arg) == 1:
obj_arg = obj_arg[0]
self._warn_event(element_type,
obj=obj_arg,
err_type=gen_MissingOptionalKeyWarning,
anc=anc_list,
obj_nex_id=id_arg,
key_list=mlist)
return True
def _validate_id_obj_list_by_schema(self, id_obj_list, vc, group_by_warning=False):
element_type = vc.curr_element_type
assert element_type is not None
schema = vc.schema
anc_list = vc.anc_list
# _LOG.debug('using schema type = ' + vc.schema_name())
using_hbf_meta = vc._using_hbf_meta # pylint: disable=W0212
_by_warn_type = {}
for obj_nex_id, obj in id_obj_list:
wrong_type = []
unrec_meta_keys = []
unrec_non_meta_keys = []
if using_hbf_meta:
for k, v in obj.items():
is_meta = k[0] == '^'
if k not in schema.ALLOWED_KEY_SET:
if is_meta:
unrec_meta_keys.append(k)
else:
unrec_non_meta_keys.append(k)
else:
correct_type, info = schema.K2VT[k](v, obj, k, vc)
if not correct_type:
wrong_type.append((k, v, info))
else:
for k, v in obj.items():
if k not in schema.ALLOWED_KEY_SET:
unrec_non_meta_keys.append(k)
else:
correct_type, info = schema.K2VT[k](v, obj, k, vc)
if not correct_type:
wrong_type.append((k, v, info))
m = self._get_list_key(obj, 'meta', vc)
if m:
# might want a flag of meta?
md = self._bf_meta_list_to_dict(m, obj, vc)
# _LOG.debug('md = ' + str(md))
mrmk = [i for i in schema.REQUIRED_META_KEY_SET if i not in md]
memk = [i for i in schema.EXPECTED_META_KEY_SET if i not in md]
if memk:
if group_by_warning:
memk.sort()
foks = frozenset(memk)
t = _by_warn_type.setdefault(foks, [[], []])
t[0].append(obj)
t[1].append(obj_nex_id)
else:
self._warn_event(element_type,
obj=obj,
err_type=gen_MissingOptionalKeyWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_list=memk)
if mrmk:
self._error_event(element_type,
obj=obj,
err_type=gen_MissingMandatoryKeyWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_list=mrmk)
msgf = 'missing mandatory meta key(s) according to {s} schema'
msg = msgf.format(s=vc.schema_name())
return errorReturn(msg)
for k, v in md.items():
if k not in schema.ALLOWED_META_KEY_SET:
unrec_meta_keys.append(k)
else:
# _LOG.debug('{k} --> "{v}"'.format(k=k, v=repr(v)))
correct_type, info = schema.K2VT[k](v, obj, k, vc)
if not correct_type:
v = extract_meta(v)
wrong_type.append((k, v, info))
if wrong_type:
self._error_event(element_type,
obj=obj,
err_type=gen_WrongValueTypeWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_val_type_list=wrong_type)
return errorReturn('wrong value type according to {s} schema'.format(s=vc.schema_name()))
if unrec_non_meta_keys:
self._warn_event(element_type,
obj=obj,
err_type=gen_UnrecognizedKeyWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_list=unrec_non_meta_keys)
if unrec_meta_keys:
# might want a flag of meta?
self._warn_event(element_type,
obj=obj,
err_type=gen_UnrecognizedKeyWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_list=unrec_meta_keys)
off_key = [k for k in schema.EXPECETED_KEY_SET if k not in obj]
if off_key:
if group_by_warning:
off_key.sort()
foks = frozenset(off_key)
t = _by_warn_type.setdefault(foks, [[], []])
t[0].append(obj)
t[1].append(obj_nex_id)
else:
self._warn_event(element_type,
obj=obj,
err_type=gen_MissingOptionalKeyWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_list=off_key)
off_key = [k for k in schema.REQUIRED_KEY_SET if k not in obj]
if off_key:
self._error_event(element_type,
obj=obj,
err_type=gen_MissingMandatoryKeyWarning,
anc=anc_list,
obj_nex_id=obj_nex_id,
key_list=off_key)
return errorReturn('missing key(s) according to {s} schema'.format(s=vc.schema_name()))
if _by_warn_type:
for ks, obj_lists in _by_warn_type.items():
mlist = list(ks)
mlist.sort()
id_arg = obj_lists[1]
if len(id_arg) == 1:
id_arg = id_arg[0]
obj_arg = obj_lists[0]
if len(obj_arg) == 1:
obj_arg = obj_arg[0]
self._warn_event(element_type,
obj=obj_arg,
err_type=gen_MissingOptionalKeyWarning,
anc=anc_list,
obj_nex_id=id_arg,
key_list=mlist)
return True
