Example #1
0
 def _validate_otu_key_if_present(self, nd_id_nd_iter, otu_by_id, vc):
     erred = False
     for nd_id, nd in nd_id_nd_iter:
         o = nd.get('@otu')
         if o is not None:
             if o not in otu_by_id:
                 erred = True
                 self._error_event(_NEXEL.NODE,
                                   obj=nd,
                                   err_type=gen_ReferencedIDNotFoundWarning,
                                   anc=vc.anc_list,
                                   obj_nex_id=nd_id,
                                   key_list=[o])
     if erred:
         errorReturn('Unknown "@otu" id')
     return True
 def _validate_otu_key_if_present(self, nd_id_nd_iter, otu_by_id, vc):
     erred = False
     for nd_id, nd in nd_id_nd_iter:
         o = nd.get('@otu')
         if o is not None:
             if o not in otu_by_id:
                 erred = True
                 self._error_event(_NEXEL.NODE,
                                   obj=nd,
                                   err_type=gen_ReferencedIDNotFoundWarning,
                                   anc=vc.anc_list,
                                   obj_nex_id=nd_id,
                                   key_list=[o])
     if erred:
         errorReturn('Unknown "@otu" id')
     return True
Example #3
0
 def _post_key_check_validate_otus_obj(self, og_nex_id, otus_group, vc):
     otu_dict = {}
     otu_list = otus_group.get('otu', [])
     if isinstance(otu_list, dict):
         otu_list = [otu_list]
     if not otu_list:
         return
     vc.push_context(_NEXEL.OTU, (otus_group, og_nex_id))
     try:
         without_id = []
         otu_tuple_list = []
         for otu in otu_list:
             oid = otu.get('@id')
             if oid is None:
                 without_id.append(otu)
             else:
                 otu_tuple_list.append((oid, otu))
             otu_dict[oid] = otu
         if without_id:
             self._error_event(_NEXEL.NEXML,
                               obj=without_id,
                               err_type=gen_MissingCrucialContentWarning,
                               anc=vc.anc_list,
                               obj_nex_id=None,
                               key_list=['@id'])
             return errorReturn('lack of "@id" in "otu"')
         r = self._validate_otu_list(otu_tuple_list, vc)
         if r:
             self._otu_by_otug[og_nex_id] = otu_dict
         return r
     except:
         vc.pop_context()
 def _post_key_check_validate_otus_obj(self, og_nex_id, otus_group, vc):
     otu_dict = {}
     otu_list = otus_group.get('otu', [])
     if isinstance(otu_list, dict):
         otu_list = [otu_list]
     if not otu_list:
         return
     vc.push_context(_NEXEL.OTU, (otus_group, og_nex_id))
     try:
         without_id = []
         otu_tuple_list = []
         for otu in otu_list:
             oid = otu.get('@id')
             if oid is None:
                 without_id.append(otu)
             else:
                 otu_tuple_list.append((oid, otu))
             otu_dict[oid] = otu
         if without_id:
             self._error_event(_NEXEL.NEXML,
                               obj=without_id,
                               err_type=gen_MissingCrucialContentWarning,
                               anc=vc.anc_list,
                               obj_nex_id=None,
                               key_list=['@id'])
             return errorReturn('lack of "@id" in "otu"')
         r = self._validate_otu_list(otu_tuple_list, vc)
         if r:
             self._otu_by_otug[og_nex_id] = otu_dict
         return r
     except:
         vc.pop_context()
 def _post_key_check_validate_tree_group(self, tg_nex_id, trees_group, vc):
     otus_el = trees_group.get('@otus')
     if otus_el not in self._otu_group_by_id:
         kl = ['@otus="{oe}"'.format(oe=otus_el)]
         self._error_event(_NEXEL.TREES,
                           obj=trees_group,
                           err_type=gen_ReferencedIDNotFoundWarning,
                           anc=vc.anc_list,
                           obj_nex_id=tg_nex_id,
                           key_list=kl)
         return errorReturn('bad "@otus" in trees group')
     tree_list = trees_group.get('tree', [])
     if isinstance(tree_list, dict):
         tree_list = [tree_list]
     elif not isinstance(tree_list, list):
         self._error_event(_NEXEL.TREES,
                           obj=trees_group,
                           err_type=gen_WrongValueTypeWarning,
                           anc=vc.anc_list,
                           obj_nex_id=tg_nex_id,
                           key_list=['tree'])
         return errorReturn('lack of "tree" in trees group')
     for tree_obj in tree_list:
         t_nex_id = tree_obj.get('@id')
         vc.push_context(_NEXEL.TREE, (tree_obj, t_nex_id))
         try:
             if t_nex_id is None:
                 self._error_event(_NEXEL.TREE,
                                   obj=tree_obj,
                                   err_type=gen_MissingCrucialContentWarning,
                                   anc=vc.anc_list,
                                   obj_nex_id=tg_nex_id,
                                   key_list=['@id'])
                 return errorReturn('no "@id" in tree')
             if not self._validate_tree(t_nex_id,
                                        tree_obj,
                                        vc,
                                        otus_group_id=otus_el):
                 return False
         finally:
             vc.pop_context()
     return True
Example #6
0
 def _post_key_check_validate_tree_group(self, tg_nex_id, trees_group, vc):
     otus_el = trees_group.get('@otus')
     if otus_el not in self._otu_group_by_id:
         kl = ['@otus="{oe}"'.format(oe=otus_el)]
         self._error_event(_NEXEL.TREES,
                           obj=trees_group,
                           err_type=gen_ReferencedIDNotFoundWarning,
                           anc=vc.anc_list,
                           obj_nex_id=tg_nex_id,
                           key_list=kl)
         return errorReturn('bad "@otus" in trees group')
     tree_list = trees_group.get('tree', [])
     if isinstance(tree_list, dict):
         tree_list = [tree_list]
     elif not isinstance(tree_list, list):
         self._error_event(_NEXEL.TREES,
                           obj=trees_group,
                           err_type=gen_WrongValueTypeWarning,
                           anc=vc.anc_list,
                           obj_nex_id=tg_nex_id,
                           key_list=['tree'])
         return errorReturn('lack of "tree" in trees group')
     for tree_obj in tree_list:
         t_nex_id = tree_obj.get('@id')
         vc.push_context(_NEXEL.TREE, (trees_group, tg_nex_id))
         try:
             if t_nex_id is None:
                 self._error_event(
                     _NEXEL.TREE,
                     obj=tree_obj,
                     err_type=gen_MissingCrucialContentWarning,
                     anc=vc.anc_list,
                     obj_nex_id=tg_nex_id,
                     key_list=['@id'])
                 return errorReturn('no "@id" in tree')
             if not self._validate_tree(
                     t_nex_id, tree_obj, vc, otus_group_id=otus_el):
                 return False
         finally:
             vc.pop_context()
     return True
Example #7
0
 def _post_key_check_validate_tree_group(self, tg_nex_id, tree_group_obj,
                                         vc):
     tree_id_order = tree_group_obj.get('^ot:treeElementOrder', [])
     tree_by_id = tree_group_obj.get('treeById', {})
     if (not isinstance(tree_by_id, dict)) \
             or (not isinstance(tree_id_order, list)):
         self._error_event(_NEXEL.TREES,
                           obj=tree_group_obj,
                           err_type=gen_WrongValueTypeWarning,
                           anc=vc.anc_list,
                           obj_nex_id=tg_nex_id,
                           key_list=['treeById', '^ot:treeElementOrder'])
         return errorReturn('no "treeById" in trees group')
     if ((not tree_by_id) and tree_id_order) or ((not tree_id_order)
                                                 and tree_by_id):
         self._error_event(_NEXEL.TREES,
                           obj=tree_group_obj,
                           err_type=gen_MissingCrucialContentWarning,
                           anc=vc.anc_list,
                           obj_nex_id=tg_nex_id,
                           key_list=['treeById', '^ot:treeElementOrder'])
         return errorReturn('no "treeById" in trees group')
     otus_el = tree_group_obj.get('@otus')
     if otus_el not in self._otu_group_by_id:
         self._error_event(_NEXEL.TREES,
                           obj=tree_group_obj,
                           err_type=gen_ReferencedIDNotFoundWarning,
                           anc=vc.anc_list,
                           obj_nex_id=tg_nex_id,
                           key_list=[otus_el])
         return errorReturn('no "@otus" in trees group')
     for t_nex_id, tree_obj in tree_by_id.items():
         vc.push_context(_NEXEL.TREE, (tree_group_obj, tg_nex_id))
         try:
             if not self._validate_tree(
                     t_nex_id, tree_obj, vc, otus_group_id=otus_el):
                 return False
         finally:
             vc.pop_context()
     return True
Example #8
0
 def _post_key_check_validate_otus_obj(self, og_nex_id, otus_group, vc):
     otu_obj = otus_group.get('otuById', {})
     if not isinstance(otu_obj, dict):
         self._error_event(_NEXEL.OTUS,
                           obj=otus_group,
                           err_type=gen_WrongValueTypeWarning,
                           anc=vc.anc_list,
                           obj_nex_id=og_nex_id,
                           key_list=['otuById'])
         return errorReturn('no "otuById" in otus group')
     self._otu_group_by_id[og_nex_id] = otus_group
     vc.push_context(_NEXEL.OTU, (otus_group, og_nex_id))
     try:
         not_dict_otu = []
         ndo_id_list = []
         otu_id_obj_list = []
         for id_obj_pair in otu_obj.items():
             if not isinstance(id_obj_pair[1], dict):
                 r = check_raw_dict(id_obj_pair[1], otus_group, None, vc)
                 assert r[0] is False
                 t = r[1]
                 not_dict_otu.append(t)
                 ndo_id_list.append(id_obj_pair[0])
             else:
                 otu_id_obj_list.append(id_obj_pair)
         if not_dict_otu:
             self._error_event(_NEXEL.OTU,
                               obj=otu_obj,
                               err_type=gen_WrongValueTypeWarning,
                               anc=vc.anc_list,
                               obj_nex_id=ndo_id_list,
                               key_val_type_list=[not_dict_otu])
             return errorReturn('otu is wrong type')
         r = self._validate_otu_list(otu_id_obj_list, vc)
         if r:
             self._otu_by_otug[og_nex_id] = otu_obj
         return r
     finally:
         vc.pop_context()
Example #9
0
 def _post_key_check_validate_otus_obj(self, og_nex_id, otus_group, vc):
     otu_obj = otus_group.get('otuById', {})
     if not isinstance(otu_obj, dict):
         self._error_event(_NEXEL.OTUS,
                           obj=otus_group,
                           err_type=gen_WrongValueTypeWarning,
                           anc=vc.anc_list,
                           obj_nex_id=og_nex_id,
                           key_list=['otuById'])
         return errorReturn('no "otuById" in otus group')
     self._otu_group_by_id[og_nex_id] = otus_group
     vc.push_context(_NEXEL.OTU, (otus_group, og_nex_id))
     try:
         not_dict_otu = []
         ndo_id_list = []
         otu_id_obj_list = []
         for id_obj_pair in otu_obj.items():
             if not isinstance(id_obj_pair[1], dict):
                 r = check_raw_dict(id_obj_pair[1], otus_group, None, vc)
                 assert r[0] is False
                 t = r[1]
                 not_dict_otu.append(t)
                 ndo_id_list.append(id_obj_pair[0])
             else:
                 otu_id_obj_list.append(id_obj_pair)
         if not_dict_otu:
             self._error_event(_NEXEL.OTU,
                               obj=otu_obj,
                               err_type=gen_WrongValueTypeWarning,
                               anc=vc.anc_list,
                               obj_nex_id=ndo_id_list,
                               key_val_type_list=[not_dict_otu])
             return errorReturn('otu is wrong type')
         r = self._validate_otu_list(otu_id_obj_list, vc)
         if r:
             self._otu_by_otug[og_nex_id] = otu_obj
         return r
     finally:
         vc.pop_context()
Example #10
0
 def _post_key_check_validate_tree_group(self, tg_nex_id, tree_group_obj, vc):
     tree_id_order = tree_group_obj.get('^ot:treeElementOrder', [])
     tree_by_id = tree_group_obj.get('treeById', {})
     if (not isinstance(tree_by_id, dict)) \
         or (not isinstance(tree_id_order, list)):
         self._error_event(_NEXEL.TREES,
                           obj=tree_group_obj,
                           err_type=gen_WrongValueTypeWarning,
                           anc=vc.anc_list,
                           obj_nex_id=tg_nex_id,
                           key_list=['treeById', '^ot:treeElementOrder'])
         return errorReturn('no "treeById" in trees group')
     if ((not tree_by_id) and tree_id_order) or ((not tree_id_order) and tree_by_id):
         self._error_event(_NEXEL.TREES,
                           obj=tree_group_obj,
                           err_type=gen_MissingCrucialContentWarning,
                           anc=vc.anc_list,
                           obj_nex_id=tg_nex_id,
                           key_list=['treeById', '^ot:treeElementOrder'])
         return errorReturn('no "treeById" in trees group')
     otus_el = tree_group_obj.get('@otus')
     if otus_el not in self._otu_group_by_id:
         self._error_event(_NEXEL.TREES,
                           obj=tree_group_obj,
                           err_type=gen_ReferencedIDNotFoundWarning,
                           anc=vc.anc_list,
                           obj_nex_id=tg_nex_id,
                           key_list=[otus_el])
         return errorReturn('no "@otus" in trees group')
     for t_nex_id, tree_obj in tree_by_id.items():
         vc.push_context(_NEXEL.TREE, (tree_group_obj, tg_nex_id))
         try:
             if not self._validate_tree(t_nex_id, tree_obj, vc, otus_group_id=otus_el):
                 return False
         finally:
             vc.pop_context()
     return True
    def _post_key_check_validate_tree(self,
                                      tree_nex_id,
                                      tree_obj,
                                      vc,
                                      otus_group_id=None):
        node_list = tree_obj.get('node')
        if isinstance(node_list, dict):
            node_list = [node_list]
        elif (not node_list) or (not isinstance(node_list, list)):
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MissingCrucialContentWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=['node',])
            return errorReturn('no "node" in "trees"')
        edge_list = tree_obj.get('edge')
        if isinstance(edge_list, dict):
            edge_list = [edge_list]
        elif (not edge_list) or (not isinstance(edge_list, list)):
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MissingCrucialContentWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=['edge',])
            return errorReturn('no "edge" in tree')
        edge_id_list = [(i.get('@id'), i) for i in edge_list]
        valid = self._validate_edge_list(edge_id_list, vc)
        if not valid:
            return False
        node_id_obj_list = [(i.get('@id'), i) for i in node_list]
        valid = self._validate_node_list(node_id_obj_list, vc)
        if not valid:
            return False
        node_dict = {}
        for i in node_id_obj_list:
            nid, nd = i
            node_dict[nid] = nd
        missing_src = []
        missing_target = []
        for el in edge_id_list:
            e = el[1]
            sid = e.get('@source')
            tid = e.get('@target')
            if sid not in node_dict:
                missing_src.append(sid)
            if tid not in node_dict:
                missing_target.append(tid)
        if missing_src:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_ReferencedIDNotFoundWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=missing_src)
            return errorReturn('no "@source" in edge')
        if missing_target:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_ReferencedIDNotFoundWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=missing_target)
            return errorReturn('no "@target" in edge')
        if otus_group_id is None:
            tree_group = vc.anc_list[-1][1]
            otus_group_id = tree_group.get('@otus')

        lowest_nodeid_set = set()
        encountered_nodes = set()
        edge_by_target = {}
        edge_by_source = {}
        multi_parent_node = []
        for e in edge_list:
            t = e.get('@target')
            if t in edge_by_target:
                multi_parent_node.append(t)
            else:
                edge_by_target[t] = e
            #_LOG.debug('e=' + str(e))
            sid = e['@source']
            edge_by_source.setdefault(sid, []).append(e)
        if multi_parent_node:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_NodeWithMultipleParents,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=multi_parent_node)
        otuid2leaf = {}
        unflagged_leaves = []
        nonleaves_with_leaf_flags = []
        with_at_root_prop = {}
        first_lowest_node = None
        for nd in node_list:
            nid = nd.get('@id')
            cycle_node, path_to_root = construct_path_to_root(nd, encountered_nodes, edge_by_target)
            if cycle_node:
                self._error_event(_NEXEL.TREE,
                                  obj=tree_obj,
                                  err_type=gen_TreeCycleWarning,
                                  anc=vc.anc_list,
                                  obj_nex_id=tree_nex_id,
                                  cycle_node=cycle_node)
                return errorReturn('"@id" in node')
            if path_to_root:
                lowest_nodeid_set.add(path_to_root[-1])
                if first_lowest_node is None:
                    first_lowest_node = path_to_root[-1]
            is_flagged_as_leaf = self._find_first_literal_meta(nd, 'ot:isLeaf')
            ch_list = edge_by_source.get(nid)
            if ch_list is None:
                otu_id = nd.get('@otu')
                if otu_id is None:
                    vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
                    try:
                        self._error_event(_NEXEL.NODE,
                                          obj=nd,
                                          err_type=gen_MissingCrucialContentWarning,
                                          anc=vc.anc_list,
                                          obj_nex_id=nid,
                                          key_list=['@otu',])
                        return errorReturn('"@otu" in leaf')
                    finally:
                        vc.pop_context()
                else:
                    if otu_id in otuid2leaf:
                        vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
                        try:
                            self._error_event(_NEXEL.NODE,
                                              obj=nd,
                                              err_type=gen_RepeatedOTUWarning,
                                              anc=vc.anc_list,
                                              obj_nex_id=nid,
                                              key_list=[otu_id])
                            return errorReturn('repeated "@otu" in leaves')
                        finally:
                            vc.pop_context()
                    otuid2leaf[otu_id] = nd
                if not is_flagged_as_leaf:
                    if not self._logger.retain_deprecated:
                        add_literal_meta(nd, 'ot:isLeaf', True, self._syntax_version)
                    else:
                        unflagged_leaves.append(nid)
            elif is_flagged_as_leaf:
                if not self._logger.retain_deprecated:
                    delete_first_literal_meta(nd, 'ot:isLeaf', self._syntax_version)
                else:
                    nonleaves_with_leaf_flags.append(nid)
            if nd.get('@root'):
                with_at_root_prop[nid] = nd
        if unflagged_leaves:
            vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
            try:
                #_LOG.debug('unflagged_leaves="{f}"'.format(f=unflagged_leaves))
                self._error_event(_NEXEL.NODE,
                                  obj=tree_obj,
                                  err_type=gen_MissingMandatoryKeyWarning,
                                  anc=vc.anc_list,
                                  obj_nex_id=unflagged_leaves,
                                  key_list=['ot:isLeaf'])
            finally:
                vc.pop_context()
        if nonleaves_with_leaf_flags:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_InvalidKeyWarning,
                              anc=vc.anc_list,
                              obj_nex_id=nonleaves_with_leaf_flags,
                              key_list=['ot:isLeaf'])
            return errorReturn('"ot:isLeaf" for internal')
        self._detect_multilabelled_tree(otus_group_id,
                                        tree_nex_id,
                                        otuid2leaf)
        if len(lowest_nodeid_set) > 1:
            lowest_nodeid_set = list(lowest_nodeid_set)
            lowest_nodeid_set.sort()
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MultipleRootsWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=lowest_nodeid_set)
            return errorReturn('multiple roots in a tree')

        root_node_id = first_lowest_node

        if root_node_id not in with_at_root_prop:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MultipleRootsWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=list(with_at_root_prop.keys()) + [root_node_id])
            return errorReturn('root without "@root"')
        elif len(with_at_root_prop) > 1:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MultipleRootsWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=list(with_at_root_prop.keys()))
            return errorReturn('Multiple nodes with "@root"')
        elif len(with_at_root_prop) == 0:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_NoRootWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id)
            return errorReturn('no node with "@root"')
        og = set([i['@id'] for i in self._otu_group_by_id[otus_group_id]['otu']])
        nli = [(i['@id'], i) for i in node_list]
        return self._validate_otu_key_if_present(nli, og, vc)
Example #12
0
 def _post_key_check_validate_nexml_obj(self, nex_obj, obj_nex_id, vc):
     otus_group_list = nex_obj.get('otus', [])
     if otus_group_list and isinstance(otus_group_list, dict):
         otus_group_list = [otus_group_list]
     if not isinstance(otus_group_list, list):
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_MissingCrucialContentWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=['otus'])
         return errorReturn('no "otus" in nexml')
     vc.push_context(_NEXEL.OTUS, (nex_obj, obj_nex_id))
     try:
         without_id = []
         og_tuple_list = []
         for og in otus_group_list:
             ogid = og.get('@id')
             if ogid is None:
                 without_id.append(None)
             else:
                 og_tuple_list.append((ogid, og))
         if without_id:
             self._error_event(_NEXEL.OTUS,
                               obj=nex_obj,
                               err_type=gen_MissingCrucialContentWarning,
                               anc=vc.anc_list,
                               obj_nex_id=None,
                               key_list=['otus[*]/@id'])
             return errorReturn('otu without "@id"')
         if not self._validate_otus_group_list(og_tuple_list, vc):
             return False
     finally:
         vc.pop_context()
     # and now the trees...
     trees_group_list = nex_obj.get('trees', [])
     if trees_group_list and isinstance(trees_group_list, dict):
         trees_group_list = [trees_group_list]
     if not isinstance(trees_group_list, list):
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_WrongValueTypeWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=['trees'])
         return errorReturn('No "trees" in nexml')
     vc.push_context(_NEXEL.TREES, (nex_obj, obj_nex_id))
     try:
         without_id = []
         tg_tuple_list = []
         for tg in trees_group_list:
             tgid = tg.get('@id')
             if tgid is None:
                 without_id.append(tg)
             else:
                 tg_tuple_list.append((tgid, tg))
         if without_id:
             self._error_event(_NEXEL.TREES,
                               obj=without_id,
                               err_type=gen_MissingCrucialContentWarning,
                               anc=vc.anc_list,
                               obj_nex_id=None,
                               key_list=['@id'])
             return errorReturn('No "@id" in trees group"')
         if not self._validate_trees_group_list(tg_tuple_list, vc):
             return False
     finally:
         vc.pop_context()
     ogid2og = {}
     for og in otus_group_list:
         ogid = og.get('@id')
         ogid2og[ogid] = og
     if not self._find_first_literal_meta(nex_obj,
                                          'ot:notIntendedForSynthesis'):
         cs = self._find_first_literal_meta(nex_obj,
                                            'ot:candidateTreeForSynthesis')
         if cs:
             if not isinstance(cs, list):
                 tree_list = [cs]
             else:
                 tree_list = cs
         else:
             tree_list = []
             for tg in trees_group_list:
                 stree_list = tg.get('tree')
                 if not isinstance(stree_list, list):
                     stree_list = [stree_list]
                 tree_list.extend([i.get('@id') for i in stree_list])
         self._generate_ott_warnings(ogid2og, tree_list,
                                     (nex_obj, obj_nex_id), vc)
     return True
Example #13
0
    def _post_key_check_validate_tree(self,
                                      tree_nex_id,
                                      tree_obj,
                                      vc,
                                      otus_group_id=None):
        # pylint: disable=R0914
        node_by_id = tree_obj.get('nodeById')
        edge_by_source = tree_obj.get('edgeBySourceId')
        root_node_id = tree_obj.get('^ot:rootNodeId')
        if (not node_by_id) or (not isinstance(node_by_id, dict)):
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MissingCrucialContentWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=[
                                  'nodeById',
                              ])
            return errorReturn('no "nodeById" in tree')
        if (not edge_by_source) or (not isinstance(edge_by_source, dict)):
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MissingCrucialContentWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=[
                                  'edgeBySourceId',
                              ])
            return errorReturn('no "edgeBySourceId" in tree')
        if not is_str_type(root_node_id):
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MissingCrucialContentWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=[
                                  '^ot:rootNodeId',
                              ])
            return errorReturn('no "^ot:rootNodeId" in tree')
        edge_dict = {}
        edge_by_target = {}
        internal_nodes = []
        if otus_group_id is None:
            tree_group = vc.anc_list[-1][1]
            otus_group_id = tree_group.get('@otus')

        bad_node_ref = []
        repeated_edge_id = []
        missing_target = []
        repeated_target = []
        reachable_nodes = set()
        for par_node_id, edge_by_id in edge_by_source.items():
            if par_node_id not in node_by_id:
                bad_node_ref.append(par_node_id)
            else:
                reachable_nodes.add(par_node_id)
                for edge_id, edge in edge_by_id.items():
                    if edge_id in edge_dict:
                        repeated_edge_id.append(edge_id)
                    else:
                        edge_dict[edge_id] = edge
                        try:
                            t = edge.get('@target')
                        except:
                            t = None
                        if t is None:
                            missing_target.append(edge_id)
                        elif t in edge_by_target:
                            repeated_target.append(t)
                        else:
                            edge_by_target[t] = edge
                            reachable_nodes.add(t)
        node_set = set(node_by_id.keys())
        if node_set != reachable_nodes:
            unreachable = list(node_set - reachable_nodes)
            unreachable.sort()
            not_in_node_by_id = list(reachable_nodes - node_set)
            not_in_node_by_id.sort()
            if unreachable:
                self._error_event(_NEXEL.TREE,
                                  obj=tree_obj,
                                  err_type=gen_UnreachableNodeWarning,
                                  anc=vc.anc_list,
                                  obj_nex_id=tree_nex_id,
                                  key_list=unreachable)
                return errorReturn('unreachable node in tree tree')
            if not_in_node_by_id:
                self._error_event(_NEXEL.TREE,
                                  obj=tree_obj,
                                  err_type=gen_ReferencedIDNotFoundWarning,
                                  anc=vc.anc_list,
                                  obj_nex_id=tree_nex_id,
                                  key_list=not_in_node_by_id)
                return errorReturn(
                    'referenced node id not in "nodeById" in tree')
        if bad_node_ref:
            bad_node_ref.sort()
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_ReferencedIDNotFoundWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=bad_node_ref)
            return errorReturn(
                'referenced parent node not in "nodeById" in tree')
        if missing_target:
            missing_target.sort()
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_ReferencedIDNotFoundWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=missing_target)
            return errorReturn('no "@target" in edge')
        if repeated_target:
            repeated_target.sort()
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_NodeWithMultipleParents,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=repeated_target)
            return errorReturn(
                'same node used as "@target" for different edges')
        if repeated_edge_id:
            repeated_edge_id.sort()
            self._error_event(_NEXEL.EDGE,
                              obj=tree_obj,
                              err_type=gen_RepeatedIDWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=repeated_edge_id)
            return errorReturn('edge "@id" repeated')
        node_set = set(edge_by_target.keys())
        internal_node_set = set(edge_by_source.keys())
        leaf_set = node_set - internal_node_set
        leaves = [(i, node_by_id[i]) for i in leaf_set]
        vc.push_context(_NEXEL.LEAF_NODE, (tree_obj, tree_nex_id))
        try:
            if not self._validate_leaf_list(leaves, vc):
                return False
        finally:
            vc.pop_context()
        internal_nodes = [(i, node_by_id[i]) for i in internal_node_set]
        with_at_root_prop = {}
        for nid, n_obj in internal_nodes:
            if n_obj.get('@root'):
                with_at_root_prop[nid] = n_obj
        if len(with_at_root_prop) > 1:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MultipleRootsWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=list(with_at_root_prop.keys()))
            return errorReturn('multiple "@root" nodes')
        if len(with_at_root_prop) == 0:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_NoRootWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id)
            return errorReturn('No node labelled as "@root"')
        if root_node_id not in with_at_root_prop:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MultipleRootsWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=list(with_at_root_prop.keys()) +
                              [root_node_id])
            return errorReturn('root node not labelled as root')
        vc.push_context(_NEXEL.INTERNAL_NODE, (tree_obj, tree_nex_id))
        try:
            if not self._validate_internal_node_list(internal_nodes, vc):
                return False
        finally:
            vc.pop_context()
        edges = [i for i in edge_dict.items()]
        vc.push_context(_NEXEL.EDGE, (tree_obj, tree_nex_id))
        try:
            if not self._validate_edge_list(edges, vc):
                return False
        finally:
            vc.pop_context()
        otuid2leaf = {}
        for nd_id, nd in leaves:
            otuid = nd['@otu']
            if otuid in otuid2leaf:
                vc.push_context(_NEXEL.LEAF_NODE, (tree_obj, tree_nex_id))
                try:
                    self._error_event(_NEXEL.LEAF_NODE,
                                      obj=nd,
                                      err_type=gen_RepeatedOTUWarning,
                                      anc=vc.anc_list,
                                      obj_nex_id=nd_id,
                                      key_list=[otuid])
                    return errorReturn('Repeated "@otu" id')
                finally:
                    vc.pop_context()
            otuid2leaf[otuid] = nd_id
        self._detect_multilabelled_tree(otus_group_id=otus_group_id,
                                        tree_id=tree_nex_id,
                                        otuid2leaf=otuid2leaf)
        og = self._otu_group_by_id[otus_group_id]['otuById']
        return self._validate_otu_key_if_present(node_by_id.items(), og, vc)
Example #14
0
    def _post_key_check_validate_tree(self,
                                      tree_nex_id,
                                      tree_obj,
                                      vc,
                                      otus_group_id=None):
        node_list = tree_obj.get('node')
        if isinstance(node_list, dict):
            node_list = [node_list]
        elif (not node_list) or (not isinstance(node_list, list)):
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MissingCrucialContentWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=[
                                  'node',
                              ])
            return errorReturn('no "node" in "trees"')
        edge_list = tree_obj.get('edge')
        if isinstance(edge_list, dict):
            edge_list = [edge_list]
        elif (not edge_list) or (not isinstance(edge_list, list)):
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MissingCrucialContentWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=[
                                  'edge',
                              ])
            return errorReturn('no "edge" in tree')
        edge_id_list = [(i.get('@id'), i) for i in edge_list]
        vc.push_context(_NEXEL.EDGE, (tree_obj, tree_nex_id))
        try:
            valid = self._validate_edge_list(edge_id_list, vc)
            if not valid:
                return False
        finally:
            vc.pop_context()
        node_id_obj_list = [(i.get('@id'), i) for i in node_list]
        vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
        try:
            valid = self._validate_node_list(node_id_obj_list, vc)
            if not valid:
                return False
        finally:
            vc.pop_context()
        node_dict = {}
        for i in node_id_obj_list:
            nid, nd = i
            node_dict[nid] = nd
        missing_src = []
        missing_target = []
        for el in edge_id_list:
            e = el[1]
            sid = e.get('@source')
            tid = e.get('@target')
            if sid not in node_dict:
                missing_src.append(sid)
            if tid not in node_dict:
                missing_target.append(tid)
        if missing_src:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_ReferencedIDNotFoundWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=missing_src)
            return errorReturn('no "@source" in edge')
        if missing_target:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_ReferencedIDNotFoundWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=missing_target)
            return errorReturn('no "@target" in edge')
        if otus_group_id is None:
            tree_group = vc.anc_list[-1][1]
            otus_group_id = tree_group.get('@otus')

        lowest_nodeid_set = set()
        encountered_nodes = set()
        edge_by_target = {}
        edge_by_source = {}
        multi_parent_node = []
        for e in edge_list:
            t = e.get('@target')
            if t in edge_by_target:
                multi_parent_node.append(t)
            else:
                edge_by_target[t] = e
            #_LOG.debug('e=' + str(e))
            sid = e['@source']
            edge_by_source.setdefault(sid, []).append(e)
        if multi_parent_node:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_NodeWithMultipleParents,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=multi_parent_node)
        otuid2leaf = {}
        unflagged_leaves = []
        nonleaves_with_leaf_flags = []
        with_at_root_prop = {}
        first_lowest_node = None
        for nd in node_list:
            nid = nd.get('@id')
            cycle_node, path_to_root = construct_path_to_root(
                nd, encountered_nodes, edge_by_target)
            if cycle_node:
                self._error_event(_NEXEL.TREE,
                                  obj=tree_obj,
                                  err_type=gen_TreeCycleWarning,
                                  anc=vc.anc_list,
                                  obj_nex_id=tree_nex_id,
                                  cycle_node=cycle_node)
                return errorReturn('"@id" in node')
            if path_to_root:
                lowest_nodeid_set.add(path_to_root[-1])
                if first_lowest_node is None:
                    first_lowest_node = path_to_root[-1]
            is_flagged_as_leaf = self._find_first_literal_meta(nd, 'ot:isLeaf')
            ch_list = edge_by_source.get(nid)
            if ch_list is None:
                otu_id = nd.get('@otu')
                if otu_id is None:
                    vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
                    try:
                        self._error_event(
                            _NEXEL.NODE,
                            obj=nd,
                            err_type=gen_MissingCrucialContentWarning,
                            anc=vc.anc_list,
                            obj_nex_id=nid,
                            key_list=[
                                '@otu',
                            ])
                        return errorReturn('"@otu" in leaf')
                    finally:
                        vc.pop_context()
                else:
                    if otu_id in otuid2leaf:
                        vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
                        try:
                            self._error_event(_NEXEL.NODE,
                                              obj=nd,
                                              err_type=gen_RepeatedOTUWarning,
                                              anc=vc.anc_list,
                                              obj_nex_id=nid,
                                              key_list=[otu_id])
                            return errorReturn('repeated "@otu" in leaves')
                        finally:
                            vc.pop_context()
                    otuid2leaf[otu_id] = nd
                if not is_flagged_as_leaf:
                    if not self._logger.retain_deprecated:
                        add_literal_meta(nd, 'ot:isLeaf', True,
                                         self._syntax_version)
                    else:
                        unflagged_leaves.append(nid)
            elif is_flagged_as_leaf:
                if not self._logger.retain_deprecated:
                    delete_first_literal_meta(nd, 'ot:isLeaf',
                                              self._syntax_version)
                else:
                    nonleaves_with_leaf_flags.append(nid)
            if nd.get('@root'):
                with_at_root_prop[nid] = nd
        if unflagged_leaves:
            vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
            try:
                #_LOG.debug('unflagged_leaves="{f}"'.format(f=unflagged_leaves))
                self._error_event(_NEXEL.NODE,
                                  obj=tree_obj,
                                  err_type=gen_MissingMandatoryKeyWarning,
                                  anc=vc.anc_list,
                                  obj_nex_id=unflagged_leaves,
                                  key_list=['ot:isLeaf'])
            finally:
                vc.pop_context()
        if nonleaves_with_leaf_flags:
            vc.push_context(_NEXEL.NODE, (tree_obj, tree_nex_id))
            try:
                self._error_event(_NEXEL.NODE,
                                  obj=tree_obj,
                                  err_type=gen_InvalidKeyWarning,
                                  anc=vc.anc_list,
                                  obj_nex_id=nonleaves_with_leaf_flags,
                                  key_list=['ot:isLeaf'])
                return errorReturn('"ot:isLeaf" for internal')
            finally:
                vc.pop_context()
        self._detect_multilabelled_tree(otus_group_id, tree_nex_id, otuid2leaf)
        if len(lowest_nodeid_set) > 1:
            lowest_nodeid_set = list(lowest_nodeid_set)
            lowest_nodeid_set.sort()
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MultipleRootsWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=lowest_nodeid_set)
            return errorReturn('multiple roots in a tree')

        root_node_id = first_lowest_node

        if root_node_id not in with_at_root_prop:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MultipleRootsWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=list(with_at_root_prop.keys()) +
                              [root_node_id])
            return errorReturn('root without "@root"')
        elif len(with_at_root_prop) > 1:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MultipleRootsWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=list(with_at_root_prop.keys()))
            return errorReturn('Multiple nodes with "@root"')
        elif len(with_at_root_prop) == 0:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_NoRootWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id)
            return errorReturn('no node with "@root"')
        og = set(
            [i['@id'] for i in self._otu_group_by_id[otus_group_id]['otu']])
        nli = [(i['@id'], i) for i in node_list]
        return self._validate_otu_key_if_present(nli, og, vc)
Example #15
0
 def _post_key_check_validate_nexml_obj(self, nex_obj, obj_nex_id, vc):
     otus = nex_obj.get('otusById', {})
     otus_order_list = nex_obj.get('^ot:otusElementOrder', [])
     if not isinstance(otus, dict) or (otus_order_list and (not otus)):
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_WrongValueTypeWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=['otusById'])
         return errorReturn('Missing "otusById"')
     if (not isinstance(otus_order_list, list)) or ((not otus_order_list)
                                                    and otus):
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_WrongValueTypeWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=['^ot:otusElementOrder'])
         return errorReturn('Missing "^ot:otusElementOrder"')
     otus_group_list = []
     missing_ogid = []
     not_dict_og = []
     for ogid in otus_order_list:
         og = otus.get(ogid)
         if og is None:
             missing_ogid.append(ogid)
         elif not isinstance(og, dict):
             r = check_raw_dict(og, otus, ogid, vc)
             assert r[0] is False
             t = r[1]
             not_dict_og.append(t)
         else:
             otus_group_list.append((ogid, og))
     if missing_ogid:
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_ReferencedIDNotFoundWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=missing_ogid)
         return errorReturn('Missing "@id" for otus group')
     vc.push_context(_NEXEL.OTUS, (nex_obj, obj_nex_id))
     try:
         if not_dict_og:
             self._error_event(_NEXEL.OTUS,
                               obj=otus,
                               err_type=gen_WrongValueTypeWarning,
                               anc=vc.anc_list,
                               obj_nex_id=None,
                               key_val_type_list=[not_dict_og])
             return errorReturn('Otus objects of the wrong type of object')
         if not self._validate_otus_group_list(otus_group_list, vc):
             return False
     finally:
         vc.pop_context()
     # and now the trees...
     trees = nex_obj.get('treesById')
     trees_order_list = nex_obj.get('^ot:treesElementOrder')
     if (not isinstance(trees, dict)) or ((not trees) and trees_order_list):
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_MissingCrucialContentWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=['treesById'])
         return errorReturn('Missing "treesById"')
     if (not isinstance(trees_order_list, list)) or ((not trees_order_list)
                                                     and trees):
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_MissingCrucialContentWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=['^ot:treesElementOrder'])
         return errorReturn('Missing "^ot:treesElementOrder"')
     trees_group_list = []
     missing_tgid = []
     not_dict_tg = []
     for tgid in trees_order_list:
         tg = trees.get(tgid)
         if tg is None:
             missing_tgid.append(tgid)
         elif not isinstance(tg, dict):
             r = check_raw_dict(tg, trees, tgid, vc)
             assert r[0] is False
             t = r[1]
             not_dict_tg.append(t)
         else:
             trees_group_list.append((tgid, tg))
     if missing_tgid:
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_ReferencedIDNotFoundWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=missing_tgid)
         return errorReturn('Missing trees group id')
     vc.push_context(_NEXEL.TREES, (nex_obj, obj_nex_id))
     try:
         if not_dict_tg:
             self._error_event(_NEXEL.TREES,
                               obj=trees,
                               err_type=gen_WrongValueTypeWarning,
                               anc=vc.anc_list,
                               obj_nex_id=None,
                               key_val_type_list=[not_dict_tg])
             return errorReturn('Trees element of the wrong type')
         if not self._validate_trees_group_list(trees_group_list, vc):
             return False
     finally:
         vc.pop_context()
     if not nex_obj.get('ot:notIntendedForSynthesis'):
         cs = nex_obj.get('ot:candidateTreeForSynthesis')
         if cs:
             if not isinstance(cs, list):
                 tree_list = [cs]
             else:
                 tree_list = cs
         else:
             tree_list = []
             for tg in trees.values():
                 tbi = tg.get('treeById', {})
                 tree_list.extend(tbi.keys())
         self._generate_ott_warnings(otus, tree_list, (nex_obj, obj_nex_id),
                                     vc)
     return True
 def _post_key_check_validate_nexml_obj(self, nex_obj, obj_nex_id, vc):
     otus_group_list = nex_obj.get('otus', [])
     if otus_group_list and isinstance(otus_group_list, dict):
         otus_group_list = [otus_group_list]
     if not isinstance(otus_group_list, list):
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_MissingCrucialContentWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=['otus'])
         return errorReturn('no "otus" in nexml')
     vc.push_context(_NEXEL.OTUS, (nex_obj, obj_nex_id))
     try:
         without_id = []
         og_tuple_list = []
         for og in otus_group_list:
             ogid = og.get('@id')
             if ogid is None:
                 without_id.append(None)
             else:
                 og_tuple_list.append((ogid, og))
         if without_id:
             self._error_event(_NEXEL.OTUS,
                               obj=nex_obj,
                               err_type=gen_MissingCrucialContentWarning,
                               anc=vc.anc_list,
                               obj_nex_id=None,
                               key_list=['otus[*]/@id'])
             return errorReturn('otu without "@id"')
         if not self._validate_otus_group_list(og_tuple_list, vc):
             return False
     finally:
         vc.pop_context()
     # and now the trees...
     trees_group_list = nex_obj.get('trees', [])
     if trees_group_list and isinstance(trees_group_list, dict):
         trees_group_list = [trees_group_list]
     if not isinstance(trees_group_list, list):
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_WrongValueTypeWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=['trees'])
         return errorReturn('No "trees" in nexml')
     vc.push_context(_NEXEL.TREES, (nex_obj, obj_nex_id))
     try:
         without_id = []
         tg_tuple_list = []
         for tg in trees_group_list:
             tgid = tg.get('@id')
             if tgid is None:
                 without_id.append(tg)
             else:
                 tg_tuple_list.append((tgid, tg))
         if without_id:
             self._error_event(_NEXEL.TREES,
                               obj=without_id,
                               err_type=gen_MissingCrucialContentWarning,
                               anc=vc.anc_list,
                               obj_nex_id=None,
                               key_list=['@id'])
             return errorReturn('No "@id" in trees group"')
         if not self._validate_trees_group_list(tg_tuple_list, vc):
             return False
     finally:
         vc.pop_context()
     ogid2og = {}
     for og in otus_group_list:
         ogid = og.get('@id')
         ogid2og[ogid] = og
     if not self._find_first_literal_meta(nex_obj, 'ot:notIntendedForSynthesis'):
         cs = self._find_first_literal_meta(nex_obj, 'ot:candidateTreeForSynthesis')
         if cs:
             if not isinstance(cs, list):
                 tree_list = [cs]
             else:
                 tree_list = cs
         else:
             tree_list = []
             for tg in trees_group_list:
                 stree_list = tg.get('tree')
                 if not isinstance(stree_list, list):
                     stree_list = [stree_list]
                 tree_list.extend([i.get('@id') for i in stree_list])
         self._generate_ott_warnings(ogid2og, tree_list, (nex_obj, obj_nex_id), vc)
     return True
Example #17
0
 def _post_key_check_validate_nexml_obj(self, nex_obj, obj_nex_id, vc):
     otus = nex_obj.get('otusById', {})
     otus_order_list = nex_obj.get('^ot:otusElementOrder', [])
     if not isinstance(otus, dict) or (otus_order_list and (not otus)):
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_WrongValueTypeWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=['otusById'])
         return errorReturn('Missing "otusById"')
     if (not isinstance(otus_order_list, list)) or ((not otus_order_list) and otus):
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_WrongValueTypeWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=['^ot:otusElementOrder'])
         return errorReturn('Missing "^ot:otusElementOrder"')
     otus_group_list = []
     missing_ogid = []
     not_dict_og = []
     for ogid in otus_order_list:
         og = otus.get(ogid)
         if og is None:
             missing_ogid.append(ogid)
         elif not isinstance(og, dict):
             r = check_raw_dict(og, otus, ogid, vc)
             assert r[0] is False
             t = r[1]
             not_dict_og.append(t)
         else:
             otus_group_list.append((ogid, og))
     if missing_ogid:
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_ReferencedIDNotFoundWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=missing_ogid)
         return errorReturn('Missing "@id" for otus group')
     vc.push_context(_NEXEL.OTUS, (nex_obj, obj_nex_id))
     try:
         if not_dict_og:
             self._error_event(_NEXEL.OTUS,
                               obj=otus,
                               err_type=gen_WrongValueTypeWarning,
                               anc=vc.anc_list,
                               obj_nex_id=None,
                               key_val_type_list=[not_dict_og])
             return errorReturn('Otus objects of the wrong type of object')
         if not self._validate_otus_group_list(otus_group_list, vc):
             return False
     finally:
         vc.pop_context()
     # and now the trees...
     trees = nex_obj.get('treesById')
     trees_order_list = nex_obj.get('^ot:treesElementOrder')
     if (not isinstance(trees, dict)) or ((not trees) and trees_order_list):
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_MissingCrucialContentWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=['treesById'])
         return errorReturn('Missing "treesById"')
     if (not isinstance(trees_order_list, list)) or ((not trees_order_list) and trees):
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_MissingCrucialContentWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=['^ot:treesElementOrder'])
         return errorReturn('Missing "^ot:treesElementOrder"')
     trees_group_list = []
     missing_tgid = []
     not_dict_tg = []
     for tgid in trees_order_list:
         tg = trees.get(tgid)
         if tg is None:
             missing_tgid.append(tgid)
         elif not isinstance(tg, dict):
             r = check_raw_dict(tg, trees, tgid, vc)
             assert r[0] is False
             t = r[1]
             not_dict_tg.append(t)
         else:
             trees_group_list.append((tgid, tg))
     if missing_tgid:
         self._error_event(_NEXEL.NEXML,
                           obj=nex_obj,
                           err_type=gen_ReferencedIDNotFoundWarning,
                           anc=vc.anc_list,
                           obj_nex_id=obj_nex_id,
                           key_list=missing_tgid)
         return errorReturn('Missing trees group id')
     vc.push_context(_NEXEL.TREES, (nex_obj, obj_nex_id))
     try:
         if not_dict_tg:
             self._error_event(_NEXEL.TREES,
                               obj=trees,
                               err_type=gen_WrongValueTypeWarning,
                               anc=vc.anc_list,
                               obj_nex_id=None,
                               key_val_type_list=[not_dict_tg])
             return errorReturn('Trees element of the wrong type')
         if not self._validate_trees_group_list(trees_group_list, vc):
             return False
     finally:
         vc.pop_context()
     if not nex_obj.get('ot:notIntendedForSynthesis'):
         cs = nex_obj.get('ot:candidateTreeForSynthesis')
         if cs:
             if not isinstance(cs, list):
                 tree_list = [cs]
             else:
                 tree_list = cs
         else:
             tree_list = []
             for tg in trees.values():
                 tbi = tg.get('treeById', {})
                 tree_list.extend(tbi.keys())
         self._generate_ott_warnings(otus, tree_list, (nex_obj, obj_nex_id), vc)
     return True
Example #18
0
    def _post_key_check_validate_tree(self,
                                      tree_nex_id,
                                      tree_obj,
                                      vc,
                                      otus_group_id=None):
        #pylint: disable=R0914
        node_by_id = tree_obj.get('nodeById')
        edge_by_source = tree_obj.get('edgeBySourceId')
        root_node_id = tree_obj.get('^ot:rootNodeId')
        if (not node_by_id) or (not isinstance(node_by_id, dict)):
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MissingCrucialContentWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=['nodeById',])
            return errorReturn('no "nodeById" in tree')
        if (not edge_by_source) or (not isinstance(edge_by_source, dict)):
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MissingCrucialContentWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=['edgeBySourceId',])
            return errorReturn('no "edgeBySourceId" in tree')
        if not is_str_type(root_node_id):
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MissingCrucialContentWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=['^ot:rootNodeId',])
            return errorReturn('no "^ot:rootNodeId" in tree')
        edge_dict = {}
        edge_by_target = {}
        internal_nodes = []
        if otus_group_id is None:
            tree_group = vc.anc_list[-1][1]
            otus_group_id = tree_group.get('@otus')

        bad_node_ref = []
        repeated_edge_id = []
        missing_target = []
        repeated_target = []
        reachable_nodes = set()
        for par_node_id, edge_by_id in edge_by_source.items():
            if par_node_id not in node_by_id:
                bad_node_ref.append(par_node_id)
            else:
                reachable_nodes.add(par_node_id)
                for edge_id, edge in edge_by_id.items():
                    if edge_id in edge_dict:
                        repeated_edge_id.append(edge_id)
                    else:
                        edge_dict[edge_id] = edge
                        try:
                            t = edge.get('@target')
                        except:
                            t = None
                        if t is None:
                            missing_target.append(edge_id)
                        elif t in edge_by_target:
                            repeated_target.append(t)
                        else:
                            edge_by_target[t] = edge
                            reachable_nodes.add(t)
        node_set = set(node_by_id.keys())
        if node_set != reachable_nodes:
            unreachable = list(node_set - reachable_nodes)
            unreachable.sort()
            not_in_node_by_id = list(reachable_nodes - node_set)
            not_in_node_by_id.sort()
            if unreachable:
                self._error_event(_NEXEL.TREE,
                                  obj=tree_obj,
                                  err_type=gen_UnreachableNodeWarning,
                                  anc=vc.anc_list,
                                  obj_nex_id=tree_nex_id,
                                  key_list=unreachable)
                return errorReturn('unreachable node in tree tree')
            if not_in_node_by_id:
                self._error_event(_NEXEL.TREE,
                                  obj=tree_obj,
                                  err_type=gen_ReferencedIDNotFoundWarning,
                                  anc=vc.anc_list,
                                  obj_nex_id=tree_nex_id,
                                  key_list=not_in_node_by_id)
                return errorReturn('referenced node id not in "nodeById" in tree')
        if bad_node_ref:
            bad_node_ref.sort()
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_ReferencedIDNotFoundWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=bad_node_ref)
            return errorReturn('referenced parent node not in "nodeById" in tree')
        if missing_target:
            missing_target.sort()
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_ReferencedIDNotFoundWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=missing_target)
            return errorReturn('no "@target" in edge')
        if repeated_target:
            repeated_target.sort()
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_NodeWithMultipleParents,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=repeated_target)
            return errorReturn('same node used as "@target" for different edges')
        if repeated_edge_id:
            repeated_edge_id.sort()
            self._error_event(_NEXEL.EDGE,
                              obj=tree_obj,
                              err_type=gen_RepeatedIDWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              key_list=repeated_edge_id)
            return errorReturn('edge "@id" repeated')
        node_set = set(edge_by_target.keys())
        internal_node_set = set(edge_by_source.keys())
        leaf_set = node_set - internal_node_set
        leaves = [(i, node_by_id[i]) for i in leaf_set]
        vc.push_context(_NEXEL.LEAF_NODE, (tree_obj, tree_nex_id))
        try:
            if not self._validate_leaf_list(leaves, vc):
                return False
        finally:
            vc.pop_context()
        internal_nodes = [(i, node_by_id[i]) for i in internal_node_set]
        with_at_root_prop = {}
        for nid, n_obj in internal_nodes:
            if n_obj.get('@root'):
                with_at_root_prop[nid] = n_obj
        if len(with_at_root_prop) > 1:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MultipleRootsWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=list(with_at_root_prop.keys()))
            return errorReturn('multiple "@root" nodes')
        if len(with_at_root_prop) == 0:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_NoRootWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id)
            return errorReturn('No node labelled as "@root"')
        if root_node_id not in with_at_root_prop:
            self._error_event(_NEXEL.TREE,
                              obj=tree_obj,
                              err_type=gen_MultipleRootsWarning,
                              anc=vc.anc_list,
                              obj_nex_id=tree_nex_id,
                              node_id_list=list(with_at_root_prop.keys()) + [root_node_id])
            return errorReturn('root node not labelled as root')
        vc.push_context(_NEXEL.INTERNAL_NODE, (tree_obj, tree_nex_id))
        try:
            if not self._validate_internal_node_list(internal_nodes, vc):
                return False
        finally:
            vc.pop_context()
        edges = [i for i in edge_dict.items()]
        vc.push_context(_NEXEL.EDGE, (tree_obj, tree_nex_id))
        try:
            if not self._validate_edge_list(edges, vc):
                return False
        finally:
            vc.pop_context()
        otuid2leaf = {}
        for nd_id, nd in leaves:
            otuid = nd['@otu']
            if otuid in otuid2leaf:
                vc.push_context(_NEXEL.LEAF_NODE, (tree_obj, tree_nex_id))
                try:
                    self._error_event(_NEXEL.LEAF_NODE,
                                      obj=nd,
                                      err_type=gen_RepeatedOTUWarning,
                                      anc=vc.anc_list,
                                      obj_nex_id=nd_id,
                                      key_list=[otuid])
                    return errorReturn('Repeated "@otu" id')
                finally:
                    vc.pop_context()
            otuid2leaf[otuid] = nd_id
        self._detect_multilabelled_tree(otus_group_id=otus_group_id,
                                        tree_id=tree_nex_id,
                                        otuid2leaf=otuid2leaf)
        og = self._otu_group_by_id[otus_group_id]['otuById']
        return self._validate_otu_key_if_present(node_by_id.items(), og, vc)
Example #19
0
    def _validate_id_obj_list_by_schema(self,
                                        id_obj_list,
                                        vc,
                                        group_by_warning=False):
        element_type = vc.curr_element_type
        assert element_type is not None
        schema = vc.schema
        anc_list = vc.anc_list
        #_LOG.debug('using schema type = ' + vc.schema_name())
        using_hbf_meta = vc._using_hbf_meta  #pylint: disable=W0212
        _by_warn_type = {}
        for obj_nex_id, obj in id_obj_list:
            wrong_type = []
            unrec_meta_keys = []
            unrec_non_meta_keys = []

            if using_hbf_meta:
                for k, v in obj.items():
                    is_meta = k[0] == '^'
                    if k not in schema.ALLOWED_KEY_SET:
                        if is_meta:
                            unrec_meta_keys.append(k)
                        else:
                            unrec_non_meta_keys.append(k)
                    else:
                        correct_type, info = schema.K2VT[k](v, obj, k, vc)
                        if not correct_type:
                            wrong_type.append((k, v, info))
            else:
                for k, v in obj.items():
                    if k not in schema.ALLOWED_KEY_SET:
                        unrec_non_meta_keys.append(k)
                    else:
                        correct_type, info = schema.K2VT[k](v, obj, k, vc)
                        if not correct_type:
                            wrong_type.append((k, v, info))
                m = self._get_list_key(obj, 'meta', vc)
                if m:
                    # might want a flag of meta?
                    md = self._bf_meta_list_to_dict(m, obj, vc)
                    #_LOG.debug('md = ' + str(md))
                    mrmk = [
                        i for i in schema.REQUIRED_META_KEY_SET if i not in md
                    ]
                    memk = [
                        i for i in schema.EXPECTED_META_KEY_SET if i not in md
                    ]
                    if memk:
                        if group_by_warning:
                            memk.sort()
                            foks = frozenset(memk)
                            t = _by_warn_type.setdefault(foks, [[], []])
                            t[0].append(obj)
                            t[1].append(obj_nex_id)
                        else:
                            self._warn_event(
                                element_type,
                                obj=obj,
                                err_type=gen_MissingOptionalKeyWarning,
                                anc=anc_list,
                                obj_nex_id=obj_nex_id,
                                key_list=memk)
                    if mrmk:
                        self._error_event(
                            element_type,
                            obj=obj,
                            err_type=gen_MissingMandatoryKeyWarning,
                            anc=anc_list,
                            obj_nex_id=obj_nex_id,
                            key_list=mrmk)
                        msgf = 'missing mandatory meta key(s) according to {s} schema'
                        msg = msgf.format(s=vc.schema_name())
                        return errorReturn(msg)
                    for k, v in md.items():
                        if k not in schema.ALLOWED_META_KEY_SET:
                            unrec_meta_keys.append(k)
                        else:
                            #_LOG.debug('{k} --> "{v}"'.format(k=k, v=repr(v)))
                            correct_type, info = schema.K2VT[k](v, obj, k, vc)
                            if not correct_type:
                                v = extract_meta(v)
                                wrong_type.append((k, v, info))
            if wrong_type:
                self._error_event(element_type,
                                  obj=obj,
                                  err_type=gen_WrongValueTypeWarning,
                                  anc=anc_list,
                                  obj_nex_id=obj_nex_id,
                                  key_val_type_list=wrong_type)
                return errorReturn(
                    'wrong value type according to {s} schema'.format(
                        s=vc.schema_name()))
            if unrec_non_meta_keys:
                self._warn_event(element_type,
                                 obj=obj,
                                 err_type=gen_UnrecognizedKeyWarning,
                                 anc=anc_list,
                                 obj_nex_id=obj_nex_id,
                                 key_list=unrec_non_meta_keys)
            if unrec_meta_keys:
                # might want a flag of meta?
                self._warn_event(element_type,
                                 obj=obj,
                                 err_type=gen_UnrecognizedKeyWarning,
                                 anc=anc_list,
                                 obj_nex_id=obj_nex_id,
                                 key_list=unrec_meta_keys)
            off_key = [k for k in schema.EXPECETED_KEY_SET if k not in obj]
            if off_key:
                if group_by_warning:
                    off_key.sort()
                    foks = frozenset(off_key)
                    t = _by_warn_type.setdefault(foks, [[], []])
                    t[0].append(obj)
                    t[1].append(obj_nex_id)
                else:
                    self._warn_event(element_type,
                                     obj=obj,
                                     err_type=gen_MissingOptionalKeyWarning,
                                     anc=anc_list,
                                     obj_nex_id=obj_nex_id,
                                     key_list=off_key)
            off_key = [k for k in schema.REQUIRED_KEY_SET if k not in obj]
            if off_key:
                self._error_event(element_type,
                                  obj=obj,
                                  err_type=gen_MissingMandatoryKeyWarning,
                                  anc=anc_list,
                                  obj_nex_id=obj_nex_id,
                                  key_list=off_key)
                return errorReturn(
                    'missing key(s) according to {s} schema'.format(
                        s=vc.schema_name()))
        if _by_warn_type:
            for ks, obj_lists in _by_warn_type.items():
                mlist = list(ks)
                mlist.sort()
                id_arg = obj_lists[1]
                if len(id_arg) == 1:
                    id_arg = id_arg[0]
                obj_arg = obj_lists[0]
                if len(obj_arg) == 1:
                    obj_arg = obj_arg[0]
                self._warn_event(element_type,
                                 obj=obj_arg,
                                 err_type=gen_MissingOptionalKeyWarning,
                                 anc=anc_list,
                                 obj_nex_id=id_arg,
                                 key_list=mlist)
        return True
Example #20
0
    def _validate_id_obj_list_by_schema(self, id_obj_list, vc, group_by_warning=False):
        element_type = vc.curr_element_type
        assert element_type is not None
        schema = vc.schema
        anc_list = vc.anc_list
        # _LOG.debug('using schema type = ' + vc.schema_name())
        using_hbf_meta = vc._using_hbf_meta  # pylint: disable=W0212
        _by_warn_type = {}
        for obj_nex_id, obj in id_obj_list:
            wrong_type = []
            unrec_meta_keys = []
            unrec_non_meta_keys = []

            if using_hbf_meta:
                for k, v in obj.items():
                    is_meta = k[0] == '^'
                    if k not in schema.ALLOWED_KEY_SET:
                        if is_meta:
                            unrec_meta_keys.append(k)
                        else:
                            unrec_non_meta_keys.append(k)
                    else:
                        correct_type, info = schema.K2VT[k](v, obj, k, vc)
                        if not correct_type:
                            wrong_type.append((k, v, info))
            else:
                for k, v in obj.items():
                    if k not in schema.ALLOWED_KEY_SET:
                        unrec_non_meta_keys.append(k)
                    else:
                        correct_type, info = schema.K2VT[k](v, obj, k, vc)
                        if not correct_type:
                            wrong_type.append((k, v, info))
                m = self._get_list_key(obj, 'meta', vc)
                if m:
                    # might want a flag of meta?
                    md = self._bf_meta_list_to_dict(m, obj, vc)
                    # _LOG.debug('md = ' + str(md))
                    mrmk = [i for i in schema.REQUIRED_META_KEY_SET if i not in md]
                    memk = [i for i in schema.EXPECTED_META_KEY_SET if i not in md]
                    if memk:
                        if group_by_warning:
                            memk.sort()
                            foks = frozenset(memk)
                            t = _by_warn_type.setdefault(foks, [[], []])
                            t[0].append(obj)
                            t[1].append(obj_nex_id)
                        else:
                            self._warn_event(element_type,
                                             obj=obj,
                                             err_type=gen_MissingOptionalKeyWarning,
                                             anc=anc_list,
                                             obj_nex_id=obj_nex_id,
                                             key_list=memk)
                    if mrmk:
                        self._error_event(element_type,
                                          obj=obj,
                                          err_type=gen_MissingMandatoryKeyWarning,
                                          anc=anc_list,
                                          obj_nex_id=obj_nex_id,
                                          key_list=mrmk)
                        msgf = 'missing mandatory meta key(s) according to {s} schema'
                        msg = msgf.format(s=vc.schema_name())
                        return errorReturn(msg)
                    for k, v in md.items():
                        if k not in schema.ALLOWED_META_KEY_SET:
                            unrec_meta_keys.append(k)
                        else:
                            # _LOG.debug('{k} --> "{v}"'.format(k=k, v=repr(v)))
                            correct_type, info = schema.K2VT[k](v, obj, k, vc)
                            if not correct_type:
                                v = extract_meta(v)
                                wrong_type.append((k, v, info))
            if wrong_type:
                self._error_event(element_type,
                                  obj=obj,
                                  err_type=gen_WrongValueTypeWarning,
                                  anc=anc_list,
                                  obj_nex_id=obj_nex_id,
                                  key_val_type_list=wrong_type)
                return errorReturn('wrong value type according to {s} schema'.format(s=vc.schema_name()))
            if unrec_non_meta_keys:
                self._warn_event(element_type,
                                 obj=obj,
                                 err_type=gen_UnrecognizedKeyWarning,
                                 anc=anc_list,
                                 obj_nex_id=obj_nex_id,
                                 key_list=unrec_non_meta_keys)
            if unrec_meta_keys:
                # might want a flag of meta?
                self._warn_event(element_type,
                                 obj=obj,
                                 err_type=gen_UnrecognizedKeyWarning,
                                 anc=anc_list,
                                 obj_nex_id=obj_nex_id,
                                 key_list=unrec_meta_keys)
            off_key = [k for k in schema.EXPECETED_KEY_SET if k not in obj]
            if off_key:
                if group_by_warning:
                    off_key.sort()
                    foks = frozenset(off_key)
                    t = _by_warn_type.setdefault(foks, [[], []])
                    t[0].append(obj)
                    t[1].append(obj_nex_id)
                else:
                    self._warn_event(element_type,
                                     obj=obj,
                                     err_type=gen_MissingOptionalKeyWarning,
                                     anc=anc_list,
                                     obj_nex_id=obj_nex_id,
                                     key_list=off_key)
            off_key = [k for k in schema.REQUIRED_KEY_SET if k not in obj]
            if off_key:
                self._error_event(element_type,
                                  obj=obj,
                                  err_type=gen_MissingMandatoryKeyWarning,
                                  anc=anc_list,
                                  obj_nex_id=obj_nex_id,
                                  key_list=off_key)
                return errorReturn('missing key(s) according to {s} schema'.format(s=vc.schema_name()))
        if _by_warn_type:
            for ks, obj_lists in _by_warn_type.items():
                mlist = list(ks)
                mlist.sort()
                id_arg = obj_lists[1]
                if len(id_arg) == 1:
                    id_arg = id_arg[0]
                obj_arg = obj_lists[0]
                if len(obj_arg) == 1:
                    obj_arg = obj_arg[0]
                self._warn_event(element_type,
                                 obj=obj_arg,
                                 err_type=gen_MissingOptionalKeyWarning,
                                 anc=anc_list,
                                 obj_nex_id=id_arg,
                                 key_list=mlist)
        return True