def testGetAdjacentCategories(self): """Test case -""" try: myIo = IoAdapter(raiseExceptions=True) self.__containerList = myIo.readFile(inputFilePath=self.__pathPdbxDictionary) dApi = DictionaryApi(containerList=self.__containerList, consolidate=True, verbose=self.__verbose) cList = dApi.getCategoryList() cI = {} for cV in cList: chL = dApi.getChildCategories(cV) pL = dApi.getParentCategories(cV) for ch in chL: if (ch, cV) not in cI: cI[(ch, cV)] = 1 else: cI[(ch, cV)] += 1 for pV in pL: if (cV, pV) not in cI: cI[(cV, pV)] = 1 else: cI[(cV, pV)] += 1 linkL = [] for tup in cI: dD = {"source": tup[0], "target": tup[1], "type": "link"} linkL.append(dD) if self.__verbose: print(json.dumps(linkL, sort_keys=True, indent=4, separators=(",", ": "))) self.assertGreater(len(linkL), 50) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def test_get_adjacent_categories(self, api_paths): myIo = IoAdapter(raiseExceptions=True) containerList = myIo.readFile( inputFilePath=str(api_paths['pathPdbxDictionary'])) dApi = DictionaryApi(containerList=containerList, consolidate=True) cList = dApi.getCategoryList() cI = {} for c in cList: chL = dApi.getChildCategories(c) pL = dApi.getParentCategories(c) for ch in chL: if (ch, c) not in cI: cI[(ch, c)] = 1 else: cI[(ch, c)] += 1 for p in pL: if (c, p) not in cI: cI[(c, p)] = 1 else: cI[(c, p)] += 1 linkL = [] for s, t in cI.keys(): d = {'source': s, 'target': t, 'type': 'link'} linkL.append(d) assert len(linkL) > 50
def testDumpEnums(self): """Test case - to verify enum ordering - """ try: myIo = IoAdapter(raiseExceptions=True) self.__containerList = myIo.readFile( inputFilePath=self.__pathPdbxDictionary) dApi = DictionaryApi(containerList=self.__containerList, consolidate=True, verbose=self.__verbose) # eList = dApi.getEnumListAlt(category="pdbx_audit_support", attribute="country") logger.debug("Item %s Enum list sorted %r\n", "country", eList) eList = dApi.getEnumListAlt(category="pdbx_audit_support", attribute="country", sortFlag=False) logger.debug("Item %s Enum list unsorted %r\n", "country", eList) eList = dApi.getEnumListAltWithDetail( category="pdbx_audit_support", attribute="country") logger.debug("Item %s Enum with detail list %r\n", "country", eList) self.assertGreater(len(eList), 100) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def testDumpDictionary(self): """Test case - dump methods for dictionary metadata """ try: myIo = IoAdapter(raiseExceptions=True) self.__containerList = myIo.readFile( inputFilePath=self.__pathPdbxDictionary) dApi = DictionaryApi(containerList=self.__containerList, consolidate=True, verbose=self.__verbose) # dApi.dumpCategoryIndex(fh=self.__lfh) # dApi.dumpEnumFeatures(fh=self.__lfh) # dApi.dumpFeatures(fh=self.__lfh) # dApi.dumpMethods(fh=self.__lfh) logger.debug( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") groupList = dApi.getCategoryGroups() logger.debug("groupList %s\n", groupList) for group in groupList: logger.debug("Group %s category list %s\n", group, dApi.getCategoryGroupCategories(groupName=group)) self.assertGreater(len(groupList), 10) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def testGetUcode(self): """Test case - Get all data items of type ucode """ print("\n") try: myIo = IoAdapter(raiseExceptions=True) self.__containerList = myIo.readFile( inputFilePath=self.__pathPdbxDictionary) dApi = DictionaryApi(containerList=self.__containerList, consolidate=True, verbose=self.__verbose) logger.debug( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") catNameList = dApi.getCategoryList() for catName in catNameList: itemNameList = dApi.getItemNameList(catName) for itemName in itemNameList: categoryName = CifName.categoryPart(itemName) attributeName = CifName.attributePart(itemName) code = dApi.getTypeCode(categoryName, attributeName) if (code == "ucode"): print("Table: ", categoryName, "\tColumn: ", attributeName, "\tType: ", code) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def __init__(self, dictContainerList, verbose=False): # self.__verbose = verbose # list of dictionary data & definition containers self.__dictContainerList = dictContainerList self.__dApi = DictionaryApi(containerList=self.__dictContainerList, consolidate=True, verbose=self.__verbose) # # Target data container list self.__dataContainerList = []
def test_dump_index(self, api_paths): myIo = IoAdapter(raiseExceptions=True) containerList = myIo.readFile( inputFilePath=str(api_paths['pathPdbxDictionary'])) dApi = DictionaryApi(containerList=containerList, consolidate=True) print("Index = %r\n" % dApi.getItemNameList('pdbx_nmr_spectral_dim')) print("Index = %r\n" % dApi.getAttributeNameList('pdbx_nmr_spectral_dim')) catIndex = dApi.getCategoryIndex() print("Index = %r\n" % catIndex['pdbx_nmr_spectral_dim']) assert catIndex['pdbx_nmr_spectral_dim'] is not None
def test_dump_dictionary(self, api_paths): myIo = IoAdapter(raiseExceptions=True) containerList = myIo.readFile( inputFilePath=str(api_paths['pathPdbxDictionary'])) dApi = DictionaryApi(containerList=containerList, consolidate=True) print('+++++++++++++++++++++++++++++++++++++++++++++++++++++++\n') groupList = dApi.getCategoryGroups() print('groupList %s\n' % groupList) for group in groupList: print('Group %s category list %s\n' % (group, dApi.getCategoryGroupCategories(groupName=group))) assert len(groupList) > 10
def testExtendedEnums(self): """Test case - to verify extended enums -""" try: myIo = IoAdapter(raiseExceptions=True) self.__containerList = myIo.readFile(inputFilePath=self.__pathPdbxDictionary) dApi = DictionaryApi(containerList=self.__containerList, consolidate=True, verbose=self.__verbose) # eList = dApi.getEnumListWithFullDetails(category="chem_comp", attribute="mon_nstd_flag") logger.debug("Item Enum list sorted %r\n", eList) self.assertGreaterEqual(len(eList), 4) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def __getApi(self, dictLocators, **kwargs): """ Return an instance of a dictionary API instance for the input dictionary locator list. """ consolidate = kwargs.get("consolidate", True) replaceDefinition = kwargs.get("replaceDefinitions", True) verbose = kwargs.get("verbose", True) # ok = self.__reload(dictLocators, self.__dirPath, useCache=self.__useCache) # dApi = None if ok: mU = MarshalUtil() containerList = [] for dictLocator in dictLocators: cacheFilePath = os.path.join( self.__dirPath, self.__fileU.getFileName(dictLocator)) containerList.extend( mU.doImport(cacheFilePath, fmt="mmcif-dict")) # dApi = DictionaryApi(containerList=containerList, consolidate=consolidate, replaceDefinition=replaceDefinition, verbose=verbose) return dApi
def test_dump_enums(self, api_paths): myIo = IoAdapter(raiseExceptions=True) containerList = myIo.readFile( inputFilePath=str(api_paths['pathPdbxDictionary'])) dApi = DictionaryApi(containerList=containerList, consolidate=True) eList = dApi.getEnumListAlt(category="pdbx_audit_support", attribute="country") print("Item %s Enum list sorted %r\n" % ('country', eList)) eList = dApi.getEnumListAlt(category="pdbx_audit_support", attribute="country", sortFlag=False) print("Item %s Enum list unsorted %r\n" % ('country', eList)) eList = dApi.getEnumListAltWithDetail(category="pdbx_audit_support", attribute="country") print("Item %s Enum with detail list %r\n" % ('country', eList)) assert len(eList) > 100
def test_consolidate_dictionary(self, api_paths): myIo = IoAdapter(raiseExceptions=True) containerList = myIo.readFile( inputFilePath=str(api_paths['pathPdbxDictionary'])) dApi = DictionaryApi(containerList=containerList, consolidate=True) for itemName in [ '_entity.id', '_entity_poly_seq.num', '_atom_site.label_asym_id', '_struct_asym.id', '_chem_comp.id', 'chem_comp_atom.comp_id', 'chem_comp_bond.comp_id' ]: categoryName = CifName.categoryPart(itemName) attributeName = CifName.attributePart(itemName) print("Full parent list for %s : %s\n" % (itemName, dApi.getFullParentList(categoryName, attributeName))) print( "Full child list for %s : %s\n" % (itemName, dApi.getFullChildList(categoryName, attributeName))) print("Ultimate parent for %s : %s\n" % (itemName, dApi.getUltimateParent(categoryName, attributeName))) print("Type code for %s : %s\n" % (itemName, dApi.getTypeCode(categoryName, attributeName))) assert dApi.getTypeCode(categoryName, attributeName) is not None
def testConsolidateDictionary(self): """Test case - dump methods for dictionary metadata""" try: myIo = IoAdapter(raiseExceptions=True) self.__containerList = myIo.readFile(inputFilePath=self.__pathPdbxDictionary) dApi = DictionaryApi(containerList=self.__containerList, consolidate=True, expandItemLinked=False, verbose=self.__verbose) for itemName in [ "_entity.id", "_entity_poly_seq.num", "_atom_site.label_asym_id", "_struct_asym.id", "_chem_comp.id", "chem_comp_atom.comp_id", "chem_comp_bond.comp_id", ]: categoryName = CifName.categoryPart(itemName) attributeName = CifName.attributePart(itemName) logger.debug("Full parent list for %s : %s\n", itemName, dApi.getFullParentList(categoryName, attributeName)) logger.debug("Full child list for %s : %s\n", itemName, dApi.getFullChildList(categoryName, attributeName)) logger.debug("Ultimate parent for %s : %s\n", itemName, dApi.getUltimateParent(categoryName, attributeName)) logger.debug("Type code for %s : %s\n", itemName, dApi.getTypeCode(categoryName, attributeName)) self.assertIsNotNone(dApi.getTypeCode(categoryName, attributeName)) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def testExtensions(self): """Test case - condition extensions""" try: myIo = IoAdapter(raiseExceptions=True) self.__containerList = myIo.readFile(inputFilePath=self.__pathPdbxDictionary) self.__containerList.extend(myIo.readFile(inputFilePath=self.__pathPdbxDictionaryExtension)) dApi = DictionaryApi(containerList=self.__containerList, consolidate=True) tD = dApi.getItemValueConditionDict() logger.debug("tD \n%s", pprint.pformat(tD)) self.assertGreaterEqual(len(tD), 2) tD = dApi.getComparisonOperatorDict() logger.debug("tD \n%s", pprint.pformat(tD)) self.assertGreaterEqual(len(tD), 5) tL = dApi.getComparisonOperators() logger.debug("tL %r", tL) self.assertGreaterEqual(len(tL), 5) # tD = dApi.getItemLinkedConditions() logger.debug("tD \n%s", pprint.pformat(tD)) self.assertGreaterEqual(len(tD), 1) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def __getApi(self, dictPath, consolidate=True, replaceDefinition=True): """""" try: mU = MarshalUtil() containerList = mU.doImport(dictPath, fmt="mmcif-dict") return DictionaryApi(containerList=containerList, consolidate=consolidate, replaceDefinition=replaceDefinition, verbose=self.__verbose) except Exception as e: logger.error( "DictionaryFileUtils.__setup() dictionary API construction failed for %s", dictPath) logger.exception("Failing with %s", str(e))
def setUp(self): # self.__pathOutputDir = os.path.join(HERE, "test-output") self.__pathTextCif = os.path.join(HERE, "data", "1bna.cif") self.__testBcifOutput = os.path.join(self.__pathOutputDir, "1bna-generated.bcif") self.__testBcifTranslated = os.path.join(self.__pathOutputDir, "1bna-generated-translated.bcif") # self.__pathPdbxDictionary = os.path.join(HERE, "data", "mmcif_pdbx_v5_next.dic") myIo = IoAdapter(raiseExceptions=True) self.__containerList = myIo.readFile(inputFilePath=self.__pathPdbxDictionary) self.__dApi = DictionaryApi(containerList=self.__containerList, consolidate=True) # self.__floatTolerance = 1.0e-10 self.__startTime = time.time() logger.debug("Running tests on version %s", __version__) logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def testDumpIndex(self): """Test case - dump methods for dictionary metadata""" try: myIo = IoAdapter(raiseExceptions=True) self.__containerList = myIo.readFile(inputFilePath=self.__pathPdbxDictionary) dApi = DictionaryApi(containerList=self.__containerList, consolidate=True, verbose=self.__verbose) if self.__verbose: dApi.dumpCategoryIndex(fh=self.__lfh) logger.debug("Index = %r\n", dApi.getItemNameList("pdbx_nmr_spectral_dim")) logger.debug("Index = %r\n", dApi.getAttributeNameList("pdbx_nmr_spectral_dim")) catIndex = dApi.getCategoryIndex() logger.debug("Index = %r\n", catIndex["pdbx_nmr_spectral_dim"]) self.assertIsNotNone(catIndex["pdbx_nmr_spectral_dim"]) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def test_gen_ddlm(self, in_tmpdir, test_files): myIo = IoAdapterPy() containerList = myIo.readFile( inputFilePath=str(test_files / 'mmcif_pdbx_v5_next.dic')) dApi = DictionaryApi(containerList=containerList, consolidate=True) parentD = dApi.getParentDictionary() # oCList = [] dDef = DataContainer('mmcif_pdbx_ddlm_auto') dc = DataCategory("dictionary") dc.appendAttribute("title") dc.appendAttribute("class") dc.appendAttribute("version") dc.appendAttribute("date") dc.appendAttribute("ddl_conformance") dc.appendAttribute("text") dc.append([ 'mmcif_pdbx_ddlm_auto', 'Instance', 'latest', '2018-03-09', 'ddlm best effort', 'Software converted PDBx dictionary using DDLm semantics' ]) dDef.append(dc) oCList.append(dDef) catIdx = dApi.getCategoryIndex() for catName in sorted(catIdx.keys()): attNameList = catIdx[catName] # created definition container - cDef = DefinitionContainer(catName) oCList.append(cDef) # dc = DataCategory("definition") dc.appendAttribute("id") dc.appendAttribute("scope") dc.appendAttribute("class") dc.appendAttribute("update") dc.append([catName, "Category", "Loop", "2018-03-09"]) cDef.append(dc) val = dApi.getCategoryDescription(category=catName) dc = DataCategory("description") dc.appendAttribute("text") dc.append([val]) cDef.append(dc) # dc = DataCategory("name") dc.appendAttribute("category_id") dc.appendAttribute("object_id") valList = dApi.getCategoryGroupList(category=catName) pcg = catName for val in valList: if val != 'inclusive_group': pcg = val break dc.append([catName, pcg]) cDef.append(dc) valList = dApi.getCategoryKeyList(category=catName) if len(valList) < 1: print("Missing caegory key for category %s\n" % catName) else: dc = DataCategory("category") dc.appendAttribute("key_id") kItemName = CifName.itemName(catName, "synthetic_key") dc.append([kItemName]) cDef.append(dc) iDef = DefinitionContainer(kItemName) self._makeKeyItem(catName, "synthetic_key", valList, iDef) oCList.append(iDef) for attName in attNameList: itemName = CifName.itemName(catName, attName) iDef = DefinitionContainer(itemName) oCList.append(iDef) # dc = DataCategory("definition") dc.appendAttribute("id") dc.appendAttribute("scope") dc.appendAttribute("class") dc.appendAttribute("update") dc.append([itemName, "Item", "Single", "2013-08-22"]) iDef.append(dc) # val = dApi.getDescription(category=catName, attribute=attName) dc = DataCategory("description") dc.appendAttribute("text") dc.append([val]) iDef.append(dc) # dc = DataCategory("name") dc.appendAttribute("category_id") dc.appendAttribute("object_id") # if itemName in parentD: dc.appendAttribute("linked_item_id") dc.append([catName, attName, parentD[itemName][0]]) else: dc.append([catName, attName]) iDef.append(dc) # # aliasList = dApi.getItemAliasList(category=catName, attribute=attName) if len(aliasList) > 0: dc = DataCategory("alias") dc.appendAttribute("definition_id") for alias in aliasList: dc.append([alias[0]]) iDef.append(dc) enList = dApi.getEnumListAltWithDetail(category=catName, attribute=attName) tC = dApi.getTypeCode(category=catName, attribute=attName) tcontainer = 'Single' purpose = 'Describe' source = 'Recorded' contents = 'Text' # if tC is None: self.__lfh.write("Missing data type attribute %s\n" % attName) elif tC in [ 'code', 'atcode', 'name', 'idname', 'symop', 'fax', 'phone', 'email', 'code30', 'ec-type' ]: purpose = 'Encode' contents = 'Text' source = 'Assigned' elif tC in ['ucode']: purpose = 'Encode' contents = 'Code' source = 'Assigned' elif tC in ['line', 'uline', 'text']: purpose = 'Describe' source = 'Recorded' contents = 'Text' elif tC in ['int']: purpose = 'Number' source = 'Recorded' contents = 'Integer' elif tC in ['int-range']: purpose = 'Number' source = 'Recorded' contents = 'Range' elif tC in ['float']: purpose = 'Measurand' source = 'Recorded' contents = 'Real' elif tC in ['float-range']: purpose = 'Measurand' source = 'Recorded' contents = 'Range' elif tC.startswith('yyyy'): source = 'Assigned' contents = 'Date' purpose = 'Describe' if len(enList) > 0: purpose = 'State' dc = DataCategory("type") dc.appendAttribute("purpose") dc.appendAttribute("source") dc.appendAttribute("contents") dc.appendAttribute("container") dc.append([purpose, source, contents, tcontainer]) iDef.append(dc) # if (len(enList) > 0): dc = DataCategory("enumeration_set") dc.appendAttribute("state") dc.appendAttribute("detail") for en in enList: dc.append([en[0], en[1]]) iDef.append(dc) dfv = dApi.getDefaultValue(category=catName, attribute=attName) bvList = dApi.getBoundaryList(category=catName, attribute=attName) if (((dfv is not None) and (dfv not in ['?', '.'])) or len(bvList) > 0): row = [] dc = DataCategory("enumeration") if dfv is not None: dc.appendAttribute("default") row.append(dfv) if len(bvList) > 0: dc.appendAttribute("range") mminVp = -1000000 mmaxVp = 10000000 mminV = mmaxVp mmaxV = mminVp for bv in bvList: minV = float(bv[0]) if bv[0] != '.' else mminVp maxV = float(bv[1]) if bv[1] != '.' else mmaxVp mminV = min(mminV, minV) mmaxV = max(mmaxV, maxV) if mminV == mminVp: mminV = '' if mmaxV == mmaxVp: mmaxV = '' row.append(str(mminV) + ":" + str(mmaxV)) dc.append(row) iDef.append(dc) myIo.writeFile(outputFilePath="mmcif_pdbx_ddlm_auto.dic", containerList=oCList)
class DictionaryData(object): def __init__(self): basedir = os.path.join(os.path.dirname(__file__), "..", "dist") self.dictPath = os.path.join(basedir, "mmcif_pdbx_v5_next.dic") def readDictionary(self): myIo = IoAdapterPy() self.__containerList = myIo.readFile(inputFilePath=self.dictPath) self._dApi = DictionaryApi(containerList=self.__containerList, consolidate=True) assert len(self.getGroups()) > 0, "Failed to load %s" % self.dictPath def getGroups(self): groupList = self._dApi.getCategoryGroups() return groupList def getTypeRegex(self, cat, attrib): return self._dApi.getTypeRegex(cat, attrib) def getTypeRegexAlt(self, cat, attrib): return self._dApi.getTypeRegexAlt(cat, attrib) def getDataTypeList(self): """ Return list of tuples containing ('code','primitive_code','construct','detail' ) """ return self._dApi.getDataTypeList() def getEnumList(self, category, attribute, sortFlag=True): return self._dApi.getEnumList(category, attribute, sortFlag) def getEnumListAlt(self, category, attribute, sortFlag=True): return self._dApi.getEnumListAlt(category, attribute, sortFlag) def getEnumListAltWithDetail(self, category, attribute): return self._dApi.getEnumListAltWithDetail(category, attribute) def getAllCategory(self): """Returns list of all categories""" return self._dApi.getCategoryList() def getCategoryList(self): return self._dApi.getCategoryList() def getItemNameList(self, category): return self._dApi.getItemNameList(category) def wwPDBContext(self, catItem): """Returns true if category or item have local context""" category, item = catItem.split('.') if category[0] == "_": category = category[1:] #print "CHECKING ", category, item, self.__dictA.getCategoryContextList(category) if self._dApi.getCategoryContextList(category) == ['WWPDB_LOCAL']: return True if self._dApi.getContextList(category, item) == ['WWPDB_LOCAL']: return True return False def getCategoryItemEnum(self, itemName): """Returns any archive enum list""" categoryName = CifName.categoryPart(itemName) attributeName = CifName.attributePart(itemName) return self._dApi.getEnumList(categoryName, attributeName) def getCategoryPdbxItemEnum(self, itemName): """Returns any DepUI enum list""" categoryName = CifName.categoryPart(itemName) attributeName = CifName.attributePart(itemName) return self._dApi.getEnumListAlt(categoryName, attributeName) def getItemRelatedList(self, itemName): categoryName = CifName.categoryPart(itemName) attributeName = CifName.attributePart(itemName) return self._dApi.getItemRelatedList(categoryName, attributeName)
def readDictionary(self): myIo = IoAdapterPy() self.__containerList = myIo.readFile(inputFilePath=self.dictPath) self._dApi = DictionaryApi(containerList=self.__containerList, consolidate=True) assert len(self.getGroups()) > 0, "Failed to load %s" % self.dictPath
def __testMarkupCategoryGroup(self, oFile, groupSelectList=None): """Test case - extract the content to be rendered -""" startTime = time.time() logger.debug("\nStarting at %s\n", time.strftime("%Y %m %d %H:%M:%S", time.localtime())) try: rL = [] myIo = IoAdapter() containerList = myIo.readFile(self.__pathPdbxV50Dictionary) dApi = DictionaryApi(containerList=containerList, consolidate=True, verbose=self.__verbose) # groupList = dApi.getCategoryGroups() logger.debug("groupList %s\n", groupList) for groupName in groupList: if groupSelectList and groupName not in groupSelectList: continue # # Goup header details # rL.append("# Category Group %s" % groupName) rL.append("") rL.append("") rL.append("%s" % dApi.getCategoryGroupDescription(groupName)) rL.append("") rL.append("---") rL.append("") catNameList = dApi.getCategoryGroupCategories( groupName=groupName) # cList = self.__sortIgnoreCase(catNameList) for catName in cList: logger.debug("Group %s category %s\n", groupName, catName) catDescription = dApi.getCategoryDescription( category=catName) catExTupList = dApi.getCategoryExampleList( category=catName) keyItemNameList = dApi.getCategoryKeyList(category=catName) keyAttNameList = [ CifName.attributePart(k) for k in keyItemNameList ] # # Category header details # rL.append("## Category %s" % catName) rL.append("") rL.append("") rL.append(" %s" % catDescription) rL.append("") rL.append("---") rL.append("") if catExTupList: rL.extend( self.__formatTupListInset(catExTupList, tab=" ")) # # summary table # rL.append("") rL.append("---") rL.append("") rL.append( "| Attribute | Key | Required | Type | Units | Enumerated | Bounded |" ) rL.append( "| --------- | --- | -------- | ---- | ----- | ---------- | ------- |" ) aList = self.__sortIgnoreCase( dApi.getAttributeNameList(category=catName)) for attName in aList: isKey = attName in keyAttNameList attDescription = dApi.getDescription(category=catName, attribute=attName) attUnits = dApi.getUnits(category=catName, attribute=attName) attMandatory = dApi.getMandatoryCode(category=catName, attribute=attName) attTypeCode = dApi.getTypeCode(category=catName, attribute=attName) enumTupList = dApi.getEnumListWithDetail( category=catName, attribute=attName) if enumTupList: isEnum = True else: isEnum = False bL = dApi.getBoundaryList(category=catName, attribute=attName) if bL: isBounded = True else: isBounded = False rL.append("| %s | %s | %s | %s | %s | %s | %s |" % (attName, self.__trB(isKey), attMandatory, attTypeCode, attUnits, self.__trB(isEnum), self.__trB(isBounded))) # rL.append("") rL.append("---") rL.append("") # for attName in aList: isKey = attName in keyAttNameList attMandatory = dApi.getMandatoryCode(category=catName, attribute=attName) # tN = "_" + catName + "." + attName if isKey: tN = tN + " (key)" elif attMandatory.upper() in ["YES", "Y"]: tN = tN + " (required)" # rL.append("#### %s\n" % tN) rL.append("") attDescription = dApi.getDescription(category=catName, attribute=attName) rL.append(" %s\n" % attDescription) rL.append("") attUnits = dApi.getUnits(category=catName, attribute=attName) attTypeCode = dApi.getTypeCode(category=catName, attribute=attName) enumTupList = dApi.getEnumListWithDetail( category=catName, attribute=attName) if enumTupList: rL.append("") rL.append("---") rL.append("") rL.append("| Allowed Values | Detail |") rL.append("| -------------- | ------ |") for tup in enumTupList: if tup[1]: rL.append("| %s | %s |" % (tup[0], tup[1])) else: rL.append("| %s | %s |" % (tup[0], " ")) rL.append("") # bL = dApi.getBoundaryList(category=catName, attribute=attName) btL = self.__processbounds(bL) if btL: tup = btL[0] rL.append("") rL.append("---") rL.append("") rL.append("| %s | %s |" % (tup[0], tup[1])) # rL.append("| ------------- | ------ |") for tup in btL[1:]: rL.append("| %s | %s |" % (tup[0], tup[1])) rL.append("") rL.append("") with open(oFile, "w") as ofh: ofh.write("\n".join(rL)) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() endTime = time.time() logger.debug("\nCompleted at %s (%.2f seconds)", time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - startTime)
def testDDLApi(self): """Test case - DDL API linkage counts""" try: myIo = IoAdapter(raiseExceptions=True) containerList = myIo.readFile( inputFilePath=self.__pathDdlGeneratedDictionary) dApi = DictionaryApi(containerList=containerList, consolidate=True, expandItemLinked=True) cL = dApi.getCategoryList() logger.info("Category length %r", len(cL)) self.assertGreaterEqual(len(cL), 63) cL = dApi.getFullChildList("category", "id") logger.info("Children of category.id (%d)", len(cL)) self.assertGreaterEqual(len(cL), 11) # cL = dApi.getFullDescendentList("category", "id") logger.info("Descendents of category.id (%d)", len(cL)) self.assertGreaterEqual(len(cL), 13) # cL = dApi.getFullChildList("item", "name") logger.info("Children of item.name (%d)", len(cL)) self.assertGreaterEqual(len(cL), 36) cL = dApi.getFullDescendentList("item", "name") logger.info("Descendents of item.name (%d)", len(cL)) self.assertGreaterEqual(len(cL), 38) # val = dApi.getDictionaryVersion() self.assertEqual(val, "2.2.2") val = dApi.getDictionaryTitle() self.assertEqual(val, "mmcif_ddl.dic") val = dApi.getDictionaryUpdate(order="reverse") self.assertEqual(val, "2020-06-05") val = dApi.getDictionaryRevisionCount() self.assertGreaterEqual(val, 78) valL = dApi.getDictionaryHistory(order="reverse") self.assertGreaterEqual(len(valL), 78) # val = dApi.getDictionaryComponentCount() self.assertGreaterEqual(val, 6) # valL = dApi.getDictionaryComponentDetails() self.assertGreaterEqual(len(valL), 6) valL = dApi.getDictionaryComponents() self.assertGreaterEqual(len(valL), 6) for dictionaryComponentId in dApi.getDictionaryComponents(): valL = dApi.getDictionaryComponentHistory( dictionaryComponentId, order="reverse") self.assertGreaterEqual(len(valL), 2) # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def __testScanRepo(self, contentType, scanType="full"): """Utility method to scan repo for data type and coverage content. Using mock repos for tests. """ try: containerList = self.__mU.doImport(self.__pathPdbxDictionary, fmt="mmcif-dict") dictApi = DictionaryApi(containerList=containerList, consolidate=True, verbose=True) failedFilePath = os.path.join( HERE, "test-output", "%s-failed-list-%s.txt" % (contentType, scanType)) savedFilePath = os.path.join( HERE, "test-output", "%s-path-list-%s.txt" % (contentType, scanType)) scanDataFilePath = os.path.join(HERE, "test-output", "%s-scan-data.pic" % (contentType)) dataCoverageFilePath = os.path.join( HERE, "test-output", "%s-scan-data-coverage-%s.json" % (contentType, scanType)) dataCoverageItemFilePath = os.path.join( HERE, "test-output", "%s-scan-data-item-coverage-%s.tdd" % (contentType, scanType)) dataTypeFilePath = os.path.join( HERE, "test-output", "%s-scan-data-type-%s.json" % (contentType, scanType)) # ### categoryList = sorted(dictApi.getCategoryList()) dictSchema = { catName: sorted(dictApi.getAttributeNameList(catName)) for catName in categoryList } attributeDataTypeD = OrderedDict() for catName in categoryList: aD = {} for atName in dictSchema[catName]: aD[atName] = dictApi.getTypeCode(catName, atName) attributeDataTypeD[catName] = aD ### # sr = ScanRepoUtil(self.__cfgOb, attributeDataTypeD=attributeDataTypeD, numProc=self.__numProc, chunkSize=self.__chunkSize, fileLimit=self.__fileLimit, workPath=self.__cachePath) ok = sr.scanContentType(contentType, scanType=scanType, inputPathList=None, scanDataFilePath=scanDataFilePath, failedFilePath=failedFilePath, saveInputFileListPath=savedFilePath) self.assertTrue(ok) ok = sr.evalScan(scanDataFilePath, dataTypeFilePath, evalType="data_type") self.assertTrue(ok) ok = sr.evalScan(scanDataFilePath, dataCoverageFilePath, evalType="data_coverage") self.assertTrue(ok) ok = sr.evalScanItem(scanDataFilePath, dataCoverageItemFilePath) self.assertTrue(ok) return ok except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
class MethodUtils(object): def __init__(self, dictContainerList, verbose=False): # self.__verbose = verbose # list of dictionary data & definition containers self.__dictContainerList = dictContainerList self.__dApi = DictionaryApi(containerList=self.__dictContainerList, consolidate=True, verbose=self.__verbose) # # Target data container list self.__dataContainerList = [] # def setDataContainerList(self, dataContainerList): self.__dataContainerList = dataContainerList def getDataContainerList(self): return self.__dataContainerList def getDictionary(self): return self.__dApi def getMethods(self): return self.__dApi.getMethodIndex() def getMethod(self, mId): return self.__dApi.getMethod(mId) def invokeMethods(self, fh=sys.stdout): _ = fh mI = self.__dApi.getMethodIndex() lenD = len(mI) i = 0 for k, mRefL in mI.items(): for mRef in mRefL: i += 1 mId = mRef.getId() mType = mRef.getType() categoryName = mRef.getCategoryName() attributeName = mRef.getAttributeName() # logger.debug("\n") logger.debug("++++++++++++++++++--------------------\n") logger.debug( "Invoking dictionary method on file object: %s (%d/%d)", k, i, lenD) logger.debug(" + Method id: %s", mId) logger.debug(" + Type: %s", mType) logger.debug(" + Category: %s", categoryName) logger.debug(" + Attribute: %s", attributeName) # if mType == "datablock": logger.debug("Invoke datablock method %s", mId) # self.invokeDataBlockMethod(type,self.__dApi.getMethod(id)) # continue # for db in self.__dataContainerList: if mType == "category": if not db.exists(categoryName): dc = DataCategory(categoryName) db.append(dc) dObj = db.getObj(categoryName) dObj.invokeCategoryMethod(mType, self.__dApi.getMethod(mId), db) elif mType == "attribute": if not db.exists(categoryName): dc = DataCategory(categoryName) db.append(dc) dObj = db.getObj(categoryName) # logger.debug("invoke - %r %r %r %r" % (attributeName, type, self.__dApi.getMethod(id), db)) dObj.invokeAttributeMethod(attributeName, mType, self.__dApi.getMethod(mId), db) elif mType == "datablock": logger.debug("Invoke datablock method %s", mId) db.invokeDataBlockMethod(mType, self.__dApi.getMethod(mId), db) else: pass def dumpMethods(self, fh=sys.stdout): self.__dApi.dumpMethods(fh) def dumpDictionary(self, fh=sys.stdout): lenD = len(self.__dictContainerList) fh.write("\n--------------------------------------------\n") fh.write("\n-----------DUMP DICTIONARY------------------\n") fh.write("Dictionary object list length is: %d\n" % lenD) i = 1 for dObj in self.__dictContainerList: if dObj.getName(): fh.write("\n") fh.write("++++++++++++++++++--------------------\n") fh.write("Dumping dictionary object named: %s (%d/%d)\n" % (dObj.getName(), i, lenD)) dObj.printIt(fh) i += 1 # def dumpDataFile(self, fh=sys.stdout): lenD = len(self.__dataContainerList) fh.write("\n--------------------------------------------\n") fh.write("\n-----------DUMP DATAFILE--------------------\n") fh.write("Data object list length is: %d\n" % lenD) i = 1 for dObj in self.__dataContainerList: fh.write("\n") fh.write("++++++++++++++++++--------------------\n") fh.write("Dumping data file object named: %s (%d/%d)\n" % (dObj.getName(), i, lenD)) dObj.printIt(fh) i += 1
def testGenDDLm(self): """Generating alternative DDLm metadata format. (starting point)""" try: myIo = IoAdapterPy(self.__verbose, self.__lfh) self.__containerList = myIo.readFile( inputFilePath=self.__pathPdbxDictionary) dApi = DictionaryApi(containerList=self.__containerList, consolidate=True, verbose=self.__verbose) parentD = dApi.getParentDictionary() # oCList = [] dDef = DataContainer("mmcif_pdbx_ddlm_auto") dc = DataCategory("dictionary") dc.appendAttribute("title") dc.appendAttribute("class") dc.appendAttribute("version") dc.appendAttribute("date") dc.appendAttribute("ddl_conformance") dc.appendAttribute("text") dc.append([ "mmcif_pdbx_ddlm_auto", "Instance", "latest", "2018-03-09", "ddlm best effort", "Software converted PDBx dictionary using DDLm semantics" ]) dDef.append(dc) oCList.append(dDef) catIdx = dApi.getCategoryIndex() for catName in sorted(catIdx.keys()): attNameList = catIdx[catName] # created definition container - cDef = DefinitionContainer(catName) oCList.append(cDef) # dc = DataCategory("definition") dc.appendAttribute("id") dc.appendAttribute("scope") dc.appendAttribute("class") dc.appendAttribute("update") dc.append([catName, "Category", "Loop", "2018-03-09"]) cDef.append(dc) val = dApi.getCategoryDescription(category=catName) dc = DataCategory("description") dc.appendAttribute("text") dc.append([val]) cDef.append(dc) # dc = DataCategory("name") dc.appendAttribute("category_id") dc.appendAttribute("object_id") valList = dApi.getCategoryGroupList(category=catName) pcg = catName for val in valList: if val != "inclusive_group": pcg = val break dc.append([catName, pcg]) cDef.append(dc) valList = dApi.getCategoryKeyList(category=catName) if not valList: self.__lfh.write("Missing caegory key for category %s\n" % catName) else: dc = DataCategory("category") dc.appendAttribute("key_id") kItemName = CifName.itemName(catName, "synthetic_key") dc.append([kItemName]) cDef.append(dc) iDef = DefinitionContainer(kItemName) self.__makeKeyItem(catName, "synthetic_key", valList, iDef) oCList.append(iDef) for attName in attNameList: itemName = CifName.itemName(catName, attName) iDef = DefinitionContainer(itemName) oCList.append(iDef) # dc = DataCategory("definition") dc.appendAttribute("id") dc.appendAttribute("scope") dc.appendAttribute("class") dc.appendAttribute("update") dc.append([itemName, "Item", "Single", "2013-08-22"]) iDef.append(dc) # val = dApi.getDescription(category=catName, attribute=attName) dc = DataCategory("description") dc.appendAttribute("text") dc.append([val]) iDef.append(dc) # dc = DataCategory("name") dc.appendAttribute("category_id") dc.appendAttribute("object_id") # if itemName in parentD: dc.appendAttribute("linked_item_id") dc.append([catName, attName, parentD[itemName][0]]) else: dc.append([catName, attName]) iDef.append(dc) # # aliasList = dApi.getItemAliasList(category=catName, attribute=attName) if aliasList: dc = DataCategory("alias") dc.appendAttribute("definition_id") for alias in aliasList: dc.append([alias[0]]) iDef.append(dc) enList = dApi.getEnumListAltWithDetail(category=catName, attribute=attName) tC = dApi.getTypeCode(category=catName, attribute=attName) tcontainer = "Single" purpose = "Describe" source = "Recorded" contents = "Text" # if tC is None: self.__lfh.write("Missing data type attribute %s\n" % attName) elif tC in [ "code", "atcode", "name", "idname", "symop", "fax", "phone", "email", "code30", "ec-type" ]: purpose = "Encode" contents = "Text" source = "Assigned" elif tC in ["ucode"]: purpose = "Encode" contents = "Code" source = "Assigned" elif tC in ["line", "uline", "text"]: purpose = "Describe" source = "Recorded" contents = "Text" elif tC in ["int"]: purpose = "Number" source = "Recorded" contents = "Integer" elif tC in ["int-range"]: purpose = "Number" source = "Recorded" contents = "Range" elif tC in ["float"]: purpose = "Measurand" source = "Recorded" contents = "Real" elif tC in ["float-range"]: purpose = "Measurand" source = "Recorded" contents = "Range" elif tC.startswith("yyyy"): source = "Assigned" contents = "Date" purpose = "Describe" if enList: purpose = "State" dc = DataCategory("type") dc.appendAttribute("purpose") dc.appendAttribute("source") dc.appendAttribute("contents") dc.appendAttribute("container") dc.append([purpose, source, contents, tcontainer]) iDef.append(dc) # if enList: dc = DataCategory("enumeration_set") dc.appendAttribute("state") dc.appendAttribute("detail") for en in enList: dc.append([en[0], en[1]]) iDef.append(dc) dfv = dApi.getDefaultValue(category=catName, attribute=attName) bvList = dApi.getBoundaryList(category=catName, attribute=attName) if ((dfv is not None) and (dfv not in ["?", "."])) or bvList: row = [] dc = DataCategory("enumeration") if dfv is not None: dc.appendAttribute("default") row.append(dfv) if bvList: dc.appendAttribute("range") mminVp = -1000000 mmaxVp = 10000000 mminV = mmaxVp mmaxV = mminVp for bv in bvList: minV = float(bv[0]) if bv[0] != "." else mminVp maxV = float(bv[1]) if bv[1] != "." else mmaxVp mminV = min(mminV, minV) mmaxV = max(mmaxV, maxV) if mminV == mminVp: mminV = "" if mmaxV == mmaxVp: mmaxV = "" row.append(str(mminV) + ":" + str(mmaxV)) dc.append(row) iDef.append(dc) myIo.writeFile(outputFilePath=os.path.join( HERE, "test-output", "mmcif_pdbx_ddlm_auto.dic"), containerList=oCList) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def test_markup_category_group(self, in_tmpdir, test_files, out_file_name, groupSelectList): oFile = out_file_name rL = [] myIo = IoAdapter() containerList = myIo.readFile( str(test_files / "mmcif_pdbx_v5_next.dic")) dApi = DictionaryApi(containerList=containerList, consolidate=True) # groupList = dApi.getCategoryGroups() print('groupList %s\n' % groupList) for groupName in groupList: if groupSelectList and groupName not in groupSelectList: continue # # Goup header details # rL.append("# Category Group %s" % groupName) rL.append("") rL.append("") rL.append("%s" % dApi.getCategoryGroupDescription(groupName)) rL.append("") rL.append("---") rL.append("") catNameList = dApi.getCategoryGroupCategories(groupName=groupName) # cList = self.__sortIgnoreCase(catNameList) for catName in cList: print('Group %s category %s\n' % (groupName, catName)) catDescription = dApi.getCategoryDescription(category=catName) catExTupList = dApi.getCategoryExampleList(category=catName) keyItemNameList = dApi.getCategoryKeyList(category=catName) keyAttNameList = [ CifName.attributePart(k) for k in keyItemNameList ] # # Category header details # rL.append("## Category %s" % catName) rL.append("") rL.append("") rL.append(" %s" % catDescription) rL.append("") rL.append("---") rL.append("") if catExTupList: rL.extend( self.__formatTupListInset(catExTupList, tab=' ')) # # summary table # rL.append("") rL.append("---") rL.append("") rL.append( "| Attribute | Key | Required | Type | Units | Enumerated | Bounded |" ) rL.append( "| --------- | --- | -------- | ---- | ----- | ---------- | ------- |" ) aList = self.__sortIgnoreCase( dApi.getAttributeNameList(category=catName)) for attName in aList: isKey = attName in keyAttNameList attDescription = dApi.getDescription(category=catName, attribute=attName) attUnits = dApi.getUnits(category=catName, attribute=attName) attMandatory = dApi.getMandatoryCode(category=catName, attribute=attName) attTypeCode = dApi.getTypeCode(category=catName, attribute=attName) enumTupList = dApi.getEnumListWithDetail(category=catName, attribute=attName) if len(enumTupList) > 0: isEnum = True else: isEnum = False bL = dApi.getBoundaryList(category=catName, attribute=attName) if len(bL) > 0: isBounded = True else: isBounded = False rL.append( '| %s | %s | %s | %s | %s | %s | %s |' % (attName, self.__trB(isKey), attMandatory, attTypeCode, attUnits, self.__trB(isEnum), self.__trB(isBounded))) # rL.append("") rL.append("---") rL.append("") # for attName in aList: isKey = attName in keyAttNameList attMandatory = dApi.getMandatoryCode(category=catName, attribute=attName) # tN = '_' + catName + '.' + attName if isKey: tN = tN + ' (key)' elif attMandatory.upper() in ['YES', 'Y']: tN = tN + ' (required)' # rL.append("#### %s\n" % tN) rL.append("") attDescription = dApi.getDescription(category=catName, attribute=attName) rL.append(" %s\n" % attDescription) rL.append("") attUnits = dApi.getUnits(category=catName, attribute=attName) attTypeCode = dApi.getTypeCode(category=catName, attribute=attName) enumTupList = dApi.getEnumListWithDetail(category=catName, attribute=attName) if len(enumTupList) > 0: rL.append("") rL.append("---") rL.append("") rL.append("| Allowed Values | Detail |") rL.append("| -------------- | ------ |") for tup in enumTupList: if tup[1] and len(tup[1]) > 0: rL.append("| %s | %s |" % (tup[0], tup[1])) else: rL.append("| %s | %s |" % (tup[0], ' ')) rL.append("") # bL = dApi.getBoundaryList(category=catName, attribute=attName) btL = self.__processbounds(bL) if len(btL) > 0: tup = btL[0] rL.append("") rL.append("---") rL.append("") rL.append("| %s | %s |" % (tup[0], tup[1])) # rL.append("| ------------- | ------ |") for tup in btL[1:]: rL.append("| %s | %s |" % (tup[0], tup[1])) rL.append("") rL.append("") with open(str(oFile), 'w') as ofh: ofh.write('\n'.join(rL))
def testClassifyByGroup(self): """Test case - organize dictionary items by classes: SAMPLE, MX, NMR, EM, STRUCTURE, and DB""" try: myIo = IoAdapter(raiseExceptions=True) self.__containerList = myIo.readFile( inputFilePath=self.__pathPdbxDictionary) dApi = DictionaryApi(containerList=self.__containerList, consolidate=True, verbose=self.__verbose) # itemList = [] groupList = dApi.getCategoryGroups() categoryList = dApi.getCategoryList() for category in categoryList: itemList.extend(dApi.getItemNameList(category)) itemList = sorted(set(itemList)) logger.info("Total category length %d", len(categoryList)) logger.info("Total definition length %d", len(itemList)) logger.info("group length %s", len(groupList)) logger.debug("groupList %r", groupList) # findUnlinked = False if findUnlinked: tSet = set(["pdbx_group", "inclusive_group"]) for category in categoryList: gList = dApi.getCategoryGroupList(category) gSet = set(gList) if gSet == tSet: logger.info("unqualified %s", category) # logger.info("%s -> %r", category, gList) if not gList: logger.info("--- No category group assignment for %s", category) # classD = {} # Add category group members - for groupName, className in self.__groupClassTupL: categoryL = dApi.getCategoryGroupCategories( groupName, followChildren=True) for category in categoryL: classD.setdefault(className, []).extend( dApi.getItemNameList(category)) # # Add unlinked categories # for category, className in self.__unlinkedCategoryClassTup: classD.setdefault(className, []).extend(dApi.getItemNameList(category)) # sumItem = 0 classItemD = {} for className, itemL in classD.items(): numItem = len(set(itemL)) sumItem += numItem logger.info("class %s items %d", className, len(set(itemL))) for item in itemL: classItemD[item] = True # logger.info("Sum classified items is %d", sumItem) logger.info("classified items %d", len(classItemD)) # logger.debug("classItemD.items() %r", list(classItemD.items())[:10]) missingGroupL = [] jj = 0 for item in itemList: if item not in classItemD: jj += 1 category = CifName.categoryPart(item) logger.info("%d item %r category %r", jj, item, category) missingGroupL.extend(dApi.getCategoryGroupList(category)) # logger.info("missing groups %r", sorted(set(missingGroupL))) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()