def __fetchReferenceEntries(self, refDbName, idList, saveText=False, fetchLimit=None): """Fetch database entries from the input reference sequence database name.""" dD = {"refDbName": refDbName, "refDbCache": {}, "matchInfo": {}} try: idList = idList[:fetchLimit] if fetchLimit else idList logger.info("Starting fetch for %d %s entries", len(idList), refDbName) if refDbName == "UNP": fobj = UniProtUtils(saveText=saveText) refD, matchD = fobj.fetchList(idList) dD = { "refDbName": refDbName, "refDbCache": refD, "matchInfo": matchD } except Exception as e: logger.exception("Failing with %s", str(e)) return dD
def __fetchReferenceEntries(self, refDbName, idList, saveText=False, fetchLimit=None): """Fetch database entries from the input reference sequence database name.""" dD = {"refDbName": refDbName, "refDbCache": {}} idD = {"matchInfo": {}, "refIdMap": {}} try: idList = idList[:fetchLimit] if fetchLimit else idList logger.info("Starting fetch for %d %s entries", len(idList), refDbName) if refDbName == "UniProt": fobj = UniProtUtils(saveText=saveText) logger.info("Maximum reference chunk size %d", self.__maxChunkSize) refD, matchD = fobj.fetchList(idList, maxChunkSize=self.__maxChunkSize) dD = {"refDbName": refDbName, "refDbCache": refD} idD = {"matchInfo": matchD} # # Check the coverage - # countD = defaultdict(int) logger.info("Match dictionary length %d", len(matchD)) for _, mD in matchD.items(): if "matched" in mD: countD[mD["matched"]] += 1 logger.info("Reference length %d match length %d coverage %r", len(refD), len(matchD), countD.items()) except Exception as e: logger.exception("Failing with %s", str(e)) return dD, idD
def testLookup(self): """Test lookup gene names""" try: uUtils = UniProtUtils(saveText=False) geneList = ["BCOR"] for gene in geneList: idList, retCode = uUtils.doLookup([gene], itemKey="GENENAME") logger.info("retCode %r rspList (%d) %r", retCode, len(idList), idList) self.assertGreaterEqual(len(idList), 500) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def testGeneLookup(self): """Test lookup gene names for human""" try: uUtils = UniProtUtils(saveText=False) geneList = ["BCOR", "BCORL1"] for gene in geneList: idList, retCode = uUtils.doGeneLookup(gene, 9606) logger.info("retCode %r rspList (%d) %r", retCode, len(idList), idList) self.assertGreaterEqual(len(idList), 1) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def testBatchFetchVariants(self): """Test batch variant entry fetch""" try: fobj = UniProtUtils(saveText=True) retD, matchD = fobj.fetchList(self.__unpIdListV, usePrimary=self.__usePrimary, retryAltApi=self.__retryAltApi) numPrimary, numSecondary, numNone = self.__matchSummary(matchD) logger.debug("%d %d %d", numPrimary, numSecondary, numNone) self.assertGreaterEqual(len(retD), len(self.__unpIdListV)) if retD and self.__export: fobj.writeUnpXml( os.path.join(self.__workPath, "variant-batch-fetch.xml")) # self.__dumpEntries(retD) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def updateList(self, dataList, procName, optionsD, workingDir): """Update the input list of reference sequence identifiers and return matching diagnostics and reference feature data. """ _ = optionsD _ = workingDir saveText = optionsD.get("saveText", False) fetchLimit = optionsD.get("fetchLimit", None) refDbName = optionsD.get("refDbName", "UniProt") maxChunkSize = optionsD.get("maxChunkSize", 50) successList = [] retList1 = [] retList2 = [] diagList = [] emptyList = [] # try: tU = TimeUtil() idList = dataList[:fetchLimit] if fetchLimit else dataList logger.info("%s starting fetch for %d %s entries", procName, len(idList), refDbName) if refDbName == "UniProt": fobj = UniProtUtils(saveText=saveText) logger.debug("Maximum reference chunk size %d", maxChunkSize) refD, matchD = fobj.fetchList(idList, maxChunkSize=maxChunkSize) if len(matchD) == len(idList): for uId, tD in matchD.items(): tD["rcsb_id"] = uId.strip() tD["rcsb_last_update"] = tU.getDateTimeObj(tU.getTimestamp()) retList1.append(tD) for uId, tD in refD.items(): tD["rcsb_id"] = uId.strip() tD["rcsb_last_update"] = tU.getDateTimeObj(tU.getTimestamp()) retList2.append(tD) successList.extend(idList) self.__updateReferenceData(self.__refDatabaseName, self.__refDataCollectionName, retList2) self.__updateReferenceData(self.__refDatabaseName, self.__refMatchDataCollectionName, retList1) else: logger.info("Failing with fetch for %d entries with matchD %r", len(idList), matchD) else: logger.error("Unsupported reference database %r", refDbName) except Exception as e: logger.exception("Failing %s for %d data items %s", procName, len(dataList), str(e)) logger.info("%s dataList length %d success length %d rst1 %d rst2 %d", procName, len(dataList), len(successList), len(retList1), len(retList2)) # return successList, emptyList, emptyList, diagList
def testValidateExchangeObject(self): """Test fetch exchange objects""" try: # sD = self.__mU.doImport(self.__jsonSchemaPath, "json") # fobj = UniProtUtils(saveText=False) idList = self.__unpIdList1 retD, _ = fobj.fetchList(idList, usePrimary=self.__usePrimary, retryAltApi=self.__retryAltApi) # exObjD = fobj.reformat(retD, formatType="exchange") if exObjD and self.__export: for rId in exObjD: self.__mU.doExport(os.path.join(self.__workPath, rId + "-exchange.json"), exObjD[rId], fmt="json", indent=3) # Draft4Validator.check_schema(sD) # valInfo = Draft4Validator(sD, format_checker=FormatChecker()) eCount = 0 for rId, dD in exObjD.items(): logger.debug("Uid %s", rId) try: cCount = 0 for error in sorted(valInfo.iter_errors(dD), key=str): logger.info("%s path %s error: %s", rId, error.path, error.message) logger.debug(">>> failing object is %r", dD) eCount += 1 cCount += 1 # logger.debug("%s errors count %d", rId, cCount) except Exception as e: logger.exception("Validator fails %s", str(e)) # logger.debug("Total errors count %d", eCount) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def testExchangeObject(self): """Test fetch exchange objects""" try: # fobj = UniProtUtils(saveText=False) idList = self.__unpIdList1 retD, _ = fobj.fetchList(idList, usePrimary=self.__usePrimary, retryAltApi=self.__retryAltApi) exObjD = fobj.reformat(retD, formatType="exchange") if exObjD and self.__export: for rId in exObjD: self.__mU.doExport(os.path.join(self.__workPath, rId + "-exchange.json"), exObjD[rId], fmt="json", indent=3) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def testBatchFetchFailureMode1(self): """Test batch entry fetch (failure mode)""" try: fobj = UniProtUtils(saveText=False, urlPrimary="http://none.none.none") idList = self.__unpIdListLong[:100] logger.info("idList length %d unique %d", len(idList), len(set(idList))) retD, matchD = fobj.fetchList(idList, maxChunkSize=len(idList), usePrimary=self.__usePrimary, retryAltApi=self.__retryAltApi) logger.info("IdList %d reference return length %d match length %d", len(idList), len(retD), len(matchD)) numPrimary, numSecondary, numNone = self.__matchSummary(matchD) logger.debug("%d %d %d", numPrimary, numSecondary, numNone) sumRet = numPrimary + numSecondary + numNone logger.info("sumRet returned %d", sumRet) self.assertGreaterEqual(sumRet, len(idList) - 1) if retD and self.__export: for rId in retD: self.__mU.doExport(os.path.join(self.__workPath, rId + ".json"), retD[rId], fmt="json", indent=3) # retD, matchD = fobj.fetchList(idList, usePrimary=False, retryAltApi=True) logger.info("IdList %d reference return length %d match length %d", len(idList), len(retD), len(matchD)) numPrimary, numSecondary, numNone = self.__matchSummary(matchD) logger.debug("%d %d %d", numPrimary, numSecondary, numNone) sumRet = numPrimary + numSecondary + numNone logger.info("sumRet returned %d", sumRet) self.assertGreaterEqual(sumRet, len(idList) - 1) # except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def testFetchSequenceList(self): """Test fetch UniProt sequence data (FASTA)""" try: # fobj = UniProtUtils(saveText=False) # Note: this list contains one obsolete entry idList = self.__unpIdList1 ok, sD = fobj.fetchSequenceList(idList, usePrimary=self.__usePrimary, retryAltApi=self.__retryAltApi) self.assertFalse(ok) self.assertEqual(len(sD), len(idList) - 1) if self.__export: self.__mU.doExport(os.path.join(self.__workPath, "data-sequences.json"), sD, fmt="json", indent=3) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def testFetchVariantIds(self): """Test individual variant entry fetch""" try: fobj = UniProtUtils(saveText=True) for tId in self.__unpIdListV: retD, matchD = fobj.fetchList([tId], usePrimary=self.__usePrimary, retryAltApi=self.__retryAltApi) numPrimary, numSecondary, numNone = self.__matchSummary(matchD) logger.debug("%d %d %d", numPrimary, numSecondary, numNone) self.assertGreaterEqual(len(retD), 1) if retD: if self.__export: fobj.writeUnpXml( os.path.join(self.__workPath, tId + ".xml")) self.__mU.doExport(os.path.join( self.__workPath, tId + ".json"), retD, fmt="json", indent=3) # for (eId, eDict) in retD.items(): if "db_isoform" in eDict and eId == tId: logger.debug( "------ sequence database code %s has key db_isoform: %r", eId, eDict["db_isoform"]) logger.debug( "------ sequence database code %s sequence length %d", eId, len(eDict["sequence"])) # logger.debug("%s\n", eDict['sequence']) elif eId == tId: logger.debug("------ No matching isoform for %s\n", tId) # for k,v in eDict.items(): # logger.info("%-25s = %s\n", k, v) else: logger.info("Fetch failed for id %s\n", tId) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def testFetchIds(self): """Test individual entry fetch""" idList = None try: fobj = UniProtUtils(saveText=True) for tId in self.__unpIdList3: idList = [tId] retD, matchD = fobj.fetchList(idList, usePrimary=self.__usePrimary, retryAltApi=self.__retryAltApi) numPrimary, numSecondary, numNone = self.__matchSummary(matchD) logger.debug("%d %d %d", numPrimary, numSecondary, numNone) # rematchD = fobj.rebuildMatchResultIndex(idList, retD) self.assertDictEqual(matchD, rematchD) # self.assertGreaterEqual(len(retD), len(idList)) if retD and self.__export: fobj.writeUnpXml( os.path.join(self.__workPath, tId + ".xml")) self.__mU.doExport(os.path.join(self.__workPath, tId + ".json"), retD, fmt="json", indent=3) except Exception as e: logger.exception("Failing with idList %r %s", idList, str(e)) self.fail()
def getDocuments(self, formatType="exchange"): fobj = UniProtUtils(saveText=False) exObjD = fobj.reformat(self.__refD, formatType=formatType) return list(exObjD.values())
def __rebuildReferenceMatchIndex(self, idList, referenceD): fobj = UniProtUtils() logger.info("Rebuilding match index on idList (%d) using reference data (%d) %r", len(idList), len(referenceD), type(referenceD)) matchD = fobj.rebuildMatchResultIndex(idList, referenceD) return matchD