def __rebuildCache(self, urlTargetPfam, urlTargetPfamFB, dirPath, useCache): pfamD = {} fmt = "json" ext = fmt if fmt == "json" else "pic" pfamDataPath = os.path.join(dirPath, "pfam-data.%s" % ext) # logger.debug("Using cache data path %s", dirPath) self.__mU.mkdir(dirPath) # if useCache and self.__mU.exists(pfamDataPath): pfamD = self.__mU.doImport(pfamDataPath, fmt=fmt) logger.debug("Pfam data length %d", len(pfamD)) elif not useCache: # ------ fU = FileUtil() logger.info("Fetch data from source %s in %s", urlTargetPfam, dirPath) fp = os.path.join(dirPath, fU.getFileName(urlTargetPfam)) ok = fU.get(urlTargetPfam, fp) if not ok: fp = os.path.join(dirPath, fU.getFileName(urlTargetPfamFB)) ok = fU.get(urlTargetPfamFB, fp) logger.info("Fetch data fallback fetch status is %r", ok) pfamD = self.__getPfamIndex(fp) ok = self.__mU.doExport(pfamDataPath, pfamD, fmt=fmt) logger.info("Caching %d in %s status %r", len(pfamD), pfamDataPath, ok) # ------ # return pfamD
def __reloadEntryIds(self, urlTarget, urlFallbackTarget, dirPath, useCache=True): idD = {} fU = FileUtil() fn = fU.getFileName(urlTarget) fp = os.path.join(dirPath, fn) self.__mU.mkdir(dirPath) # if useCache and self.__mU.exists(fp): tdL = self.__mU.doImport(fp, fmt="json") logger.debug("Reading cached IDs list (%d)", len(tdL)) else: logger.info("Fetch ID list from %s", urlTarget) ok = fU.get(urlTarget, fp) if not ok: ok = fU.get(urlFallbackTarget, fp) # if ok: tdL = self.__mU.doImport(fp, fmt="json") # for td in tdL: for k, v in td.items(): try: idD[k] = datetime.datetime.fromisoformat(v) except Exception as e: logger.error("Date processing failing for %r %r with %s", k, v, str(e)) # sTupL = sorted(idD.items(), key=lambda item: item[1]) return {k: v for k, v in sTupL}
def __rebuildCache(self, urlTargetIsoLtwa, dirPath, useCache): """Rebuild the cache of ISO abbreviation term data Args: urlTargetIsoLtwa (str): URL for ISO4 LTWA title word abbreviations dirPath (str): cache path useCache (bool): flag to use cached files Returns: tuple: (dict) title word abbreviations (dict) language conflict dictionary (list) multi-word abbreviation targets Notes: ISO source file (tab delimited UTF-16LE) is maintained at the ISSN site - https://www.issn.org/wp-content/uploads/2013/09/LTWA_20160915.txt """ aD = {} mU = MarshalUtil(workPath=dirPath) fmt = "json" ext = fmt if fmt == "json" else "pic" isoLtwaNamePath = os.path.join(dirPath, "iso-ltwa.%s" % ext) logger.debug("Using cache data path %s", dirPath) mU.mkdir(dirPath) if not useCache: for fp in [isoLtwaNamePath]: try: os.remove(fp) except Exception: pass # if useCache and mU.exists(isoLtwaNamePath): aD = mU.doImport(isoLtwaNamePath, fmt=fmt) logger.debug("Abbreviation name length %d", len(aD["abbrev"])) elif not useCache: # ------ fU = FileUtil() logger.info("Fetch data from source %s in %s", urlTargetIsoLtwa, dirPath) fp = os.path.join(dirPath, fU.getFileName(urlTargetIsoLtwa)) ok = fU.get(urlTargetIsoLtwa, fp) aD = self.__getLtwaTerms(dirPath, fp) ok = mU.doExport(isoLtwaNamePath, aD, fmt=fmt) logger.debug("abbrevD keys %r", list(aD.keys())) logger.debug("Caching %d ISO LTWA in %s status %r", len(aD["abbrev"]), isoLtwaNamePath, ok) # abbrevD = aD["abbrev"] if "abbrev" in aD else {} conflictD = aD["conflicts"] if "conflicts" in aD else {} multiWordTermL = aD[ "multi_word_abbrev"] if "multi_word_abbrev" in aD else [] # return abbrevD, conflictD, multiWordTermL
def __rebuildCache(self, urlTargetInterPro, urlTargetInterProFB, urlTargetInterProParent, urlTargetInterProParentFB, dirPath, useCache): fmt = "json" ext = fmt if fmt == "json" else "pic" interProDataPath = os.path.join(dirPath, "interPro-data.%s" % ext) # logger.debug("Using cache data path %s", dirPath) self.__mU.mkdir(dirPath) # if useCache and self.__mU.exists(interProDataPath): rD = self.__mU.doImport(interProDataPath, fmt=fmt) interProD = rD["index"] interProParentD = rD["parents"] logger.debug("InterPro index length %d parent length %d", len(interProD), len(interProParentD)) else: # ------ fU = FileUtil() logger.info("Fetch data from source %s in %s", urlTargetInterPro, dirPath) fp = os.path.join(dirPath, fU.getFileName(urlTargetInterPro)) ok = fU.get(urlTargetInterPro, fp) if not ok: fp = os.path.join(dirPath, fU.getFileName(urlTargetInterProFB)) ok = fU.get(urlTargetInterProFB, fp) logger.info("Fetch data fallback fetch status is %r", ok) interProD = self.__getInterProIndex(fp) logger.info("Caching %d in %s status %r", len(interProD), interProDataPath, ok) # ------ logger.info("Fetch data from source %s in %s", urlTargetInterProParent, dirPath) fp = os.path.join(dirPath, fU.getFileName(urlTargetInterProParent)) ok = fU.get(urlTargetInterProParent, fp) if not ok: fp = os.path.join(dirPath, fU.getFileName(urlTargetInterProParentFB)) ok = fU.get(urlTargetInterProParentFB, fp) logger.info("Fetch data fallback fetch status is %r", ok) interProParentD = self.__getInterProParents(fp) # ok = self.__mU.doExport(interProDataPath, {"index": interProD, "parents": interProParentD}, fmt=fmt) # return interProD, interProParentD
def __fetchUrl(self, urlTarget, dirPath, useCache=False): fU = FileUtil() fn = fU.getFileName(urlTarget) filePath = os.path.join(dirPath, fn) if not (useCache and fU.exists(filePath)): startTime = time.time() ok2 = fU.get(urlTarget, filePath) endTime = time.time() if ok2: logger.info("Fetched %s for resource file %s (status = %r) (%.4f seconds)", urlTarget, filePath, ok2, endTime - startTime) else: logger.error("Failing fetch for %s for resource file %s (status = %r) (%.4f seconds)", urlTarget, filePath, ok2, endTime - startTime) # return filePath
def __fetchFromSource(self, urlTarget): """Fetch the classification names and domain assignments from the ECOD repo.""" fU = FileUtil() fn = fU.getFileName(urlTarget) fp = os.path.join(self.__dirPath, fn) if not fU.exists(fp): fU.get(urlTarget, fp) # with open(fp, "r", encoding="utf-8") as ifh: line = ifh.readline() line = ifh.readline() line = ifh.readline() ff = line[:-1].split() self.__version = ff[-1] # nmL = self.__mU.doImport(fp, fmt="list", uncomment=True) fU.remove(fp) # return nmL
def __processAppendedSections(self, appendConfigOption, cachePath, useCache=True): """Fetch and append configuration assets assigned to input configuration option. Args: appendConfigOption (str): reserved configuration option to hold a list of configuration asset locators cachePath (str): path to store cached copies configuration assets useCache (bool, optional): use existing cached configuration assets. Defaults to True. Returns: bool: True for success of False otherwise """ try: ret = True appendLocL = self.getList(appendConfigOption, sectionName=self.__defaultSectionName) logger.debug("appendLocL is %r", appendLocL) if appendLocL: cP = os.path.join(cachePath, "config") fU = FileUtil(workPath=cP) logger.debug("Fetching append sections from %r", appendLocL) for appendLoc in appendLocL: fn = fU.getFileName(appendLoc) fp = os.path.join(cP, fn) okF = True if not (useCache and fU.exists(fp)): # get a fresh copy from source okF = fU.get(appendLoc, fp) logger.debug("Fetched %r to %r", appendLoc, fp) ok = self.appendConfig(fp) ret = ret and ok and okF except Exception as e: logger.exception("Failing for option %r cachePath %r with %s", appendConfigOption, cachePath, str(e)) ret = False # if not ret: logger.error("Fetching appended sections failing %r", appendLocL) return ret
def __reload(self, urlTarget, urlFallbackTarget, dirPath, useCache=True): invD = {} fU = FileUtil() fn = fU.getFileName(urlTarget) fp = os.path.join(dirPath, fn) self.__mU.mkdir(dirPath) # if useCache and self.__mU.exists(fp): invD = self.__mU.doImport(fp, fmt="json") logger.debug("Reading cached inventory (%d)", len(invD)) else: logger.info("Fetch inventory from %s", urlTarget) ok = fU.get(urlTarget, fp) if not ok: ok = fU.get(urlFallbackTarget, fp) # if ok: invD = self.__mU.doImport(fp, fmt="json") # return invD
def __reload(self, urlTarget, dirPath, useCache=True): """Reload local cache of mapping resources to support validation report reader and translator. Args: urlTarget (list, str): URL for schema mapping file dirPath (str): path to the directory containing cache files useCache (bool, optional): flag to use cached files. Defaults to True. Returns: (object): instance of ValidationReportReader() """ mapD = {} # mU = MarshalUtil() fU = FileUtil() fn = fU.getFileName(urlTarget) mappingFilePath = os.path.join(dirPath, fn) mU.mkdir(dirPath) # # if not useCache: # for fp in [mappingFilePath]: # try: # os.remove(fp) # except Exception: # pass # # logger.debug("Loading validation mapping data in %s (useCache %r)", fn, useCache) if useCache and fU.exists(mappingFilePath): mapD = mU.doImport(mappingFilePath, fmt="json") else: logger.info("Fetching url %s to resource file %s", urlTarget, mappingFilePath) tS = uuid.uuid4().hex tP = os.path.join(dirPath, "._" + tS) ok = fU.get(urlTarget, tP) if ok: mapD = mU.doImport(tP, fmt="json") os.replace(tP, mappingFilePath) return mapD
def __reload(self, urlTarget, dirPath, useCache=True): """ Reload input GO OBO ontology file and return a nx graph object. ' Returns: dictionary[goId] = {'name_list': ... , 'id_list': ... 'depth_list': ... } """ goGraph = None # # mU = MarshalUtil() fU = FileUtil() fn = fU.getFileName(urlTarget) oboFilePath = os.path.join(dirPath, fn) fU.mkdir(dirPath) # if not useCache: for fp in [oboFilePath]: try: os.remove(fp) except Exception: pass # if useCache and fU.exists(oboFilePath): goGraph = obonet.read_obo(oboFilePath) else: logger.info("Fetching url %s to resource file %s", urlTarget, oboFilePath) ok = fU.get(urlTarget, oboFilePath) if ok: goGraph = obonet.read_obo(oboFilePath) if goGraph: logger.info("Reading %d nodes and %d edges", len(goGraph), goGraph.number_of_edges()) else: logger.info("Go graph construction failing") # return goGraph
class CODModelSearch(object): def __init__(self, cachePath, **kwargs): self.__cachePath = cachePath # self.__useCache = kwargs.get("useCache", True) self.__ccUrlTarget = kwargs.get("ccUrlTarget", None) self.__birdUrlTarget = kwargs.get("birdUrlTarget", None) self.__descriptorUrlTarget = kwargs.get( "descriptorUrlTarget", "http://www.crystallography.net/cod/smi/allcod.smi") self.__prefix = kwargs.get("prefix", None) self.__numProc = kwargs.get("numProc", 4) self.__chunkSize = kwargs.get("chunkSize", 50) self.__ccFileNamePrefix = "cc-%s" % self.__prefix if self.__prefix else "cc-full" self.__fU = FileUtil() # self.__ccmG = ChemCompModelGen(self.__cachePath, self.__prefix) def getResultIndex(self): mU = MarshalUtil(workPath=self.__cachePath) cD = mU.doImport(self.getResultFilePath(), fmt="json") return cD def getResultDetails(self, codId): mU = MarshalUtil(workPath=self.__cachePath) dD = mU.doImport(self.__getCodDetailsFilePath(codId), fmt="json") return dD def storeResultIndex(self, cD): mU = MarshalUtil(workPath=self.__cachePath) ok = mU.doExport(self.getResultFilePath(), cD, fmt="json", indent=3) return ok def getResultDirFilePath(self): dN = "cod-%s-result-files" % self.__prefix if self.__prefix else "cod-result-files" return os.path.join(self.__cachePath, dN) def getRawResultFilePath(self): dN = "cod-%s-result-files" % self.__prefix if self.__prefix else "cod-search-files" return os.path.join(self.__cachePath, dN, "cod-raw-result-file-index.json") def getResultFilePath(self): dN = "cod-%s-result-files" % self.__prefix if self.__prefix else "cod-search-files" return os.path.join(self.__cachePath, dN, "cod-result-file-index.json") def getDescriptorPath(self): fn = self.__fU.getFileName(self.__descriptorUrlTarget) dirPath = self.getResultDirFilePath() filePath = os.path.join(dirPath, fn) return filePath def updateDescriptors(self): self.__fetchUrl(self.__descriptorUrlTarget, filePath=self.getDescriptorPath(), useCache=False) def __fetchUrl(self, urlTarget, filePath, useCache=False, noRetry=False): ok = False try: if not (useCache and self.__fU.exists(filePath)): startTime = time.time() ok = self.__fU.get(urlTarget, filePath, noRetry=noRetry) endTime = time.time() if ok: logger.debug( "Fetched %s for resource file %s (status = %r) (%.4f seconds)", urlTarget, filePath, ok, endTime - startTime) else: logger.error( "Failing fetch for %s for resource file %s (status = %r) (%.4f seconds)", urlTarget, filePath, ok, endTime - startTime) else: ok = True logger.debug("Using cached data for %s", urlTarget) # except Exception as e: logger.exception("Failing for %r with %s", urlTarget, str(e)) return ok def search(self, molLimit=None): try: bsw = BatchChemSearch( useCache=self.__useCache, ccUrlTarget=self.__ccUrlTarget, birdUrlTarget=self.__birdUrlTarget, ccFileNamePrefix=self.__ccFileNamePrefix, cachePath=self.__cachePath, numProc=self.__numProc, chunkSize=self.__chunkSize, ) smiPath = self.getDescriptorPath() smiL = bsw.fetchDescriptorList(smiPath, swap=True) logger.info("Query length (%d)", len(smiL)) # smiL = bsw.splitSmiles(smiL) retL = bsw.doQuery(smiL[:molLimit], "SMILES", matchOpts="graph-exact") logger.info("Result length (%d)", len(retL)) # for ii, ret in enumerate(retL, 1): logger.debug("%5d %8s %4s (%.3f) %s: %s", ii, ret.queryId, ret.ccId, ret.fpScore, ret.queryType, ret.query) # fp = self.getRawResultFilePath() ok = bsw.storeMatchList(fp, retL) return len(retL) if ok else 0 except Exception as e: logger.exception("Failing with %s", str(e)) def __getSearchResults(self): """Read search results and convert to a chemical component dictionary.""" fp = self.getRawResultFilePath() mU = MarshalUtil(workPath=self.__cachePath) rawL = mU.doImport(fp, fmt="json") rD = {} for cD in rawL: rD.setdefault(cD["ccId"], []).append(cD) return rD def __getCodEntryUrl(self, codId): # Template Examples: # https://molecules.crystallography.net/cod/sdf/1/00/00/1000098.sdf # https://molecules.crystallography.net/cod/sdf/6/00/05/6000557.sdf # baseUrl = "https://molecules.crystallography.net/cod/sdf" url = os.path.join(baseUrl, codId[0:1], codId[1:3], codId[3:5], codId + ".sdf") return url def __getCodDetailsUrl(self, codId): baseUrl = "http://www.crystallography.net/cod/optimade/structures" url = os.path.join(baseUrl, codId) return url def __getCodDetailsFilePath(self, codId): dirPath = self.getResultDirFilePath() fp = os.path.join(dirPath, "cod-data", codId[0:1], codId[1:3], codId[3:5], codId + ".json") return fp def __getCodEntryFilePath(self, codId): dirPath = self.getResultDirFilePath() fp = os.path.join(dirPath, "cod-data", codId[0:1], codId[1:3], codId[3:5], codId + ".sdf") return fp def fetchMatchedData(self, useCache=True): """Fetch COD matched entries and metadata and update the raw search index with essential COD data attrbutes. Args: useCache (bool, optional): use any cached COD data. Defaults to True. Returns: int: search result count """ eCount = 0 eSkip = 0 rcD = {} cD = self.__getSearchResults() # for ccId, qDL in cD.items(): # cifPath = self.__ccmG.getChemCompPath(ccId) # if not cifPath: # logger.info("No CIF for %s skipping", ccId) # continue parentId = ccId.split("|")[0] rqDL = [] for qD in qDL: codId = qD["queryId"] codEntryFilePath = self.__getCodEntryFilePath(codId) codDetailsFilePath = self.__getCodDetailsFilePath(codId) ok1 = self.__fetchUrl(self.__getCodEntryUrl(codId), self.__getCodEntryFilePath(codId), useCache=useCache, noRetry=True) ok2 = self.__fetchUrl(self.__getCodDetailsUrl(codId), self.__getCodDetailsFilePath(codId), useCache=useCache, noRetry=True) tD = self.getResultDetails(codId) dD = tD["data"][ "attributes"] if "data" in tD and "attributes" in tD[ "data"] else {} mD = tD["meta"][ "implementation"] if "meta" in tD and "implementation" in tD[ "meta"] else {} if ok1 & ok2: logger.info("Fetched COD entry and details for %s (%r)", codId, ok1 & ok2) eCount += 1 qD["codEntryFilePath"] = codEntryFilePath qD["codDetailsFilePath"] = codDetailsFilePath # qD["cifPath"] = cifPath qD["parentId"] = parentId qD["chemicalName"] = dD[ "_cod_commonname"] if "_cod_commonname" in dD else None qD["chemicalName"] = dD[ "_cod_chemname"] if "_cod_chemname" in dD else qD[ "chemicalName"] qD["rValue"] = dD[ "_cod_Robs"] if "_cod_Robs" in dD else None qD["diffrnTemp"] = dD[ "_cod_diffrtemp"] if "_cod_diffrtemp" in dD else None qD["radiationSource"] = dD[ "_cod_radType"] if "_cod_radType" in dD else None qD["publicationDOI"] = dD[ "_cod_doi"] if "_cod_doi" in dD else None qD["version"] = mD["version"] if "version" in mD else None qD["hasDisorder"] = "N" rqDL.append(qD) else: logger.info("Skipping entry missing data for %r at %r", codId, self.__getCodEntryUrl(codId)) eSkip += 1 if rqDL: rcD[ccId] = rqDL # ok = self.storeResultIndex(rcD) logger.info( "Final match result (w/sdf and metadata) (%d/%d) cod hits (%d) skipped (%d)", len(rcD), len(cD), eCount, eSkip) return eCount if ok else 0 def fetchMatchedDataMp(self, numProc=6, chunkSize=5, useCache=True): rcD = {} cD = self.__getSearchResults() idList = list(cD.keys()) # --- mpu = MultiProcUtil(verbose=True) mpu.setWorkingDir(self.__cachePath) mpu.setOptions(optionsD={ "resultPath": self.__cachePath, "cD": cD, "useCache": useCache }) mpu.set(workerObj=self, workerMethod="fetchDataWorker") ok, failList, resultList, _ = mpu.runMulti(dataList=idList, numProc=numProc, numResults=1, chunkSize=chunkSize) logger.info("Run ended with status %r success count %d failures %r", ok, len(resultList[0]), len(failList)) for rTup in resultList[0]: rcD[rTup[0]] = rTup[1] # --- ok = self.storeResultIndex(rcD) logger.info("Final match result (w/sdf and metadata) (%d/%d)", len(rcD), len(cD)) return True def fetchDataWorker(self, dataList, procName, optionsD, workingDir): """Worker method to fetch COD data for matched entries Args: dataList (list): list of mol2 file paths to be searched procName (str): processName optionsD (dict): dictionary of options workingDir (str): path to working directory (not used) Returns: (successList, resultList, []): success and result lists of mol2 paths with CCDC matches """ resultPath = optionsD["resultPath"] cD = optionsD["cD"] useCache = optionsD["useCache"] _ = workingDir resultList = [] successList = [] startTime = time.time() logger.info("starting %s at %s", procName, time.strftime("%Y %m %d %H:%M:%S", time.localtime())) # eCount = 0 eSkip = 0 try: stopPath = os.path.join(resultPath, "STOP") logger.info("%s starting search data length %d", procName, len(dataList)) if self.__checkStop(stopPath): logger.info("%s stopping", procName) return resultList, resultList, [] # # for ccId, qDL in cD.items(): for ccId in dataList: if ccId in cD: qDL = cD[ccId] # parentId = ccId.split("|")[0] rqDL = [] for qD in qDL: codId = qD["queryId"] codEntryFilePath = self.__getCodEntryFilePath(codId) codDetailsFilePath = self.__getCodDetailsFilePath(codId) ok1 = self.__fetchUrl(self.__getCodEntryUrl(codId), self.__getCodEntryFilePath(codId), useCache=useCache, noRetry=True) ok2 = self.__fetchUrl(self.__getCodDetailsUrl(codId), self.__getCodDetailsFilePath(codId), useCache=useCache, noRetry=True) tD = self.getResultDetails(codId) dD = tD["data"][ "attributes"] if "data" in tD and "attributes" in tD[ "data"] else {} mD = tD["meta"][ "implementation"] if "meta" in tD and "implementation" in tD[ "meta"] else {} if ok1 & ok2: logger.info( "Fetched COD entry and details for %s (%r)", codId, ok1 & ok2) eCount += 1 qD["codEntryFilePath"] = codEntryFilePath qD["codDetailsFilePath"] = codDetailsFilePath # qD["cifPath"] = cifPath qD["parentId"] = parentId qD["chemicalName"] = dD[ "_cod_commonname"] if "_cod_commonname" in dD else None qD["chemicalName"] = dD[ "_cod_chemname"] if "_cod_chemname" in dD else qD[ "chemicalName"] qD["rValue"] = dD[ "_cod_Robs"] if "_cod_Robs" in dD else None qD["diffrnTemp"] = dD[ "_cod_diffrtemp"] if "_cod_diffrtemp" in dD else None qD["radiationSource"] = dD[ "_cod_radType"] if "_cod_radType" in dD else None qD["publicationDOI"] = dD[ "_cod_doi"] if "_cod_doi" in dD else None qD["version"] = mD[ "version"] if "version" in mD else None qD["hasDisorder"] = "N" rqDL.append(qD) else: logger.info("Skipping entry missing data for %r at %r", codId, self.__getCodEntryUrl(codId)) eSkip += 1 if rqDL: resultList.append((ccId, rqDL)) successList.append(ccId) except Exception as e: logger.exception("Failing with %s", str(e)) endTime = time.time() logger.info( "%s (entries %d skipped %d) (ccId result length %d) completed at %s (%.2f seconds)", procName, eCount, eSkip, len(successList), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - startTime, ) return successList, resultList, [] def __checkStop(self, path): try: if os.access(path, os.F_OK): return True except Exception: pass return False
def __reload(self, dirPath, useCache=False, imgtDumpUrl=None, testList=None, maxCount=None): imgtD = {} startTime = time.time() fU = FileUtil() fU.mkdir(dirPath) # imgtDataPath = os.path.join(self.__dirPath, "imgt-data.json") # logger.info("useCache %r imgtFeaturePath %r", useCache, imgtDataPath) if useCache and self.__mU.exists(imgtDataPath): imgtD = self.__mU.doImport(imgtDataPath, fmt="json") self.__version = imgtD["version"] else: imgtDumpUrl = imgtDumpUrl if imgtDumpUrl else "http://www.imgt.org/download/3Dstructure-DB/IMGT3DFlatFiles.tgz" imgtReadmeUrl = "http://www.imgt.org/download/3Dstructure-DB/RELEASE" imgtDumpFileName = fU.getFileName(imgtDumpUrl) imgtDumpPath = os.path.join(dirPath, imgtDumpFileName) imgtReleasePath = os.path.join(dirPath, "IMGT-release.txt") _, fn = os.path.split(imgtDumpUrl) imgtFlatFilePath = os.path.join(self.__dirPath, fn[:-4]) # logger.info("Fetching url %s path %s", imgtDumpUrl, imgtDumpPath) ok1 = fU.get(imgtDumpUrl, imgtDumpPath) ok2 = fU.get(imgtReadmeUrl, imgtReleasePath) fU.unbundleTarfile(imgtDumpPath, dirPath=dirPath) logger.info("Completed fetch (%r) at %s (%.4f seconds)", ok1 and ok2, time.strftime("%Y %m %d %H:%M:%S", time.localtime()), time.time() - startTime) # --- readmeLines = self.__mU.doImport(imgtReleasePath, fmt="list") self.__version = readmeLines[0].strip() if readmeLines else None logger.info("IMGT version %r", self.__version) # --- chainD, rawD = self.__imgtFlatFileProcessor(imgtFlatFilePath, maxCount=maxCount, testList=testList) # --- tS = datetime.datetime.now().isoformat() # vS = datetime.datetime.now().strftime("%Y-%m-%d") if testList: imgtD = { "version": self.__version, "date": tS, "chains": chainD, "raw": rawD } else: imgtD = { "version": self.__version, "date": tS, "chains": chainD } ok = self.__mU.doExport(imgtDataPath, imgtD, fmt="json", indent=3) logger.info("Completed flatfile prep (%r) at %s (%.4f seconds)", ok, time.strftime("%Y %m %d %H:%M:%S", time.localtime()), time.time() - startTime) return imgtD
def exportFasta(self, withGaps=False): """ Example: The IMGT/GENE-DB FASTA header contains 15 fields separated by '|': 1. IMGT/LIGM-DB accession number(s) 2. IMGT gene and allele name 3. species (may be followed by an "_" and the name of the strain, breed or isolate, if defined) 4. IMGT gene and allele functionality 5. exon(s), region name(s), or extracted label(s) 6. start and end positions in the IMGT/LIGM-DB accession number(s) 7. number of nucleotides in the IMGT/LIGM-DB accession number(s) 8. codon start, or 'NR' (not relevant) for non coding labels 9. +n: number of nucleotides (nt) added in 5' compared to the corresponding label extracted from IMGT/LIGM-DB 10. +n or -n: number of nucleotides (nt) added or removed in 3' compared to the corresponding label extracted from IMGT/LIGM-DB 11. +n, -n, and/or nS: number of added, deleted, and/or substituted nucleotides to correct sequencing errors, or 'not corrected' if non corrected sequencing errors 12. number of amino acids (AA): this field indicates that the sequence is in amino acids 13. number of characters in the sequence: nt (or AA)+IMGT gaps=total 14. partial (if it is) 15. reverse complementary (if it is) """ # -- fU = FileUtil() fU.mkdir(self.__dirPath) if withGaps: imgtTargetUrl = "http://www.imgt.org/download/GENE-DB/IMGTGENEDB-ReferenceSequences.fasta-AA-WithGaps-F+ORF+inframeP" else: imgtTargetUrl = "http://www.imgt.org/download/GENE-DB/IMGTGENEDB-ReferenceSequences.fasta-AA-WithoutGaps-F+ORF+inframeP" imgtTargetFileName = fU.getFileName(imgtTargetUrl) rawFastaPath = os.path.join(self.__dirPath, imgtTargetFileName) # -- logger.debug("Fetching url %s path %s", imgtTargetUrl, rawFastaPath) ok = fU.get(imgtTargetUrl, rawFastaPath) logger.info("Fetch status (%r) url %s path %s", ok, imgtTargetUrl, rawFastaPath) # -- fastaPath = os.path.join(self.__dirPath, "imgt-reference.fa") taxonPath = os.path.join(self.__dirPath, "imgt-reference-taxon.tdd") tP = TaxonomyProvider(cachePath=self.__cachePath, useCache=True) ok = tP.testCache() if not ok: tP = TaxonomyProvider(cachePath=self.__cachePath, useCache=False) rawQD = self.__mU.doImport(rawFastaPath, fmt="fasta", commentStyle="default") oD = {} taxonL = [] for queryId, sD in rawQD.items(): qL = queryId.split("|") tL = qL[2].split("_") taxName = tL[0] taxVar = tL[1].replace(" ", "_") if len(tL) > 1 else None taxId = tP.getTaxId(taxName) if taxId: tD = { "seqId": qL[0], "imgtGene": qL[1], "functionality": qL[3], "labels": qL[4], "taxId": taxId } if taxVar: tD["taxVar"] = taxVar sD.update(tD) else: logger.info("Unknown taxonomy %r (taxName=%r)", queryId, taxName) sD["sequence"].replace(".", "-") seqId = "" cL = [] for k, v in sD.items(): if k in ["sequence"]: continue cL.append(str(v)) cL.append(str(k)) seqId = "|".join(cL) oD[seqId] = sD taxonL.append("%s\t%s" % (seqId, taxId)) # ok1 = self.__mU.doExport(taxonPath, taxonL, fmt="list") ok2 = self.__mU.doExport(fastaPath, oD, fmt="fasta", makeComment=True) return ok1 and ok2
class DataTypeApiProvider(SingletonClass): """ Data type application and instance information provider. """ def __init__(self, cfgOb, cachePath, useCache=True, **kwargs): """Data type application and instance information provider. Args: cfgOb (object): ConfigInfo() object instance cachePath (str): path to hold the cache directory useCache (bool, optional): flag to use cached files. Defaults to True. """ self.__cfgOb = cfgOb self.__configName = self.__cfgOb.getDefaultSectionName() self.__useCache = useCache self.__cachePath = cachePath # self.__contentInfoConfigName = "content_info_helper_configuration" self.__fileU = FileUtil() self.__contentDefHelper = self.__cfgOb.getHelper( "CONTENT_DEF_HELPER_MODULE", sectionName=self.__configName, cfgOb=self.__cfgOb) self.__dirPath = os.path.join( cachePath, self.__cfgOb.get("DATA_TYPE_INFO_CACHE_DIR", sectionName=self.__configName)) self.__kwargs = kwargs # logger.debug("Leaving constructor") def getDataTypeInstanceApi(self, databaseName, **kwargs): """Return instance of DataTypeInstanceInfo(). Args: databaseName (str): database name Returns: (object): Instance of DataTypeInstanceInfo() """ _ = kwargs dataTypeInstanceLocatorPath = self.__cfgOb.getPath( "INSTANCE_DATA_TYPE_INFO_LOCATOR_PATH", sectionName=self.__configName) dataTypeInstanceFile = self.__contentDefHelper.getDataTypeInstanceFile( databaseName) if self.__contentDefHelper else None if dataTypeInstanceLocatorPath and dataTypeInstanceFile: loc = os.path.join(dataTypeInstanceLocatorPath, dataTypeInstanceFile) filePath = self.__reload(loc, self.__dirPath, useCache=self.__useCache) dtApi = DataTypeInstanceInfo(filePath) else: # DataTypeInstanceInfo() provides an internal by-pass mode where no coverage data is available. dtApi = DataTypeInstanceInfo(None) logger.debug("No data coverage available for database %s", databaseName) return dtApi def getDataTypeApplicationApi(self, appName, **kwargs): """Return instance of DataTypeApplicationInfo. Args: appName (str): application name (e.g., SQL, ANY) Returns: (object): Instance of DataTypeApplicationInfo() """ _ = kwargs dataTypeApplicationLocator = self.__cfgOb.getPath( "APP_DATA_TYPE_INFO_LOCATOR", sectionName=self.__configName) filePath = self.__reload(dataTypeApplicationLocator, self.__dirPath, useCache=self.__useCache) dtApi = DataTypeApplicationInfo( filePath, dataTyping=appName, workPath=self.__dirPath) if filePath else None return dtApi def __reload(self, urlTarget, dirPath, useCache=True): # fn = self.__fileU.getFileName(urlTarget) filePath = os.path.join(dirPath, fn) logger.debug("Using cache path %s", dirPath) self.__fileU.mkdir(dirPath) if not useCache: try: os.remove(filePath) except Exception: pass # if useCache and self.__fileU.exists(filePath): ok = True else: logger.debug("Fetch data from source %s", urlTarget) ok = self.__fileU.get(urlTarget, os.path.join(dirPath, fn)) return filePath if ok else None
class DictionaryApiProvider(SingletonClass): """ Resource provider for dictionary APIs. """ def __init__(self, dirPath, useCache=True): """Resource provider for dictionary APIs. Args: dirPath (str): path to the directory containing cache files useCache (bool, optional): flag to use cached files. Defaults to True. """ self.__apiMap = {} self.__dirPath = dirPath self.__useCache = useCache # self.__fileU = FileUtil(workPath=self.__dirPath) logger.debug("Leaving constructor") def __reload(self, dictLocators, dirPath, useCache=True): """Reload local cache of dictionary resources and return a dictionary API instance. Args: dictLocators (list, str): list of locators for dictionary resource files dirPath (str): path to the directory containing cache files useCache (bool, optional): flag to use cached files. Defaults to True. Returns: (object): instance of dictionary API """ # # verify the exitence of the cache directory ... self.__fileU.mkdir(dirPath) if not useCache: for dictLocator in dictLocators: try: fn = self.__fileU.getFileName(dictLocator) os.remove(os.path.join(dirPath, fn)) except Exception: pass # ret = True for dictLocator in dictLocators: cacheFilePath = os.path.join(dirPath, self.__fileU.getFileName(dictLocator)) if useCache and self.__fileU.exists(cacheFilePath): # nothing to do continue logger.debug("Fetching url %s caching in %s", dictLocator, cacheFilePath) ok = self.__fileU.get(dictLocator, cacheFilePath) ret = ret and ok return ret def getApi(self, dictLocators, **kwargs): """Return a dictionary API object of the input dictioaries. Arguments: dictLocators {list str} -- list of dictionary locator paths Returns: [object] -- returns DictionaryApi() object for input dictionaries """ dictFileNames = [ self.__fileU.getFileName(dictLocator) for dictLocator in dictLocators ] dictTup = tuple(dictFileNames) dApi = self.__apiMap[ dictTup] if dictTup in self.__apiMap else self.__getApi( dictLocators, **kwargs) self.__apiMap[dictTup] = dApi return dApi def __getApi(self, dictLocators, **kwargs): """ Return an instance of a dictionary API instance for the input dictionary locator list. """ consolidate = kwargs.get("consolidate", True) replaceDefinition = kwargs.get("replaceDefinitions", True) verbose = kwargs.get("verbose", True) # ok = self.__reload(dictLocators, self.__dirPath, useCache=self.__useCache) # dApi = None if ok: mU = MarshalUtil() containerList = [] for dictLocator in dictLocators: cacheFilePath = os.path.join( self.__dirPath, self.__fileU.getFileName(dictLocator)) containerList.extend( mU.doImport(cacheFilePath, fmt="mmcif-dict")) # dApi = DictionaryApi(containerList=containerList, consolidate=consolidate, replaceDefinition=replaceDefinition, verbose=verbose) return dApi
class FileUtilTests(unittest.TestCase): def setUp(self): self.__verbose = True self.__pathPdbxDictionaryFile = os.path.join(TOPDIR, "rcsb", "mock-data", "dictionaries", "mmcif_pdbx_v5_next.dic") self.__pathTaxonomyFile = os.path.join(TOPDIR, "rcsb", "mock-data", "NCBI", "names.dmp.gz") self.__zipFileUrl = "https://inventory.data.gov/dataset/794cd3d7-4d28-4408-8f7d-84b820dbf7f2/resource/6b78ec0c-4980-4ad8-9cbd-2d6eb9eda8e7/download/myfoodapediadata.zip" self.__xzFile = os.path.join(TOPDIR, "rcsb", "mock-data", "MOCK_MODBASE_MODELS", "NP_001030614.1_1.pdb.xz") # self.__ftpFileUrl = "ftp://ftp.wwpdb.org/pub/pdb/data/component-models/complete/chem_comp_model.cif.gz" self.__httpsFileUrl = "https://ftp.wwpdb.org/pub/pdb/data/component-models/complete/chem_comp_model.cif.gz" # self.__workPath = os.path.join(HERE, "test-output") self.__inpDirPath = os.path.join(HERE, "test-data") self.__fileU = FileUtil() self.__startTime = time.time() logger.debug("Running tests on version %s", __version__) logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime())) def tearDown(self): endTime = time.time() logger.debug("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime) def testTarBundling(self): """Test case for tarfile bundling and unbundling""" try: tP = os.path.join(self.__workPath, "t0.tar.gz") dirPath = os.path.join(self.__inpDirPath, "topdir") ok = self.__fileU.bundleTarfile(tP, [dirPath], mode="w:gz", recursive=True) self.assertTrue(ok) numBytes = self.__fileU.size(tP) self.assertGreaterEqual(numBytes, 250) # md5 = self.__fileU.hash(tP, hashType="md5") self.assertTrue(md5 is not None) # ok = self.__fileU.unbundleTarfile(tP, dirPath=self.__workPath) self.assertTrue(ok) # tP = os.path.join(self.__workPath, "t1.tar.gz") dirPathList = [ os.path.join(self.__inpDirPath, "topdir", "subdirA"), os.path.join(self.__inpDirPath, "topdir", "subdirB") ] ok = self.__fileU.bundleTarfile(tP, dirPathList, mode="w:gz", recursive=True) self.assertTrue(ok) # ok = self.__fileU.unbundleTarfile(tP, dirPath=self.__workPath) self.assertTrue(ok) tP = os.path.join(self.__workPath, "t2.tar") dirPathList = [ os.path.join(self.__inpDirPath, "topdir", "subdirA"), os.path.join(self.__inpDirPath, "topdir", "subdirB") ] ok = self.__fileU.bundleTarfile(tP, dirPathList, mode="w", recursive=True) self.assertTrue(ok) # ok = self.__fileU.unbundleTarfile(tP, dirPath=self.__workPath) self.assertTrue(ok) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testGetFile(self): """Test case for a local files and directories""" try: remoteLocator = self.__pathPdbxDictionaryFile fn = self.__fileU.getFileName(remoteLocator) # _, fn = os.path.split(remoteLocator) lPath = os.path.join(self.__workPath, fn) ok = self.__fileU.get(remoteLocator, lPath) self.assertTrue(ok) ok = self.__fileU.exists(lPath) self.assertTrue(ok) ok = self.__fileU.isLocal(lPath) self.assertTrue(ok) tPath = self.__fileU.getFilePath(lPath) self.assertEqual(lPath, tPath) ok = self.__fileU.remove(lPath) self.assertTrue(ok) dPath = os.path.join(self.__workPath, "tdir") ok = self.__fileU.mkdir(dPath) self.assertTrue(ok) ok = self.__fileU.remove(dPath) self.assertTrue(ok) ok = self.__fileU.remove(";lakdjf") self.assertTrue(ok) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testMoveAndCopyFile(self): """Test case for copying ("put") and moving ("replace") local files""" try: remoteLocator = self.__pathPdbxDictionaryFile fn = self.__fileU.getFileName(remoteLocator) # _, fn = os.path.split(remoteLocator) lPath = os.path.join(self.__workPath, fn) ok = self.__fileU.get(remoteLocator, lPath) self.assertTrue(ok) # Test copy file dPath2 = os.path.join(self.__workPath, "tdir") ok = self.__fileU.mkdir(dPath2) self.assertTrue(ok) lPath2 = os.path.join(dPath2, fn) ok = self.__fileU.put(lPath, lPath2) self.assertTrue(ok) ok = self.__fileU.exists(lPath) self.assertTrue(ok) ok = self.__fileU.exists(lPath2) self.assertTrue(ok) # Remove copied file (to test moving file next) ok = self.__fileU.remove(lPath2) self.assertTrue(ok) ok = self.__fileU.exists(lPath2) self.assertFalse(ok) # Test move file ok = self.__fileU.replace(lPath, lPath2) self.assertTrue(ok) ok = self.__fileU.exists(lPath) self.assertFalse(ok) ok = self.__fileU.exists(lPath2) self.assertTrue(ok) # Now clean up files and dirs ok = self.__fileU.remove(lPath) self.assertTrue(ok) ok = self.__fileU.remove(dPath2) self.assertTrue(ok) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testZipUrl(self): """Test case for downloading remote zip file and extracting contents.""" try: remoteLocator = self.__zipFileUrl # fn = self.__fileU.getFileName(remoteLocator) ok = self.__fileU.isLocal(remoteLocator) self.assertFalse(ok) # lPath = os.path.join(self.__workPath, self.__fileU.getFileName(self.__zipFileUrl)) ok = self.__fileU.get(remoteLocator, lPath) self.assertTrue(ok) ok = self.__fileU.exists(lPath) self.assertTrue(ok) ok = self.__fileU.isLocal(lPath) self.assertTrue(ok) tPath = self.__fileU.getFilePath(lPath) self.assertEqual(lPath, tPath) fp = self.__fileU.uncompress(lPath, outputDir=self.__workPath) ok = fp.endswith("Food_Display_Table.xlsx") self.assertTrue(ok) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testFtpUrl(self): """Test case for downloading remote file ftp protocol and extracting contents.""" try: remoteLocator = self.__ftpFileUrl # fn = self.__fileU.getFileName(remoteLocator) ok = self.__fileU.isLocal(remoteLocator) self.assertFalse(ok) # dirPath = os.path.join(self.__workPath, "chem_comp_models") lPath = os.path.join(dirPath, self.__fileU.getFileName(self.__ftpFileUrl)) ok = self.__fileU.get(remoteLocator, lPath) self.assertTrue(ok) ok = self.__fileU.exists(lPath) self.assertTrue(ok) ok = self.__fileU.isLocal(lPath) self.assertTrue(ok) tPath = self.__fileU.getFilePath(lPath) self.assertEqual(lPath, tPath) fp = self.__fileU.uncompress(lPath, outputDir=dirPath) ok = fp.endswith("chem_comp_model.cif") self.assertTrue(ok) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testRemote(self): """Test case remote status""" try: remoteLocator = self.__httpsFileUrl ok = self.__fileU.isLocal(remoteLocator) self.assertFalse(ok) # ok = self.__fileU.exists(remoteLocator) self.assertTrue(ok) size = self.__fileU.size(remoteLocator) self.assertGreaterEqual(size, 1000) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() @unittest.skip("DrugBank example -- skipping") def testGetDrugBankUrl(self): """Test case for downloading drugbank master xml file""" try: remoteLocator = "https://www.drugbank.ca/releases/latest/downloads/all-full-database" un = "username" pw = "password" # fn = self.__fileU.getFileName(remoteLocator) ok = self.__fileU.isLocal(remoteLocator) self.assertFalse(ok) # lPath = os.path.join(self.__workPath, "db-download.zip") ok = self.__fileU.get(remoteLocator, lPath, username=un, password=pw) self.assertTrue(ok) ok = self.__fileU.exists(lPath) self.assertTrue(ok) ok = self.__fileU.isLocal(lPath) self.assertTrue(ok) tPath = self.__fileU.getFilePath(lPath) self.assertEqual(lPath, tPath) self.__fileU.uncompress(lPath, outputDir=self.__workPath) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail() def testXzFile(self): """Test case for extracting contents from xz file""" try: remoteLocator = self.__xzFile fn = self.__fileU.getFileName(remoteLocator) lPath = os.path.join(self.__workPath, fn) ok = self.__fileU.get(remoteLocator, lPath) self.assertTrue(ok) ok = self.__fileU.exists(lPath) self.assertTrue(ok) ok = self.__fileU.isLocal(lPath) self.assertTrue(ok) tPath = self.__fileU.getFilePath(lPath) self.assertEqual(lPath, tPath) fp = self.__fileU.uncompress(lPath, outputDir=self.__workPath) ok = fp.endswith(".pdb") self.assertTrue(ok) except Exception as e: logger.exception("Failing with %s", str(e)) self.fail()
def __reloadFasta(self, dirPath, **kwargs): """Reload DrugBank target FASTA data files. Args: dirPath (str, optional): path to DrugBank cache directory useCache (bool, optional): flag to use cached files. Defaults to True. Returns: """ startTime = time.time() logger.info("Starting db reload at %s", time.strftime("%Y %m %d %H:%M:%S", time.localtime())) retFilePathList = [] urlTargetL = [ "https://go.drugbank.com/releases/latest/downloads/target-all-polypeptide-sequences", "https://go.drugbank.com/releases/latest/downloads/enzyme-all-polypeptide-sequences", "https://go.drugbank.com/releases/latest/downloads/carrier-all-polypeptide-sequences", "https://go.drugbank.com/releases/latest/downloads/transporter-all-polypeptide-sequences", ] useCache = kwargs.get("useCache", True) username = kwargs.get("username", None) password = kwargs.get("password", None) # if not username or not password: return retFilePathList # fU = FileUtil() fU.mkdir(dirPath) # if not useCache: # Clear any cached files for urlTarget in urlTargetL: baseFileName = fU.getFileName(urlTarget) zipFileName = baseFileName + ".fasta.zip" retFileName = baseFileName + ".fa" for fn in [baseFileName, zipFileName, retFileName]: try: fp = os.path.join(dirPath, fn) os.remove(fp) except Exception: pass # ok = False if useCache: ok = True for urlTarget in urlTargetL: baseFileName = fU.getFileName(urlTarget) retFileName = baseFileName + ".fa" retFilePath = os.path.join(dirPath, retFileName) ok = fU.exists(retFilePath) if not ok: break retFilePathList.append(retFilePath) # logger.info("Using cached files %r", ok) if not useCache or not ok: if not username or not password: logger.warning( "Missing credentials for DrugBank file download...") for urlTarget in urlTargetL: baseFileName = fU.getFileName(urlTarget) zipFileName = baseFileName + ".fasta.zip" retFileName = baseFileName + ".fa" zipFilePath = os.path.join(dirPath, zipFileName) retFilePath = os.path.join(dirPath, retFileName) basePath = os.path.join(dirPath, baseFileName) logger.info("Fetching url %s for FASTA target file %s", urlTarget, baseFileName) ok = fU.get(urlTarget, zipFilePath, username=username, password=password) endTime = time.time() logger.info( "Completed db fetch at %s (%.4f seconds)", time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - startTime) # ok = fU.unbundleZipfile(zipFilePath, dirPath=basePath) fU.put(os.path.join(basePath, "protein.fasta"), retFilePath) endTime = time.time() logger.info( "Completed unzip at %s (%.4f seconds)", time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - startTime) retFilePathList.append(retFilePath) return retFilePathList
class ProvenanceProvider(SingletonClass): """Utilities to access and update provenance details.""" def __init__(self, cfgOb, cachePath, useCache=True, **kwargs): """Utilities to access and update provenance details. Args: cfgOb ([type]): ConfigInfo() instance cachePath ([type]): path to directory containing schema useCache (bool, optional): use cached schema. Defaults to True. """ self.__cfgOb = cfgOb self.__configName = self.__cfgOb.getDefaultSectionName() self.__cachePath = cachePath self.__useCache = useCache # self.__workPath = os.path.join(self.__cachePath, "work") self.__provenanceCachePath = os.path.join( self.__cachePath, self.__cfgOb.get("PROVENANCE_INFO_CACHE_DIR", sectionName=self.__configName)) self.__provenanceLocator = self.__cfgOb.getPath( "PROVENANCE_INFO_LOCATOR", sectionName=self.__configName) # self.__fileU = FileUtil(workPath=self.__workPath) self.__fileU.mkdir(self.__provenanceCachePath) self.__kwargs = kwargs # def __reload(self, locator, dirPath, useCache=True): # fn = self.__fileU.getFileName(locator) filePath = os.path.join(dirPath, fn) logger.debug("Using cache path %s", dirPath) self.__fileU.mkdir(dirPath) if not useCache: try: os.remove(filePath) except Exception: pass # if useCache and self.__fileU.exists(filePath): ok = True else: logger.debug("Fetch data from source %s", locator) ok = self.__fileU.get(locator, filePath) return filePath if ok else None def fetch(self): try: provenanceFileCachePath = self.__reload(self.__provenanceLocator, self.__provenanceCachePath, useCache=self.__useCache) mU = MarshalUtil(workPath=self.__workPath) return mU.doImport(provenanceFileCachePath, fmt="json") except Exception as e: logger.exception("Failed retreiving provenance with %s", str(e)) return {} def update(self, provD): ok = False try: provenanceFileCachePath = self.__reload(self.__provenanceLocator, self.__provenanceCachePath, useCache=self.__useCache) mU = MarshalUtil(workPath=self.__workPath) tD = mU.doImport(provenanceFileCachePath, fmt="json") tD.update(provD) ok = mU.doExport(provenanceFileCachePath, tD, fmt="json") except Exception as e: logger.exception("Failed updating provenance with %s", str(e)) return ok def store(self, provD): ok = False try: provenanceFileCachePath = self.__reload(self.__provenanceLocator, self.__provenanceCachePath, useCache=self.__useCache) mU = MarshalUtil(workPath=self.__workPath) ok = mU.doExport(provenanceFileCachePath, provD, fmt="json") except Exception as e: logger.exception("Failed storing provenance with %s", str(e)) return ok
class MarshalUtil(object): """Wrapper for serialization and deserialization methods.""" def __init__(self, **kwargs): self.__workPath = kwargs.get("workPath", ".") self.__workDirSuffix = kwargs.get("workDirSuffix", "marshall_") self.__workDirPrefix = kwargs.get("workDirSuffix", "_tempdir") # self.__fileU = FileUtil(workPath=self.__workPath) self.__ioU = IoUtil() def doExport(self, locator, obj, fmt="list", marshalHelper=None, numParts=None, **kwargs): """Serialize the input object at locator path in specified format. The input object is optionally preprocessed by the helper method. Args: locator (str): target path or URI obj (object): data to be serialized fmt (str, optional): format for serialization (mmcif, tdd, csv, list). Defaults to "list". marshalHelper (method, optional): pre-processor method applied to input data object. Defaults to None. numParts (int, optional): serialize the data in parts. Defaults to None. (json and pickle formats) Returns: bool: True for sucess or False otherwise """ try: ret = False localFlag = self.__fileU.isLocal(locator) if marshalHelper: myObj = marshalHelper(obj, **kwargs) else: myObj = obj # if localFlag and numParts and fmt in ["json", "pickle"]: localFilePath = self.__fileU.getFilePath(locator) ret = self.__ioU.serializeInParts(localFilePath, myObj, numParts, fmt=fmt, **kwargs) elif localFlag: localFilePath = self.__fileU.getFilePath(locator) ret = self.__ioU.serialize(localFilePath, myObj, fmt=fmt, workPath=self.__workPath, **kwargs) else: with tempfile.TemporaryDirectory( suffix=self.__workDirSuffix, prefix=self.__workDirPrefix, dir=self.__workPath) as tmpDirName: # write a local copy then copy to destination - # localFilePath = os.path.join( self.__workPath, tmpDirName, self.__fileU.getFileName(locator)) ok1 = self.__ioU.serialize(localFilePath, myObj, fmt=fmt, workPath=self.__workPath, **kwargs) ok2 = True if ok1: ok2 = self.__fileU.put(localFilePath, locator, **kwargs) ret = ok1 and ok2 except Exception as e: logger.exception("Exporting locator %r failing with %s", locator, str(e)) return ret def doImport(self, locator, fmt="list", marshalHelper=None, numParts=None, **kwargs): """Deserialize data at the target locator in specified format. The deserialized data is optionally post-processed by the input helper method. Args: locator (str): path or URI to input data fmt (str, optional): format for deserialization (mmcif, tdd, csv, list). Defaults to "list". marshalHelper (method, optional): post-processor method applied to deserialized data object. Defaults to None. numParts (int, optional): deserialize the data in parts. Defaults to None. (json and pickle formats) tarMember (str, optional): name of a member of tar file bundle. Defaults to None. (tar file format) Returns: Any: format specific return type """ try: tarMember = kwargs.get("tarMember", None) localFlag = self.__fileU.isLocal(locator) and not tarMember # if localFlag and numParts and fmt in ["json", "pickle"]: filePath = self.__fileU.getFilePath(locator) ret = self.__ioU.deserializeInParts(filePath, numParts, fmt=fmt, **kwargs) elif localFlag: filePath = self.__fileU.getFilePath(locator) ret = self.__ioU.deserialize(filePath, fmt=fmt, workPath=self.__workPath, **kwargs) else: # if fmt == "mmcif": ret = self.__ioU.deserialize(locator, fmt=fmt, workPath=self.__workPath, **kwargs) else: with tempfile.TemporaryDirectory( suffix=self.__workDirSuffix, prefix=self.__workDirPrefix, dir=self.__workPath) as tmpDirName: # # Fetch first then read a local copy - # if tarMember: localFilePath = os.path.join( self.__workPath, tmpDirName, tarMember) else: localFilePath = os.path.join( self.__workPath, tmpDirName, self.__fileU.getFileName(locator)) # --- Local copy approach --- self.__fileU.get(locator, localFilePath, **kwargs) ret = self.__ioU.deserialize(localFilePath, fmt=fmt, workPath=self.__workPath, **kwargs) if marshalHelper: ret = marshalHelper(ret, **kwargs) except Exception as e: logger.exception("Importing locator %r failing with %s", locator, str(e)) ret = None return ret def exists(self, filePath, mode=os.R_OK): return self.__fileU.exists(filePath, mode=mode) def mkdir(self, dirPath, mode=0o755): return self.__fileU.mkdir(dirPath, mode=mode) def remove(self, pth): return self.__fileU.remove(pth)
def __rebuildCache(self, targetUrl, mapNameL, outDirPath, rawDirPath, fmt="pickle", useCache=True): """Fetch the UniProt selected id mapping resource file and extract UniProt Acc to 'mapIndex' mapping. Serialize the mapping as required. Args: targetUrl (str): source URL of the remote index file mapNameL (list): list of key mapping names to extract from the index outDirPath (str): directory path for raw and processed mapping files fmt (str, optional): output format (pickle|json) . Defaults to "pickle". useCache (bool, optional): use cached files. Defaults to True. Returns: dict: od[uniprotId] = mapped value idmapping_selected.tab 1. UniProtKB-AC 2. UniProtKB-ID 3. GeneID (EntrezGene) 4. RefSeq 5. GI 6. PDB 7. GO 8. UniRef100 9. UniRef90 10. UniRef50 11. UniParc 12. PIR 13. NCBI-taxon 14. MIM 15. UniGene 16. PubMed 17. EMBL 18. EMBL-CDS 19. Ensembl 20. Ensembl_TRS 21. Ensembl_PRO 22. Additional PubMed """ startTime = time.time() nL = mapNameL oD = {} try: fileU = FileUtil() fExt = "pic" if fmt == "pickle" else "json" fExt = "tdd" if fmt == "tdd" else fExt fN, _ = os.path.splitext(fileU.getFileName(targetUrl)) mapFileName = fN + "-map." + fExt idMapPath = os.path.join(outDirPath, mapFileName) mU = MarshalUtil() if useCache and mU.exists(idMapPath): logger.info("Reading cached serialized file %r", idMapPath) if fmt in ["pickle", "json"]: tD = mU.doImport(idMapPath, fmt=fmt) nL = list(set(tD["idNameList"])) oD = tD["uniprotMapD"] logger.info("keys %r", list(oD.keys())[:10]) logger.info("nL %r", nL) ok = True elif fmt == "tdd": ioU = IoUtil() it = ioU.deserializeCsvIter(idMapPath, delimiter="\t", rowFormat="list", encodingErrors="ignore") tL = next(it, []) nL = tL[1:] if len(nL) == 1: for row in it: oD[row[0]] = row[1] else: for row in it: oD[row[0]] = row[1:] ok = True else: idPath = os.path.join(rawDirPath, fileU.getFileName(targetUrl)) if not fileU.exists(idPath): logger.info( "Fetching selected UniProt idmapping data from %r in %r", targetUrl, outDirPath) ok = fileU.get(targetUrl, idPath) if not ok: logger.error("Failed to downlowd %r", targetUrl) return oD else: logger.info("Using cached mapping file %r", idPath) # --- ioU = IoUtil() if fmt in ["pickle", "json"]: if len(mapNameL) == 1: for row in ioU.deserializeCsvIter( idPath, delimiter="\t", rowFormat="list", encodingErrors="ignore"): oD[row[0]] = str( row[self.__mapRecordD[mapNameL[0]] - 1]) else: for row in ioU.deserializeCsvIter( idPath, delimiter="\t", rowFormat="list", encodingErrors="ignore"): for mapName in mapNameL: oD.setdefault(row[0], []).append( str(row[self.__mapRecordD[mapName] - 1])) logger.info("Writing serialized mapping file %r", idMapPath) ok = mU.doExport(idMapPath, { "idNameList": mapNameL, "uniprotMapD": oD }, fmt=fmt) elif fmt == "tdd": # logger.info("Writing serialized mapping file %r", idMapPath) fU = FileUtil() fU.mkdirForFile(idMapPath) colNameL = [] colNameL.append("UniProtId") colNameL.extend(mapNameL) with open(idMapPath, "w", encoding="utf-8") as ofh: ofh.write("%s\n" % "\t".join(colNameL)) if len(mapNameL) == 1: idx = self.__mapRecordD[mapNameL[0]] - 1 for row in ioU.deserializeCsvIter( idPath, delimiter="\t", rowFormat="list", encodingErrors="ignore"): ofh.write("%s\t%s\n" % (row[0], row[idx])) else: idxL = [0] idxL.extend([ self.__mapRecordD[mapName] - 1 for mapName in mapNameL ]) for row in ioU.deserializeCsvIter( idPath, delimiter="\t", rowFormat="list", encodingErrors="ignore"): ofh.write( "%s\n" % "\t".join([str(row[idx]) for idx in idxL])) # nL, oD = self.__rebuildCache(targetUrl, mapNameL, outDirPath, rawDirPath, fmt=fmt, useCache=True) ok = True if nL and oD else False logger.info("Completed reload (%r) at %s (%.4f seconds)", ok, time.strftime("%Y %m %d %H:%M:%S", time.localtime()), time.time() - startTime) except Exception as e: logger.exception("Failing with %s", str(e)) # return nL, oD
class SchemaProvider(SingletonClass): """ A collection of schema build and caching methods. Static cache worflow: <authorative source> <-- <cache dir> <- client API Compute workflow: <dependent resource files, config file, dictionaries> -> [schema builder] --> <schema def> --> <Json schema> """ def __init__(self, cfgOb, cachePath, useCache=True, rebuildFlag=False, **kwargs): """A collection of schema build and caching methods. Args: cfgOb (object): ConfigInfo() instance cachePath (str): path to directory containing schema useCache (bool, optional): use cached schema. Defaults to True. rebuildFlag (bool, optional): on-the-fly rebuild and cache schema """ self.__cfgOb = cfgOb self.__configName = self.__cfgOb.getDefaultSectionName() self.__cachePath = os.path.abspath(cachePath) self.__useCache = useCache self.__rebuildFlag = rebuildFlag self.__useCache = rebuildFlag if rebuildFlag else useCache # self.__workPath = os.path.join(self.__cachePath, "work") self.__fileU = FileUtil(workPath=os.path.join(self.__cachePath, "work")) self.__schemaCachePath = os.path.join(self.__cachePath, self.__cfgOb.get("SCHEMA_DEFINITION_CACHE_DIR", sectionName=self.__configName)) self.__jsonSchemaCachePath = os.path.join(self.__cachePath, self.__cfgOb.get("JSON_SCHEMA_DEFINITION_CACHE_DIR", sectionName=self.__configName)) self.__fileU.mkdir(self.__schemaCachePath) self.__fileU.mkdir(self.__jsonSchemaCachePath) self.__kwargs = kwargs def getSchemaOptions(self, schemaLevel, extraOpts=None): opts = extraOpts + "|" if extraOpts else "" if schemaLevel == "full": return opts + "mandatoryKeys|mandatoryAttributes|bounds|enums|rcsb" elif schemaLevel in ["min", "minimum"]: return opts + "mandatoryKeys|enums|rcsb" else: return opts def getSchemaInfo(self, databaseName, dataTyping="ANY"): """Convenience method to return essential schema details for the input repository content type. Args: databaseName (str): schema name (e.g. pdbx, bird, chem_comp, ...) dataTyping (str, optional): Application name for the target schema (e.g. ANY, SQL, ...) Returns: tuple: SchemaDefAccess(object), target database name, target collection name list, primary index attribute list """ sd = None dbName = None collectionNameList = [] docIndexD = {} try: mU = MarshalUtil(workPath=self.__workPath) schemaLocator = self.__getSchemaDefLocator(databaseName, dataTyping=dataTyping) if self.__rebuildFlag: filePath = os.path.join(self.__schemaCachePath, self.__fileU.getFileName(schemaLocator)) self.makeSchemaDef(databaseName, dataTyping=dataTyping, saveSchema=True) else: filePath = self.__reload(schemaLocator, self.__schemaCachePath, useCache=self.__useCache) if not filePath: logger.error("Unable to recover schema %s (%s)", databaseName, dataTyping) logger.debug("ContentType %r dataTyping %r schemaLocator %r", databaseName, dataTyping, schemaLocator) schemaDef = mU.doImport(filePath, fmt="json") if schemaDef: logger.debug("Using cached schema definition for %s application %s", databaseName, dataTyping) sd = SchemaDefAccess(schemaDef) if sd: dbName = sd.getDatabaseName() collectionInfoList = sd.getCollectionInfo() logger.debug("Schema %s database name %s collections %r", databaseName, dbName, collectionInfoList) for cd in collectionInfoList: collectionName = cd["NAME"] collectionNameList.append(collectionName) docIndexD[collectionName] = sd.getDocumentIndices(collectionName) except Exception as e: logger.exception("Retreiving schema %s for %s failing with %s", databaseName, dataTyping, str(e)) return sd, dbName, collectionNameList, docIndexD def schemaDefCompare(self, databaseName, dataTyping="ANY"): """Compare computed schema defintion with current source/cached version. Args: databaseName (str): schema definition name for comparison dataTyping (str, optional): data type conventions for the schema comparison. Defaults to "ANY". Returns: (str): file path for schema difference or None """ mU = MarshalUtil(workPath=self.__workPath) schemaDiffPath = os.path.join(self.__cachePath, "schema_diff") mU.mkdir(schemaDiffPath) schemaPath = self.__getSchemaDefLocator(databaseName, dataTyping=dataTyping) fn = self.__fileU.getFileName(schemaPath) sD = self.makeSchemaDef(databaseName, dataTyping=dataTyping) v2 = sD["DATABASE_VERSION"] # ---- # tPath = os.path.join(self.__schemaCachePath, self.__fileU.getFileName(schemaPath) + "-test") # logger.info("Exporting schema def to %s", tPath) # mU.doExport(tPath, sD, fmt="json", indent=3) # sD = mU.doImport(tPath, fmt="json") # ---- cPath = os.path.join(self.__schemaCachePath, self.__fileU.getFileName(schemaPath)) sDCache = mU.doImport(cPath, fmt="json") v1 = sDCache["DATABASE_VERSION"] # numDiff, difD = self.schemaCompare(sDCache, sD) # # jD = diff(sDCache, sD, syntax="explicit", marshal=True) diffPath = None if numDiff: bn, _ = os.path.splitext(fn) diffPath = os.path.join(schemaDiffPath, bn + "-" + v1 + "-" + v2 + "-diff.json") # logger.info("diff for %s %s = \n%s", databaseName, dataTyping, pprint.pformat(difD, indent=3, width=100)) mU.doExport(diffPath, difD, fmt="json", indent=3) # return diffPath def jsonSchemaCompare(self, databaseName, collectionName, encodingType, level, extraOpts=None): """Compare computed JSON schema defintion with current source/cached version. Args: databaseName (str): schema name collectionName (str): collection name encodingType (str): schema data type conventions (JSON|BSON) level (str): metadata level (min|full) extraOpts (str): extra schema construction options Returns: (str): path to the difference file or None """ mU = MarshalUtil(workPath=self.__workPath) schemaDiffPath = os.path.join(self.__cachePath, "schema_diff") mU.mkdir(schemaDiffPath) schemaLocator = self.__getJsonSchemaLocator(databaseName, collectionName, encodingType, level) fn = self.__fileU.getFileName(schemaLocator) schemaPath = os.path.join(self.__jsonSchemaCachePath, fn) # sD = self.makeSchema(databaseName, collectionName, encodingType=encodingType, level=level, saveSchema=False, extraOpts=extraOpts) v2 = self.__getSchemaVersion(sD) # ---- # tPath = os.path.join(self.__jsonSchemaCachePath, self.__fileU.getFileName(schemaPath) + "-test") # logger.info("Exporting json schema to %s", tPath) # mU.doExport(tPath, sD, fmt="json", indent=3) # ---- # sDCache = mU.doImport(schemaPath, fmt="json") v1 = self.__getSchemaVersion(sDCache) if not v1: logger.error("no version for %s - %s %s", schemaLocator, databaseName, collectionName) # numDiff, difD = self.schemaCompare(sDCache, sD) # jD = diff(sDCache, sD, marshal=True, syntax="explicit") diffPath = None if numDiff: logger.debug("diff for %s %s %s %s = \n%s", databaseName, collectionName, encodingType, level, pprint.pformat(difD, indent=3, width=100)) bn, _ = os.path.splitext(fn) diffPath = os.path.join(schemaDiffPath, bn + "-" + v1 + "-" + v2 + "-diff.json") mU.doExport(diffPath, difD, fmt="json", indent=3) return diffPath def __getSchemaVersion(self, jsonSchema): try: comment = jsonSchema["$comment"] if "$comment" in jsonSchema else "" ff = comment.split(":") version = ff[1].strip() return version except Exception as e: logger.exception("Failing for with %s", str(e)) return "" def __getSchemaDefLocator(self, databaseName, dataTyping="ANY"): """Internal method returning schema definition path for the input content type and application. Defines schema definition naming convention - Args: databaseName (str): schema name (e.g. pdbx, bird, chem_comp, ...) dataTyping (str, optional): Application name for the target schema (e.g. ANY, SQL, ...) Returns: str: schema definition file locator """ schemaLocator = None try: locPath = self.__cfgOb.get("SCHEMA_DEFINITION_LOCATOR_PATH", sectionName=self.__configName) fn = "schema_def-%s-%s.json" % (databaseName, dataTyping.upper()) schemaLocator = os.path.join(locPath, fn) except Exception as e: logger.exception("Retreiving schema definition path %s for %s failing with %s", databaseName, dataTyping, str(e)) return schemaLocator def __getJsonSchemaLocator(self, databaseName, collectionName, encodingType="BSON", level="full"): """Internal method returning JSON schema path for the input collection data type convention and level. Defines the JSON/BSON schema naming convention - Args: databaseName (str): database name in the document store collectionName (str): collection name in document store encodingType (str, optional): data type convention (BSON|JSON) level (str, optional): Completeness of the schema (e.g. min or full) Returns: str: schema file locator """ schemaLocator = None try: sdType = None sLevel = None schemaLocator = None if encodingType.upper() in ["JSON", "BSON"]: sdType = encodingType.lower() if level.lower() in ["min", "minimun"]: sLevel = "min" elif level.lower() in ["full"]: sLevel = level.lower() # if sdType and sLevel: locPath = self.__cfgOb.get("JSON_SCHEMA_DEFINITION_LOCATOR_PATH", sectionName=self.__configName) fn = "%s-%s-db-%s-col-%s.json" % (sdType, sLevel, databaseName, collectionName) schemaLocator = os.path.join(locPath, fn) else: logger.error("Unsupported schema options: %s level %r type %r", collectionName, level, encodingType) schemaLocator = None except Exception as e: logger.debug("Retreiving JSON schema definition for %s type %s failing with %s", collectionName, encodingType, str(e)) # return schemaLocator def __reload(self, locator, dirPath, useCache=True): # fn = self.__fileU.getFileName(locator) filePath = os.path.join(dirPath, fn) logger.debug("Target cache filePath %s", filePath) self.__fileU.mkdir(dirPath) if not useCache: try: os.remove(filePath) except Exception: pass # if useCache and self.__fileU.exists(filePath): ok = True else: logger.info("Fetch data from source %s to %s", locator, filePath) ok = self.__fileU.get(locator, filePath) return filePath if ok else None def getJsonSchema(self, databaseName, collectionName, encodingType="BSON", level="full", extraOpts=None): """Return JSON schema (w/ BSON types) object for the input collection and level.and Args: databaseName (str): database name collectionName (str): collection name in document store encodingType (str, optional): data type convention (BSON|JSON) level (str, optional): Completeness of the schema (e.g. min or full) Returns: dict: Schema object """ sObj = None schemaLocator = self.__getJsonSchemaLocator(databaseName, collectionName, encodingType=encodingType, level=level) # if self.__rebuildFlag: filePath = os.path.join(self.__schemaCachePath, self.__fileU.getFileName(schemaLocator)) self.makeSchema(databaseName, collectionName, encodingType=encodingType, level=level, extraOpts=extraOpts) else: filePath = self.__reload(schemaLocator, self.__jsonSchemaCachePath, useCache=self.__useCache) mU = MarshalUtil(workPath=self.__workPath) if filePath and mU.exists(filePath): mU = MarshalUtil(workPath=self.__workPath) sObj = mU.doImport(filePath, fmt="json") else: logger.debug("Failed to read schema for %s %r", collectionName, level) return sObj def makeSchema(self, databaseName, collectionName, encodingType="BSON", level="full", saveSchema=False, extraOpts=None): try: smb = SchemaDefBuild(databaseName, self.__cfgOb, cachePath=self.__cachePath) # cD = None stU = encodingType.upper() cD = smb.build(collectionName, dataTyping=stU, encodingType=stU, enforceOpts=self.getSchemaOptions(level, extraOpts=extraOpts)) if cD and saveSchema: schemaLocator = self.__getJsonSchemaLocator(databaseName, collectionName, encodingType=encodingType, level=level) localPath = os.path.join(self.__jsonSchemaCachePath, self.__fileU.getFileName(schemaLocator)) mU = MarshalUtil(workPath=self.__workPath) mU.doExport(localPath, cD, fmt="json", indent=3, enforceAscii=False) except Exception as e: logger.exception("Building schema %s collection %s failing with %s", databaseName, collectionName, str(e)) return cD def makeSchemaDef(self, databaseName, dataTyping="ANY", saveSchema=False): schemaDef = None try: smb = SchemaDefBuild(databaseName, self.__cfgOb, cachePath=self.__cachePath) schemaDef = smb.build(dataTyping=dataTyping, encodingType="rcsb") if schemaDef and saveSchema: schemaLocator = self.__getSchemaDefLocator(databaseName, dataTyping=dataTyping) localPath = os.path.join(self.__schemaCachePath, self.__fileU.getFileName(schemaLocator)) mU = MarshalUtil(workPath=self.__workPath) mU.doExport(localPath, schemaDef, fmt="json", indent=3, enforceAscii=False) except Exception as e: logger.exception("Building schema %s failing with %s", databaseName, str(e)) return schemaDef def schemaCompare(self, orgD, newD): """ Compute the difference of nested dictionaries. """ fOrgD = self.__flatten(orgD) fNewD = self.__flatten(newD) if len(fOrgD) != len(fNewD): logger.debug("Schema lengths differ: org %d new %d", len(fOrgD), len(fNewD)) # addedD = {k: fNewD[k] for k in set(fNewD) - set(fOrgD)} removedD = {k: fOrgD[k] for k in set(fOrgD) - set(fNewD)} changedOrgD = {k: fOrgD[k] for k in set(fOrgD) & set(fNewD) if fOrgD[k] != fNewD[k]} changedNewD = {k: fNewD[k] for k in set(fOrgD) & set(fNewD) if fOrgD[k] != fNewD[k]} chD = {} for ky in changedOrgD: kyS = ".".join(ky) vOrg = changedOrgD[ky] vNew = changedNewD[ky] if isinstance(vOrg, (list, tuple)) and isinstance(vNew, (list, tuple)): # logger.info(" >> %r vOrg %r vNew %r", ky, vOrg, vNew) dV = list(set(vNew) - set(vOrg)) if dV: chD[kyS] = {"diff": dV} else: chD[kyS] = {"from": vOrg, "to": vNew} # nT = len(addedD) + len(removedD) + len(chD) diffD = {"added": [".".join(kk) for kk in addedD.keys()], "removed": [".".join(kk) for kk in removedD.keys()], "changed": chD} return nT, diffD def __flatten(self, inpDict, prefix=None): prefix = prefix[:] if prefix else [] outDict = {} for key, value in inpDict.items(): if isinstance(value, dict) and value: deeper = self.__flatten(value, prefix + [key]) outDict.update({tuple(key2): val2 for key2, val2 in deeper.items()}) elif isinstance(value, (list, tuple)) and value: for index, sublist in enumerate(value, start=1): if isinstance(sublist, dict) and sublist: deeper = self.__flatten(sublist, prefix + [key] + [str(index)]) outDict.update({tuple(key2): val2 for key2, val2 in deeper.items()}) else: outDict[tuple(prefix + [key] + [str(index)])] = value else: outDict[tuple(prefix + [key])] = value return outDict def __flattenX(self, inpDict, prefix=None): prefix = prefix[:] if prefix else [] # separator = "." outDict = {} for key, value in inpDict.items(): if isinstance(value, dict) and value: deeper = self.__flatten(value, prefix + [key]) outDict.update({tuple(key2): val2 for key2, val2 in deeper.items()}) elif isinstance(value, list) and value: for index, sublist in enumerate(value, start=1): if isinstance(sublist, dict) and sublist: deeper = self.__flatten(sublist, prefix + [key] + [str(index)]) outDict.update({tuple(key2): val2 for key2, val2 in deeper.items()}) else: outDict[tuple(prefix + [key] + [str(index)])] = value else: outDict[tuple(prefix + [key])] = value return outDict def __flattenOrg(self, inpDict, separator=".", prefix=""): outDict = {} for key, value in inpDict.items(): if isinstance(value, dict) and value: deeper = self.__flattenOrg(value, separator, prefix + key + separator) outDict.update({key2: val2 for key2, val2 in deeper.items()}) elif isinstance(value, list) and value: for index, sublist in enumerate(value, start=1): if isinstance(sublist, dict) and sublist: deeper = self.__flattenOrg(sublist, separator, prefix + key + separator + str(index) + separator) outDict.update({key2: val2 for key2, val2 in deeper.items()}) else: outDict[prefix + key + separator + str(index)] = value else: outDict[prefix + key] = value return outDict def __dictGen(self, indict, pre=None): pre = pre[:] if pre else [] if isinstance(indict, dict): for key, value in indict.items(): if isinstance(value, dict): for dD in self.__dictGen(value, pre + [key]): yield dD elif isinstance(value, list) or isinstance(value, tuple): for v in value: for dD in self.__dictGen(v, pre + [key]): yield dD else: yield pre + [key, value] else: yield indict
def __reload(self, dirPath, reloadDb=False, fromDb=False, useCache=False, pharosDumpUrl=None, mysqlUser=None, mysqlPassword=None): startTime = time.time() pharosSelectedTables = ["drug_activity", "cmpd_activity", "target", "protein", "t2tc"] pharosDumpUrl = pharosDumpUrl if pharosDumpUrl else "http://juniper.health.unm.edu/tcrd/download/latest.sql.gz" pharosReadmeUrl = "http://juniper.health.unm.edu/tcrd/download/latest.README" ok = False fU = FileUtil() pharosDumpFileName = fU.getFileName(pharosDumpUrl) pharosDumpPath = os.path.join(dirPath, pharosDumpFileName) pharosUpdatePath = os.path.join(dirPath, "pharos-update.sql") pharosReadmePath = os.path.join(dirPath, "pharos-readme.txt") logPath = os.path.join(dirPath, "pharosLoad.log") # fU.mkdir(dirPath) # exU = ExecUtils() # if reloadDb: logger.info("useCache %r pharosDumpPath %r", useCache, pharosDumpPath) if useCache and self.__mU.exists(pharosDumpPath): ok = True else: logger.info("Fetching url %s path %s", pharosDumpUrl, pharosDumpPath) ok1 = fU.get(pharosDumpUrl, pharosDumpPath) ok2 = fU.get(pharosReadmeUrl, pharosReadmePath) logger.info("Completed fetch (%r) at %s (%.4f seconds)", ok1 and ok2, time.strftime("%Y %m %d %H:%M:%S", time.localtime()), time.time() - startTime) # --- readmeLines = self.__mU.doImport(pharosReadmePath, fmt="list") self.__version = readmeLines[0].split(" ")[1][1:] if readmeLines else "6" # --- logger.info("Filtering SQL dump %r for selected tables %r", pharosDumpFileName, pharosSelectedTables) doWrite = True # Note: the pharos dump file latest.sql.gz is not gzipped with open(pharosDumpPath, "r", encoding="utf-8") as ifh, open(pharosUpdatePath, "w", encoding="utf-8") as ofh: for line in ifh: if line.startswith("-- Table structure for table"): tN = line.split(" ")[-1][1:-2] doWrite = True if tN in pharosSelectedTables else False if doWrite: ofh.write(line) # --- ok = exU.run( "mysql", execArgList=["-v", "-u", mysqlUser, "--password=%s" % mysqlPassword, "-e", "create database if not exists tcrd6;"], outPath=logPath, outAppend=False, timeOut=None, ) # ok = exU.run( # "mysql", # execArgList=["-u", mysqlUser, "--password=%s" % mysqlPassword, "tcrd6"], # outPath=logPath, # inpPath=pharosDumpPath, # outAppend=True, # timeOut=None, # ) shellCmd = 'trap "" SIGHUP SIGINT SIGTERM; nohup mysql -u %s --password=%s tcrd6 < %s >& %s' % (mysqlUser, mysqlPassword, pharosUpdatePath, logPath) ok = exU.runShell( shellCmd, outPath=None, inpPath=None, outAppend=True, timeOut=None, ) logger.info("SQL dump restore status %r", ok) # -- if fromDb: for tbl in pharosSelectedTables: outPath = os.path.join(dirPath, "%s.tdd" % tbl) # if useCache and self.__mU.exists(outPath): # continue ok = exU.run( "mysql", execArgList=["-u", mysqlUser, "--password=%s" % mysqlPassword, "-e", "use tcrd6; select * from %s;" % tbl], outPath=outPath, outAppend=False, timeOut=None, suppressStderr=True, ) logger.info("SQL table %s export status %r", tbl, ok) return ok