async def startupEvent():
logger.info("Startup - loading search dependencies")
#
ccsw = ChemCompSearchWrapper()
#
clDataUrl = os.environ.get("CHEM_SEARCH_DATA_HOSTNAME", None)
clDataPath = os.environ.get("CHEM_SEARCH_DATA_PATH", None)
clChannel = os.environ.get("CHEM_SEARCH_UPDATE_CHANNEL", None)
#
logger.info("Dependency data host %r path %r update channel %r", clDataUrl,
clDataPath, clChannel)
if clDataUrl and clDataPath and clChannel in ["A", "B", "a", "b"]:
ccsw.restoreDependencies("http://" + clDataUrl,
clDataPath,
bundleLabel=clChannel.upper())
#
ok1 = ccsw.readConfig()
ok2 = ccsw.updateChemCompIndex(useCache=True)
ok3 = ccsw.reloadSearchDatabase()
ok4 = ccsw.updateSearchIndex(useCache=True)
#
logger.info("Completed - loading search dependencies status %r", ok1
and ok2 and ok3 and ok4)
ccdw = ChemCompDepictWrapper()
ok1 = ccdw.readConfig()
logger.info("Completed - loading depict dependencies status %r", ok1)
#
ccsw.status()
def toMolFile(self,
identifier,
identifierType,
molfilePath=None,
fmt="mol",
**kwargs):
"""Create molfile (fmt) from InChI, SMILES descriptors or PDB identifier."""
try:
molfilePath = molfilePath if molfilePath else self.__makeMolfilePath(
fmt=fmt)
oeio = OeIoUtils()
if identifierType.lower() in ["smiles"]:
oeMol = oeio.smilesToMol(identifier)
oeMol.SetTitle("From SMILES")
elif identifierType.lower() in ["inchi"]:
oeMol = oeio.inchiToMol(identifier)
oeMol.SetTitle("From InChI")
elif identifierType.lower() in ["identifierpdb"]:
ccsw = ChemCompSearchWrapper()
oesmP = ccsw.getSearchMoleculeProvider()
oeMol = oesmP.getMol(identifier)
#
ok = self.__toMolFile(oeMol, molfilePath, **kwargs)
return molfilePath if ok else None
except Exception as e:
logger.exception("Failing with %s", str(e))
return None
def __reload(self):
ccsw = ChemCompSearchWrapper(cachePath=self.__cachePath)
ok1 = ccsw.setConfig(
self.__ccUrlTarget, self.__birdUrlTarget, ccFileNamePrefix=self.__ccFileNamePrefix, useCache=self.__useCache, numProc=self.__numProc, maxChunkSize=self.__chunkSize
)
ok2 = ccsw.readConfig()
ok3 = ccsw.reloadSearchDatabase()
return ccsw if ok1 and ok2 and ok3 else None
def testAReadConfig(self):
"""Test read/access configuration"""
try:
ccsw = ChemCompSearchWrapper()
ok = ccsw.readConfig()
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testBUpdateChemCompIndex(self):
"""Test update chemical component/Bird basic index."""
try:
ccsw = ChemCompSearchWrapper()
ok = ccsw.readConfig()
self.assertTrue(ok)
ok = ccsw.updateChemCompIndex()
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testDUpdateSearchMoleculeProvider(self):
"""Test update of the search molecule provider."""
try:
ccsw = ChemCompSearchWrapper()
ok = ccsw.readConfig()
self.assertTrue(ok)
ok = ccsw.updateSearchMoleculeProvider()
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testEReloadSearchDatabase(self):
"""Test reload search databases."""
try:
ccsw = ChemCompSearchWrapper()
ok = ccsw.readConfig()
self.assertTrue(ok)
ok = ccsw.reloadSearchDatabase()
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testCUpdateSearchIndex(self):
"""Test update search index."""
try:
ccsw = ChemCompSearchWrapper()
ok = ccsw.readConfig()
self.assertTrue(ok)
ok = ccsw.updateSearchIndex()
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def matchRangePostQuery(query: FormulaRangeQuery):
logger.debug("Got %r", query)
qD = jsonable_encoder(query)
logger.debug("qD %r", qD)
# ---
ccsw = ChemCompSearchWrapper()
retStatus, matchResultL = ccsw.matchByFormulaRange(qD["query"],
qD["matchSubset"])
logger.info("Results (%r) rL (%d)", retStatus, len(matchResultL))
rL = [mr.ccId for mr in matchResultL]
# ---
return {"query": qD["query"], "matchedIdList": rL}
def __init__(self, **kwargs):
""" Module entry point for chemical component and BIRD search index generation workflow.
Args:
cachePath(str, optional): cache directory to store indices and temporary files (default: '.')
licenseFilePath (str, optional) = path to OpenEye license text file (default: use enviroment OE_LICENSE setting)
ccFilNamePrefix (str, optional) = index prefix (default: "cc-full")
"""
self.__licensePath = kwargs.get("licenseFilePath", "oe_license.txt")
cachePath = kwargs.get("cachePath", ".")
self.__cachePath = os.path.abspath(cachePath)
self.__ccFileNamePrefix = kwargs.get("ccFilNamePrefix", "cc-full")
self.__ccsw = ChemCompSearchWrapper(
cachePath=self.__cachePath,
ccFileNamePrefix=self.__ccFileNamePrefix)
def testAABuildDependencies(self):
"""Test case - build all dependencies (convenience method)"""
try:
ccsw = ChemCompSearchWrapper()
ccUrlTarget = os.path.join(
self.__dataPath,
"components-abbrev.cif") if not self.__testFlagFull else None
birdUrlTarget = os.path.join(
self.__dataPath,
"prdcc-abbrev.cif") if not self.__testFlagFull else None
ok = ccsw.buildDependenices(ccUrlTarget=ccUrlTarget,
birdUrlTarget=birdUrlTarget)
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testAABuildConfiguration(self):
"""Test case - build configuration -"""
try:
ccsw = ChemCompSearchWrapper()
ccUrlTarget = os.path.join(
self.__dataPath,
"components-abbrev.cif") if not self.__testFlagFull else None
birdUrlTarget = os.path.join(
self.__dataPath,
"prdcc-abbrev.cif") if not self.__testFlagFull else None
ok = ccsw.setConfig(ccUrlTarget=ccUrlTarget,
birdUrlTarget=birdUrlTarget)
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testZoomSubStructSearch(self):
"""Test substructure search"""
try:
numMolsTest = self.__numMolsTest
ccsw = ChemCompSearchWrapper()
ok = ccsw.readConfig()
self.assertTrue(ok)
ok = ccsw.updateChemCompIndex(useCache=True)
self.assertTrue(ok)
ccIdx = ccsw.getChemCompIndex()
ok = ccsw.updateSearchIndex(useCache=True)
self.assertTrue(ok)
ok = ccsw.reloadSearchDatabase()
self.assertTrue(ok)
#
logger.debug("ccIdx (%d) keys %r entry %r", len(ccIdx),
list(ccIdx.keys())[:10], ccIdx["000"])
#
logger.info(
"Dependencies loaded - Starting search test scan of (limit=%r)",
numMolsTest)
for ii, (ccId, ccD) in enumerate(ccIdx.items(), 1):
if numMolsTest and ii > numMolsTest:
break
for buildType in self.__buildTypeList:
if buildType in ccD:
startTime = time.time()
retStatus, ssL, _ = ccsw.searchByDescriptor(
ccD[buildType],
buildType,
matchOpts="sub-struct-graph-relaxed")
if retStatus == -100:
logger.warning("Descriptor error continuing...")
continue
mOk = self.__resultContains(ccId, ssL)
self.assertTrue(mOk)
#
ssCcIdList = list(
set([t.ccId.split("|")[0] for t in ssL]))
logger.info(
"%s (%d) for buildType %s (%d) (%r) %r (%.4f secs)",
ccId,
ii,
buildType,
len(ssCcIdList),
mOk and retStatus == 0,
ssCcIdList,
time.time() - startTime,
)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def buildConfiguration(self, ccUrlTarget=None, birdUrlTarget=None):
"""Build bootstrap configuration files"""
try:
configFlagFull = self.__ccFileNamePrefix == "cc-full"
logger.info("Update configuration for prefix %r configFlagFull %r",
self.__ccFileNamePrefix, configFlagFull)
logger.info("Data source cc %r bird %r", ccUrlTarget,
birdUrlTarget)
ccsw = ChemCompSearchWrapper(
cachePath=self.__cachePath,
ccFileNamePrefix=self.__ccFileNamePrefix)
ok1 = ccsw.setConfig(ccUrlTarget=ccUrlTarget,
birdUrlTarget=birdUrlTarget)
ccdw = ChemCompDepictWrapper()
ok2 = ccdw.setConfig(self.__cachePath)
return ok1 and ok2
except Exception as e:
logger.exception("Failing with %s", str(e))
return False
def alignMoleculePair(self,
refIdentifier,
refIdentifierType,
fitIdentifier,
fitIdentifierType,
imagePath=None,
**kwargs):
"""Create aligned depiction for a target molecule InChI, SMILES descriptors or PDB identifier."""
try:
imagePath = imagePath if imagePath else self.__makeImagePath()
oeio = OeIoUtils()
ccsw = ChemCompSearchWrapper()
oesmP = ccsw.getSearchMoleculeProvider()
# ---
if refIdentifierType.lower() in ["smiles"]:
oeMolRef = oeio.smilesToMol(refIdentifier)
elif refIdentifierType.lower() in ["inchi"]:
oeMolRef = oeio.inchiToMol(refIdentifier)
elif refIdentifierType.lower() in ["identifierpdb"]:
oeMolRef = oesmP.getMol(refIdentifier)
#
if fitIdentifierType.lower() in ["smiles"]:
oeMolFit = oeio.smilesToMol(fitIdentifier)
elif fitIdentifierType.lower() in ["inchi"]:
oeMolFit = oeio.inchiToMol(fitIdentifier)
elif fitIdentifierType.lower() in ["identifierpdb"]:
oeMolFit = oesmP.getMol(fitIdentifier)
# ---
logger.info("oeMolRef atoms %r", oeMolRef.NumAtoms())
logger.info("oeMolFit atoms %r", oeMolFit.NumAtoms())
displayIdRef = "Ref"
displayIdFit = "Fit"
ok = self.__depictAlignedPair(oeMolRef, displayIdRef, oeMolFit,
displayIdFit, imagePath, **kwargs)
return imagePath if ok else None
except Exception as e:
logger.exception("Failing with %s", str(e))
return None
def matchGetQuery(
query: str = Query(None,
title="Molecular formula",
description="Molecular formula (ex. C8H9NO2)",
example="C8H9NO2"),
matchSubset: bool = Query(
False,
title="Formula subsets",
description=
"Find formulas satisfying only the subset of query the conditions",
example=False),
):
logger.debug("Got %r", query)
# ---
ccsw = ChemCompSearchWrapper()
logger.debug("matchSubset %r", matchSubset)
retStatus, matchResultL = ccsw.matchByFormula(query,
matchSubset=matchSubset)
logger.info("Results (%r) rL (%d)", retStatus, len(matchResultL))
rL = [mr.ccId for mr in matchResultL]
# ---
return {"query": query, "matchedIdList": rL}
def depictMolecule(self,
identifier,
identifierType,
imagePath=None,
**kwargs):
"""Create depiction from InChI, SMILES descriptors or PDB identifier."""
try:
imagePath = imagePath if imagePath else self.__makeImagePath()
oeio = OeIoUtils()
if identifierType.lower() in ["smiles"]:
oeMol = oeio.smilesToMol(identifier)
elif identifierType.lower() in ["inchi"]:
oeMol = oeio.inchiToMol(identifier)
elif identifierType.lower() in ["identifierpdb"]:
ccsw = ChemCompSearchWrapper()
oesmP = ccsw.getSearchMoleculeProvider()
oeMol = oesmP.getMol(identifier)
#
ok = self.__depictOne(oeMol, imagePath, **kwargs)
return imagePath if ok else None
except Exception as e:
logger.exception("Failing with %s", str(e))
return None
def matchGetQuery(
query: str = Query(None, title="Descriptor string", description="SMILES or InChI chemical descriptor", example="c1ccc(cc1)[C@@H](C(=O)O)N"),
matchType: DescriptorMatchType = Query(
"graph-relaxed", title="Query match type", description="Qualitative graph matching or fingerprint comparison criteria", example="graph-relaxed"
),
descriptorType: DescriptorType = Path(..., title="Descriptor type", description="Type of chemical descriptor (SMILES or InChI)", example="SMILES"),
):
matchType = matchType if matchType else "graph-relaxed"
logger.info("Got %r %r %r", descriptorType, query, matchType)
# ---
ccsw = ChemCompSearchWrapper()
retStatus, ssL, fpL = ccsw.searchByDescriptor(query, descriptorType, matchOpts=matchType)
logger.info("Result (%r) ssL (%d) fpL (%d)", retStatus, len(ssL), len(fpL))
qL = fpL if matchType in ["fingerprint-similarity"] else ssL
rD = {}
for mr in qL:
ccId = mr.ccId.split("|")[0]
rD[ccId] = max(rD[ccId], mr.fpScore) if ccId in rD else mr.fpScore
rTupL = sorted(rD.items(), key=lambda kv: kv[1], reverse=True)
rL = [rTup[0] for rTup in rTupL]
scoreL = [rTup[1] for rTup in rTupL]
# ---
return {"query": query, "descriptorType": descriptorType, "matchedIdList": rL, "matchedScoreList": scoreL}
def matchPostQuery(
query: DescriptorQuery,
descriptorType: DescriptorType = Path(..., title="Descriptor type", description="Type of chemical descriptor (SMILES or InChI)", example="SMILES"),
):
logger.info("Got %r %r", descriptorType, query)
qD = jsonable_encoder(query)
logger.debug("qD %r", qD)
matchType = qD["matchType"] if "matchType" in qD and qD["matchType"] else "graph-relaxed"
# ---
ccsw = ChemCompSearchWrapper()
retStatus, ssL, fpL = ccsw.searchByDescriptor(qD["query"], descriptorType, matchOpts=matchType)
logger.info("Results (%r) ssL (%d) fpL (%d)", retStatus, len(ssL), len(fpL))
#
qL = fpL if matchType in ["fingerprint-similarity"] else ssL
rD = {}
for mr in qL:
ccId = mr.ccId.split("|")[0]
rD[ccId] = max(rD[ccId], mr.fpScore) if ccId in rD else mr.fpScore
rTupL = sorted(rD.items(), key=lambda kv: kv[1], reverse=True)
rL = [rTup[0] for rTup in rTupL]
scoreL = [rTup[1] for rTup in rTupL]
# ---
return {"query": query.query, "descriptorType": descriptorType, "matchedIdList": rL, "matchedScoreList": scoreL}
def testZoomFingerprintRepeat(self):
"""Test substructure search"""
try:
ccsw = ChemCompSearchWrapper()
ok = ccsw.readConfig()
self.assertTrue(ok)
ok = ccsw.updateChemCompIndex(useCache=True)
self.assertTrue(ok)
ccIdx = ccsw.getChemCompIndex()
ok = ccsw.updateSearchIndex(useCache=True)
self.assertTrue(ok)
ok = ccsw.reloadSearchDatabase()
self.assertTrue(ok)
#
logger.debug("ccIdx (%d)", len(ccIdx))
#
fpL = []
descr = "InChI=1S/C9H15N5O3/c1-3(15)6(16)4-2-11-7-5(12-4)8(17)14-9(10)13-7/h3-4,6,12,15-16H,2H2,1H3,(H4,10,11,13,14,17)/t3-,4-,6-/m1/s1"
for ii in range(100):
startTime = time.time()
retStatus, _, fpL = ccsw.searchByDescriptor(
descr, "InChI", matchOpts="fingerprint-similarity")
if retStatus == -100:
logger.warning("Descriptor error continuing...")
continue
rD = {}
for mr in fpL:
ccId = mr.ccId.split("|")[0]
rD[ccId] = max(rD[ccId],
mr.fpScore) if ccId in rD else mr.fpScore
rTupL = sorted(rD.items(), key=lambda kv: kv[1], reverse=True)
rL = [rTup[0] for rTup in rTupL]
scoreL = [rTup[1] for rTup in rTupL]
logger.info("%4d (%3d) %r (%.4f secs)", ii, len(rL),
retStatus == 0,
time.time() - startTime)
logger.info("rL %r", rL)
logger.info("scoreL %r", scoreL)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def updateDependencies(self):
"""Rebuild search indices using configuration files."""
try:
logger.info("Starting update %r in %r", self.__ccFileNamePrefix,
self.__cachePath)
ccsw = ChemCompSearchWrapper(
cachePath=self.__cachePath,
ccFileNamePrefix=self.__ccFileNamePrefix)
ok1 = ccsw.readConfig()
ok2 = ccsw.updateChemCompIndex()
ok3 = ccsw.updateSearchIndex()
ok4 = ccsw.updateSearchMoleculeProvider()
# verify access -
ok5 = ccsw.reloadSearchDatabase()
return ok1 and ok2 and ok3 and ok4 and ok5
except Exception as e:
logger.exception("Failing with %s", str(e))
return False
def testAABuildDependenciesAndStash(self):
"""Test case - build, stash and restore dependencies -"""
try:
ccsw = ChemCompSearchWrapper()
ccUrlTarget = os.path.join(
self.__dataPath,
"components-abbrev.cif") if not self.__testFlagFull else None
birdUrlTarget = os.path.join(
self.__dataPath,
"prdcc-abbrev.cif") if not self.__testFlagFull else None
ok = ccsw.buildDependenices(ccUrlTarget=ccUrlTarget,
birdUrlTarget=birdUrlTarget)
self.assertTrue(ok)
#
if self.__testStash:
url = "sftp://bl-east.rcsb.org"
userName = ""
pw = ""
dirPath = "4-coastal"
ok = ccsw.stashDependencies(url,
dirPath,
userName=userName,
pw=pw)
self.assertTrue(ok)
#
fileU = FileUtil()
fileU.remove(self.__cachePath)
#
url = "http://bl-east.rcsb.org"
ok = ccsw.restoreDependencies(url, dirPath)
#
fileU.remove(self.__cachePath)
#
url = "sftp://bl-east.rcsb.org"
ok = ccsw.restoreDependencies(url,
dirPath,
userName=userName,
pw=pw)
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testZoomMatchFormula(self):
"""Test formula matching"""
try:
numMolsTest = self.__numMolsTest
ccsw = ChemCompSearchWrapper()
ok = ccsw.readConfig()
self.assertTrue(ok)
ok = ccsw.updateChemCompIndex(useCache=True)
self.assertTrue(ok)
ccIdx = ccsw.getChemCompIndex()
#
logger.debug("ccIdx (%d) keys %r entry %r", len(ccIdx),
list(ccIdx.keys())[:10], ccIdx["000"])
#
logger.info(
"Dependencies loaded - Starting formula test scan of (limit=%r)",
numMolsTest)
for ii, (ccId, idxD) in enumerate(ccIdx.items(), 1):
if numMolsTest and ii > numMolsTest:
break
#
startTime = time.time()
elementRangeD = {
el: {
"min": eCount,
"max": eCount
}
for el, eCount in idxD["type-counts"].items()
}
retStatus, rL = ccsw.matchByFormulaRange(elementRangeD, ccId)
mOk = self.__resultContains(ccId, rL)
self.assertTrue(mOk)
logger.info(
"%s (%d) (%d) (%r) (%.4f secs)",
ccId,
ii,
len(rL),
mOk and retStatus == 0,
time.time() - startTime,
)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
class ChemCompSearchIndexWorkflow(object):
def __init__(self, **kwargs):
""" Module entry point for chemical component and BIRD search index generation workflow.
Args:
cachePath(str, optional): cache directory to store indices and temporary files (default: '.')
licenseFilePath (str, optional) = path to OpenEye license text file (default: use enviroment OE_LICENSE setting)
ccFilNamePrefix (str, optional) = index prefix (default: "cc-full")
"""
self.__licensePath = kwargs.get("licenseFilePath", "oe_license.txt")
cachePath = kwargs.get("cachePath", ".")
self.__cachePath = os.path.abspath(cachePath)
self.__ccFileNamePrefix = kwargs.get("ccFilNamePrefix", "cc-full")
self.__ccsw = ChemCompSearchWrapper(
cachePath=self.__cachePath,
ccFileNamePrefix=self.__ccFileNamePrefix)
def __setLicense(self, licensePath):
ok = False
try:
if os.environ.get("OE_LICENSE") and os.access(
os.environ["OE_LICENSE"], os.R_OK):
logger.info("Using license from environment %r",
os.environ["OE_LICENSE"])
ok = True
elif os.access(licensePath, os.R_OK):
os.environ["OE_LICENSE"] = licensePath
logger.info("Setting environmenal variable OE_LICENSE to %r",
os.environ["OE_LICENSE"])
ok = True
except Exception as e:
logger.error("Setting Openeye license file %r failing %s",
licensePath, str(e))
return ok
def testCache(self):
return True
def makeIndices(self, ccUrlTarget, birdUrlTarget, **kwargs):
"""Build chemical component and BIRD search indices.
Args:
ccUrlTarget (str): URL or path for concatenated chemical component dictionary
birdUrlTarget (str): URL or path for concatenated BIRD dictionary
Other arguments may be supplied to change defaults for index generators (testing/troubleshooting)
Returns:
bool: True for success or False otherwise
"""
ok = False
if not self.__setLicense(self.__licensePath):
logger.error("Invalid OpenEye license details - exiting")
return ok
#
try:
ok = self.__ccsw.buildDependenices(ccUrlTarget, birdUrlTarget,
**kwargs)
except Exception as e:
logger.exception("Failing with %s", str(e))
return ok
def stashIndices(self,
url,
dirPath,
bundleLabel="A",
userName=None,
pw=None):
""" Store a copy of the bundled search dependencies remotely -
Args:
url (str): URL string for the destination host (e.g. sftp://myserver.net or None for a local file)
dirPath (str): directory path on the remote resource
bundleLabel (str, optional): optional label preppended to the stashed dependency bundle artifact (default='A')
userName (str, optional): optional access information. Defaults to None.
password (str, optional): optional access information. Defaults to None.
Returns:
bool: True for success or False otherwise
"""
#
ok = False
try:
ok = self.__ccsw.stashDependencies(url,
dirPath,
bundleLabel=bundleLabel,
userName=userName,
pw=pw)
except Exception as e:
logger.exception("Failing with %s", str(e))
return ok
def serverStatus():
ccsw = ChemCompSearchWrapper()
ccsw.status()
return {"msg": "Service is up!"}