def testBuildResourceCache(self):
"""Fixture - generate and check selected resource caches"""
try:
resourceNameL = [
"AtcProvider instance",
"DrugBankProvider instance",
"PubChemProvider instance",
"CitationReferenceProvider instance",
"JournalTitleAbbreviationProvider instance",
"EnzymeDatabaseProvider instance",
"PfamProvider instance",
"SiftsSummaryProvider instance",
"CathProvider instance",
"ScopProvider instance",
"EcodProvider instance",
"Scop2Provider instance",
"TaxonomyProvider instance",
]
rP = DictMethodResourceProvider(self.__cfgOb, configName=self.__configName, cachePath=self.__cachePath, restoreUseStash=False, restoreUseGit=True)
for resourceName in resourceNameL:
rP.getResource(resourceName, useCache=True, default=None, doRestore=True, doBackup=False)
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testRecoverResourceCache(self):
"""Fixture - generate and check resource caches"""
try:
rp = DictMethodResourceProvider(self.__cfgOb, configName=self.__configName, cachePath=self.__cachePath)
ret = rp.cacheResources(useCache=True)
self.assertTrue(ret)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def buildResourceCache(cfgOb, configName, cachePath, rebuildCache=False):
"""Generate and cache resource dependencies."""
ret = False
try:
rp = DictMethodResourceProvider(cfgOb,
configName=configName,
cachePath=cachePath)
ret = rp.cacheResources(useCache=not rebuildCache)
except Exception as e:
logger.exception("Failing with %s", str(e))
return ret
def testRecoverResourceCache(self):
"""Fixture - reload and check resource caches"""
try:
rp = DictMethodResourceProvider(
self.__cfgOb,
configName=self.__configName,
cachePath=self.__cachePath,
restoreUseGit=True,
restoreUseStash=False,
providerTypeExclude=self.__excludeType,
)
ret = rp.cacheResources(useCache=True)
self.assertTrue(ret)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def buildResourceCache(self, rebuildCache=False, providerTypeExclude=None, restoreUseGit=True, restoreUseStash=True):
"""Generate and cache resource dependencies."""
ret = False
try:
rp = DictMethodResourceProvider(
self.__cfgOb,
configName=self.__configName,
cachePath=self.__cachePath,
providerTypeExclude=providerTypeExclude,
restoreUseGit=restoreUseGit,
restoreUseStash=restoreUseStash,
)
ret = rp.cacheResources(useCache=not rebuildCache, doBackup=False)
except Exception as e:
logger.exception("Failing with %s", str(e))
return ret
def __runContentType(self, contentType, mockLength, mergeContent):
"""Read and process test fixture data files from the input content type."""
try:
dP = DictionaryApiProviderWrapper(self.__cfgOb,
self.__cachePath,
useCache=True)
dictApi = dP.getApiByName(contentType)
rP = DictMethodResourceProvider(self.__cfgOb,
configName=self.__configName,
cachePath=self.__cachePath,
siftsAbbreviated="TEST")
dmh = DictMethodRunner(dictApi,
modulePathMap=self.__modulePathMap,
resourceProvider=rP)
locatorObjList = self.__rpP.getLocatorObjList(
contentType=contentType, mergeContentTypes=mergeContent)
containerList = self.__rpP.getContainerList(locatorObjList)
#
logger.debug("Length of locator list %d\n", len(locatorObjList))
self.assertGreaterEqual(len(locatorObjList), mockLength)
for container in containerList:
cName = container.getName()
#
# if cName not in ["1B5F"]:
# continue
logger.debug("Processing container %s", cName)
dmh.apply(container)
if self.__export:
savePath = os.path.join(HERE, "test-output",
cName + "-with-method.cif")
self.__mU.doExport(savePath, [container], fmt="mmcif")
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def buildResourceCache(self, rebuildCache=False):
"""Generate and cache resource dependencies."""
ret = False
try:
useCache = not rebuildCache
rP = DictMethodResourceProvider(
self.__cfgOb,
configName=self.__configName,
cachePath=self.__cachePath,
restoreUseStash=True,
restoreUseGit=True,
providerTypeExclude=None,
)
ret = rP.cacheResources(useCache=useCache,
doBackup=False,
useStash=False,
useGit=False)
logger.info("useCache %r cache reload status (%r)", useCache, ret)
except Exception as e:
logger.exception("Failing with %s", str(e))
return ret
def testMethodRunnerSetup(self):
"""Test the setup methods for method runner class"""
try:
dP = DictionaryApiProviderWrapper(self.__cfgOb,
self.__cachePath,
useCache=True)
dictApi = dP.getApiByName("pdbx")
rP = DictMethodResourceProvider(self.__cfgOb,
configName=self.__configName,
cachePath=self.__cachePath,
siftsAbbreviated="TEST")
dmh = DictMethodRunner(dictApi,
modulePathMap=self.__modulePathMap,
resourceProvider=rP)
ok = dmh is not None
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def __testPrepDocumentsFromContainers(
self,
inputPathList,
databaseName,
collectionName,
styleType="rowwise_by_name_with_cardinality",
mergeContentTypes=None):
"""Test case - create loadable PDBx data from repository files"""
try:
sd, _, _, _ = self.__schP.getSchemaInfo(databaseName)
#
dP = DictionaryApiProviderWrapper(self.__cachePath,
cfgOb=self.__cfgOb,
configName=self.__configName,
useCache=True)
dictApi = dP.getApiByName(databaseName)
rP = DictMethodResourceProvider(
self.__cfgOb,
configName=self.__configName,
cachePath=self.__cachePath,
restoreUseStash=False,
restoreUseGit=True,
providerTypeExclude=self.__excludeType,
)
dmh = DictMethodRunner(dictApi,
modulePathMap=self.__modulePathMap,
resourceProvider=rP)
#
dtf = DataTransformFactory(schemaDefAccessObj=sd,
filterType=self.__fTypeRow)
sdp = SchemaDefDataPrep(schemaDefAccessObj=sd,
dtObj=dtf,
workPath=self.__cachePath,
verbose=self.__verbose)
containerList = self.__rpP.getContainerList(inputPathList)
for container in containerList:
cName = container.getName()
logger.debug("Processing container %s", cName)
dmh.apply(container)
if self.__export:
savePath = os.path.join(HERE, "test-output",
cName + "-with-method.cif")
self.__mU.doExport(savePath, [container], fmt="mmcif")
#
tableIdExcludeList = sd.getCollectionExcluded(collectionName)
tableIdIncludeList = sd.getCollectionSelected(collectionName)
sliceFilter = sd.getCollectionSliceFilter(collectionName)
sdp.setSchemaIdExcludeList(tableIdExcludeList)
sdp.setSchemaIdIncludeList(tableIdIncludeList)
#
logger.debug("%s (%r) exclude list %r", collectionName,
sliceFilter, tableIdExcludeList)
logger.debug("%s (%r) include list %r", collectionName,
sliceFilter, tableIdIncludeList)
docList, containerNameList, _ = sdp.processDocuments(
containerList,
styleType=styleType,
filterType=self.__fTypeRow,
dataSelectors=["PUBLIC_RELEASE"],
sliceFilter=sliceFilter,
collectionName=collectionName)
docList = sdp.addDocumentPrivateAttributes(docList, collectionName)
docList = sdp.addDocumentSubCategoryAggregates(
docList, collectionName)
#
mergeS = "-".join(mergeContentTypes) if mergeContentTypes else ""
if self.__export and docList:
fp = os.path.join(
HERE, "test-output", "prep-%s-%s-%s.json" %
(databaseName, collectionName, mergeS))
self.__mU.doExport(fp, docList, fmt="json", indent=3)
logger.debug("Exported %r", fp)
#
return docList, containerNameList
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def __init__(
self,
cfgOb,
schemaDefObj,
cfgSectionName="site_info_configuration",
dbCon=None,
cachePath=".",
workPath=".",
cleanUp=False,
warnings="default",
verbose=True,
restoreUseStash=True,
restoreUseGit=True,
providerTypeExclude=True,
):
self.__verbose = verbose
self.__debug = False
self.__cfgOb = cfgOb
sectionName = cfgSectionName
self.__sD = schemaDefObj
#
self.__dbCon = dbCon
self.__cachePath = cachePath
self.__workPath = workPath
self.__pathList = []
self.__cleanUp = cleanUp
#
self.__colSep = "&##&\t"
self.__rowSep = "$##$\n"
#
#
self.__fTypeRow = "skip-max-width"
self.__fTypeCol = "skip-max-width"
#
self.__warningAction = warnings
dtf = DataTransformFactory(schemaDefAccessObj=self.__sD,
filterType=self.__fTypeRow)
self.__sdp = SchemaDefDataPrep(schemaDefAccessObj=self.__sD,
dtObj=dtf,
workPath=self.__cachePath,
verbose=self.__verbose)
self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb,
cachePath=self.__cachePath)
#
schemaName = self.__sD.getName()
modulePathMap = self.__cfgOb.get("DICT_METHOD_HELPER_MODULE_PATH_MAP",
sectionName=sectionName)
dP = DictionaryApiProviderWrapper(self.__cachePath,
cfgOb=self.__cfgOb,
configName=sectionName,
useCache=True)
dictApi = dP.getApiByName(schemaName)
rP = DictMethodResourceProvider(
self.__cfgOb,
cachePath=self.__cachePath,
restoreUseStash=restoreUseStash,
restoreUseGit=restoreUseGit,
providerTypeExclude=providerTypeExclude)
self.__dmh = DictMethodRunner(dictApi,
modulePathMap=modulePathMap,
resourceProvider=rP)
def __fullSchemaDataPrep(self,
contentType,
filterType,
styleType,
mockLength,
rejectLength=0,
dataSelectors=None,
mergeContentTypes=None,
excludeExtras=None):
"""Internal method for preparing file-based data requiring dynamic methods, slicing, or key injection.
Args:
contentType (str): Content type name
filterType (str): List of data processing options (separated by '|') (e.g. "drop-empty-attributes|drop-empty-tables|skip-max-width|...)
styleType (str): organization of output document (e.g. rowise-by-name)
mockLength (int): Expected length of the test data for the input content type
rejectLength (int, optional): number of input data sets rejected by the dataselection criteria. Defaults to 0.
dataSelectors (list of str, optional): data selection criteria. Defaults to None.
mergeContentTypes (list of str, optional): list content types to merge with the input data set. Defaults to None. (e.g. ['vrpt'])
"""
try:
excludeExtras = excludeExtras if excludeExtras else []
_ = mockLength
_ = rejectLength
dD = self.__schP.makeSchemaDef(contentType,
dataTyping="ANY",
saveSchema=True)
_ = SchemaDefAccess(dD)
inputPathList = self.__rpP.getLocatorObjList(
contentType=contentType, mergeContentTypes=mergeContentTypes)
sd, _, collectionNameList, _ = self.__schP.getSchemaInfo(
databaseName=contentType, dataTyping="ANY")
#
dP = DictionaryApiProviderWrapper(self.__cachePath,
cfgOb=self.__cfgOb,
configName=self.__configName,
useCache=True)
dictApi = dP.getApiByName(contentType)
#
rP = DictMethodResourceProvider(
self.__cfgOb,
configName=self.__configName,
cachePath=self.__cachePath,
restoreUseStash=False,
restoreUseGit=True,
providerTypeExclude=self.__excludeType,
)
dmh = DictMethodRunner(dictApi,
modulePathMap=self.__modulePathMap,
resourceProvider=rP)
#
dtf = DataTransformFactory(schemaDefAccessObj=sd,
filterType=filterType)
sdp = SchemaDefDataPrep(schemaDefAccessObj=sd,
dtObj=dtf,
workPath=self.__cachePath,
verbose=self.__verbose)
containerList = self.__rpP.getContainerList(inputPathList)
for container in containerList:
cName = container.getName()
logger.debug("Processing container %s", cName)
dmh.apply(container)
#
for collectionName in collectionNameList:
tableIdExcludeList = sd.getCollectionExcluded(collectionName)
tableIdIncludeList = sd.getCollectionSelected(collectionName)
sliceFilter = sd.getCollectionSliceFilter(collectionName)
sdp.setSchemaIdExcludeList(tableIdExcludeList)
sdp.setSchemaIdIncludeList(tableIdIncludeList)
#
docList, _, _ = sdp.processDocuments(
containerList,
styleType=styleType,
sliceFilter=sliceFilter,
filterType=filterType,
dataSelectors=dataSelectors,
collectionName=collectionName)
docList = sdp.addDocumentPrivateAttributes(
docList, collectionName)
docList = sdp.addDocumentSubCategoryAggregates(
docList, collectionName)
# Special exclusions for the test harness. (removes timestamped data items to allow diffs.)
self.__filterDocuments(docList, excludeExtras)
mergeS = "-".join(
mergeContentTypes) if mergeContentTypes else ""
fName = "full-prep-%s-%s-%s-%s.json" % (
contentType, collectionName, mergeS, styleType)
if self.__exportFlag:
self.__logDocumentOrder(docList)
fPath = os.path.join(self.__outputPath, fName)
self.__mU.doExport(fPath, docList, fmt="json", indent=3)
logger.debug("Exported %r", fPath)
#
if self.__diffFlag:
fPath = os.path.join(self.__savedOutputPath, fName)
refDocList = self.__mU.doImport(fPath, fmt="json")
self.assertEqual(len(refDocList), len(docList))
logger.debug("For %s %s len refDocList %d", contentType,
collectionName, len(refDocList))
logger.debug("For %s %s len docList %d", contentType,
collectionName, len(docList))
jD = diff(refDocList,
docList,
syntax="explicit",
marshal=True)
if jD:
_, fn = os.path.split(fPath)
bn, _ = os.path.splitext(fn)
fPath = os.path.join(self.__outputPath,
bn + "-diff.json")
logger.debug("jsondiff for %s %s = \n%s", contentType,
collectionName,
pprint.pformat(jD, indent=3, width=100))
self.__mU.doExport(fPath, jD, fmt="json", indent=3)
self.assertEqual(len(jD), 0)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()