class SchemaDataPrepValidateTests(unittest.TestCase):
def setUp(self):
self.__numProc = 2
# self.__fileLimit = 200
self.__fileLimit = None
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
self.__cachePath = os.path.join(TOPDIR, "CACHE")
self.__configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example-ihm.yml")
configName = "site_info_configuration"
self.__configName = configName
self.__cfgOb = ConfigUtil(configPath=self.__configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath)
self.__mU = MarshalUtil(workPath=self.__cachePath)
#self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=False, rebuildFlag=True)
self.__schP = SchemaProvider(self.__cfgOb, self.__cachePath, useCache=True)
self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb, numProc=self.__numProc, fileLimit=self.__fileLimit, cachePath=self.__cachePath)
#
self.__birdRepoPath = self.__cfgOb.getPath("BIRD_REPO_PATH", sectionName=configName)
#
self.__fTypeRow = "drop-empty-attributes|drop-empty-tables|skip-max-width|convert-iterables|normalize-enums|translateXMLCharRefs"
self.__fTypeCol = "drop-empty-tables|skip-max-width|convert-iterables|normalize-enums|translateXMLCharRefs"
self.__verbose = True
#
self.__modulePathMap = self.__cfgOb.get("DICT_METHOD_HELPER_MODULE_PATH_MAP", sectionName=configName)
self.__testDirPath = os.path.join(HERE, "test-output", "pdbx-files")
self.__testIhmDirPath = os.path.join(HERE, "test-output", "ihm-files")
self.__export = True
#
#self.__extraOpts = None
# The following for extended parent/child info -
self.__extraOpts = 'addParentRefs|addPrimaryKey'
#
self.__alldatabaseNameD = {
"ihm_dev": ["ihm_dev"],
"pdbx": ["pdbx", "pdbx_ext"],
"pdbx_core": ["pdbx_core_entity", "pdbx_core_entry", "pdbx_core_assembly", "pdbx_core_entity_instance", "pdbx_core_entity_instance_validation"],
"bird": ["bird"],
"bird_family": ["family"],
"chem_comp": ["chem_comp"],
"bird_chem_comp": ["bird_chem_comp"],
"bird_chem_comp_core": ["bird_chem_comp_core"],
}
self.__databaseNameD = {
"pdbx_core": ["pdbx_core_entity", "pdbx_core_entry", "pdbx_core_assembly", "pdbx_core_entity_instance", "pdbx_core_entity_instance_validation"],
"bird_chem_comp_core": ["bird_chem_comp_core"],
}
self.__mergeContentTypeD = {"pdbx_core": ["vrpt"]}
# self.__databaseNameD = {"chem_comp_core": ["chem_comp_core"], "bird_chem_comp_core": ["bird_chem_comp_core"]}
# self.__databaseNameD = {"ihm_dev_full": ["ihm_dev_full"]}
# self.__databaseNameD = {"pdbx_core": ["pdbx_core_entity_instance_validation"]}
# self.__databaseNameD = {"pdbx_core": ["pdbx_core_entity_monomer"]}
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def tearDown(self):
endTime = time.time()
logger.debug("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime)
def testValidateOptsRepo(self):
# schemaLevel = "min"
schemaLevel = "full"
inputPathList = None
eCount = self.__testValidateOpts(databaseNameD=self.__databaseNameD, inputPathList=inputPathList, schemaLevel=schemaLevel, mergeContentTypeD=self.__mergeContentTypeD)
logger.info("Total validation errors schema level %s : %d", schemaLevel, eCount)
self.assertLessEqual(eCount, 1)
@unittest.skip("Disable troubleshooting test")
def testValidateOptsList(self):
schemaLevel = "min"
inputPathList = self.__mU.doImport(os.path.join(HERE, "test-output", "failed-path.list"), "list")
# inputPathList = glob.glob(self.__testDirPath + "/*.cif")
if not inputPathList:
return True
databaseNameD = {"pdbx_core": ["pdbx_core_entity", "pdbx_core_entry", "pdbx_core_entity_instance", "pdbx_core_entity_instance_validation"]}
for ii, subList in enumerate(chunkList(inputPathList[::-1], 40)):
if ii < 5:
continue
eCount = self.__testValidateOpts(databaseNameD=databaseNameD, inputPathList=subList, schemaLevel=schemaLevel, mergeContentTypeD=self.__mergeContentTypeD)
logger.info("Chunk %d total validation errors schema level %s : %d", ii, schemaLevel, eCount)
# self.assertGreaterEqual(eCount, 20)
#@unittest.skip("Disable IHM troubleshooting test")
def testValidateOptsIhmRepo(self):
schemaLevel = "min"
inputPathList = None
self.__export = True
databaseNameD = {"ihm_dev_full": ["ihm_dev_full"]}
databaseNameD = {"ihm_dev": ["ihm_dev"]}
eCount = self.__testValidateOpts(databaseNameD=databaseNameD, inputPathList=inputPathList, schemaLevel=schemaLevel, mergeContentTypeD=self.__mergeContentTypeD)
logger.info("Total validation errors schema level %s : %d", schemaLevel, eCount)
# self.assertGreaterEqual(eCount, 20)
#
#@unittest.skip("Disable IHM troubleshooting test")
def testValidateOptsIhmList(self):
#schemaLevel = "full"
schemaLevel = "min"
inputPathList = glob.glob(self.__testIhmDirPath + "/*.cif")
if not inputPathList:
return True
#databaseNameD = {"ihm_dev_full": ["ihm_dev_full"]}
databaseNameD = {"ihm_dev": ["ihm_dev"]}
eCount = self.__testValidateOpts(databaseNameD=databaseNameD, inputPathList=inputPathList, schemaLevel=schemaLevel, mergeContentTypeD=self.__mergeContentTypeD)
logger.info("Total validation errors schema level %s : %d", schemaLevel, eCount)
# self.assertGreaterEqual(eCount, 20)
#
def __testValidateOpts(self, databaseNameD, inputPathList=None, schemaLevel="full", mergeContentTypeD=None):
#
eCount = 0
for databaseName in databaseNameD:
mergeContentTypes = mergeContentTypeD[databaseName] if databaseName in mergeContentTypeD else None
_ = self.__schP.makeSchemaDef(databaseName, dataTyping="ANY", saveSchema=True)
pthList = inputPathList if inputPathList else self.__rpP.getLocatorObjList(databaseName, mergeContentTypes=mergeContentTypes)
for collectionName in databaseNameD[databaseName]:
cD = self.__schP.makeSchema(databaseName, collectionName, encodingType="JSON", level=schemaLevel, saveSchema=True, extraOpts=self.__extraOpts)
#
dL, cnL = self.__testPrepDocumentsFromContainers(
pthList, databaseName, collectionName, styleType="rowwise_by_name_with_cardinality", mergeContentTypes=mergeContentTypes
)
# Raises exceptions for schema compliance.
try:
Draft4Validator.check_schema(cD)
except Exception as e:
logger.error("%s %s schema validation fails with %s", databaseName, collectionName, str(e))
#
valInfo = Draft4Validator(cD, format_checker=FormatChecker())
logger.info("Validating %d documents from %s %s", len(dL), databaseName, collectionName)
for ii, dD in enumerate(dL):
logger.debug("Schema %s collection %s document %d", databaseName, collectionName, ii)
try:
cCount = 0
#for error in sorted(valInfo.iter_errors(dD), key=str):
# logger.info("schema %s collection %s (%s) path %s error: %s", databaseName, collectionName, cnL[ii], error.path, error.message)
# logger.debug("Failing document %d : %r", ii, list(dD.items()))
# eCount += 1
# cCount += 1
#if cCount > 0:
# logger.info("schema %s collection %s container %s error count %d", databaseName, collectionName, cnL[ii], cCount)
except Exception as e:
logger.exception("Validation processing error %s", str(e))
return eCount
def __testPrepDocumentsFromContainers(self, inputPathList, databaseName, collectionName, styleType="rowwise_by_name_with_cardinality", mergeContentTypes=None):
"""Test case - create loadable PDBx data from repository files
"""
try:
sd, _, _, _ = self.__schP.getSchemaInfo(databaseName)
#
dP = DictionaryApiProviderWrapper(self.__cfgOb, self.__cachePath, useCache=False)
dictApi = dP.getApiByName(databaseName)
rP = DictMethodResourceProvider(self.__cfgOb, configName=self.__configName, cachePath=self.__cachePath, siftsAbbreviated="TEST")
dmh = DictMethodRunner(dictApi, modulePathMap=self.__modulePathMap, resourceProvider=rP)
#
dtf = DataTransformFactory(schemaDefAccessObj=sd, filterType=self.__fTypeRow)
sdp = SchemaDefDataPrep(schemaDefAccessObj=sd, dtObj=dtf, workPath=self.__cachePath, verbose=self.__verbose)
containerList = self.__rpP.getContainerList(inputPathList)
for container in containerList:
cName = container.getName()
logger.debug("Processing container %s", cName)
dmh.apply(container)
if self.__export:
savePath = os.path.join(HERE, "test-output", cName + "-with-method.cif")
#self.__mU.doExport(savePath, [container], fmt="mmcif")
#
tableIdExcludeList = sd.getCollectionExcluded(collectionName)
tableIdIncludeList = sd.getCollectionSelected(collectionName)
sliceFilter = sd.getCollectionSliceFilter(collectionName)
sdp.setSchemaIdExcludeList(tableIdExcludeList)
sdp.setSchemaIdIncludeList(tableIdIncludeList)
#
docList, containerNameList, _ = sdp.processDocuments(
containerList, styleType=styleType, filterType=self.__fTypeRow, dataSelectors=["PUBLIC_RELEASE"], sliceFilter=sliceFilter, collectionName=collectionName
)
docList = sdp.addDocumentPrivateAttributes(docList, collectionName)
docList = sdp.addDocumentSubCategoryAggregates(docList, collectionName)
#
mergeS = "-".join(mergeContentTypes) if mergeContentTypes else ""
if self.__export and docList:
# for ii, doc in enumerate(docList[:1]):
for ii, doc in enumerate(docList):
cn = containerNameList[ii]
fp = os.path.join(HERE, "test-output", "prep-%s-%s-%s-%s.json" % (cn, databaseName, collectionName, mergeS))
self.__mU.doExport(fp, [doc], fmt="json", indent=3)
logger.debug("Exported %r", fp)
#
return docList, containerNameList
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
class SchemaDefLoadercrateDbMultiTests(unittest.TestCase):
def __init__(self, methodName="runTest"):
super(SchemaDefLoadercrateDbMultiTests, self).__init__(methodName)
self.__verbose = True
self.__createFlag = True
def setUp(self):
self.__verbose = True
self.__numProc = 2
self.__fileLimit = 100
self.__chunkSize = 0
self.__workPath = os.path.join(HERE, "test-output")
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName)
self.__resourceName = "CRATE_DB"
self.__schP = SchemaProvider(self.__cfgOb,
self.__workPath,
useCache=True)
self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb,
numProc=self.__numProc,
fileLimit=self.__fileLimit,
cachePath=self.__workPath)
#
#
self.__tableIdSkipD = {
"ATOM_SITE": True,
"ATOM_SITE_ANISOTROP": True,
"__LOAD_STATUS__": True
}
self.__ioObj = IoAdapter(verbose=self.__verbose)
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def tearDown(self):
endTime = time.time()
logger.debug("Completed %s at %s (%.4f seconds)", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()),
endTime - self.__startTime)
def testConnection(self):
try:
with Connection(cfgOb=self.__cfgOb,
resourceName=self.__resourceName) as client:
self.assertNotEqual(client, None)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testSchemaCreate(self):
"""Create table schema (live) for BIRD, chemical component, and PDBx data."""
try:
sd, _, _, _ = self.__schP.getSchemaInfo("bird")
ret = self.__schemaCreate(schemaDefObj=sd)
self.assertEqual(ret, True)
#
sd, _, _, _ = self.__schP.getSchemaInfo("chem_comp")
ret = self.__schemaCreate(schemaDefObj=sd)
self.assertEqual(ret, True)
#
sd, _, _, _ = self.__schP.getSchemaInfo("pdbx")
ret = self.__schemaCreate(schemaDefObj=sd)
self.assertEqual(ret, True)
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testSchemaRemove(self):
"""Remove table schema (live) for BIRD, chemical component, and PDBx data."""
try:
sd, _, _, _ = self.__schP.getSchemaInfo("bird")
ret = self.__schemaRemove(schemaDefObj=sd)
self.assertEqual(ret, True)
#
sd, _, _, _ = self.__schP.getSchemaInfo("chem_comp")
ret = self.__schemaRemove(schemaDefObj=sd)
self.assertEqual(ret, True)
#
sd, _, _, _ = self.__schP.getSchemaInfo("pdbx")
ret = self.__schemaRemove(schemaDefObj=sd)
self.assertEqual(ret, True)
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testLoadChemCompMulti(self):
self.__testLoadFilesMulti("chem_comp")
def testLoadBirdMulti(self):
self.__testLoadFilesMulti("bird")
def testLoadPdbxMulti(self):
self.__testLoadFilesMulti("pdbx")
def __getPathList(self, fType):
pathList = []
if fType == "chem_comp":
pathList = self.__rpP.getLocatorObjList("chem_comp")
elif fType == "bird":
pathList = self.__rpP.getLocatorObjList("bird")
pathList.extend(self.__rpP.getLocatorObjList("bird_family"))
elif fType == "pdbx":
pathList = self.__rpP.getLocatorObjList("pdbx")
return pathList
def loadInsertMany(self, dataList, procName, optionsD, workingDir):
try:
_ = workingDir
ret = None
sd = optionsD["sd"]
skipD = optionsD["skip"]
ioObj = IoAdapter(verbose=self.__verbose)
logger.debug("%s pathlist %r", procName, dataList)
with Connection(cfgOb=self.__cfgOb,
resourceName=self.__resourceName) as client:
sdl = CrateDbLoader(schemaDefObj=sd,
ioObj=ioObj,
dbCon=client,
workPath=self.__workPath,
cleanUp=False,
warnings="default",
verbose=self.__verbose)
ret = sdl.load(inputPathList=dataList,
loadType="crate-insert-many",
deleteOpt="selected",
tableIdSkipD=skipD)
# all or nothing here
if ret:
return dataList, dataList, []
else:
return [], [], []
except Exception as e:
logger.info("Failing with dataList %r", dataList)
logger.exception("Failing with %s", str(e))
return [], [], []
def __testLoadFilesMulti(self, contentType):
"""Test case - create load w/insert-many all chemical component definition data files - (multiproc test)"""
numProc = self.__numProc
chunkSize = self.__chunkSize
try:
#
sd, _, _, _ = self.__schP.getSchemaInfo(contentType)
if self.__createFlag:
self.__schemaCreate(schemaDefObj=sd)
optD = {}
optD["sd"] = sd
if contentType == "pdbx":
optD["skip"] = self.__tableIdSkipD
else:
optD["skip"] = {}
#
pathList = self.__getPathList(fType=contentType)
logger.debug("Input path list %r", pathList)
mpu = MultiProcUtil(verbose=True)
mpu.setOptions(optionsD=optD)
mpu.set(workerObj=self, workerMethod="loadInsertMany")
ok, _, _, _ = mpu.runMulti(dataList=pathList,
numProc=numProc,
numResults=1,
chunkSize=chunkSize)
self.assertEqual(ok, True)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def __schemaCreate(self, schemaDefObj):
"""Test case - create table schema using schema definition"""
ret = 0
try:
tableIdList = schemaDefObj.getTableIdList()
sqlGen = SqlGenAdmin(self.__verbose, serverType="cratedb")
sqlL = []
for tableId in tableIdList:
if tableId in self.__tableIdSkipD:
continue
tableDefObj = schemaDefObj.getTable(tableId)
sqlL.extend(
sqlGen.createTableSQL(
databaseName=schemaDefObj.getVersionedDatabaseName(),
tableDefObj=tableDefObj))
logger.debug("Schema creation SQL string\n %s\n\n",
"\n".join(sqlL))
logger.info("Creating schema using database %s",
schemaDefObj.getVersionedDatabaseName())
#
with Connection(cfgOb=self.__cfgOb,
resourceName=self.__resourceName) as client:
crQ = CrateDbQuery(dbcon=client, verbose=self.__verbose)
ret = crQ.sqlCommandList(sqlCommandList=sqlL)
logger.debug("Schema create command returns %r\n", ret)
return ret
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def __schemaRemove(self, schemaDefObj):
"""Test case - remove table schema using schema definition"""
ret = 0
try:
tableIdList = schemaDefObj.getTableIdList()
sqlGen = SqlGenAdmin(self.__verbose, serverType="cratedb")
sqlL = []
for tableId in tableIdList:
if tableId in self.__tableIdSkipD:
continue
tableDefObj = schemaDefObj.getTable(tableId)
sqlL.extend(
sqlGen.dropTableSQL(
databaseName=schemaDefObj.getVersionedDatabaseName(),
tableDefObj=tableDefObj))
sqlL.extend(
sqlGen.dropTableSQL(
databaseName=schemaDefObj.getDatabaseName(),
tableDefObj=tableDefObj))
logger.debug("Schema Remove SQL string\n %s", "\n".join(sqlL))
with Connection(cfgOb=self.__cfgOb,
resourceName=self.__resourceName) as client:
crQ = CrateDbQuery(dbcon=client, verbose=self.__verbose)
ret = crQ.sqlCommandList(sqlCommandList=sqlL)
logger.debug("Schema remove command returns %r\n", ret)
return ret
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
class ChemRefEtlWorker(object):
"""Prepare and load chemical reference data collections."""
def __init__(self,
cfgOb,
cachePath,
useCache=True,
numProc=2,
chunkSize=10,
readBackCheck=False,
documentLimit=None,
verbose=False):
self.__cfgOb = cfgOb
self.__cachePath = cachePath
self.__useCache = useCache
self.__readBackCheck = readBackCheck
self.__numProc = numProc
self.__chunkSize = chunkSize
self.__documentLimit = documentLimit
#
self.__resourceName = "MONGO_DB"
self.__verbose = verbose
self.__statusList = []
self.__schP = SchemaProvider(self.__cfgOb,
self.__cachePath,
useCache=self.__useCache)
#
def __updateStatus(self, updateId, databaseName, collectionName, status,
startTimestamp):
try:
sFlag = "Y" if status else "N"
desp = DataExchangeStatus()
desp.setStartTime(tS=startTimestamp)
desp.setObject(databaseName, collectionName)
desp.setStatus(updateId=updateId, successFlag=sFlag)
desp.setEndTime()
self.__statusList.append(desp.getStatus())
return True
except Exception as e:
logger.exception("Failing with %s", str(e))
return False
def load(self, updateId, extResource, loadType="full"):
"""Load chemical reference integrated data for the input external resource-"""
try:
self.__statusList = []
desp = DataExchangeStatus()
statusStartTimestamp = desp.setStartTime()
#
if extResource == "DrugBank":
databaseName = "drugbank_core"
configName = self.__cfgOb.getDefaultSectionName()
user = self.__cfgOb.get("_DRUGBANK_AUTH_USERNAME",
sectionName=configName)
pw = self.__cfgOb.get("_DRUGBANK_AUTH_PASSWORD",
sectionName=configName)
#
dbP = DrugBankProvider(cachePath=self.__cachePath,
useCache=self.__useCache,
username=user,
password=pw)
#
crExt = ChemRefExtractor(self.__cfgOb)
idD = crExt.getChemCompAccessionMapping(extResource)
dList = dbP.getDocuments(mapD=idD)
#
logger.info("Resource %r extracted mapped document length %d",
extResource, len(dList))
logger.debug("Objects %r", dList[:2])
sD, _, collectionList, _ = self.__schP.getSchemaInfo(
databaseName)
collectionName = collectionList[
0] if collectionList else "unassigned"
indexL = sD.getDocumentIndex(collectionName, "primary")
logger.info("Database %r collection %r index attributes %r",
databaseName, collectionName, indexL)
#
collectionVersion = sD.getCollectionVersion(collectionName)
addValues = {"_schema_version": collectionVersion}
#
addValues = {}
#
dl = DocumentLoader(
self.__cfgOb,
self.__cachePath,
self.__resourceName,
numProc=self.__numProc,
chunkSize=self.__chunkSize,
documentLimit=self.__documentLimit,
verbose=self.__verbose,
readBackCheck=self.__readBackCheck,
)
#
ok = dl.load(databaseName,
collectionName,
loadType=loadType,
documentList=dList,
indexAttributeList=indexL,
keyNames=None,
addValues=addValues)
self.__updateStatus(updateId, databaseName, collectionName, ok,
statusStartTimestamp)
return True
except Exception as e:
logger.exception("Failing with %s", str(e))
return False
def getLoadStatus(self):
return self.__statusList
class CockroachDbLoaderCockroachDbTests(unittest.TestCase):
def __init__(self, methodName="runTest"):
super(CockroachDbLoaderCockroachDbTests, self).__init__(methodName)
self.__verbose = True
self.__createFlag = False
def setUp(self):
self.__verbose = True
self.__numProc = 2
self.__fileLimit = 100
self.__workPath = os.path.join(HERE, "test-output")
self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
configPath = os.path.join(TOPDIR, "rcsb", "db", "config", "exdb-config-example.yml")
configName = "site_info_configuration"
self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName)
self.__resourceName = "COCKROACH_DB"
self.__schP = SchemaProvider(self.__cfgOb, self.__workPath, useCache=True)
self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb, numProc=self.__numProc, fileLimit=self.__fileLimit, cachePath=self.__workPath)
#
self.__tableIdSkipD = {"ATOM_SITE": True, "ATOM_SITE_ANISOTROP": True}
self.__ioObj = IoAdapter(verbose=self.__verbose)
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def tearDown(self):
endTime = time.time()
logger.debug("Completed %s at %s (%.4f seconds)\n", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime)
def testConnection(self):
try:
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
self.assertNotEqual(client, None)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testSchemaCreate(self):
"""Create table schema (live) for BIRD, chemical component, and PDBx data."""
try:
sd, _, _, _ = self.__schP.getSchemaInfo("bird")
ret = self.__schemaCreate(schemaDefObj=sd)
self.assertEqual(ret, True)
#
sd, _, _, _ = self.__schP.getSchemaInfo("chem_comp")
ret = self.__schemaCreate(schemaDefObj=sd)
self.assertEqual(ret, True)
#
sd, _, _, _ = self.__schP.getSchemaInfo("pdbx")
ret = self.__schemaCreate(schemaDefObj=sd)
self.assertEqual(ret, True)
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testSchemaRemove(self):
"""Remove table schema (live) for BIRD, chemical component, and PDBx data."""
try:
sd, _, _, _ = self.__schP.getSchemaInfo("bird")
ret = self.__schemaRemove(schemaDefObj=sd)
self.assertEqual(ret, True)
#
sd, _, _, _ = self.__schP.getSchemaInfo("chem_comp")
ret = self.__schemaRemove(schemaDefObj=sd)
self.assertEqual(ret, True)
#
sd, _, _, _ = self.__schP.getSchemaInfo("pdbx")
ret = self.__schemaRemove(schemaDefObj=sd)
self.assertEqual(ret, True)
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testLoadInsertBirdReference(self):
try:
sd, _, _, _ = self.__schP.getSchemaInfo("bird")
if self.__createFlag:
self.__schemaCreate(schemaDefObj=sd)
inputPathList = self.__rpP.getLocatorObjList("bird")
inputPathList.extend(self.__rpP.getLocatorObjList("bird_family"))
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
sdl = CockroachDbLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose)
ret = sdl.load(inputPathList=inputPathList, loadType="cockroach-insert", deleteOpt="selected")
self.assertEqual(ret, True)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testLoadInsertManyBirdReference(self):
try:
sd, _, _, _ = self.__schP.getSchemaInfo("bird")
if self.__createFlag:
self.__schemaCreate(schemaDefObj=sd)
inputPathList = self.__rpP.getLocatorObjList("bird")
inputPathList.extend(self.__rpP.getLocatorObjList("bird_family"))
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
sdl = CockroachDbLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose)
ret = sdl.load(inputPathList=inputPathList, loadType="cockroach-insert-many", deleteOpt="selected")
self.assertEqual(ret, True)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testLoadInsertChemCompReference(self):
try:
sd, _, _, _ = self.__schP.getSchemaInfo("chem_comp")
if self.__createFlag:
self.__schemaCreate(schemaDefObj=sd)
inputPathList = self.__rpP.getLocatorObjList("chem_comp")
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
sdl = CockroachDbLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose)
ret = sdl.load(inputPathList=inputPathList, loadType="cockroach-insert", deleteOpt="selected")
self.assertEqual(ret, True)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testLoadInsertManyChemCompReference(self):
try:
sd, _, _, _ = self.__schP.getSchemaInfo("chem_comp")
if self.__createFlag:
self.__schemaCreate(schemaDefObj=sd)
inputPathList = self.__rpP.getLocatorObjList("chem_comp")
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
sdl = CockroachDbLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose)
ret = sdl.load(inputPathList=inputPathList, loadType="cockroach-insert-many", deleteOpt="selected")
self.assertEqual(ret, True)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testLoadInsertPdbxExampleFiles(self):
try:
sd, _, _, _ = self.__schP.getSchemaInfo("pdbx")
if self.__createFlag:
self.__schemaCreate(schemaDefObj=sd)
inputPathList = self.__rpP.getLocatorObjList("pdbx")
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
sdl = CockroachDbLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose)
ret = sdl.load(inputPathList=inputPathList, loadType="cockroach-insert", deleteOpt="selected", tableIdSkipD=self.__tableIdSkipD)
self.assertEqual(ret, True)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testLoadInsertManyPdbxExampleFiles(self):
try:
sd, _, _, _ = self.__schP.getSchemaInfo("pdbx")
if self.__createFlag:
self.__schemaCreate(schemaDefObj=sd)
inputPathList = self.__rpP.getLocatorObjList("pdbx")
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
sdl = CockroachDbLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=client, workPath=self.__workPath, cleanUp=False, warnings="default", verbose=self.__verbose)
ret = sdl.load(inputPathList=inputPathList, loadType="cockroach-insert-many", deleteOpt="selected", tableIdSkipD=self.__tableIdSkipD)
self.assertEqual(ret, True)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def __schemaCreateSQL(self, schemaDefObj):
"""Test case - create table schema using schema definition"""
sqlL = []
try:
tableIdList = schemaDefObj.getTableIdList()
sqlGen = SqlGenAdmin(self.__verbose, serverType="CockroachDb")
dbName = schemaDefObj.getVersionedDatabaseName()
sqlL = sqlGen.createDatabaseSQL(dbName)
for tableId in tableIdList:
tableDefObj = schemaDefObj.getTable(tableId)
sqlL.extend(sqlGen.createTableSQL(databaseName=schemaDefObj.getVersionedDatabaseName(), tableDefObj=tableDefObj))
logger.debug("\nSchema creation SQL string\n %s\n\n", "\n".join(sqlL))
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
return sqlL
def __schemaCreate(self, schemaDefObj):
"""Test case - create table schema using schema definition"""
ret = 0
try:
tableIdList = schemaDefObj.getTableIdList()
sqlGen = SqlGenAdmin(self.__verbose, serverType="CockroachDb")
dbName = schemaDefObj.getVersionedDatabaseName()
sqlL = sqlGen.createDatabaseSQL(dbName)
for tableId in tableIdList:
tableDefObj = schemaDefObj.getTable(tableId)
sqlL.extend(sqlGen.createTableSQL(databaseName=schemaDefObj.getVersionedDatabaseName(), tableDefObj=tableDefObj))
logger.debug("\nSchema creation SQL string\n %s\n\n", "\n".join(sqlL))
logger.info("Creating schema using database %s", schemaDefObj.getVersionedDatabaseName())
#
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
crQ = CockroachDbQuery(dbcon=client, verbose=self.__verbose)
ret = crQ.sqlCommandList(sqlCommandList=sqlL)
# ret = crQ.sqlCommand(' '.join(sqlL))
logger.info("Schema create command returns %r\n", ret)
return ret
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def __schemaRemove(self, schemaDefObj):
"""Test case - remove table schema using schema definition"""
ret = 0
try:
dbName = schemaDefObj.getVersionedDatabaseName()
sqlGen = SqlGenAdmin(self.__verbose, serverType="CockroachDb")
sqlL = sqlGen.removeDatabaseSQL(dbName)
logger.debug("Schema Remove SQL string\n %s", "\n".join(sqlL))
with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
crQ = CockroachDbQuery(dbcon=client, verbose=self.__verbose)
ret = crQ.sqlCommandList(sqlCommandList=sqlL)
# ret = crQ.sqlCommand(' '.join(sqlL))
logger.debug("Schema remove command returns %r\n", ret)
return ret
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
class SchemaDefDataPrepTests(unittest.TestCase):
def __init__(self, methodName="runTest"):
super(SchemaDefDataPrepTests, self).__init__(methodName)
self.__loadPathList = []
self.__verbose = True
def setUp(self):
self.__isMac = platform.system() == "Darwin"
self.__excludeType = None if self.__isMac else "optional"
self.__numProc = 2
mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
self.__cachePath = os.path.join(TOPDIR, "CACHE")
self.__outputPath = os.path.join(HERE, "test-output")
self.__savedOutputPath = os.path.join(HERE, "test-saved-output")
configPath = os.path.join(TOPDIR, "rcsb", "db", "config",
"exdb-config-example.yml")
configName = "site_info_configuration"
self.__configName = configName
self.__cfgOb = ConfigUtil(configPath=configPath,
defaultSectionName=configName,
mockTopPath=mockTopPath)
self.__mU = MarshalUtil(workPath=self.__cachePath)
self.__discoveryMode = self.__cfgOb.get("DISCOVERY_MODE",
sectionName=configName,
default="local")
self.__fileLimit = 100 if self.__discoveryMode == "local" else 10
self.__schP = SchemaProvider(self.__cfgOb,
self.__cachePath,
useCache=True)
self.__rpP = RepositoryProvider(cfgOb=self.__cfgOb,
numProc=self.__numProc,
fileLimit=self.__fileLimit,
cachePath=self.__cachePath)
#
#
self.__fTypeRow = "drop-empty-attributes|drop-empty-tables|skip-max-width|convert-iterables|normalize-enums|translateXMLCharRefs"
self.__fTypeCol = "drop-empty-tables|skip-max-width|convert-iterables|normalize-enums|translateXMLCharRefs"
self.__chemCompMockLen = 24
self.__pdbxMockLen = 30
# removes timestamped data items to allow diffs.)
excludeExtras = ["rcsb_load_status"]
# excludeExtras = []
#
self.__verbose = True
self.__modulePathMap = self.__cfgOb.get(
"DICT_METHOD_HELPER_MODULE_PATH_MAP", sectionName=configName)
#
self.__exportFlag = True
self.__diffFlag = False
#
self.__simpleTestCaseList = [
{
"contentType": "chem_comp",
"mockLength": self.__chemCompMockLen,
"filterType": self.__fTypeRow,
"styleType": "rowwise_by_name",
"mergeContentTypes": None,
"rejectLength": 2,
},
{
"contentType": "chem_comp",
"mockLength": self.__chemCompMockLen,
"filterType": self.__fTypeRow,
"styleType": "rowwise_no_name",
"mergeContentTypes": None,
"rejectLength": 2,
},
{
"contentType": "chem_comp",
"mockLength": self.__chemCompMockLen,
"filterType": self.__fTypeCol,
"styleType": "columnwise_by_name",
"mergeContentTypes": None,
"rejectLength": 2,
},
{
"contentType": "chem_comp",
"mockLength": self.__chemCompMockLen,
"filterType": self.__fTypeRow,
"styleType": "rowwise_by_name",
"mergeContentTypes": None,
"rejectLength": 2,
},
{
"contentType": "pdbx_core",
"mockLength": self.__pdbxMockLen,
"filterType": self.__fTypeRow,
"styleType": "rowwise_by_name",
"mergeContentTypes": None,
"rejectLength": 0,
},
]
#
self.__fullTestCaseList = [
{
"contentType": "pdbx_core",
"mockLength": self.__pdbxMockLen,
"filterType": self.__fTypeRow,
"styleType": "rowwise_by_name_with_cardinality",
"mergeContentTypes": ["vrpt"],
"rejectLength": 0,
"excludeExtras": excludeExtras,
},
{
"contentType": "bird_chem_comp_core",
"mockLength": self.__chemCompMockLen,
"filterType": self.__fTypeRow,
"styleType": "rowwise_by_name_with_cardinality",
"mergeContentTypes": None,
"rejectLength": 2,
"excludeExtras": excludeExtras,
},
]
#
self.__fullTestCaseListA = [
{
"contentType": "pdbx_core",
"mockLength": self.__pdbxMockLen,
"filterType": self.__fTypeRow,
"styleType": "rowwise_by_name_with_cardinality",
"mergeContentTypes": ["vrpt"],
"rejectLength": 0,
"excludeExtras": excludeExtras,
},
]
#
self.__startTime = time.time()
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def tearDown(self):
unitS = "MB" if platform.system() == "Darwin" else "GB"
rusageMax = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
logger.info("Maximum resident memory size %.4f %s", rusageMax / 1.0e6,
unitS)
endTime = time.time()
logger.info("Completed %s at %s (%.4f seconds)", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()),
endTime - self.__startTime)
def __timeStep(self, msg):
endTime = time.time()
logger.info("Completed %s at %s (%.4f seconds)", msg,
time.strftime("%Y %m %d %H:%M:%S", time.localtime()),
endTime - self.__startTime)
def testSimpleSchemaDefDataPrep(self):
for tcD in self.__simpleTestCaseList:
rejectLength = 0 if self.__discoveryMode == "remote" else tcD[
"rejectLength"]
mockLength = self.__fileLimit if self.__discoveryMode == "remote" else tcD[
"mockLength"]
if tcD["contentType"] == "bird_chem_comp_core" and self.__discoveryMode == "remote":
logger.info("Skipping %r in discovery mode %r",
tcD["contentType"], self.__discoveryMode)
continue
self.__simpleSchemaDataPrep(
tcD["contentType"],
tcD["filterType"],
tcD["styleType"],
mockLength,
rejectLength=rejectLength,
mergeContentTypes=tcD["mergeContentTypes"])
def testFullSchemaDefDataPrep(self):
for tcD in self.__fullTestCaseList:
rejectLength = 0 if self.__discoveryMode == "remote" else tcD[
"rejectLength"]
mockLength = self.__fileLimit if self.__discoveryMode == "remote" else tcD[
"mockLength"]
if tcD["contentType"] == "bird_chem_comp_core" and self.__discoveryMode == "remote":
logger.info("Skipping %r in discovery mode %r",
tcD["contentType"], self.__discoveryMode)
continue
self.__fullSchemaDataPrep(
tcD["contentType"],
tcD["filterType"],
tcD["styleType"],
mockLength,
rejectLength=rejectLength,
mergeContentTypes=tcD["mergeContentTypes"],
excludeExtras=tcD["excludeExtras"],
)
def __simpleSchemaDataPrep(self,
contentType,
filterType,
styleType,
mockLength,
rejectLength=0,
dataSelectors=None,
mergeContentTypes=None):
"""Internal method for preparing file-based data NOT requiring dynamic methods, slicing, or key injection.
Args:
contentType (str): Content type name
filterType (str): List of data processing options (separated by '|') (e.g. "drop-empty-attributes|drop-empty-tables|skip-max-width|...)
styleType (str): organization of output document (e.g. rowise-by-name)
mockLength (int): Expected length of the test data for the input content type
rejectLength (int, optional): number of input data sets rejected by the dataselection criteria. Defaults to 0.
dataSelectors (list of str, optional): data selection criteria. Defaults to None.
mergeContentTypes (list of str, optional): list content types to merge with the input data set. Defaults to None. (e.g. ['vrpt'])
"""
try:
dataSelectors = dataSelectors if dataSelectors else [
"PUBLIC_RELEASE"
]
dD = self.__schP.makeSchemaDef(contentType,
dataTyping="ANY",
saveSchema=True)
_ = SchemaDefAccess(dD)
inputPathList = self.__rpP.getLocatorObjList(
contentType=contentType, mergeContentTypes=mergeContentTypes)
sd, _, _, _ = self.__schP.getSchemaInfo(databaseName=contentType,
dataTyping="ANY")
dtf = DataTransformFactory(schemaDefAccessObj=sd,
filterType=filterType)
sdp = SchemaDefDataPrep(schemaDefAccessObj=sd,
dtObj=dtf,
workPath=self.__cachePath,
verbose=self.__verbose)
#
logger.debug("For %s mock length %d length of path list %d\n",
contentType, mockLength, len(inputPathList))
self.assertGreaterEqual(len(inputPathList), mockLength)
tableDataDictList, containerNameList, rejectList = sdp.fetchDocuments(
inputPathList,
styleType=styleType,
filterType=filterType,
dataSelectors=dataSelectors)
logger.debug(
"For %s mock length %d reject length %d length of tddl list %d\n",
contentType, mockLength, rejectLength, len(tableDataDictList))
self.assertGreaterEqual(len(tableDataDictList),
mockLength - rejectLength)
self.assertGreaterEqual(len(containerNameList),
mockLength - rejectLength)
if rejectList:
logger.debug("For %s rejecting components %r", contentType,
rejectList)
#
self.assertEqual(len(rejectList), rejectLength)
fName = "simple-prep-%s-%s.json" % (contentType, styleType)
if self.__exportFlag:
fPath = os.path.join(self.__outputPath, fName)
self.__mU.doExport(fPath,
tableDataDictList,
fmt="json",
indent=3)
if self.__diffFlag:
fPath = os.path.join(self.__savedOutputPath, fName)
refDocList = self.__mU.doImport(fPath, fmt="json")
self.assertEqual(len(refDocList), len(tableDataDictList))
#
jD = diff(refDocList,
tableDataDictList,
syntax="explicit",
marshal=True)
if jD:
_, fn = os.path.split(fPath)
bn, _ = os.path.splitext(fn)
fPath = os.path.join(self.__outputPath, bn + "-diff.json")
logger.debug("jsondiff for %s %s = \n%s", contentType,
styleType,
pprint.pformat(jD, indent=3, width=100))
self.__mU.doExport(fPath, jD, fmt="json", indent=3)
self.assertEqual(len(jD), 0)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def __logDocumentOrder(self, docList):
for doc in docList:
logger.debug("keys %r", list(doc.keys()))
def __filterDocuments(self, docList, excludeList=None):
excludeList = excludeList if excludeList else []
for doc in docList:
for excl in excludeList:
if excl in doc:
del doc[excl]
def __fullSchemaDataPrep(self,
contentType,
filterType,
styleType,
mockLength,
rejectLength=0,
dataSelectors=None,
mergeContentTypes=None,
excludeExtras=None):
"""Internal method for preparing file-based data requiring dynamic methods, slicing, or key injection.
Args:
contentType (str): Content type name
filterType (str): List of data processing options (separated by '|') (e.g. "drop-empty-attributes|drop-empty-tables|skip-max-width|...)
styleType (str): organization of output document (e.g. rowise-by-name)
mockLength (int): Expected length of the test data for the input content type
rejectLength (int, optional): number of input data sets rejected by the dataselection criteria. Defaults to 0.
dataSelectors (list of str, optional): data selection criteria. Defaults to None.
mergeContentTypes (list of str, optional): list content types to merge with the input data set. Defaults to None. (e.g. ['vrpt'])
"""
try:
excludeExtras = excludeExtras if excludeExtras else []
_ = mockLength
_ = rejectLength
dD = self.__schP.makeSchemaDef(contentType,
dataTyping="ANY",
saveSchema=True)
_ = SchemaDefAccess(dD)
inputPathList = self.__rpP.getLocatorObjList(
contentType=contentType, mergeContentTypes=mergeContentTypes)
sd, _, collectionNameList, _ = self.__schP.getSchemaInfo(
databaseName=contentType, dataTyping="ANY")
#
dP = DictionaryApiProviderWrapper(self.__cachePath,
cfgOb=self.__cfgOb,
configName=self.__configName,
useCache=True)
dictApi = dP.getApiByName(contentType)
#
rP = DictMethodResourceProvider(
self.__cfgOb,
configName=self.__configName,
cachePath=self.__cachePath,
restoreUseStash=False,
restoreUseGit=True,
providerTypeExclude=self.__excludeType,
)
dmh = DictMethodRunner(dictApi,
modulePathMap=self.__modulePathMap,
resourceProvider=rP)
#
dtf = DataTransformFactory(schemaDefAccessObj=sd,
filterType=filterType)
sdp = SchemaDefDataPrep(schemaDefAccessObj=sd,
dtObj=dtf,
workPath=self.__cachePath,
verbose=self.__verbose)
containerList = self.__rpP.getContainerList(inputPathList)
for container in containerList:
cName = container.getName()
logger.debug("Processing container %s", cName)
dmh.apply(container)
#
for collectionName in collectionNameList:
tableIdExcludeList = sd.getCollectionExcluded(collectionName)
tableIdIncludeList = sd.getCollectionSelected(collectionName)
sliceFilter = sd.getCollectionSliceFilter(collectionName)
sdp.setSchemaIdExcludeList(tableIdExcludeList)
sdp.setSchemaIdIncludeList(tableIdIncludeList)
#
docList, _, _ = sdp.processDocuments(
containerList,
styleType=styleType,
sliceFilter=sliceFilter,
filterType=filterType,
dataSelectors=dataSelectors,
collectionName=collectionName)
docList = sdp.addDocumentPrivateAttributes(
docList, collectionName)
docList = sdp.addDocumentSubCategoryAggregates(
docList, collectionName)
# Special exclusions for the test harness. (removes timestamped data items to allow diffs.)
self.__filterDocuments(docList, excludeExtras)
mergeS = "-".join(
mergeContentTypes) if mergeContentTypes else ""
fName = "full-prep-%s-%s-%s-%s.json" % (
contentType, collectionName, mergeS, styleType)
if self.__exportFlag:
self.__logDocumentOrder(docList)
fPath = os.path.join(self.__outputPath, fName)
self.__mU.doExport(fPath, docList, fmt="json", indent=3)
logger.debug("Exported %r", fPath)
#
if self.__diffFlag:
fPath = os.path.join(self.__savedOutputPath, fName)
refDocList = self.__mU.doImport(fPath, fmt="json")
self.assertEqual(len(refDocList), len(docList))
logger.debug("For %s %s len refDocList %d", contentType,
collectionName, len(refDocList))
logger.debug("For %s %s len docList %d", contentType,
collectionName, len(docList))
jD = diff(refDocList,
docList,
syntax="explicit",
marshal=True)
if jD:
_, fn = os.path.split(fPath)
bn, _ = os.path.splitext(fn)
fPath = os.path.join(self.__outputPath,
bn + "-diff.json")
logger.debug("jsondiff for %s %s = \n%s", contentType,
collectionName,
pprint.pformat(jD, indent=3, width=100))
self.__mU.doExport(fPath, jD, fmt="json", indent=3)
self.assertEqual(len(jD), 0)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()