def testBirdBatchInsertImport(self):
"""Test case - import loadable data via SQL inserts -"""
startTime = time.time()
self.__lfh.write(
"\nStarting BirdLoaderTests testBirdBatchInsertImport at %s\n" %
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
try:
self.testPrdPathList()
bsd = BirdSchemaDef()
sml = SchemaDefLoader(schemaDefObj=bsd,
verbose=self.__verbose,
log=self.__lfh)
self.__lfh.write("Length of path list %d\n" %
len(self.__loadPathList))
#
tableDataDict, _containerNameList = sml.fetch(self.__loadPathList)
databaseName = bsd.getDatabaseName()
tableIdList = bsd.getTableIdList()
myQ = MyDbQuery(dbcon=self.__dbCon,
verbose=self.__verbose,
log=self.__lfh)
myAd = MyDbAdminSqlGen(self.__verbose, self.__lfh)
#
for tableId in tableIdList:
tableDefObj = bsd.getTable(tableId)
tableName = tableDefObj.getName()
tableAttributeIdList = tableDefObj.getAttributeIdList()
tableAttributeNameList = tableDefObj.getAttributeNameList()
sqlL = []
if tableId in tableDataDict:
rowList = tableDataDict[tableId]
for row in rowList:
vList = []
aList = []
for tid, nm in zip(tableAttributeIdList,
tableAttributeNameList):
if len(row[tid]) > 0 and row[id] != r"\N":
vList.append(row[tid])
aList.append(nm)
sqlL.append(
(myAd.insertTemplateSQL(databaseName, tableName,
aList), vList))
ok = myQ.sqlBatchTemplateCommand(sqlL)
if self.__verbose and not ok:
# self.__lfh.write("\n\n+ERROR batch insert fails for table %s row %r\n" % (tableName,sqlL))
pass
except: # noqa: E722 pylint: disable=bare-except
traceback.print_exc(file=self.__lfh)
self.fail()
endTime = time.time()
self.__lfh.write(
"\nCompleted BirdLoaderTests testBirdBatchInsertImport at %s (%.2f seconds)\n"
% (time.strftime("%Y %m %d %H:%M:%S",
time.localtime()), endTime - startTime))
def testMakeLoadPrdFiles(self):
"""Test case - for loading BIRD definition data files"""
self.__lfh.write("\nStarting BirdLoaderTests resrMakeLoadPrdFiles\n")
startTime = time.time()
try:
self.testPrdPathList()
bsd = BirdSchemaDef()
sml = SchemaDefLoader(schemaDefObj=bsd,
verbose=self.__verbose,
log=self.__lfh)
self.__lfh.write("Length of path list %d\n" %
len(self.__loadPathList))
_containerNameList, self.__tddFileList = sml.makeLoadFiles(
self.__loadPathList)
for tId, tPath in self.__tddFileList:
self.__lfh.write("\nCreate loadable file %s %s\n" %
(tId, tPath))
except: # noqa: E722 pylint: disable=bare-except
traceback.print_exc(file=self.__lfh)
self.fail()
endTime = time.time()
self.__lfh.write(
"\nCompleted BirdLoaderTests testMakeLoadPrdFiles at %s (%.2f seconds)\n"
% (time.strftime("%Y %m %d %H:%M:%S",
time.localtime()), endTime - startTime))
def testLoadPdbxFiles(self):
startTime = time.time()
self.__lfh.write("\nStarting SchemaDefLoaderDbTest testLoadPdbxFiles at %s\n" % time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
try:
sd = PdbxSchemaDef()
self.__schemaCreate(schemaDefObj=sd)
inputPathList = self.__getPdbxPathList()
sdl = SchemaDefLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=self.__dbCon, workPath=".", cleanUp=False, warnings="default", verbose=self.__verbose, log=self.__lfh)
sdl.load(inputPathList=inputPathList, loadType="batch-insert", deleteOpt="all")
except: # noqa: E722 pylint: disable=bare-except
traceback.print_exc(file=self.__lfh)
self.fail()
endTime = time.time()
self.__lfh.write("\nCompleted SchemaDefLoaderDbTest testLoadPdbxFiles at %s (%.2f seconds)\n" % (time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - startTime))
def loadBatchFilesMulti(self, dataList, procName, optionsD, workingDir): # pylint: disable=unused-argument
ccsd = ChemCompSchemaDef()
self.open(dbName=ccsd.getDatabaseName())
sdl = SchemaDefLoader(schemaDefObj=ccsd,
ioObj=self.__ioObj,
dbCon=self.__dbCon,
workPath=".",
cleanUp=False,
warnings="default",
verbose=self.__verbose,
log=self.__lfh)
#
sdl.loadBatchFiles(loadList=dataList,
containerNameList=None,
deleteOpt=None)
self.close()
return dataList, dataList, []
def testLoadBirdReferenceWithSequence(self):
startTime = time.time()
self.__lfh.write("\nStarting SchemaDefLoaderDbTest testLoadBirdReferenceWithSequence at %s\n" % time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
try:
sd = BirdSchemaDef()
self.__schemaCreate(schemaDefObj=sd)
#
prd = PdbxPrdIo(verbose=self.__verbose, log=self.__lfh)
prd.setCachePath(self.__birdCachePath)
pathList = prd.makeDefinitionPathList()
#
for pth in pathList:
prd.setFilePath(pth)
self.__lfh.write("PRD repository read completed\n")
#
prdU = PdbxPrdUtils(prd, verbose=self.__verbose, log=self.__lfh)
_rD = prdU.getComponentSequences(addCategory=True) # noqa: F841
#
#
prdFam = PdbxFamilyIo(verbose=self.__verbose, log=self.__lfh)
prdFam.setCachePath(self.__birdFamilyCachePath)
familyPathList = prdFam.makeDefinitionPathList()
#
for pth in familyPathList:
prdFam.setFilePath(pth)
self.__lfh.write("Family repository read completed\n")
#
# combine containers -
containerList = prd.getCurrentContainerList()
containerList.extend(prdFam.getCurrentContainerList())
#
# Run loader on container list --
#
sdl = SchemaDefLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=self.__dbCon, workPath=".", cleanUp=False, warnings="error", verbose=self.__verbose, log=self.__lfh)
sdl.load(containerList=containerList, loadType="batch-file", deleteOpt="selected")
except: # noqa: E722 pylint: disable=bare-except
traceback.print_exc(file=self.__lfh)
self.fail()
endTime = time.time()
self.__lfh.write(
"\nCompleted SchemaDefLoaderDbTest testLoadBirdReferenceWithSequence at %s (%.2f seconds)\n"
% (time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - startTime)
)
def testLoadFile(self):
"""Test case - for loading BIRD definition data files"""
self.__lfh.write("\nStarting SchemaDefLoaderTests testLoadFile\n")
try:
bsd = BirdSchemaDef()
sml = SchemaDefLoader(schemaDefObj=bsd,
ioObj=self.__ioObj,
dbCon=None,
workPath=TESTOUTPUT,
cleanUp=False,
warnings="default",
verbose=self.__verbose,
log=self.__lfh)
containerNameList, tList = sml.makeLoadFiles(self.__loadPathList)
except: # noqa: E722 pylint: disable=bare-except # pragma: no cover
traceback.print_exc(file=self.__lfh)
self.fail()
self.assertNotEqual(containerNameList, [],
"Loading files %s" % self.__loadPathList)
for tId, fn in tList:
self.__lfh.write("\nCreated table %s load file %s\n" % (tId, fn))
def testLoadFiles(self):
"""Test case - create batch load files for all chemical component definition data files -"""
self.__lfh.write("\nStarting ChemCompLoaderTests testLoadFiles\n")
startTime = time.time()
try:
ccsd = ChemCompSchemaDef()
sml = SchemaDefLoader(schemaDefObj=ccsd,
ioObj=self.__ioObj,
dbCon=None,
workPath=".",
cleanUp=False,
warnings="default",
verbose=self.__verbose,
log=self.__lfh)
pathList = self.__makeComponentPathList()
containerNameList, tList = sml.makeLoadFiles(pathList,
append=False)
for tId, fn in tList:
self.__lfh.write("\nCreated table %s load file %s\n" %
(tId, fn))
#
endTime1 = time.time()
self.__lfh.write("\nBatch files created in %.2f seconds\n" %
(endTime1 - startTime))
self.open(dbName=ccsd.getDatabaseName())
sdl = SchemaDefLoader(schemaDefObj=ccsd,
ioObj=self.__ioObj,
dbCon=self.__dbCon,
workPath=".",
cleanUp=False,
warnings="default",
verbose=self.__verbose,
log=self.__lfh)
sdl.loadBatchFiles(loadList=tList,
containerNameList=containerNameList,
deleteOpt="all")
self.close()
endTime2 = time.time()
self.__lfh.write("\nLoad completed in %.2f seconds\n" %
(endTime2 - endTime1))
except: # noqa: E722 pylint: disable=bare-except
traceback.print_exc(file=self.__lfh)
self.fail()
endTime = time.time()
self.__lfh.write(
"\nCompleted ChemCompLoaderTests testLoadFiles at %s (%.2f seconds)\n"
% (time.strftime("%Y %m %d %H:%M:%S",
time.localtime()), endTime - startTime))
def testReloadBirdReference(self):
startTime = time.time()
self.__lfh.write("\nStarting SchemaDefLoaderDbTest testReloadBirdReference at %s\n" % time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
try:
sd = BirdSchemaDef()
self.__schemaCreate(schemaDefObj=sd)
inputPathList = self.__getPrdPathList()
sdl = SchemaDefLoader(schemaDefObj=sd, ioObj=self.__ioObj, dbCon=self.__dbCon, workPath=".", cleanUp=False, warnings="default", verbose=self.__verbose, log=self.__lfh)
sdl.load(inputPathList=inputPathList, loadType="batch-file")
#
self.__lfh.write("\n\n\n+INFO BATCH FILE RELOAD TEST --------------------------------------------\n")
sdl.load(inputPathList=inputPathList, loadType="batch-file", deleteOpt="all")
self.__lfh.write("\n\n\n+INFO BATCH INSERT RELOAD TEST --------------------------------------------\n")
sdl.load(inputPathList=inputPathList, loadType="batch-file", deleteOpt="selected")
except: # noqa: E722 pylint: disable=bare-except
traceback.print_exc(file=self.__lfh)
self.fail()
endTime = time.time()
self.__lfh.write(
"\nCompleted SchemaDefLoaderDbTest testReloadBirdReference at %s (%.2f seconds)\n" % (time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - startTime)
)
def testLoadFilesMulti(self):
"""Test case - create batch load files for all chemical component definition data files - (multiproc test)"""
self.__lfh.write("\nStarting ChemCompLoaderTests testLoadFilesMulti\n")
startTime = time.time()
numProc = 8
try:
dataS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
dataList = [a for a in dataS]
mpu = MultiProcUtil(verbose=True)
mpu.set(workerObj=self, workerMethod="makeComponentPathListMulti")
_ok, _failList, retLists, _diagList = mpu.runMulti(
dataList=dataList, numProc=numProc, numResults=1)
pathList = retLists[0]
endTime0 = time.time()
self.__lfh.write("\nPath list length %d in %.2f seconds\n" %
(len(pathList), endTime0 - startTime))
# self.__lfh.write("\nPath list %r\n" % pathList[:20])
# pathList=self.__makeComponentPathList()
ccsd = ChemCompSchemaDef()
sml = SchemaDefLoader(schemaDefObj=ccsd,
ioObj=self.__ioObj,
dbCon=None,
workPath=".",
cleanUp=False,
warnings="default",
verbose=self.__verbose,
log=self.__lfh)
#
mpu = MultiProcUtil(verbose=True)
mpu.set(workerObj=sml, workerMethod="makeLoadFilesMulti")
_ok, _failList, retLists, _diagList = mpu.runMulti(
dataList=pathList, numProc=numProc, numResults=2)
#
containerNameList = retLists[0]
tList = retLists[1]
for tId, fn in tList:
self.__lfh.write("\nCreated table %s load file %s\n" %
(tId, fn))
#
endTime1 = time.time()
self.__lfh.write("\nBatch files created in %.2f seconds\n" %
(endTime1 - endTime0))
self.open(dbName=ccsd.getDatabaseName())
sdl = SchemaDefLoader(schemaDefObj=ccsd,
ioObj=self.__ioObj,
dbCon=self.__dbCon,
workPath=".",
cleanUp=False,
warnings="default",
verbose=self.__verbose,
log=self.__lfh)
#
for tId, fn in tList:
sdl.delete(tId,
containerNameList=containerNameList,
deleteOpt="all")
self.close()
#
mpu = MultiProcUtil(verbose=True)
mpu.set(workerObj=self, workerMethod="loadBatchFilesMulti")
_ok, _failList, retLists, _diagList = mpu.runMulti(dataList=tList,
numProc=numProc,
numResults=1)
endTime2 = time.time()
self.__lfh.write("\nLoad completed in %.2f seconds\n" %
(endTime2 - endTime1))
except: # noqa: E722 pylint: disable=bare-except
traceback.print_exc(file=self.__lfh)
self.fail()
endTime = time.time()
self.__lfh.write(
"\nCompleted ChemCompLoaderTests testLoadFilesMulti at %s (%.2f seconds)\n"
% (time.strftime("%Y %m %d %H:%M:%S",
time.localtime()), endTime - startTime))
## # File: BirdLoaderTests.py # Author: J. Westbrook # Date: 9-Jan-2013 # Version: 0.001 # # Updates: # 11-Jan-2013 jdw revise treatment of null values in inserts. # # ## """ Tests for creating and loading BIRD rdbms database using PDBx/mmCIF data files and external schema definition. These test database connections deferring to authentication details defined in the environment. See class MyDbConnect() for the environment requirements. SchemaDefLoader() uses default native Python IoAdapter. """ __docformat__ = "restructuredtext en" __author__ = "John Westbrook" __email__ = "*****@*****.**" __license__ = "Creative Commons Attribution 3.0 Unported" __version__ = "V0.01" import sys import os import time