def main(entryId, *extraArgList):
"""inputDir may be a directory or a url. A url needs to start with http://.
"""
fastestTest = False # default: False
htmlOnly = False # default: False but enable it for faster runs without some actual data.
doWhatif = True # disables whatif actual run
doProcheck = True
doWattos = True
doQueeny = True
doTalos = True
tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten.
# NB leave this set to True or modify code below.
removeCcpnDirectory = 1 # perhaps not so in the future.
modelCount = None # default setting is None
# ranges = None
if fastestTest:
modelCount = 2 # if this is more and there is only one model present it leads to an error message.
htmlOnly = True
doWhatif = False
doProcheck = False
doWattos = False
doQueeny = False
doTalos = False
forceRedo = True
forceRetrieveInput = True
nTmessage(header)
nTmessage(getStartMessage())
# Sync below code with nrgCing#createToposTokens
expectedArgumentList = """
verbosity inputDir outputDir
pdbConvention restraintsConvention archiveType projectType
storeCING2db ranges filterTopViolations filterVasco
singleCoreOperation
""".split()
expectedNumberOfArguments = len(expectedArgumentList)
if len(extraArgList) != expectedNumberOfArguments:
nTmessage("consider updating code to include all sequential parameters: %s" % str(expectedArgumentList))
if len(extraArgList) > expectedNumberOfArguments:
nTerror("Got arguments: " + str(extraArgList))
nTerror("Failed to get expected number of arguments: %d got %d" % (
expectedNumberOfArguments, len(extraArgList)))
nTerror("Expected arguments: %s" % expectedArgumentList)
return True
# end if
# end if
entryCodeChar2and3 = entryId[1:3]
cing.verbosity = int( extraArgList[IDX_VERBOSITY] )
inputDir = extraArgList[IDX_INPUT]
outputDir = os.path.join(extraArgList[IDX_OUTPUT], DATA_STR, entryCodeChar2and3, entryId)
pdbConvention = extraArgList[IDX_PDB] #@UnusedVariable
restraintsConvention = extraArgList[IDX_RESTRAINTS]
archiveType = extraArgList[IDX_ARCHIVE] # Only used for deriving the input location not the output.
projectType = extraArgList[IDX_PROJECT_TYPE]
storeCING2db = stringMeansBooleanTrue( getDeepByKeysOrAttributes(extraArgList, IDX_STORE_DB))
ranges = getDeepByKeysOrAttributes(extraArgList, IDX_RANGES)
filterTopViolations = getDeepByKeysOrAttributes(extraArgList, IDX_FILTER_TOP)
if filterTopViolations:
filterTopViolations = int(filterTopViolations) # change '0' to 0
filterVasco = getDeepByKeysOrAttributes(extraArgList, IDX_FILTER_VASCO)
if filterVasco:
filterVasco = int(filterVasco)
else:
filterVasco = 1 # Default should be True
# end if
singleCoreOperation = getDeepByKeysOrAttributes(extraArgList, IDX_SINGLE_CORE_OPERATION )
if singleCoreOperation:
singleCoreOperation = int(singleCoreOperation)
else:
singleCoreOperation = 0 # Default should be True
# end if
if archiveType == ARCHIVE_TYPE_FLAT:
pass # default
elif archiveType == ARCHIVE_TYPE_BY_ENTRY:
inputDir = os.path.join(inputDir, entryId)
elif archiveType == ARCHIVE_TYPE_BY_CH23:
inputDir = os.path.join(inputDir, entryCodeChar2and3)
elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY:
inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId)
isRemoteOutputDir = False
if '@' in outputDir:
isRemoteOutputDir = True
# end if
# vc = vCing('.') # argument is a fake master_ssh_url not needed here.
archive_id = getArchiveIdFromDirectoryName( outputDir )
nTdebug("Using program arguments:")
nTdebug("inputDir: %s" % inputDir)
nTdebug("outputDir: %s" % outputDir)
nTdebug("pdbConvention: %s" % pdbConvention)
nTdebug("restraintsConvention: %s" % restraintsConvention)
nTdebug("archiveType: %s" % archiveType)
nTdebug("projectType: %s" % projectType)
nTdebug("storeCING2db: %s" % storeCING2db)
nTdebug("ranges: %s" % ranges)
nTdebug("filterTopViolations: %s" % filterTopViolations)
nTdebug("filterVasco: %s" % filterVasco)
nTdebug("singleCoreOperation: %s" % singleCoreOperation)
nTdebug("")
nTdebug("Using derived settings:")
nTdebug("modelCount: %s" % modelCount)
nTdebug("isRemoteOutputDir: %s" % isRemoteOutputDir)
nTdebug("archive_id: %s" % archive_id)
# For NMR_REDO required as most efficient.
if singleCoreOperation:
setToSingleCoreOperation()
# presume the directory still needs to be created.
cingEntryDir = entryId + ".cing"
if os.path.isdir(cingEntryDir):
if forceRedo:
nTmessage("Enforcing a redo")
rmtree(cingEntryDir)
else:
mainIndexFile = os.path.join(cingEntryDir, "index.html")
isDone = os.path.isfile(mainIndexFile)
if isDone:
nTmessage("SKIPPING ENTRY ALREADY DONE")
return
nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.")
rmtree(cingEntryDir)
# end if.
# end if.
if isRemoteOutputDir:
os.chdir(cingDirTmp)
else:
os.chdir(outputDir)
project = Project(entryId)
if project.removeFromDisk():
nTerror("Failed to remove existing project (if present)")
return True
# end if.
formatFileName = '%s.tgz'
if projectType == PROJECT_TYPE_CING:
formatFileName = '%s.cing.tgz'
elif projectType == PROJECT_TYPE_PDB:
formatFileName = 'pdb%s.ent.gz'
fileNameTgz = formatFileName % entryId
# nTdebug("fileNameTgz: %s" % fileNameTgz)
allowedInputProtocolList = 'http file ssh'.split()
inputProtocal = string.split( inputDir, ':' )[0]
if inputProtocal in allowedInputProtocolList:
stillToRetrieve = False
if os.path.exists(fileNameTgz):
if forceRetrieveInput:
os.unlink(fileNameTgz)
stillToRetrieve = True
# end if
else:
stillToRetrieve = True
# end if
if stillToRetrieve:
retrieveTgzFromUrl(entryId, inputDir, archiveType=archiveType, formatFileName=formatFileName)
# end if
if not os.path.exists(fileNameTgz):
nTerror("Tgz should already have been present skipping entry")
return
# end if
else:
nTdebug("Entry not retrieved which might be normal in some situations.")
# end if.
if projectType == PROJECT_TYPE_CING:
# Needs to be copied because the open method doesn't take a directory argument..
# fullFileNameTgz = os.path.join(inputDir, fileNameTgz)
# shutil.copy(fullFileNameTgz, '.')
project = Project.open(entryId, status='old')
if not project:
nTerror("Failed to init old project")
return True
elif projectType == PROJECT_TYPE_CCPN:
project = Project.open(entryId, status='new')
if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount):
nTerror("Failed to init project from ccpn")
return True
elif projectType == PROJECT_TYPE_PDB:
project = Project.open(entryId, status='new')
pdbFilePath = entryId + ".pdb"
gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True)
project.initPDB(pdbFile=pdbFilePath, convention=IUPAC, nmodels=modelCount)
# if tmpPdbFile:
if True:
nTdebug("Removing tmp: %s" % pdbFilePath)
os.unlink(pdbFilePath)
# end if
if ranges is not None:
project.molecule.setRanges(ranges)
# end if
if archive_id:
project.molecule.setArchiveId(archive_id)
# end if
project.molecule.superpose(ranges=ranges)
if filterTopViolations and not project.filterHighRestraintViol():
nTerror("Failed to filterHighRestraintViol")
####> MAIN UTILITY HERE
if 0: # DEFAULT 0
project.save()
if project.validate(htmlOnly=htmlOnly, ranges=ranges, doProcheck=doProcheck, doWhatif=doWhatif,
doWattos=doWattos, doQueeny = doQueeny, doTalos=doTalos, filterVasco = filterVasco ):
nTerror("Failed to validate project read")
return True
# end if filterVasco
# Write a single PDB file containing all models
# according to IUPAC conventions
project.export2PDB()
project.save()
if storeCING2db and archive_id:
# Does require:
#from cing.PluginCode.sqlAlchemy import csqlAlchemy
# and should never crash run.
# archive_id = ARCHIVE_DEV_NRG_ID
# if isProduction:
# archive_id = ARCHIVE_NRG_ID
try:
if doStoreCING2db( entryId, archive_id, project=project):
nTerror("Failed to store CING project's data to DB but continuing.")
except:
nTtracebackError()
nTerror("Failed to store CING project's data due to above traceback error.")
if projectType == PROJECT_TYPE_CCPN:
# fileNameTgz = entryId + '.tgz'
os.unlink(fileNameTgz) # temporary ccpn tgz
if removeCcpnDirectory:
rmdir(entryId) # ccpn dir may contain vasco info.
if tgzCing:
directoryNameCing = entryId + ".cing"
tgzFileNameCing = directoryNameCing + ".tgz"
if os.path.exists(tgzFileNameCing):
nTwarning("Overwriting: " + tgzFileNameCing)
cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing)
nTdebug("cmd: %s" % cmd)
# do_cmd(cmd)
status, result = commands.getstatusoutput(cmd)
if status:
nTerror("Failed to tar status: %s with result %s" % (status, result))
return True
if isRemoteOutputDir:
if putFileBySsh(tgzFileNameCing, outputDir, ntriesMax = 2):
nTerror("Failed to send File By Scp status: %s with result %s" % (status, result))
nTerror("Maintaining results.")
return True
# end if
nTmessage("Removing tgz result: %s" % tgzFileNameCing)
os.remove(tgzFileNameCing)
nTmessage("Removing cing dir itself: %s" % directoryNameCing)
rmdir(directoryNameCing)
else: # do NOT remove local copy
pass
def test_ccpn(self):
# cing.verbosity = verbosityDebug
# if you have a local copy you can use it; make sure to adjust the path setting below.
fastestTest = True # DEFAULT: True. Not passed to the validate routine in order to customize checks for speed.
modelCount = 99 # DEFAULT: 99
redoFromCingProject = False # DEFAULT: False
htmlOnly = False # DEFAULT: False # default is False but enable it for faster runs without some actual data.
doWhatif = True # DEFAULT: True # disables whatif actual run
doProcheck = True # DEFAULT: True
doWattos = True # DEFAULT: True
doQueeny = True # DEFAULT: True
doTalos = True # DEFAULT: True
filterVasco = True # DEFAULT: True
filterTopViolations = True # DEFAULT: True
useNrgArchive = False # DEFAULT: False
ranges = CV_STR # DEFAULT: CV_STR
# ranges='173-177'
# ranges='6-13,29-45' # 1bus
doSwapCheck = False
doRestoreCheck = False
doStoreCheck = False # DEFAULT: False Requires sqlAlchemy
doSave = not redoFromCingProject # DEFAULT: False Requires sqlAlchemy
if fastestTest:
modelCount = 2 # DEFAULT 2
# redoFromCingProject = False
htmlOnly = True # DEFAULT: True
doWhatif = False # DEFAULT: False
doProcheck = False # DEFAULT: False
doWattos = False # DEFAULT: False
doQueeny = False # DEFAULT: False
doTalos = False # DEFAULT: False
filterVasco = False # DEFAULT: False
doRestoreCheck = False # DEFAULT: False
doStoreCheck = False # DEFAULT: False
if redoFromCingProject:
useNrgArchive = False
doWhatif = False
doProcheck = False
doWattos = False
doTalos = False
cingDirTmpTest = os.path.join(cingDirTmp, getCallerName())
mkdirs(cingDirTmpTest)
self.failIf(os.chdir(cingDirTmpTest),
msg="Failed to change to test directory for files: " +
cingDirTmpTest)
for i, entryId in enumerate(AllChecks.entryList):
if i:
nTmessage('\n\n')
if redoFromCingProject:
project = Project.open(entryId, status='old')
else:
project = Project.open(entryId, status='new')
self.assertTrue(project, 'Failed opening project: ' + entryId)
if useNrgArchive: # default is False
# inputArchiveDir = os.path.join('/Library/WebServer/Documents/NRG-CING/recoordSync', entryId)
# Mounted from nmr.cmbi.ru.nl
# inputArchiveDir = os.path.join('/Volumes/tria1/Library/WebServer/Documents/NRG-CING/recoordSync', entryId)
inputArchiveDir = os.path.join(
'/Users/jd/ccpn_tmp/data/recoord', entryId)
else:
inputArchiveDir = os.path.join(cingDirTestsData, "ccpn")
ccpnFile = os.path.join(inputArchiveDir, entryId + ".tgz")
if not os.path.exists(ccpnFile):
ccpnFile = os.path.join(inputArchiveDir,
entryId + ".tar.gz")
if not os.path.exists(ccpnFile):
self.fail("Neither %s or the .tgz exist" % ccpnFile)
if doSwapCheck: # Need to start with ccpn without loading into CING api yet.
if os.path.exists(entryId):
nTmessage("Removing previous directory: %s" % entryId)
rmtree(entryId)
do_cmd("tar -xzf " +
ccpnFile) # will extract to local dir.
if os.path.exists('linkNmrStarData'):
nTmessage(
"Renaming standard directory linkNmrStarData to entry: %s"
% entryId)
os.rename('linkNmrStarData', entryId)
ccpnProject = loadProject(entryId)
if not ccpnProject:
self.fail("Failed to read project: %s" % entryId)
# constraintsHandler = ConstraintsHandler()
nmrConstraintStore = ccpnProject.findFirstNmrConstraintStore(
)
structureEnsemble = ccpnProject.findFirstStructureEnsemble(
)
numSwapCheckRuns = 2
if nmrConstraintStore:
if structureEnsemble:
swapCheck(nmrConstraintStore, structureEnsemble,
numSwapCheckRuns)
else:
nTmessage(
"Failed to find structureEnsemble; skipping swapCheck"
)
else:
nTmessage(
"Failed to find nmrConstraintStore; skipping swapCheck"
)
# constraintsHandler.swapCheck(nmrConstraintStore, structureEnsemble, numSwapCheckRuns)
nTmessage('Saving to new path')
# checkValid=True,
saveProject(ccpnProject,
newPath=entryId,
removeExisting=True)
ccpnFile = entryId # set to local dir now.
# end if doSwapCheck
self.assertTrue(
project.initCcpn(ccpnFolder=ccpnFile,
modelCount=modelCount))
# if doSave:
# self.assertTrue(project.save())
if False:
ranges = "173-183"
# residueOfInterest = range(171,174)
# for residue in project.molecule.A.allResidues():
# if residue.resNum not in residueOfInterest:
# # nTmessage("Removing residue of no interest")
# project.molecule.A.removeResidue(residue)
if False:
ccpnProject = project.ccpn
printSequenceFromCcpnProject(ccpnProject)
if True:
self.assertFalse(project.molecule.setRanges(ranges))
nTdebug('In test_ccpn.py: ranges: %s' %
str(project.molecule.ranges))
project.molecule.rangesToMmCifRanges(ranges)
if True:
self.assertFalse(
project.validate(
htmlOnly=htmlOnly,
ranges=ranges,
# fastestTest=fastestTest, # disabled
doProcheck=doProcheck,
doWhatif=doWhatif,
doWattos=doWattos,
doQueeny=doQueeny,
doTalos=doTalos,
filterVasco=filterVasco,
filterTopViolations=filterTopViolations))
if doWattos and False:
mol = project.molecule
completenessMol = mol.getDeepByKeys(
WATTOS_STR, COMPLCHK_STR, VALUE_LIST_STR)
nTdebug("completenessMol: %s" % completenessMol)
for res in mol.allResidues():
completenessRes = res.getDeepByKeys(
WATTOS_STR, COMPLCHK_STR, VALUE_LIST_STR)
nTdebug("%s: %s" % (res, completenessRes))
# end for
# end if
color = project.molecule.getRogColor()
nTdebug("The color of this molecule is: %s" % color)
self.assertFalse(color == COLOR_RED) # ;-)
# end if
# self.assertTrue(project.exportValidation2ccpn())
# self.assertFalse(project.removeCcpnReferences())
# Do not leave the old CCPN directory laying around since it might get added to by another test.
# if os.path.exists(entryId):
# self.assertFalse(shutil.rmtree(entryId))
if False:
self.assertTrue(project.save())
self.assertTrue(project.saveCcpn(entryId))
if doRestoreCheck:
del project
project = Project.open(entryId, status='old')
self.assertTrue(project,
'Failed reopening project: ' + entryId)
if doStoreCheck:
# # Does require below import which is used here to trigger import warning in case it's not installed.
# pylint: disable=W0612
from cing.PluginCode.sqlAlchemy import CsqlAlchemy #@UnusedImport # pylint: disable=W0404
if doStoreCING2db(entryId, ARCHIVE_NRG_ID, project=project):
nTerror(
"Failed to store CING project's data to DB but continuing."
)
if doSave:
self.assertTrue(project.save())
def main(entryId, *extraArgList):
"""inputDir may be a directory or a url. A url needs to start with http://.
"""
fastestTest = True # default: False
# ranges=AUTO_STR # default is None retrieved from DBMS csv files.
htmlOnly = False # default: False but enable it for faster runs without some actual data.
doWhatif = True # disables whatif actual run
doProcheck = True
doWattos = True
doTalos = True
tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten.
modelCount = None # default setting is None
if fastestTest:
modelCount = 3
htmlOnly = True
doWhatif = False
doProcheck = False
doWattos = False
doTalos = False
force_redo = True
force_retrieve_input = True
nTmessage(cing.cingDefinitions.getHeaderString())
nTmessage(cing.systemDefinitions.getStartMessage())
expectedArgumentList = [ 'inputDir', 'outputDir', 'pdbConvention', 'restraintsConvention', 'archiveType','projectType','storeCING2db']
expectedNumberOfArguments = len(expectedArgumentList)
if len(extraArgList) != expectedNumberOfArguments:
nTerror("Got arguments: " + repr(extraArgList))
nTerror("Failed to get expected number of arguments: %d got %d" % (
expectedNumberOfArguments, len(extraArgList)))
nTerror("Expected arguments: %s" % expectedArgumentList)
return True
entryCodeChar2and3 = entryId[1:3]
inputDir = extraArgList[0]
outputDir = os.path.join(extraArgList[1], DATA_STR, entryCodeChar2and3, entryId)
pdbConvention = extraArgList[2] #@UnusedVariable
restraintsConvention = extraArgList[3]
archiveType = extraArgList[4]
projectType = extraArgList[5]
storeCING2db = False
if len(extraArgList) >= expectedNumberOfArguments:
storeCING2db = extraArgList[6]
if archiveType == ARCHIVE_TYPE_FLAT:
pass
# default
elif archiveType == ARCHIVE_TYPE_BY_ENTRY:
inputDir = os.path.join(inputDir, entryId)
elif archiveType == ARCHIVE_TYPE_BY_CH23:
inputDir = os.path.join(inputDir, entryCodeChar2and3)
elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY:
inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId)
ranges = None
# targetId = getTargetForFullEntryName(entryId)
# if not targetId:
# nTerror("Failed to getTargetForFullEntryName for entryId: %s" % entryId)
# return True
# ranges = getRangesForTarget(targetId)
# if ranges == None:
# nTerror("Failed to getRangesForTarget for targetId: %s" % targetId)
# return True
nTdebug("Using:")
nTdebug("inputDir: %s" % inputDir)
nTdebug("outputDir: %s" % outputDir)
nTdebug("pdbConvention: %s" % pdbConvention)
nTdebug("restraintsConvention: %s" % restraintsConvention)
nTdebug("archiveType: %s" % archiveType)
nTdebug("projectType: %s" % projectType)
nTdebug("modelCount: %s" % modelCount)
nTdebug("storeCING2db: %s" % storeCING2db)
nTdebug("ranges: %s" % ranges)
# presume the directory still needs to be created.
cingEntryDir = entryId + ".cing"
if os.path.isdir(cingEntryDir):
if force_redo:
nTmessage("Enforcing a redo")
rmtree(cingEntryDir)
else:
mainIndexFile = os.path.join(cingEntryDir, "index.html")
isDone = os.path.isfile(mainIndexFile)
if isDone:
nTmessage("SKIPPING ENTRY ALREADY DONE")
return
nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.")
rmtree(cingEntryDir)
# end if.
# end if.
os.chdir(outputDir)
project = Project(entryId)
if project.removeFromDisk():
nTerror("Failed to remove existing project (if present)")
return True
# end if.
# extension = '.tgz'
formatFileName = '%s.tgz'
# fileNameTgz = entryId + '.tgz'
if projectType == PROJECT_TYPE_CING:
# fileNameTgz = entryId + '.cing.tgz'
formatFileName = '%s.cing.tgz'
elif projectType == PROJECT_TYPE_PDB:
formatFileName = 'pdb%s.ent.gz'
fileNameTgz = formatFileName % entryId
# nTdebug("fileNameTgz: %s" % fileNameTgz)
# if true will do retrieveTgzFromUrl.
if inputDir.startswith("http") or inputDir.startswith("file"):
stillToRetrieve = False
if os.path.exists(fileNameTgz):
if force_retrieve_input:
os.unlink(fileNameTgz)
stillToRetrieve = True
# end if
else:
stillToRetrieve = True
# end if
if stillToRetrieve:
retrieveTgzFromUrl(entryId, inputDir, archiveType=archiveType, formatFileName=formatFileName)
# end if
if not os.path.exists(fileNameTgz):
nTerror("Tgz should already have been present skipping entry")
return
# end if
# end if.
# retrieveTgzFromUrl(entryId, inputDir)
if projectType == PROJECT_TYPE_CING:
# Needs to be copied because the open method doesn't take a directory argument..
fullFileNameTgz = os.path.join(inputDir, fileNameTgz)
shutil.copy(fullFileNameTgz, '.')
project = Project.open(entryId, status='old')
if not project:
nTerror("Failed to init old project")
return True
elif projectType == PROJECT_TYPE_CCPN:
project = Project.open(entryId, status='new')
if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount):
nTerror("Failed to init project from ccpn")
return True
elif projectType == PROJECT_TYPE_PDB:
project = Project.open(entryId, status='new')
# pdbFileFormats = [ entryId + ".pdb", "pdb" + entryId + ".ent.gz" ]
# for pdbFileName in pdbFileFormats:
# pdbFileName = "pdb" + entryId + ".ent.gz"
# # pdbFilePath = os.path.join( inputDir, pdbFileName)
# pdbFilePath = os.path.join(inputDir, pdbFileName)
# if os.path.exists(pdbFilePath):
# break
# tmpPdbFile = None
# if pdbFileName.endswith('.gz'):
pdbFilePath = entryId + ".pdb"
# tmpPdbFile = pdbFilePath
# if os.path.exists(pdbFilePath):
# os.unlink(pdbFilePath)
gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True)
project.initPDB(pdbFile=pdbFilePath, convention=IUPAC, nmodels=modelCount)
# if tmpPdbFile:
if True:
nTdebug("Removing tmp: %s" % pdbFilePath)
os.unlink(pdbFilePath)
# if inputDirOrg == inputDirCASD_NMR:
# if True: # Default is False for this is specific to CASD-NMR
# nTmessage("Renaming molecule name to entry id: %s" % entryId)
# project.molecule.name = entryId # insufficient since all data is already initialized to disk.
# project.molecule.rename( entryId )
# project.save()
# project.molecule.ranges = ranges # JFD: this doesn't seem to be set there exactly.
project.molecule.superpose(ranges=ranges)
if True:
if project.validate(htmlOnly=htmlOnly, ranges=ranges, doProcheck=doProcheck, doWhatif=doWhatif,
doWattos=doWattos, doTalos=doTalos):
nTerror("Failed to validate project read")
return True
if storeCING2db:
# Does require:
#from cing.PluginCode.sqlAlchemy import csqlAlchemy
# and should never crash run.
try:
if doStoreCING2db( entryId, ARCHIVE_CASP_ID, project=project):
nTerror("Failed to store CING project's data to DB but continuing.")
except:
nTtracebackError()
nTerror("Failed to store CING project's data due to above traceback error.")
project.save()
if projectType == PROJECT_TYPE_CCPN:
# fileNameTgz = entryId + '.tgz'
# os.unlink(fileNameTgz) # temporary ccpn tgz
rmdir(entryId) # temporary ccpn dir
if tgzCing:
directoryNameCing = entryId + ".cing"
tgzFileNameCing = directoryNameCing + ".tgz"
if os.path.exists(tgzFileNameCing):
nTwarning("Overwriting: " + tgzFileNameCing)
cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing)
do_cmd(cmd)
def test_queeny(self):
runQueeny = 1 # DEFAULT: 1
doStoreCheck = False # DEFAULT: False Requires sqlAlchemy
doValueCheck = 1 # DEFAULT: 1 Requires 1brv
entryList = "1brv_cs_pk_2mdl".split(
) # DEFAULT because it contains many data types and is small/fast to run.
# entryList = "1a24 1a4d 1afp 1ai0 1b4y 1brv 1brv 1bus 1cjg 1hue 1ieh 1iv6 1kr8 2cka 2hgh 2jzn 2k0e 8psh".split()
expectedInfoList = [
(158, 0.0),
(159, 0.0),
(160, 0.0),
(161, 0.0),
(162, 0.0),
(163, 0.0),
(164, 0.0),
(165, 0.000141),
(166, 0.006669),
(167, 0.047112),
(168, 0.159379),
(169, 0.309577),
(170, 0.578983),
(
171, 0.838257
), # First residue with restraints. Previously none zero elements are due to window averaging JFD assumes.
(172, 1.029047),
(173, 1.063531),
(174, 0.969195),
(175, 1.027744),
(176, 0.954469),
(177, 0.794625),
(178, 0.822723),
(179, 0.845598),
(180, 0.828139),
(181, 0.971881),
(182, 1.141096),
(183, 1.063613),
(184, 0.974891),
(185, 1.116405),
(186, 1.113886),
(187, 0.945552),
(188, 0.731124),
(189, 0.411986),
]
cingDirTmpTest = os.path.join(cingDirTmp, getCallerName())
mkdirs(cingDirTmpTest)
self.failIf(os.chdir(cingDirTmpTest),
msg="Failed to change to test directory for files: " +
cingDirTmpTest)
for entryId in entryList:
project = Project.open(entryId, status='new')
self.assertTrue(project, 'Failed opening project: ' + entryId)
inputArchiveDir = os.path.join(cingDirTestsData, "ccpn")
ccpnFile = os.path.join(inputArchiveDir, entryId + ".tgz")
if not os.path.exists(ccpnFile):
ccpnFile = os.path.join(inputArchiveDir, entryId + ".tar.gz")
if not os.path.exists(ccpnFile):
self.fail("Neither %s or the .tgz exist" % ccpnFile)
self.assertTrue(project.initCcpn(ccpnFolder=ccpnFile,
modelCount=1))
if runQueeny:
self.assertFalse(project.runQueeny())
if doValueCheck:
for i, res in enumerate(project.molecule.allResidues()):
expected = expectedInfoList[i][1]
information = getDeepByKeysOrAttributes(
res, QUEENY_INFORMATION_STR)
uncertainty1 = getDeepByKeysOrAttributes(
res, QUEENY_UNCERTAINTY1_STR)
uncertainty2 = getDeepByKeysOrAttributes(
res, QUEENY_UNCERTAINTY2_STR)
nTdebug(
"Comparing expected versus inf/unc1/unc2 for %20s with %8.3f %8.3f %8.3f %8.3f"
% (res, expected, information, uncertainty1,
uncertainty2))
# self.assertAlmostEqual(expected,information,3)
if doStoreCheck:
# # Does require:
try:
from cing.PluginCode.sqlAlchemy import CsqlAlchemy #@UnusedImport # pylint: disable=W0612,W0404
pdbEntryId = entryId[0:4]
if doStoreCING2db(pdbEntryId,
ARCHIVE_DEV_NRG_ID,
project=project):
nTerror(
"Failed to store CING project's data to DB but continuing."
)
except:
nTwarning("Failed to doStoreCING2db from test_queeny.py")
def main(entryId, *extraArgList):
"""inputDir may be a directory or a url. A url needs to start with http://.
"""
fastestTest = False # default: False
htmlOnly = False # default: False but enable it for faster runs without some actual data.
doWhatif = True # disables whatif actual run
doProcheck = True
doWattos = True
doQueeny = True
doTalos = True
tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten.
# NB leave this set to True or modify code below.
removeCcpnDirectory = 1 # perhaps not so in the future.
modelCount = None # default setting is None
# ranges = None
if fastestTest:
modelCount = 2 # if this is more and there is only one model present it leads to an error message.
htmlOnly = True
doWhatif = False
doProcheck = False
doWattos = False
doQueeny = False
doTalos = False
forceRedo = True
forceRetrieveInput = True
nTmessage(header)
nTmessage(getStartMessage())
# Sync below code with nrgCing#createToposTokens
expectedArgumentList = """
verbosity inputDir outputDir
pdbConvention restraintsConvention archiveType projectType
storeCING2db ranges filterTopViolations filterVasco
singleCoreOperation
""".split()
expectedNumberOfArguments = len(expectedArgumentList)
if len(extraArgList) != expectedNumberOfArguments:
nTmessage(
"consider updating code to include all sequential parameters: %s" %
str(expectedArgumentList))
if len(extraArgList) > expectedNumberOfArguments:
nTerror("Got arguments: " + str(extraArgList))
nTerror("Failed to get expected number of arguments: %d got %d" %
(expectedNumberOfArguments, len(extraArgList)))
nTerror("Expected arguments: %s" % expectedArgumentList)
return True
# end if
# end if
entryCodeChar2and3 = entryId[1:3]
cing.verbosity = int(extraArgList[IDX_VERBOSITY])
inputDir = extraArgList[IDX_INPUT]
outputDir = os.path.join(extraArgList[IDX_OUTPUT], DATA_STR,
entryCodeChar2and3, entryId)
pdbConvention = extraArgList[IDX_PDB] #@UnusedVariable
restraintsConvention = extraArgList[IDX_RESTRAINTS]
archiveType = extraArgList[
IDX_ARCHIVE] # Only used for deriving the input location not the output.
projectType = extraArgList[IDX_PROJECT_TYPE]
storeCING2db = stringMeansBooleanTrue(
getDeepByKeysOrAttributes(extraArgList, IDX_STORE_DB))
ranges = getDeepByKeysOrAttributes(extraArgList, IDX_RANGES)
filterTopViolations = getDeepByKeysOrAttributes(extraArgList,
IDX_FILTER_TOP)
if filterTopViolations:
filterTopViolations = int(filterTopViolations) # change '0' to 0
filterVasco = getDeepByKeysOrAttributes(extraArgList, IDX_FILTER_VASCO)
if filterVasco:
filterVasco = int(filterVasco)
else:
filterVasco = 1 # Default should be True
# end if
singleCoreOperation = getDeepByKeysOrAttributes(extraArgList,
IDX_SINGLE_CORE_OPERATION)
if singleCoreOperation:
singleCoreOperation = int(singleCoreOperation)
else:
singleCoreOperation = 0 # Default should be True
# end if
if archiveType == ARCHIVE_TYPE_FLAT:
pass # default
elif archiveType == ARCHIVE_TYPE_BY_ENTRY:
inputDir = os.path.join(inputDir, entryId)
elif archiveType == ARCHIVE_TYPE_BY_CH23:
inputDir = os.path.join(inputDir, entryCodeChar2and3)
elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY:
inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId)
isRemoteOutputDir = False
if '@' in outputDir:
isRemoteOutputDir = True
# end if
# vc = vCing('.') # argument is a fake master_ssh_url not needed here.
archive_id = getArchiveIdFromDirectoryName(outputDir)
nTdebug("Using program arguments:")
nTdebug("inputDir: %s" % inputDir)
nTdebug("outputDir: %s" % outputDir)
nTdebug("pdbConvention: %s" % pdbConvention)
nTdebug("restraintsConvention: %s" % restraintsConvention)
nTdebug("archiveType: %s" % archiveType)
nTdebug("projectType: %s" % projectType)
nTdebug("storeCING2db: %s" % storeCING2db)
nTdebug("ranges: %s" % ranges)
nTdebug("filterTopViolations: %s" % filterTopViolations)
nTdebug("filterVasco: %s" % filterVasco)
nTdebug("singleCoreOperation: %s" % singleCoreOperation)
nTdebug("")
nTdebug("Using derived settings:")
nTdebug("modelCount: %s" % modelCount)
nTdebug("isRemoteOutputDir: %s" % isRemoteOutputDir)
nTdebug("archive_id: %s" % archive_id)
# For NMR_REDO required as most efficient.
if singleCoreOperation:
setToSingleCoreOperation()
# presume the directory still needs to be created.
cingEntryDir = entryId + ".cing"
if os.path.isdir(cingEntryDir):
if forceRedo:
nTmessage("Enforcing a redo")
rmtree(cingEntryDir)
else:
mainIndexFile = os.path.join(cingEntryDir, "index.html")
isDone = os.path.isfile(mainIndexFile)
if isDone:
nTmessage("SKIPPING ENTRY ALREADY DONE")
return
nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.")
rmtree(cingEntryDir)
# end if.
# end if.
if isRemoteOutputDir:
os.chdir(cingDirTmp)
else:
os.chdir(outputDir)
project = Project(entryId)
if project.removeFromDisk():
nTerror("Failed to remove existing project (if present)")
return True
# end if.
formatFileName = '%s.tgz'
if projectType == PROJECT_TYPE_CING:
formatFileName = '%s.cing.tgz'
elif projectType == PROJECT_TYPE_PDB:
formatFileName = 'pdb%s.ent.gz'
fileNameTgz = formatFileName % entryId
# nTdebug("fileNameTgz: %s" % fileNameTgz)
allowedInputProtocolList = 'http file ssh'.split()
inputProtocal = string.split(inputDir, ':')[0]
if inputProtocal in allowedInputProtocolList:
stillToRetrieve = False
if os.path.exists(fileNameTgz):
if forceRetrieveInput:
os.unlink(fileNameTgz)
stillToRetrieve = True
# end if
else:
stillToRetrieve = True
# end if
if stillToRetrieve:
retrieveTgzFromUrl(entryId,
inputDir,
archiveType=archiveType,
formatFileName=formatFileName)
# end if
if not os.path.exists(fileNameTgz):
nTerror("Tgz should already have been present skipping entry")
return
# end if
else:
nTdebug(
"Entry not retrieved which might be normal in some situations.")
# end if.
if projectType == PROJECT_TYPE_CING:
# Needs to be copied because the open method doesn't take a directory argument..
# fullFileNameTgz = os.path.join(inputDir, fileNameTgz)
# shutil.copy(fullFileNameTgz, '.')
project = Project.open(entryId, status='old')
if not project:
nTerror("Failed to init old project")
return True
elif projectType == PROJECT_TYPE_CCPN:
project = Project.open(entryId, status='new')
if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount):
nTerror("Failed to init project from ccpn")
return True
elif projectType == PROJECT_TYPE_PDB:
project = Project.open(entryId, status='new')
pdbFilePath = entryId + ".pdb"
gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True)
project.initPDB(pdbFile=pdbFilePath,
convention=IUPAC,
nmodels=modelCount)
# if tmpPdbFile:
if True:
nTdebug("Removing tmp: %s" % pdbFilePath)
os.unlink(pdbFilePath)
# end if
if ranges is not None:
project.molecule.setRanges(ranges)
# end if
if archive_id:
project.molecule.setArchiveId(archive_id)
# end if
project.molecule.superpose(ranges=ranges)
if filterTopViolations and not project.filterHighRestraintViol():
nTerror("Failed to filterHighRestraintViol")
####> MAIN UTILITY HERE
if 0: # DEFAULT 0
project.save()
if project.validate(htmlOnly=htmlOnly,
ranges=ranges,
doProcheck=doProcheck,
doWhatif=doWhatif,
doWattos=doWattos,
doQueeny=doQueeny,
doTalos=doTalos,
filterVasco=filterVasco):
nTerror("Failed to validate project read")
return True
# end if filterVasco
# Write a single PDB file containing all models
# according to IUPAC conventions
project.export2PDB()
project.save()
if storeCING2db and archive_id:
# Does require:
#from cing.PluginCode.sqlAlchemy import csqlAlchemy
# and should never crash run.
# archive_id = ARCHIVE_DEV_NRG_ID
# if isProduction:
# archive_id = ARCHIVE_NRG_ID
try:
if doStoreCING2db(entryId, archive_id, project=project):
nTerror(
"Failed to store CING project's data to DB but continuing."
)
except:
nTtracebackError()
nTerror(
"Failed to store CING project's data due to above traceback error."
)
if projectType == PROJECT_TYPE_CCPN:
# fileNameTgz = entryId + '.tgz'
os.unlink(fileNameTgz) # temporary ccpn tgz
if removeCcpnDirectory:
rmdir(entryId) # ccpn dir may contain vasco info.
if tgzCing:
directoryNameCing = entryId + ".cing"
tgzFileNameCing = directoryNameCing + ".tgz"
if os.path.exists(tgzFileNameCing):
nTwarning("Overwriting: " + tgzFileNameCing)
cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing)
nTdebug("cmd: %s" % cmd)
# do_cmd(cmd)
status, result = commands.getstatusoutput(cmd)
if status:
nTerror("Failed to tar status: %s with result %s" %
(status, result))
return True
if isRemoteOutputDir:
if putFileBySsh(tgzFileNameCing, outputDir, ntriesMax=2):
nTerror(
"Failed to send File By Scp status: %s with result %s" %
(status, result))
nTerror("Maintaining results.")
return True
# end if
nTmessage("Removing tgz result: %s" % tgzFileNameCing)
os.remove(tgzFileNameCing)
nTmessage("Removing cing dir itself: %s" % directoryNameCing)
rmdir(directoryNameCing)
else: # do NOT remove local copy
pass
def test_ccpn(self):
# cing.verbosity = verbosityDebug
# if you have a local copy you can use it; make sure to adjust the path setting below.
fastestTest = True # DEFAULT: True. Not passed to the validate routine in order to customize checks for speed.
modelCount=99 # DEFAULT: 99
redoFromCingProject = False # DEFAULT: False
htmlOnly = False # DEFAULT: False # default is False but enable it for faster runs without some actual data.
doWhatif = True # DEFAULT: True # disables whatif actual run
doProcheck = True # DEFAULT: True
doWattos = True # DEFAULT: True
doQueeny = True # DEFAULT: True
doTalos = True # DEFAULT: True
filterVasco = True # DEFAULT: True
filterTopViolations = True # DEFAULT: True
useNrgArchive = False # DEFAULT: False
ranges = CV_STR # DEFAULT: CV_STR
# ranges='173-177'
# ranges='6-13,29-45' # 1bus
doSwapCheck = False
doRestoreCheck = False
doStoreCheck = False # DEFAULT: False Requires sqlAlchemy
doSave = not redoFromCingProject # DEFAULT: False Requires sqlAlchemy
if fastestTest:
modelCount=2 # DEFAULT 2
# redoFromCingProject = False
htmlOnly = True # DEFAULT: True
doWhatif = False # DEFAULT: False
doProcheck = False # DEFAULT: False
doWattos = False # DEFAULT: False
doQueeny = False # DEFAULT: False
doTalos = False # DEFAULT: False
filterVasco = False # DEFAULT: False
doRestoreCheck = False # DEFAULT: False
doStoreCheck = False # DEFAULT: False
if redoFromCingProject:
useNrgArchive = False
doWhatif = False
doProcheck = False
doWattos = False
doTalos = False
cingDirTmpTest = os.path.join( cingDirTmp, getCallerName() )
mkdirs( cingDirTmpTest )
self.failIf(os.chdir(cingDirTmpTest), msg =
"Failed to change to test directory for files: " + cingDirTmpTest)
for i,entryId in enumerate(AllChecks.entryList):
if i:
nTmessage('\n\n')
if redoFromCingProject:
project = Project.open(entryId, status = 'old')
else:
project = Project.open(entryId, status = 'new')
self.assertTrue(project, 'Failed opening project: ' + entryId)
if useNrgArchive: # default is False
# inputArchiveDir = os.path.join('/Library/WebServer/Documents/NRG-CING/recoordSync', entryId)
# Mounted from nmr.cmbi.ru.nl
# inputArchiveDir = os.path.join('/Volumes/tria1/Library/WebServer/Documents/NRG-CING/recoordSync', entryId)
inputArchiveDir = os.path.join('/Users/jd/ccpn_tmp/data/recoord', entryId)
else:
inputArchiveDir = os.path.join(cingDirTestsData, "ccpn")
ccpnFile = os.path.join(inputArchiveDir, entryId + ".tgz")
if not os.path.exists(ccpnFile):
ccpnFile = os.path.join(inputArchiveDir, entryId + ".tar.gz")
if not os.path.exists(ccpnFile):
self.fail("Neither %s or the .tgz exist" % ccpnFile)
if doSwapCheck: # Need to start with ccpn without loading into CING api yet.
if os.path.exists(entryId):
nTmessage("Removing previous directory: %s" % entryId)
rmtree(entryId)
do_cmd("tar -xzf " + ccpnFile) # will extract to local dir.
if os.path.exists('linkNmrStarData'):
nTmessage("Renaming standard directory linkNmrStarData to entry: %s" % entryId)
os.rename('linkNmrStarData', entryId)
ccpnProject = loadProject(entryId)
if not ccpnProject:
self.fail("Failed to read project: %s" % entryId)
# constraintsHandler = ConstraintsHandler()
nmrConstraintStore = ccpnProject.findFirstNmrConstraintStore()
structureEnsemble = ccpnProject.findFirstStructureEnsemble()
numSwapCheckRuns = 2
if nmrConstraintStore:
if structureEnsemble:
swapCheck(nmrConstraintStore, structureEnsemble, numSwapCheckRuns)
else:
nTmessage("Failed to find structureEnsemble; skipping swapCheck")
else:
nTmessage("Failed to find nmrConstraintStore; skipping swapCheck")
# constraintsHandler.swapCheck(nmrConstraintStore, structureEnsemble, numSwapCheckRuns)
nTmessage('Saving to new path')
# checkValid=True,
saveProject(ccpnProject, newPath=entryId, removeExisting=True)
ccpnFile = entryId # set to local dir now.
# end if doSwapCheck
self.assertTrue(project.initCcpn(ccpnFolder = ccpnFile, modelCount=modelCount))
# if doSave:
# self.assertTrue(project.save())
if False:
ranges = "173-183"
# residueOfInterest = range(171,174)
# for residue in project.molecule.A.allResidues():
# if residue.resNum not in residueOfInterest:
# # nTmessage("Removing residue of no interest")
# project.molecule.A.removeResidue(residue)
if False:
ccpnProject = project.ccpn
printSequenceFromCcpnProject(ccpnProject)
if True:
self.assertFalse(project.molecule.setRanges(ranges))
nTdebug('In test_ccpn.py: ranges: %s' % str(project.molecule.ranges))
project.molecule.rangesToMmCifRanges(ranges)
if True:
self.assertFalse(project.validate(htmlOnly = htmlOnly,
ranges=ranges,
# fastestTest=fastestTest, # disabled
doProcheck = doProcheck,
doWhatif = doWhatif,
doWattos=doWattos,
doQueeny = doQueeny,
doTalos=doTalos,
filterVasco=filterVasco,
filterTopViolations = filterTopViolations
))
if doWattos and False:
mol = project.molecule
completenessMol = mol.getDeepByKeys( WATTOS_STR, COMPLCHK_STR, VALUE_LIST_STR)
nTdebug("completenessMol: %s" % completenessMol)
for res in mol.allResidues():
completenessRes = res.getDeepByKeys( WATTOS_STR, COMPLCHK_STR, VALUE_LIST_STR)
nTdebug("%s: %s" % (res, completenessRes))
# end for
# end if
color = project.molecule.getRogColor()
nTdebug("The color of this molecule is: %s" % color)
self.assertFalse( color == COLOR_RED ) # ;-)
# end if
# self.assertTrue(project.exportValidation2ccpn())
# self.assertFalse(project.removeCcpnReferences())
# Do not leave the old CCPN directory laying around since it might get added to by another test.
# if os.path.exists(entryId):
# self.assertFalse(shutil.rmtree(entryId))
if False:
self.assertTrue(project.save())
self.assertTrue(project.saveCcpn(entryId))
if doRestoreCheck:
del project
project = Project.open(entryId, status = 'old')
self.assertTrue(project, 'Failed reopening project: ' + entryId)
if doStoreCheck:
# # Does require below import which is used here to trigger import warning in case it's not installed.
# pylint: disable=W0612
from cing.PluginCode.sqlAlchemy import CsqlAlchemy #@UnusedImport # pylint: disable=W0404
if doStoreCING2db( entryId, ARCHIVE_NRG_ID, project=project):
nTerror("Failed to store CING project's data to DB but continuing.")
if doSave:
self.assertTrue(project.save())
from cing.NRG import ARCHIVE_CASD_ID from cing.NRG.storeCING2db import doStoreCING2db from cing.core.classes import Project import cing import sys cing.verbosity = cing.verbosityDebug pname = sys.argv[1] project = Project.open(pname, status = 'old') project.runQueeny() project.save() doStoreCING2db( project.name, ARCHIVE_CASD_ID, project=project)
def test_queeny(self):
runQueeny = 1 # DEFAULT: 1
doStoreCheck = False # DEFAULT: False Requires sqlAlchemy
doValueCheck = 1 # DEFAULT: 1 Requires 1brv
entryList = "1brv_cs_pk_2mdl".split() # DEFAULT because it contains many data types and is small/fast to run.
# entryList = "1a24 1a4d 1afp 1ai0 1b4y 1brv 1brv 1bus 1cjg 1hue 1ieh 1iv6 1kr8 2cka 2hgh 2jzn 2k0e 8psh".split()
expectedInfoList = [
(158, 0.0 ),
(159, 0.0 ),
(160, 0.0 ),
(161, 0.0 ),
(162, 0.0 ),
(163, 0.0 ),
(164, 0.0 ),
(165, 0.000141),
(166, 0.006669),
(167, 0.047112),
(168, 0.159379),
(169, 0.309577),
(170, 0.578983),
(171, 0.838257), # First residue with restraints. Previously none zero elements are due to window averaging JFD assumes.
(172, 1.029047),
(173, 1.063531),
(174, 0.969195),
(175, 1.027744),
(176, 0.954469),
(177, 0.794625),
(178, 0.822723),
(179, 0.845598),
(180, 0.828139),
(181, 0.971881),
(182, 1.141096),
(183, 1.063613),
(184, 0.974891),
(185, 1.116405),
(186, 1.113886),
(187, 0.945552),
(188, 0.731124),
(189, 0.411986),
]
cingDirTmpTest = os.path.join( cingDirTmp, getCallerName() )
mkdirs( cingDirTmpTest )
self.failIf(os.chdir(cingDirTmpTest), msg =
"Failed to change to test directory for files: " + cingDirTmpTest)
for entryId in entryList:
project = Project.open(entryId, status = 'new')
self.assertTrue(project, 'Failed opening project: ' + entryId)
inputArchiveDir = os.path.join(cingDirTestsData, "ccpn")
ccpnFile = os.path.join(inputArchiveDir, entryId + ".tgz")
if not os.path.exists(ccpnFile):
ccpnFile = os.path.join(inputArchiveDir, entryId + ".tar.gz")
if not os.path.exists(ccpnFile):
self.fail("Neither %s or the .tgz exist" % ccpnFile)
self.assertTrue(project.initCcpn(ccpnFolder = ccpnFile, modelCount=1))
if runQueeny:
self.assertFalse(project.runQueeny())
if doValueCheck:
for i,res in enumerate( project.molecule.allResidues()):
expected = expectedInfoList[i][1]
information = getDeepByKeysOrAttributes(res, QUEENY_INFORMATION_STR )
uncertainty1 = getDeepByKeysOrAttributes(res, QUEENY_UNCERTAINTY1_STR )
uncertainty2 = getDeepByKeysOrAttributes(res, QUEENY_UNCERTAINTY2_STR )
nTdebug("Comparing expected versus inf/unc1/unc2 for %20s with %8.3f %8.3f %8.3f %8.3f" % (
res, expected, information, uncertainty1, uncertainty2))
# self.assertAlmostEqual(expected,information,3)
if doStoreCheck:
# # Does require:
try:
from cing.PluginCode.sqlAlchemy import CsqlAlchemy #@UnusedImport # pylint: disable=W0612,W0404
pdbEntryId = entryId[0:4]
if doStoreCING2db( pdbEntryId, ARCHIVE_DEV_NRG_ID, project=project):
nTerror("Failed to store CING project's data to DB but continuing.")
except:
nTwarning("Failed to doStoreCING2db from test_queeny.py")
def main(entryId, *extraArgList):
"""inputDir may be a directory or a url. A url needs to start with http://.
"""
fastestTest = True # default: False
# ranges=AUTO_STR # default is None retrieved from DBMS csv files.
htmlOnly = False # default: False but enable it for faster runs without some actual data.
doWhatif = True # disables whatif actual run
doProcheck = True
doWattos = True
doTalos = True
tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten.
modelCount = None # default setting is None
if fastestTest:
modelCount = 3
htmlOnly = True
doWhatif = False
doProcheck = False
doWattos = False
doTalos = False
force_redo = True
force_retrieve_input = True
nTmessage(header)
nTmessage(getStartMessage())
expectedArgumentList = [
'inputDir', 'outputDir', 'pdbConvention', 'restraintsConvention',
'archiveType', 'projectType', 'storeCING2db'
]
expectedNumberOfArguments = len(expectedArgumentList)
if len(extraArgList) != expectedNumberOfArguments:
nTerror("Got arguments: " + repr(extraArgList))
nTerror("Failed to get expected number of arguments: %d got %d" %
(expectedNumberOfArguments, len(extraArgList)))
nTerror("Expected arguments: %s" % expectedArgumentList)
return True
entryCodeChar2and3 = entryId[1:3]
inputDir = extraArgList[0]
outputDir = os.path.join(extraArgList[1], DATA_STR, entryCodeChar2and3,
entryId)
pdbConvention = extraArgList[2] #@UnusedVariable
restraintsConvention = extraArgList[3]
archiveType = extraArgList[4]
projectType = extraArgList[5]
storeCING2db = False
if len(extraArgList) >= expectedNumberOfArguments:
storeCING2db = extraArgList[6]
if archiveType == ARCHIVE_TYPE_FLAT:
pass
# default
elif archiveType == ARCHIVE_TYPE_BY_ENTRY:
inputDir = os.path.join(inputDir, entryId)
elif archiveType == ARCHIVE_TYPE_BY_CH23:
inputDir = os.path.join(inputDir, entryCodeChar2and3)
elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY:
inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId)
ranges = None
# targetId = getTargetForFullEntryName(entryId)
# if not targetId:
# nTerror("Failed to getTargetForFullEntryName for entryId: %s" % entryId)
# return True
# ranges = getRangesForTarget(targetId)
# if ranges == None:
# nTerror("Failed to getRangesForTarget for targetId: %s" % targetId)
# return True
nTdebug("Using:")
nTdebug("inputDir: %s" % inputDir)
nTdebug("outputDir: %s" % outputDir)
nTdebug("pdbConvention: %s" % pdbConvention)
nTdebug("restraintsConvention: %s" % restraintsConvention)
nTdebug("archiveType: %s" % archiveType)
nTdebug("projectType: %s" % projectType)
nTdebug("modelCount: %s" % modelCount)
nTdebug("storeCING2db: %s" % storeCING2db)
nTdebug("ranges: %s" % ranges)
# presume the directory still needs to be created.
cingEntryDir = entryId + ".cing"
if os.path.isdir(cingEntryDir):
if force_redo:
nTmessage("Enforcing a redo")
rmtree(cingEntryDir)
else:
mainIndexFile = os.path.join(cingEntryDir, "index.html")
isDone = os.path.isfile(mainIndexFile)
if isDone:
nTmessage("SKIPPING ENTRY ALREADY DONE")
return
nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.")
rmtree(cingEntryDir)
# end if.
# end if.
os.chdir(outputDir)
project = Project(entryId)
if project.removeFromDisk():
nTerror("Failed to remove existing project (if present)")
return True
# end if.
# extension = '.tgz'
formatFileName = '%s.tgz'
# fileNameTgz = entryId + '.tgz'
if projectType == PROJECT_TYPE_CING:
# fileNameTgz = entryId + '.cing.tgz'
formatFileName = '%s.cing.tgz'
elif projectType == PROJECT_TYPE_PDB:
formatFileName = 'pdb%s.ent.gz'
fileNameTgz = formatFileName % entryId
# nTdebug("fileNameTgz: %s" % fileNameTgz)
# if true will do retrieveTgzFromUrl.
if inputDir.startswith("http") or inputDir.startswith("file"):
stillToRetrieve = False
if os.path.exists(fileNameTgz):
if force_retrieve_input:
os.unlink(fileNameTgz)
stillToRetrieve = True
# end if
else:
stillToRetrieve = True
# end if
if stillToRetrieve:
retrieveTgzFromUrl(entryId,
inputDir,
archiveType=archiveType,
formatFileName=formatFileName)
# end if
if not os.path.exists(fileNameTgz):
nTerror("Tgz should already have been present skipping entry")
return
# end if
# end if.
# retrieveTgzFromUrl(entryId, inputDir)
if projectType == PROJECT_TYPE_CING:
# Needs to be copied because the open method doesn't take a directory argument..
fullFileNameTgz = os.path.join(inputDir, fileNameTgz)
shutil.copy(fullFileNameTgz, '.')
project = Project.open(entryId, status='old')
if not project:
nTerror("Failed to init old project")
return True
elif projectType == PROJECT_TYPE_CCPN:
project = Project.open(entryId, status='new')
if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount):
nTerror("Failed to init project from ccpn")
return True
elif projectType == PROJECT_TYPE_PDB:
project = Project.open(entryId, status='new')
# pdbFileFormats = [ entryId + ".pdb", "pdb" + entryId + ".ent.gz" ]
# for pdbFileName in pdbFileFormats:
# pdbFileName = "pdb" + entryId + ".ent.gz"
# # pdbFilePath = os.path.join( inputDir, pdbFileName)
# pdbFilePath = os.path.join(inputDir, pdbFileName)
# if os.path.exists(pdbFilePath):
# break
# tmpPdbFile = None
# if pdbFileName.endswith('.gz'):
pdbFilePath = entryId + ".pdb"
# tmpPdbFile = pdbFilePath
# if os.path.exists(pdbFilePath):
# os.unlink(pdbFilePath)
gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True)
project.initPDB(pdbFile=pdbFilePath,
convention=IUPAC,
nmodels=modelCount)
# if tmpPdbFile:
if True:
nTdebug("Removing tmp: %s" % pdbFilePath)
os.unlink(pdbFilePath)
# if inputDirOrg == inputDirCASD_NMR:
# if True: # Default is False for this is specific to CASD-NMR
# nTmessage("Renaming molecule name to entry id: %s" % entryId)
# project.molecule.name = entryId # insufficient since all data is already initialized to disk.
# project.updateProject()
# project.molecule.rename( entryId )
# project.save()
# project.molecule.ranges = ranges # JFD: this doesn't seem to be set there exactly.
project.molecule.superpose(ranges=ranges)
if True:
if project.validate(htmlOnly=htmlOnly,
ranges=ranges,
doProcheck=doProcheck,
doWhatif=doWhatif,
doWattos=doWattos,
doTalos=doTalos):
nTerror("Failed to validate project read")
return True
if storeCING2db:
# Does require:
#from cing.PluginCode.sqlAlchemy import csqlAlchemy
# and should never crash run.
try:
if doStoreCING2db(entryId, ARCHIVE_CASP_ID, project=project):
nTerror(
"Failed to store CING project's data to DB but continuing."
)
except:
nTtracebackError()
nTerror(
"Failed to store CING project's data due to above traceback error."
)
project.save()
if projectType == PROJECT_TYPE_CCPN:
# fileNameTgz = entryId + '.tgz'
# os.unlink(fileNameTgz) # temporary ccpn tgz
rmdir(entryId) # temporary ccpn dir
if tgzCing:
directoryNameCing = entryId + ".cing"
tgzFileNameCing = directoryNameCing + ".tgz"
if os.path.exists(tgzFileNameCing):
nTwarning("Overwriting: " + tgzFileNameCing)
cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing)
do_cmd(cmd)
