def test_NrgUtils(
self
): #DEFAULT disabled because it's a specific test for services not commonly used.
inputList = """
.
http://dodos.dyndns.org/NRG-CING/input/pc
/var/NMR_REDO/x
""".split()
expectedList = [None, ARCHIVE_NRG_ID, ARCHIVE_NMR_REDO_ID]
for i, inputStr in enumerate(inputList):
archive_id = getArchiveIdFromDirectoryName(inputStr)
# nTdebug("Found on iteration %s with input: %s the archive_id %s and expected %s" % ( i, inputStr, archive_id, expectedList[i]))
self.assertEqual(archive_id, expectedList[i])
def test_NrgUtils(self): #DEFAULT disabled because it's a specific test for services not commonly used.
inputList = """
.
http://dodos.dyndns.org/NRG-CING/input/pc
/var/NMR_REDO/x
""".split()
expectedList = [
None,
ARCHIVE_NRG_ID,
ARCHIVE_NMR_REDO_ID
]
for i, inputStr in enumerate(inputList):
archive_id = getArchiveIdFromDirectoryName(inputStr)
# nTdebug("Found on iteration %s with input: %s the archive_id %s and expected %s" % ( i, inputStr, archive_id, expectedList[i]))
self.assertEqual( archive_id, expectedList[i])
def updateProjectHtml(pdb_id, extraArgListStr):
# extraArgList = extraArgListStr.split()
htmlOnly = True
project = Project.open(pdb_id, status='old')
nTmessage("Opened existing project")
if not project:
nTerror("Failed to init old project")
return True
# end if
archive_id = getArchiveIdFromDirectoryName( os.getcwd() )
# Derives the related entry codes in PDB and BMRB.
project.molecule.setArchiveId( archive_id )
if True: # DEFAULT: True
nTmessage("Updating project html")
# project.runCingChecks(toFile=True, ranges=ranges)
project.setupHtml()
project.generateHtml(htmlOnly = htmlOnly)
project.renderHtml()
# end if
project.save()
nTmessage("Done with updateProjectHtml")
def updateProjectHtml(pdb_id, extraArgListStr):
# extraArgList = extraArgListStr.split()
htmlOnly = True
project = Project.open(pdb_id, status='old')
nTmessage("Opened existing project")
if not project:
nTerror("Failed to init old project")
return True
# end if
archive_id = getArchiveIdFromDirectoryName(os.getcwd())
# Derives the related entry codes in PDB and BMRB.
project.molecule.setArchiveId(archive_id)
if True: # DEFAULT: True
nTmessage("Updating project html")
# project.runCingChecks(toFile=True, ranges=ranges)
project.setupHtml()
project.generateHtml(htmlOnly=htmlOnly)
project.renderHtml()
# end if
project.save()
nTmessage("Done with updateProjectHtml")
def main(entryId, *extraArgList):
"""inputDir may be a directory or a url. A url needs to start with http://.
"""
fastestTest = False # default: False
htmlOnly = False # default: False but enable it for faster runs without some actual data.
doWhatif = True # disables whatif actual run
doProcheck = True
doWattos = True
doQueeny = True
doTalos = True
tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten.
# NB leave this set to True or modify code below.
removeCcpnDirectory = 1 # perhaps not so in the future.
modelCount = None # default setting is None
# ranges = None
if fastestTest:
modelCount = 2 # if this is more and there is only one model present it leads to an error message.
htmlOnly = True
doWhatif = False
doProcheck = False
doWattos = False
doQueeny = False
doTalos = False
forceRedo = True
forceRetrieveInput = True
nTmessage(header)
nTmessage(getStartMessage())
# Sync below code with nrgCing#createToposTokens
expectedArgumentList = """
verbosity inputDir outputDir
pdbConvention restraintsConvention archiveType projectType
storeCING2db ranges filterTopViolations filterVasco
singleCoreOperation
""".split()
expectedNumberOfArguments = len(expectedArgumentList)
if len(extraArgList) != expectedNumberOfArguments:
nTmessage("consider updating code to include all sequential parameters: %s" % str(expectedArgumentList))
if len(extraArgList) > expectedNumberOfArguments:
nTerror("Got arguments: " + str(extraArgList))
nTerror("Failed to get expected number of arguments: %d got %d" % (
expectedNumberOfArguments, len(extraArgList)))
nTerror("Expected arguments: %s" % expectedArgumentList)
return True
# end if
# end if
entryCodeChar2and3 = entryId[1:3]
cing.verbosity = int( extraArgList[IDX_VERBOSITY] )
inputDir = extraArgList[IDX_INPUT]
outputDir = os.path.join(extraArgList[IDX_OUTPUT], DATA_STR, entryCodeChar2and3, entryId)
pdbConvention = extraArgList[IDX_PDB] #@UnusedVariable
restraintsConvention = extraArgList[IDX_RESTRAINTS]
archiveType = extraArgList[IDX_ARCHIVE] # Only used for deriving the input location not the output.
projectType = extraArgList[IDX_PROJECT_TYPE]
storeCING2db = stringMeansBooleanTrue( getDeepByKeysOrAttributes(extraArgList, IDX_STORE_DB))
ranges = getDeepByKeysOrAttributes(extraArgList, IDX_RANGES)
filterTopViolations = getDeepByKeysOrAttributes(extraArgList, IDX_FILTER_TOP)
if filterTopViolations:
filterTopViolations = int(filterTopViolations) # change '0' to 0
filterVasco = getDeepByKeysOrAttributes(extraArgList, IDX_FILTER_VASCO)
if filterVasco:
filterVasco = int(filterVasco)
else:
filterVasco = 1 # Default should be True
# end if
singleCoreOperation = getDeepByKeysOrAttributes(extraArgList, IDX_SINGLE_CORE_OPERATION )
if singleCoreOperation:
singleCoreOperation = int(singleCoreOperation)
else:
singleCoreOperation = 0 # Default should be True
# end if
if archiveType == ARCHIVE_TYPE_FLAT:
pass # default
elif archiveType == ARCHIVE_TYPE_BY_ENTRY:
inputDir = os.path.join(inputDir, entryId)
elif archiveType == ARCHIVE_TYPE_BY_CH23:
inputDir = os.path.join(inputDir, entryCodeChar2and3)
elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY:
inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId)
isRemoteOutputDir = False
if '@' in outputDir:
isRemoteOutputDir = True
# end if
# vc = vCing('.') # argument is a fake master_ssh_url not needed here.
archive_id = getArchiveIdFromDirectoryName( outputDir )
nTdebug("Using program arguments:")
nTdebug("inputDir: %s" % inputDir)
nTdebug("outputDir: %s" % outputDir)
nTdebug("pdbConvention: %s" % pdbConvention)
nTdebug("restraintsConvention: %s" % restraintsConvention)
nTdebug("archiveType: %s" % archiveType)
nTdebug("projectType: %s" % projectType)
nTdebug("storeCING2db: %s" % storeCING2db)
nTdebug("ranges: %s" % ranges)
nTdebug("filterTopViolations: %s" % filterTopViolations)
nTdebug("filterVasco: %s" % filterVasco)
nTdebug("singleCoreOperation: %s" % singleCoreOperation)
nTdebug("")
nTdebug("Using derived settings:")
nTdebug("modelCount: %s" % modelCount)
nTdebug("isRemoteOutputDir: %s" % isRemoteOutputDir)
nTdebug("archive_id: %s" % archive_id)
# For NMR_REDO required as most efficient.
if singleCoreOperation:
setToSingleCoreOperation()
# presume the directory still needs to be created.
cingEntryDir = entryId + ".cing"
if os.path.isdir(cingEntryDir):
if forceRedo:
nTmessage("Enforcing a redo")
rmtree(cingEntryDir)
else:
mainIndexFile = os.path.join(cingEntryDir, "index.html")
isDone = os.path.isfile(mainIndexFile)
if isDone:
nTmessage("SKIPPING ENTRY ALREADY DONE")
return
nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.")
rmtree(cingEntryDir)
# end if.
# end if.
if isRemoteOutputDir:
os.chdir(cingDirTmp)
else:
os.chdir(outputDir)
project = Project(entryId)
if project.removeFromDisk():
nTerror("Failed to remove existing project (if present)")
return True
# end if.
formatFileName = '%s.tgz'
if projectType == PROJECT_TYPE_CING:
formatFileName = '%s.cing.tgz'
elif projectType == PROJECT_TYPE_PDB:
formatFileName = 'pdb%s.ent.gz'
fileNameTgz = formatFileName % entryId
# nTdebug("fileNameTgz: %s" % fileNameTgz)
allowedInputProtocolList = 'http file ssh'.split()
inputProtocal = string.split( inputDir, ':' )[0]
if inputProtocal in allowedInputProtocolList:
stillToRetrieve = False
if os.path.exists(fileNameTgz):
if forceRetrieveInput:
os.unlink(fileNameTgz)
stillToRetrieve = True
# end if
else:
stillToRetrieve = True
# end if
if stillToRetrieve:
retrieveTgzFromUrl(entryId, inputDir, archiveType=archiveType, formatFileName=formatFileName)
# end if
if not os.path.exists(fileNameTgz):
nTerror("Tgz should already have been present skipping entry")
return
# end if
else:
nTdebug("Entry not retrieved which might be normal in some situations.")
# end if.
if projectType == PROJECT_TYPE_CING:
# Needs to be copied because the open method doesn't take a directory argument..
# fullFileNameTgz = os.path.join(inputDir, fileNameTgz)
# shutil.copy(fullFileNameTgz, '.')
project = Project.open(entryId, status='old')
if not project:
nTerror("Failed to init old project")
return True
elif projectType == PROJECT_TYPE_CCPN:
project = Project.open(entryId, status='new')
if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount):
nTerror("Failed to init project from ccpn")
return True
elif projectType == PROJECT_TYPE_PDB:
project = Project.open(entryId, status='new')
pdbFilePath = entryId + ".pdb"
gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True)
project.initPDB(pdbFile=pdbFilePath, convention=IUPAC, nmodels=modelCount)
# if tmpPdbFile:
if True:
nTdebug("Removing tmp: %s" % pdbFilePath)
os.unlink(pdbFilePath)
# end if
if ranges is not None:
project.molecule.setRanges(ranges)
# end if
if archive_id:
project.molecule.setArchiveId(archive_id)
# end if
project.molecule.superpose(ranges=ranges)
if filterTopViolations and not project.filterHighRestraintViol():
nTerror("Failed to filterHighRestraintViol")
####> MAIN UTILITY HERE
if 0: # DEFAULT 0
project.save()
if project.validate(htmlOnly=htmlOnly, ranges=ranges, doProcheck=doProcheck, doWhatif=doWhatif,
doWattos=doWattos, doQueeny = doQueeny, doTalos=doTalos, filterVasco = filterVasco ):
nTerror("Failed to validate project read")
return True
# end if filterVasco
# Write a single PDB file containing all models
# according to IUPAC conventions
project.export2PDB()
project.save()
if storeCING2db and archive_id:
# Does require:
#from cing.PluginCode.sqlAlchemy import csqlAlchemy
# and should never crash run.
# archive_id = ARCHIVE_DEV_NRG_ID
# if isProduction:
# archive_id = ARCHIVE_NRG_ID
try:
if doStoreCING2db( entryId, archive_id, project=project):
nTerror("Failed to store CING project's data to DB but continuing.")
except:
nTtracebackError()
nTerror("Failed to store CING project's data due to above traceback error.")
if projectType == PROJECT_TYPE_CCPN:
# fileNameTgz = entryId + '.tgz'
os.unlink(fileNameTgz) # temporary ccpn tgz
if removeCcpnDirectory:
rmdir(entryId) # ccpn dir may contain vasco info.
if tgzCing:
directoryNameCing = entryId + ".cing"
tgzFileNameCing = directoryNameCing + ".tgz"
if os.path.exists(tgzFileNameCing):
nTwarning("Overwriting: " + tgzFileNameCing)
cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing)
nTdebug("cmd: %s" % cmd)
# do_cmd(cmd)
status, result = commands.getstatusoutput(cmd)
if status:
nTerror("Failed to tar status: %s with result %s" % (status, result))
return True
if isRemoteOutputDir:
if putFileBySsh(tgzFileNameCing, outputDir, ntriesMax = 2):
nTerror("Failed to send File By Scp status: %s with result %s" % (status, result))
nTerror("Maintaining results.")
return True
# end if
nTmessage("Removing tgz result: %s" % tgzFileNameCing)
os.remove(tgzFileNameCing)
nTmessage("Removing cing dir itself: %s" % directoryNameCing)
rmdir(directoryNameCing)
else: # do NOT remove local copy
pass
def main(entryId, *extraArgList):
"""inputDir may be a directory or a url. A url needs to start with http://.
"""
fastestTest = False # default: False
htmlOnly = False # default: False but enable it for faster runs without some actual data.
doWhatif = True # disables whatif actual run
doProcheck = True
doWattos = True
doQueeny = True
doTalos = True
tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten.
# NB leave this set to True or modify code below.
removeCcpnDirectory = 1 # perhaps not so in the future.
modelCount = None # default setting is None
# ranges = None
if fastestTest:
modelCount = 2 # if this is more and there is only one model present it leads to an error message.
htmlOnly = True
doWhatif = False
doProcheck = False
doWattos = False
doQueeny = False
doTalos = False
forceRedo = True
forceRetrieveInput = True
nTmessage(header)
nTmessage(getStartMessage())
# Sync below code with nrgCing#createToposTokens
expectedArgumentList = """
verbosity inputDir outputDir
pdbConvention restraintsConvention archiveType projectType
storeCING2db ranges filterTopViolations filterVasco
singleCoreOperation
""".split()
expectedNumberOfArguments = len(expectedArgumentList)
if len(extraArgList) != expectedNumberOfArguments:
nTmessage(
"consider updating code to include all sequential parameters: %s" %
str(expectedArgumentList))
if len(extraArgList) > expectedNumberOfArguments:
nTerror("Got arguments: " + str(extraArgList))
nTerror("Failed to get expected number of arguments: %d got %d" %
(expectedNumberOfArguments, len(extraArgList)))
nTerror("Expected arguments: %s" % expectedArgumentList)
return True
# end if
# end if
entryCodeChar2and3 = entryId[1:3]
cing.verbosity = int(extraArgList[IDX_VERBOSITY])
inputDir = extraArgList[IDX_INPUT]
outputDir = os.path.join(extraArgList[IDX_OUTPUT], DATA_STR,
entryCodeChar2and3, entryId)
pdbConvention = extraArgList[IDX_PDB] #@UnusedVariable
restraintsConvention = extraArgList[IDX_RESTRAINTS]
archiveType = extraArgList[
IDX_ARCHIVE] # Only used for deriving the input location not the output.
projectType = extraArgList[IDX_PROJECT_TYPE]
storeCING2db = stringMeansBooleanTrue(
getDeepByKeysOrAttributes(extraArgList, IDX_STORE_DB))
ranges = getDeepByKeysOrAttributes(extraArgList, IDX_RANGES)
filterTopViolations = getDeepByKeysOrAttributes(extraArgList,
IDX_FILTER_TOP)
if filterTopViolations:
filterTopViolations = int(filterTopViolations) # change '0' to 0
filterVasco = getDeepByKeysOrAttributes(extraArgList, IDX_FILTER_VASCO)
if filterVasco:
filterVasco = int(filterVasco)
else:
filterVasco = 1 # Default should be True
# end if
singleCoreOperation = getDeepByKeysOrAttributes(extraArgList,
IDX_SINGLE_CORE_OPERATION)
if singleCoreOperation:
singleCoreOperation = int(singleCoreOperation)
else:
singleCoreOperation = 0 # Default should be True
# end if
if archiveType == ARCHIVE_TYPE_FLAT:
pass # default
elif archiveType == ARCHIVE_TYPE_BY_ENTRY:
inputDir = os.path.join(inputDir, entryId)
elif archiveType == ARCHIVE_TYPE_BY_CH23:
inputDir = os.path.join(inputDir, entryCodeChar2and3)
elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY:
inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId)
isRemoteOutputDir = False
if '@' in outputDir:
isRemoteOutputDir = True
# end if
# vc = vCing('.') # argument is a fake master_ssh_url not needed here.
archive_id = getArchiveIdFromDirectoryName(outputDir)
nTdebug("Using program arguments:")
nTdebug("inputDir: %s" % inputDir)
nTdebug("outputDir: %s" % outputDir)
nTdebug("pdbConvention: %s" % pdbConvention)
nTdebug("restraintsConvention: %s" % restraintsConvention)
nTdebug("archiveType: %s" % archiveType)
nTdebug("projectType: %s" % projectType)
nTdebug("storeCING2db: %s" % storeCING2db)
nTdebug("ranges: %s" % ranges)
nTdebug("filterTopViolations: %s" % filterTopViolations)
nTdebug("filterVasco: %s" % filterVasco)
nTdebug("singleCoreOperation: %s" % singleCoreOperation)
nTdebug("")
nTdebug("Using derived settings:")
nTdebug("modelCount: %s" % modelCount)
nTdebug("isRemoteOutputDir: %s" % isRemoteOutputDir)
nTdebug("archive_id: %s" % archive_id)
# For NMR_REDO required as most efficient.
if singleCoreOperation:
setToSingleCoreOperation()
# presume the directory still needs to be created.
cingEntryDir = entryId + ".cing"
if os.path.isdir(cingEntryDir):
if forceRedo:
nTmessage("Enforcing a redo")
rmtree(cingEntryDir)
else:
mainIndexFile = os.path.join(cingEntryDir, "index.html")
isDone = os.path.isfile(mainIndexFile)
if isDone:
nTmessage("SKIPPING ENTRY ALREADY DONE")
return
nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.")
rmtree(cingEntryDir)
# end if.
# end if.
if isRemoteOutputDir:
os.chdir(cingDirTmp)
else:
os.chdir(outputDir)
project = Project(entryId)
if project.removeFromDisk():
nTerror("Failed to remove existing project (if present)")
return True
# end if.
formatFileName = '%s.tgz'
if projectType == PROJECT_TYPE_CING:
formatFileName = '%s.cing.tgz'
elif projectType == PROJECT_TYPE_PDB:
formatFileName = 'pdb%s.ent.gz'
fileNameTgz = formatFileName % entryId
# nTdebug("fileNameTgz: %s" % fileNameTgz)
allowedInputProtocolList = 'http file ssh'.split()
inputProtocal = string.split(inputDir, ':')[0]
if inputProtocal in allowedInputProtocolList:
stillToRetrieve = False
if os.path.exists(fileNameTgz):
if forceRetrieveInput:
os.unlink(fileNameTgz)
stillToRetrieve = True
# end if
else:
stillToRetrieve = True
# end if
if stillToRetrieve:
retrieveTgzFromUrl(entryId,
inputDir,
archiveType=archiveType,
formatFileName=formatFileName)
# end if
if not os.path.exists(fileNameTgz):
nTerror("Tgz should already have been present skipping entry")
return
# end if
else:
nTdebug(
"Entry not retrieved which might be normal in some situations.")
# end if.
if projectType == PROJECT_TYPE_CING:
# Needs to be copied because the open method doesn't take a directory argument..
# fullFileNameTgz = os.path.join(inputDir, fileNameTgz)
# shutil.copy(fullFileNameTgz, '.')
project = Project.open(entryId, status='old')
if not project:
nTerror("Failed to init old project")
return True
elif projectType == PROJECT_TYPE_CCPN:
project = Project.open(entryId, status='new')
if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount):
nTerror("Failed to init project from ccpn")
return True
elif projectType == PROJECT_TYPE_PDB:
project = Project.open(entryId, status='new')
pdbFilePath = entryId + ".pdb"
gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True)
project.initPDB(pdbFile=pdbFilePath,
convention=IUPAC,
nmodels=modelCount)
# if tmpPdbFile:
if True:
nTdebug("Removing tmp: %s" % pdbFilePath)
os.unlink(pdbFilePath)
# end if
if ranges is not None:
project.molecule.setRanges(ranges)
# end if
if archive_id:
project.molecule.setArchiveId(archive_id)
# end if
project.molecule.superpose(ranges=ranges)
if filterTopViolations and not project.filterHighRestraintViol():
nTerror("Failed to filterHighRestraintViol")
####> MAIN UTILITY HERE
if 0: # DEFAULT 0
project.save()
if project.validate(htmlOnly=htmlOnly,
ranges=ranges,
doProcheck=doProcheck,
doWhatif=doWhatif,
doWattos=doWattos,
doQueeny=doQueeny,
doTalos=doTalos,
filterVasco=filterVasco):
nTerror("Failed to validate project read")
return True
# end if filterVasco
# Write a single PDB file containing all models
# according to IUPAC conventions
project.export2PDB()
project.save()
if storeCING2db and archive_id:
# Does require:
#from cing.PluginCode.sqlAlchemy import csqlAlchemy
# and should never crash run.
# archive_id = ARCHIVE_DEV_NRG_ID
# if isProduction:
# archive_id = ARCHIVE_NRG_ID
try:
if doStoreCING2db(entryId, archive_id, project=project):
nTerror(
"Failed to store CING project's data to DB but continuing."
)
except:
nTtracebackError()
nTerror(
"Failed to store CING project's data due to above traceback error."
)
if projectType == PROJECT_TYPE_CCPN:
# fileNameTgz = entryId + '.tgz'
os.unlink(fileNameTgz) # temporary ccpn tgz
if removeCcpnDirectory:
rmdir(entryId) # ccpn dir may contain vasco info.
if tgzCing:
directoryNameCing = entryId + ".cing"
tgzFileNameCing = directoryNameCing + ".tgz"
if os.path.exists(tgzFileNameCing):
nTwarning("Overwriting: " + tgzFileNameCing)
cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing)
nTdebug("cmd: %s" % cmd)
# do_cmd(cmd)
status, result = commands.getstatusoutput(cmd)
if status:
nTerror("Failed to tar status: %s with result %s" %
(status, result))
return True
if isRemoteOutputDir:
if putFileBySsh(tgzFileNameCing, outputDir, ntriesMax=2):
nTerror(
"Failed to send File By Scp status: %s with result %s" %
(status, result))
nTerror("Maintaining results.")
return True
# end if
nTmessage("Removing tgz result: %s" % tgzFileNameCing)
os.remove(tgzFileNameCing)
nTmessage("Removing cing dir itself: %s" % directoryNameCing)
rmdir(directoryNameCing)
else: # do NOT remove local copy
pass
