def _test_RetrieveTgzFromUrl(self):
cingDirTmpTest = os.path.join( cingDirTmp, getCallerName() )
mkdirs( cingDirTmpTest )
self.failIf(os.chdir(cingDirTmpTest), msg =
"Failed to change to test directory for files: " + cingDirTmpTest)
# url = 'http://restraintsgrid.bmrb.wisc.edu/servlet_data/NRG_ccpn_tmp'
url = 'file:///Library/WebServer/Documents/NRG-CING/recoordSync'
# url = 'file:/%s/ccpn' % (cingDirTestsData)
entryId = '108d'
# entryId = '1brv'
fileNameTgz = entryId + '.tgz'
if os.path.exists(fileNameTgz):
os.unlink(fileNameTgz)
# self.assertFalse(retrieveTgzFromUrl(entryId, url, archiveType = ARCHIVE_TYPE_FLAT))
self.assertFalse(retrieveTgzFromUrl(entryId, url, archiveType = ARCHIVE_TYPE_BY_ENTRY))
def _test_RetrieveTgzFromUrl(self):
cingDirTmpTest = os.path.join(cingDirTmp, getCallerName())
mkdirs(cingDirTmpTest)
self.failIf(os.chdir(cingDirTmpTest),
msg="Failed to change to test directory for files: " +
cingDirTmpTest)
# url = 'http://restraintsgrid.bmrb.wisc.edu/servlet_data/NRG_ccpn_tmp'
url = 'file:///Library/WebServer/Documents/NRG-CING/recoordSync'
# url = 'file:/%s/ccpn' % (cingDirTestsData)
entryId = '108d'
# entryId = '1brv'
fileNameTgz = entryId + '.tgz'
if os.path.exists(fileNameTgz):
os.unlink(fileNameTgz)
# self.assertFalse(retrieveTgzFromUrl(entryId, url, archiveType = ARCHIVE_TYPE_FLAT))
self.assertFalse(
retrieveTgzFromUrl(entryId, url,
archiveType=ARCHIVE_TYPE_BY_ENTRY))
def main(entryId, *extraArgList):
"""inputDir may be a directory or a url. A url needs to start with http://.
"""
fastestTest = True # default: False
# ranges=AUTO_STR # default is None retrieved from DBMS csv files.
htmlOnly = False # default: False but enable it for faster runs without some actual data.
doWhatif = True # disables whatif actual run
doProcheck = True
doWattos = True
doTalos = True
tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten.
modelCount = None # default setting is None
if fastestTest:
modelCount = 3
htmlOnly = True
doWhatif = False
doProcheck = False
doWattos = False
doTalos = False
force_redo = True
force_retrieve_input = True
nTmessage(cing.cingDefinitions.getHeaderString())
nTmessage(cing.systemDefinitions.getStartMessage())
expectedArgumentList = [ 'inputDir', 'outputDir', 'pdbConvention', 'restraintsConvention', 'archiveType','projectType','storeCING2db']
expectedNumberOfArguments = len(expectedArgumentList)
if len(extraArgList) != expectedNumberOfArguments:
nTerror("Got arguments: " + repr(extraArgList))
nTerror("Failed to get expected number of arguments: %d got %d" % (
expectedNumberOfArguments, len(extraArgList)))
nTerror("Expected arguments: %s" % expectedArgumentList)
return True
entryCodeChar2and3 = entryId[1:3]
inputDir = extraArgList[0]
outputDir = os.path.join(extraArgList[1], DATA_STR, entryCodeChar2and3, entryId)
pdbConvention = extraArgList[2] #@UnusedVariable
restraintsConvention = extraArgList[3]
archiveType = extraArgList[4]
projectType = extraArgList[5]
storeCING2db = False
if len(extraArgList) >= expectedNumberOfArguments:
storeCING2db = extraArgList[6]
if archiveType == ARCHIVE_TYPE_FLAT:
pass
# default
elif archiveType == ARCHIVE_TYPE_BY_ENTRY:
inputDir = os.path.join(inputDir, entryId)
elif archiveType == ARCHIVE_TYPE_BY_CH23:
inputDir = os.path.join(inputDir, entryCodeChar2and3)
elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY:
inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId)
ranges = None
# targetId = getTargetForFullEntryName(entryId)
# if not targetId:
# nTerror("Failed to getTargetForFullEntryName for entryId: %s" % entryId)
# return True
# ranges = getRangesForTarget(targetId)
# if ranges == None:
# nTerror("Failed to getRangesForTarget for targetId: %s" % targetId)
# return True
nTdebug("Using:")
nTdebug("inputDir: %s" % inputDir)
nTdebug("outputDir: %s" % outputDir)
nTdebug("pdbConvention: %s" % pdbConvention)
nTdebug("restraintsConvention: %s" % restraintsConvention)
nTdebug("archiveType: %s" % archiveType)
nTdebug("projectType: %s" % projectType)
nTdebug("modelCount: %s" % modelCount)
nTdebug("storeCING2db: %s" % storeCING2db)
nTdebug("ranges: %s" % ranges)
# presume the directory still needs to be created.
cingEntryDir = entryId + ".cing"
if os.path.isdir(cingEntryDir):
if force_redo:
nTmessage("Enforcing a redo")
rmtree(cingEntryDir)
else:
mainIndexFile = os.path.join(cingEntryDir, "index.html")
isDone = os.path.isfile(mainIndexFile)
if isDone:
nTmessage("SKIPPING ENTRY ALREADY DONE")
return
nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.")
rmtree(cingEntryDir)
# end if.
# end if.
os.chdir(outputDir)
project = Project(entryId)
if project.removeFromDisk():
nTerror("Failed to remove existing project (if present)")
return True
# end if.
# extension = '.tgz'
formatFileName = '%s.tgz'
# fileNameTgz = entryId + '.tgz'
if projectType == PROJECT_TYPE_CING:
# fileNameTgz = entryId + '.cing.tgz'
formatFileName = '%s.cing.tgz'
elif projectType == PROJECT_TYPE_PDB:
formatFileName = 'pdb%s.ent.gz'
fileNameTgz = formatFileName % entryId
# nTdebug("fileNameTgz: %s" % fileNameTgz)
# if true will do retrieveTgzFromUrl.
if inputDir.startswith("http") or inputDir.startswith("file"):
stillToRetrieve = False
if os.path.exists(fileNameTgz):
if force_retrieve_input:
os.unlink(fileNameTgz)
stillToRetrieve = True
# end if
else:
stillToRetrieve = True
# end if
if stillToRetrieve:
retrieveTgzFromUrl(entryId, inputDir, archiveType=archiveType, formatFileName=formatFileName)
# end if
if not os.path.exists(fileNameTgz):
nTerror("Tgz should already have been present skipping entry")
return
# end if
# end if.
# retrieveTgzFromUrl(entryId, inputDir)
if projectType == PROJECT_TYPE_CING:
# Needs to be copied because the open method doesn't take a directory argument..
fullFileNameTgz = os.path.join(inputDir, fileNameTgz)
shutil.copy(fullFileNameTgz, '.')
project = Project.open(entryId, status='old')
if not project:
nTerror("Failed to init old project")
return True
elif projectType == PROJECT_TYPE_CCPN:
project = Project.open(entryId, status='new')
if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount):
nTerror("Failed to init project from ccpn")
return True
elif projectType == PROJECT_TYPE_PDB:
project = Project.open(entryId, status='new')
# pdbFileFormats = [ entryId + ".pdb", "pdb" + entryId + ".ent.gz" ]
# for pdbFileName in pdbFileFormats:
# pdbFileName = "pdb" + entryId + ".ent.gz"
# # pdbFilePath = os.path.join( inputDir, pdbFileName)
# pdbFilePath = os.path.join(inputDir, pdbFileName)
# if os.path.exists(pdbFilePath):
# break
# tmpPdbFile = None
# if pdbFileName.endswith('.gz'):
pdbFilePath = entryId + ".pdb"
# tmpPdbFile = pdbFilePath
# if os.path.exists(pdbFilePath):
# os.unlink(pdbFilePath)
gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True)
project.initPDB(pdbFile=pdbFilePath, convention=IUPAC, nmodels=modelCount)
# if tmpPdbFile:
if True:
nTdebug("Removing tmp: %s" % pdbFilePath)
os.unlink(pdbFilePath)
# if inputDirOrg == inputDirCASD_NMR:
# if True: # Default is False for this is specific to CASD-NMR
# nTmessage("Renaming molecule name to entry id: %s" % entryId)
# project.molecule.name = entryId # insufficient since all data is already initialized to disk.
# project.molecule.rename( entryId )
# project.save()
# project.molecule.ranges = ranges # JFD: this doesn't seem to be set there exactly.
project.molecule.superpose(ranges=ranges)
if True:
if project.validate(htmlOnly=htmlOnly, ranges=ranges, doProcheck=doProcheck, doWhatif=doWhatif,
doWattos=doWattos, doTalos=doTalos):
nTerror("Failed to validate project read")
return True
if storeCING2db:
# Does require:
#from cing.PluginCode.sqlAlchemy import csqlAlchemy
# and should never crash run.
try:
if doStoreCING2db( entryId, ARCHIVE_CASP_ID, project=project):
nTerror("Failed to store CING project's data to DB but continuing.")
except:
nTtracebackError()
nTerror("Failed to store CING project's data due to above traceback error.")
project.save()
if projectType == PROJECT_TYPE_CCPN:
# fileNameTgz = entryId + '.tgz'
# os.unlink(fileNameTgz) # temporary ccpn tgz
rmdir(entryId) # temporary ccpn dir
if tgzCing:
directoryNameCing = entryId + ".cing"
tgzFileNameCing = directoryNameCing + ".tgz"
if os.path.exists(tgzFileNameCing):
nTwarning("Overwriting: " + tgzFileNameCing)
cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing)
do_cmd(cmd)
def main(entryId, *extraArgList):
"""inputDir may be a directory or a url. A url needs to start with http://.
"""
fastestTest = True # default: False
# ranges=AUTO_STR # default is None retrieved from DBMS csv files.
htmlOnly = False # default: False but enable it for faster runs without some actual data.
doWhatif = True # disables whatif actual run
doProcheck = True
doWattos = True
doTalos = True
tgzCing = True # default: True # Create a tgz for the cing project. In case of a CING project input it will be overwritten.
modelCount = None # default setting is None
if fastestTest:
modelCount = 3
htmlOnly = True
doWhatif = False
doProcheck = False
doWattos = False
doTalos = False
force_redo = True
force_retrieve_input = True
nTmessage(header)
nTmessage(getStartMessage())
expectedArgumentList = [
'inputDir', 'outputDir', 'pdbConvention', 'restraintsConvention',
'archiveType', 'projectType', 'storeCING2db'
]
expectedNumberOfArguments = len(expectedArgumentList)
if len(extraArgList) != expectedNumberOfArguments:
nTerror("Got arguments: " + repr(extraArgList))
nTerror("Failed to get expected number of arguments: %d got %d" %
(expectedNumberOfArguments, len(extraArgList)))
nTerror("Expected arguments: %s" % expectedArgumentList)
return True
entryCodeChar2and3 = entryId[1:3]
inputDir = extraArgList[0]
outputDir = os.path.join(extraArgList[1], DATA_STR, entryCodeChar2and3,
entryId)
pdbConvention = extraArgList[2] #@UnusedVariable
restraintsConvention = extraArgList[3]
archiveType = extraArgList[4]
projectType = extraArgList[5]
storeCING2db = False
if len(extraArgList) >= expectedNumberOfArguments:
storeCING2db = extraArgList[6]
if archiveType == ARCHIVE_TYPE_FLAT:
pass
# default
elif archiveType == ARCHIVE_TYPE_BY_ENTRY:
inputDir = os.path.join(inputDir, entryId)
elif archiveType == ARCHIVE_TYPE_BY_CH23:
inputDir = os.path.join(inputDir, entryCodeChar2and3)
elif archiveType == ARCHIVE_TYPE_BY_CH23_BY_ENTRY:
inputDir = os.path.join(inputDir, entryCodeChar2and3, entryId)
ranges = None
# targetId = getTargetForFullEntryName(entryId)
# if not targetId:
# nTerror("Failed to getTargetForFullEntryName for entryId: %s" % entryId)
# return True
# ranges = getRangesForTarget(targetId)
# if ranges == None:
# nTerror("Failed to getRangesForTarget for targetId: %s" % targetId)
# return True
nTdebug("Using:")
nTdebug("inputDir: %s" % inputDir)
nTdebug("outputDir: %s" % outputDir)
nTdebug("pdbConvention: %s" % pdbConvention)
nTdebug("restraintsConvention: %s" % restraintsConvention)
nTdebug("archiveType: %s" % archiveType)
nTdebug("projectType: %s" % projectType)
nTdebug("modelCount: %s" % modelCount)
nTdebug("storeCING2db: %s" % storeCING2db)
nTdebug("ranges: %s" % ranges)
# presume the directory still needs to be created.
cingEntryDir = entryId + ".cing"
if os.path.isdir(cingEntryDir):
if force_redo:
nTmessage("Enforcing a redo")
rmtree(cingEntryDir)
else:
mainIndexFile = os.path.join(cingEntryDir, "index.html")
isDone = os.path.isfile(mainIndexFile)
if isDone:
nTmessage("SKIPPING ENTRY ALREADY DONE")
return
nTmessage("REDOING BECAUSE VALIDATION CONSIDERED NOT DONE.")
rmtree(cingEntryDir)
# end if.
# end if.
os.chdir(outputDir)
project = Project(entryId)
if project.removeFromDisk():
nTerror("Failed to remove existing project (if present)")
return True
# end if.
# extension = '.tgz'
formatFileName = '%s.tgz'
# fileNameTgz = entryId + '.tgz'
if projectType == PROJECT_TYPE_CING:
# fileNameTgz = entryId + '.cing.tgz'
formatFileName = '%s.cing.tgz'
elif projectType == PROJECT_TYPE_PDB:
formatFileName = 'pdb%s.ent.gz'
fileNameTgz = formatFileName % entryId
# nTdebug("fileNameTgz: %s" % fileNameTgz)
# if true will do retrieveTgzFromUrl.
if inputDir.startswith("http") or inputDir.startswith("file"):
stillToRetrieve = False
if os.path.exists(fileNameTgz):
if force_retrieve_input:
os.unlink(fileNameTgz)
stillToRetrieve = True
# end if
else:
stillToRetrieve = True
# end if
if stillToRetrieve:
retrieveTgzFromUrl(entryId,
inputDir,
archiveType=archiveType,
formatFileName=formatFileName)
# end if
if not os.path.exists(fileNameTgz):
nTerror("Tgz should already have been present skipping entry")
return
# end if
# end if.
# retrieveTgzFromUrl(entryId, inputDir)
if projectType == PROJECT_TYPE_CING:
# Needs to be copied because the open method doesn't take a directory argument..
fullFileNameTgz = os.path.join(inputDir, fileNameTgz)
shutil.copy(fullFileNameTgz, '.')
project = Project.open(entryId, status='old')
if not project:
nTerror("Failed to init old project")
return True
elif projectType == PROJECT_TYPE_CCPN:
project = Project.open(entryId, status='new')
if not project.initCcpn(ccpnFolder=fileNameTgz, modelCount=modelCount):
nTerror("Failed to init project from ccpn")
return True
elif projectType == PROJECT_TYPE_PDB:
project = Project.open(entryId, status='new')
# pdbFileFormats = [ entryId + ".pdb", "pdb" + entryId + ".ent.gz" ]
# for pdbFileName in pdbFileFormats:
# pdbFileName = "pdb" + entryId + ".ent.gz"
# # pdbFilePath = os.path.join( inputDir, pdbFileName)
# pdbFilePath = os.path.join(inputDir, pdbFileName)
# if os.path.exists(pdbFilePath):
# break
# tmpPdbFile = None
# if pdbFileName.endswith('.gz'):
pdbFilePath = entryId + ".pdb"
# tmpPdbFile = pdbFilePath
# if os.path.exists(pdbFilePath):
# os.unlink(pdbFilePath)
gunzip(fileNameTgz, outputFileName=pdbFilePath, removeOriginal=True)
project.initPDB(pdbFile=pdbFilePath,
convention=IUPAC,
nmodels=modelCount)
# if tmpPdbFile:
if True:
nTdebug("Removing tmp: %s" % pdbFilePath)
os.unlink(pdbFilePath)
# if inputDirOrg == inputDirCASD_NMR:
# if True: # Default is False for this is specific to CASD-NMR
# nTmessage("Renaming molecule name to entry id: %s" % entryId)
# project.molecule.name = entryId # insufficient since all data is already initialized to disk.
# project.updateProject()
# project.molecule.rename( entryId )
# project.save()
# project.molecule.ranges = ranges # JFD: this doesn't seem to be set there exactly.
project.molecule.superpose(ranges=ranges)
if True:
if project.validate(htmlOnly=htmlOnly,
ranges=ranges,
doProcheck=doProcheck,
doWhatif=doWhatif,
doWattos=doWattos,
doTalos=doTalos):
nTerror("Failed to validate project read")
return True
if storeCING2db:
# Does require:
#from cing.PluginCode.sqlAlchemy import csqlAlchemy
# and should never crash run.
try:
if doStoreCING2db(entryId, ARCHIVE_CASP_ID, project=project):
nTerror(
"Failed to store CING project's data to DB but continuing."
)
except:
nTtracebackError()
nTerror(
"Failed to store CING project's data due to above traceback error."
)
project.save()
if projectType == PROJECT_TYPE_CCPN:
# fileNameTgz = entryId + '.tgz'
# os.unlink(fileNameTgz) # temporary ccpn tgz
rmdir(entryId) # temporary ccpn dir
if tgzCing:
directoryNameCing = entryId + ".cing"
tgzFileNameCing = directoryNameCing + ".tgz"
if os.path.exists(tgzFileNameCing):
nTwarning("Overwriting: " + tgzFileNameCing)
cmd = "tar -czf %s %s" % (tgzFileNameCing, directoryNameCing)
do_cmd(cmd)