def analyze(config, project, parameters, doPrint=0):
'''Analyze a run
Returns True on failure.
'''
nTmessage("\n-- analyze --")
inPath = config.directories.converted
modelList = asci2list(parameters.models)
if getDeepByKeysOrAttributes(parameters, USE_ANNEALED_STR):
inPath = config.directories.annealed
modelList = asci2list(parameters.modelsAnneal)
nTdebug('inPath: %s' % inPath)
nTdebug('modelList: %s' % modelList)
# first create the jobs, run later
analyzeJobs = []
for i in modelList:
job = Analyze(
config,
parameters,
project=project,
fileNum=i,
molecules=[
NTdict(psfFile=parameters.psfFile,
pdbFile=parameters.baseName % i,
selectionCode='(not resn TIP3 and not resn ANI)'),
],
inPath=inPath,
outPath=config.directories.analyzed,
jobName='analyze_%d' % i,
)
analyzeJobs.append(job)
#end for
job_list = []
for job in analyzeJobs:
if job.createScript():
nTerror(
"In refine#analyze failed to create at least one job's script."
)
return True
if doPrint:
job.printScript()
#end if
# job.runScript()
job_list.append((job.runScript, ))
#end for
f = ForkOff(processes_max=cing.ncpus,
max_time_to_wait=600,
verbosity=cing.verbosity)
done_list = f.forkoff_start(job_list, 0) # delay 0 second between jobs.
nTmessage("Finished ids: %s", done_list)
if not done_list:
notDone = parameters.modelCountAnneal - len(done_list)
nTerror("Failed to analyze %s", notDone)
return True
def anneal(config, project, parameters, doPrint=0):
nTmessage("\n-- anneal --")
# first create the jobs, run later
annealJobs = []
for i in range(parameters.modelCountAnneal):
annealJobs.append(
Anneal(
config,
parameters,
project = project,
fileNum=i,
molecules=[
NTdict(
psfFile=parameters.psfFile,
pdbFile=parameters.baseName % i,
selectionCode='(all)'
),
],
inPath = config.directories.template,
outPath = config.directories.annealed,
templateFile = TEMPLATE_FILE_NAME,
jobName='anneal_%d' % i,
)
)
job_list = []
for job in annealJobs:
job.createScript()
if doPrint:
job.printScript()
job_list.append((job.runScript,) )
# job.runScript()
#end for
f = ForkOff(
processes_max=cing.ncpus,
max_time_to_wait=600,
verbosity=cing.verbosity
)
done_list = f.forkoff_start(job_list, 0) # delay 0 second between jobs.
nTmessage("Finished ids: %s", done_list)
notDone = parameters.modelCountAnneal - len(done_list)
if notDone > 0:
nTwarning("Failed to anneal %s", notDone) # We print how many models failed to anneal, and try to continue without them
nTdebug("Only finished ids will be kept")
parameters.modelsAnneal = list2asci(done_list)
# Only if the list is empty, we want to return True
if not done_list:
return True
def anneal(config, project, parameters, doPrint=0):
nTmessage("\n-- anneal --")
# first create the jobs, run later
annealJobs = []
for i in range(parameters.modelCountAnneal):
annealJobs.append(
Anneal(
config,
parameters,
project=project,
fileNum=i,
molecules=[
NTdict(psfFile=parameters.psfFile,
pdbFile=parameters.baseName % i,
selectionCode='(all)'),
],
inPath=config.directories.template,
outPath=config.directories.annealed,
templateFile=TEMPLATE_FILE_NAME,
jobName='anneal_%d' % i,
))
job_list = []
for job in annealJobs:
job.createScript()
if doPrint:
job.printScript()
job_list.append((job.runScript, ))
# job.runScript()
#end for
f = ForkOff(processes_max=cing.ncpus,
max_time_to_wait=600,
verbosity=cing.verbosity)
done_list = f.forkoff_start(job_list, 0) # delay 0 second between jobs.
nTmessage("Finished ids: %s", done_list)
notDone = parameters.modelCountAnneal - len(done_list)
if notDone > 0:
nTwarning(
"Failed to anneal %s", notDone
) # We print how many models failed to anneal, and try to continue without them
nTdebug("Only finished ids will be kept")
parameters.modelsAnneal = list2asci(done_list)
# Only if the list is empty, we want to return True
if not done_list:
return True
def testOverall(namepattern):
'Use silent testing from top level.'
# cing.verbosity = verbosityError
# Add the ones you don't want to test (perhaps you know they don't work yet)
excludedModuleList = [
# cingPythonDir + "/Cython*",
cingPythonDir + "/cyana2cing*",
# cingPythonDir + "/cing.PluginCode",
# cingPythonDir + "/cing.PluginCode.test.test_Whatif",
# cingPythonDir + "/cing.Scripts.test.test_cyana2cing",
# cingPythonDir + "/cing.STAR.FileTest",
]
startdir = cingPythonDir
nameList = findFiles(namepattern, startdir, exclude=excludedModuleList)
# enable next line(s) to do a single check only.
# nameList = ['/Users/jd/workspace35/cing/python/cing/Libs/test/test_Imagery.py']
# nameList = [
# '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_NmrStar.py',
# '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_ccpn.py'
# ]
nTdebug('Will unit check: ' + repr(nameList))
# nameList = nameList[0:5]
# namepattern = "*Test.py"
# nameList2 = findFiles(namepattern, startdir)
# for name in nameList2:
# nameList.append(name)
# translate: '/Users/jd/workspace/cing/python/cing/Libs/test/test_NTplot.py'
# to: cing.Libs.test.test_NTplot
f = ForkOff(
# processes_max=1, # use 1 if you really want to read it line by line.
processes_max=cing.ncpus,
max_time_to_wait=600, # on a very slow setup
verbosity=cing.verbosity
)
job_list = []
for name in nameList:
job_list.append( (testByName, (name, excludedModuleList)) )
done_list = f.forkoff_start(job_list, 0)
nTmessage("Finished ids: %s", done_list)
# Exit with timer info anywho. After this CING should exit so the tweak shouldn't break anything.
if cing.verbosity <= cing.verbosityError:
cing.verbosity = cing.verbosityOutput
def testOverall(namepattern):
'Use silent testing from top level.'
# cing.verbosity = verbosityError
# Add the ones you don't want to test (perhaps you know they don't work yet)
excludedModuleList = [
# cingPythonDir + "/Cython*",
cingPythonDir + "/cyana2cing*",
# cingPythonDir + "/cing.PluginCode",
# cingPythonDir + "/cing.PluginCode.test.test_Whatif",
# cingPythonDir + "/cing.Scripts.test.test_cyana2cing",
# cingPythonDir + "/cing.STAR.FileTest",
]
startdir = cingPythonDir
nameList = findFiles(namepattern, startdir, exclude=excludedModuleList)
# enable next line(s) to do a single check only.
# nameList = ['/Users/jd/workspace35/cing/python/cing/Libs/test/test_Imagery.py']
# nameList = [
# '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_NmrStar.py',
# '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_ccpn.py'
# ]
nTdebug('Will unit check: ' + repr(nameList))
# nameList = nameList[0:5]
# namepattern = "*Test.py"
# nameList2 = findFiles(namepattern, startdir)
# for name in nameList2:
# nameList.append(name)
# translate: '/Users/jd/workspace/cing/python/cing/Libs/test/test_NTplot.py'
# to: cing.Libs.test.test_NTplot
f = ForkOff(
# processes_max=1, # use 1 if you really want to read it line by line.
processes_max=cing.ncpus,
max_time_to_wait=600, # on a very slow setup
verbosity=cing.verbosity)
job_list = []
for name in nameList:
job_list.append((testByName, (name, excludedModuleList)))
done_list = f.forkoff_start(job_list, 0)
nTmessage("Finished ids: %s", done_list)
# Exit with timer info anywho. After this CING should exit so the tweak shouldn't break anything.
if cing.verbosity <= cing.verbosityError:
cing.verbosity = cing.verbosityOutput
def refine(config, project, parameters, doPrint=0):
nTmessage("\n-- refine --")
# first create the jobs, run later
refineJobs = []
for i in asci2list(parameters.models):
refineJobs.append(
WaterRefine(
config,
parameters,
project = project,
fileNum=i,
molecules=[
NTdict(
psfFile=parameters.psfFile,
pdbFile=parameters.baseName % i,
selectionCode='(not resn TIP3 and not resn ANI)'
),
],
inPath = config.directories.analyzed,
outPath = config.directories.refined,
jobName='refine_%d' % i,
)
)
job_list = []
for job in refineJobs:
job.createScript()
if doPrint:
job.printScript()
# job.runScript()
job_list.append((job.runScript,) )
#end for
f = ForkOff(
processes_max=cing.ncpus,
max_time_to_wait=6000,
verbosity=cing.verbosity
)
done_list = f.forkoff_start(job_list, 0) # delay 0 second between jobs.
nTmessage("Finished ids: %s", done_list)
if not done_list:
notDone = parameters.modelCountAnneal - len(done_list)
nTerror("Failed to refine %s", notDone)
return True
def refine(config, project, parameters, doPrint=0):
nTmessage("\n-- refine --")
# first create the jobs, run later
refineJobs = []
for i in asci2list(parameters.models):
refineJobs.append(
WaterRefine(
config,
parameters,
project=project,
fileNum=i,
molecules=[
NTdict(psfFile=parameters.psfFile,
pdbFile=parameters.baseName % i,
selectionCode='(not resn TIP3 and not resn ANI)'),
],
inPath=config.directories.analyzed,
outPath=config.directories.refined,
jobName='refine_%d' % i,
))
job_list = []
for job in refineJobs:
job.createScript()
if doPrint:
job.printScript()
# job.runScript()
job_list.append((job.runScript, ))
#end for
f = ForkOff(processes_max=cing.ncpus,
max_time_to_wait=6000,
verbosity=cing.verbosity)
done_list = f.forkoff_start(job_list, 0) # delay 0 second between jobs.
nTmessage("Finished ids: %s", done_list)
if not done_list:
notDone = parameters.modelCountAnneal - len(done_list)
nTerror("Failed to refine %s", notDone)
return True
def doPylintOverall(pylintFileName='pylint.txt'):
"Add the ones you don't want to pylint (perhaps you know they don't work yet)"
namepattern = "*.py"
pylintDir = os.path.join(cingDirTmp, 'pylint')
# pylintFileName = os.path.join( pylintDir, 'pylint.log')
if os.path.exists(pylintDir):
rmdir(pylintDir)
mkdirs(pylintDir)
if os.path.exists(pylintFileName):
os.unlink(pylintFileName)
excludedModuleList = [
# enable exclusions for quick testing.
# cingPythonDir + "/cing/core*",
# cingPythonDir + "/cing/Database*",
# cingPythonDir + "/cing/Libs*",
# cingPythonDir + "/cing/NRG*",
# cingPythonDir + "/cing/PluginCode*",
# cingPythonDir + "/cing/Scripts*",
# cingPythonDir + "/cing/STAR*",
# cingPythonDir + "/cing/Talos*",
# Normal set:
cingPythonDir + "/cing/Database/CCPN*",
cingPythonDir + "/cyana2cing*",
cingPythonDir + "/pdbe2*",
cingPythonDir + "/queen*",
cingPythonDir + "/Refine*",
cingPythonDir + "/UtilsAnalysis*",
cingPythonDir + "/xplorcing*"
]
startdir = cingPythonDir
nameList = findFiles(namepattern, startdir, exclude=excludedModuleList)
# enable next line(s) to do a couple of checks only.
# nameList = ['/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_NmrStar.py',
# '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_ccpn.py',
# '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_ccpn_2.py']
nTdebug('Will unit check: ' + repr(nameList))
f = ForkOff(
processes_max=cing.ncpus,
max_time_to_wait=600, # on a very slow setup
verbosity=2)
job_list = []
for name in nameList:
job_list.append((doPylintByName, (name, excludedModuleList)))
done_list = f.forkoff_start(job_list, 0)
nTmessage("Finished ids: %s", done_list)
for name in nameList:
mod_name = pathToModuleName(name)
if mod_name in excludedModuleList:
print "Skipping module: " + mod_name
return
pylintOutputFileName = os.path.join(pylintDir, mod_name + '.log')
if not os.path.exists(pylintDir):
nTerror("Failed to find pylint output: " + pylintOutputFileName)
continue
if appendTextFileToFile(pylintOutputFileName, pylintFileName):
nTerror("Failed to appendTextFileToFile")
# nTdebug("Done appending from: %s" % pylintOutputFileName)
# end for
doPylintOverallSummary(pylintFileName=pylintFileName)
nTmessage("Done with pylint")
def doPylintOverall(pylintFileName='pylint.txt'):
"Add the ones you don't want to pylint (perhaps you know they don't work yet)"
namepattern = "*.py"
pylintDir = os.path.join( cingDirTmp, 'pylint' )
# pylintFileName = os.path.join( pylintDir, 'pylint.log')
if os.path.exists( pylintDir ):
rmdir( pylintDir )
mkdirs( pylintDir )
if os.path.exists( pylintFileName ):
os.unlink(pylintFileName)
excludedModuleList = [
# enable exclusions for quick testing.
# cingPythonDir + "/cing/core*",
# cingPythonDir + "/cing/Database*",
# cingPythonDir + "/cing/Libs*",
# cingPythonDir + "/cing/NRG*",
# cingPythonDir + "/cing/PluginCode*",
# cingPythonDir + "/cing/Scripts*",
# cingPythonDir + "/cing/STAR*",
# cingPythonDir + "/cing/Talos*",
# Normal set:
cingPythonDir + "/cing/Database/CCPN*",
cingPythonDir + "/cyana2cing*",
cingPythonDir + "/pdbe2*",
cingPythonDir + "/queen*",
cingPythonDir + "/Refine*",
cingPythonDir + "/UtilsAnalysis*",
cingPythonDir + "/xplorcing*"
]
startdir = cingPythonDir
nameList = findFiles(namepattern, startdir, exclude=excludedModuleList)
# enable next line(s) to do a couple of checks only.
# nameList = ['/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_NmrStar.py',
# '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_ccpn.py',
# '/Users/jd/workspace35/cing/python/cing/PluginCode/test/test_ccpn_2.py']
nTdebug('Will unit check: ' + repr(nameList))
f = ForkOff(
processes_max=cing.ncpus,
max_time_to_wait=600, # on a very slow setup
verbosity=2
)
job_list = []
for name in nameList:
job_list.append( (doPylintByName, (name, excludedModuleList)) )
done_list = f.forkoff_start(job_list, 0)
nTmessage("Finished ids: %s", done_list)
for name in nameList:
mod_name = pathToModuleName( name )
if mod_name in excludedModuleList:
print "Skipping module: " + mod_name
return
pylintOutputFileName = os.path.join( pylintDir, mod_name + '.log')
if not os.path.exists( pylintDir ):
nTerror("Failed to find pylint output: " + pylintOutputFileName)
continue
if appendTextFileToFile( pylintOutputFileName, pylintFileName):
nTerror("Failed to appendTextFileToFile")
# nTdebug("Done appending from: %s" % pylintOutputFileName)
# end for
doPylintOverallSummary(pylintFileName=pylintFileName)
nTmessage("Done with pylint")
def doScriptOnEntryList(pythonScriptFileName,
entryListFileName,
startDir ='.',
processes_max = 3, # default 3
max_time_to_wait = 600,
delay_between_submitting_jobs = 1,
extraArgList = None,
start_entry_id = start_entry_id,
max_entries_todo = max_entries_todo,
expectPdbEntryList = True,
shuffleBeforeSelecting = False, # fails for chain ids when included.
entryList = () # as an alternative to a file.
):
"""Return True on error"""
if True: # DEFAULT: True
pid = os.getpid()
nTmessage("Use kill -2 %s (sending a INT (interrupt) to this Process ID) twice to kill all child processes." % pid)
nTmessage("entryListFileName : %s" % entryListFileName)
nTmessage("startDir : %s" % startDir )
nTmessage("processes_max : %s" % processes_max )
nTmessage("max_time_to_wait : %s" % max_time_to_wait )
nTmessage("delay_between_submitting_jobs: %s" % delay_between_submitting_jobs)
nTmessage("extraArgList : %s" % str(extraArgList ))
nTmessage("start_entry_id : %s" % start_entry_id )
nTmessage("max_entries_todo : %s" % max_entries_todo )
nTmessage("expectPdbEntryList : %s" % expectPdbEntryList )
nTmessage("shuffleBeforeSelecting : %s" % shuffleBeforeSelecting )
# end if
# if os.chdir(cingDirTmp):
# raise SetupError("Failed to change to directory for temporary test files: "+cingDirTmp)
# Empty list means no filtering done.
entryCodeListFilter = []
# entryCodeListFilter = string.split("1n62")
if entryListFileName:
entryList = [line.strip() for line in open(entryListFileName)]
entryList = [x for x in entryList if x]
entryCountTotal = len(entryList)
if expectPdbEntryList:
chainCodeList = []
entryCodeList = []
for ss in entryList:
entryCode = ss[:4].lower()
if entryCode not in entryCodeListFilter:
if len(ss) > 4:
chainCode = ss[4].upper()
else:
chainCode = ''
entryCodeList.append(entryCode)
chainCodeList.append(chainCode)
entryCountSelected = len(entryCodeList)
else:
#Non-PDB
entryCodeList = [x for x in entryList if x not in entryCodeListFilter]
entryCountSelected = len(entryCodeList)
chainCodeList = [''] * entryCountSelected
# lastEntryId is id of last entry excluding the entry itself.
lastEntryId = min(len(entryCodeList), start_entry_id+max_entries_todo)
if shuffleBeforeSelecting:
nTmessage("Shuffling entry list before selecting entries.")
entryCodeListCopy = entryCodeList[:]
shuffle(entryCodeListCopy)
entryCodeList = entryCodeListCopy[start_entry_id:lastEntryId] # no sense in starting at zero here; they're random.
else:
entryCodeList = entryCodeList[start_entry_id:lastEntryId] # no sense in starting at zero here; they're random.
entryCodeList.sort()
chainCodeList = chainCodeList[start_entry_id:lastEntryId]
nTmessage('Read %05d entries ' % entryCountTotal)
nTmessage('Selected %05d entries ' % entryCountSelected)
nTmessage('Sliced %05d entries: %s' % (len(entryCodeList), entryCodeList ))
# nTmessage('Sliced %05d chains: %s' % (len(chainCodeList), chainCodeList ))
useAnyCmd = False
if pythonScriptFileName.endswith('.py'):
(_directory, pythonScriptFileNameRoot, _ext) = nTpath(pythonScriptFileName)
else:
useAnyCmd = True
pythonScriptFileNameRoot = pythonScriptFileName.split()[0] # get first word of eg.
if pythonScriptFileNameRoot.count('/'):
pythonScriptFileNameRoot = pythonScriptFileNameRoot.split('/')[-1]
if pythonScriptFileNameRoot.count('.'):
pythonScriptFileNameRoot = pythonScriptFileNameRoot.split('.')[0]
# 'cing -v 9 -n $x --initPDB $PDB/pdb$x.ent.gz --validateFastest --ranges cv')
mkSubDirStructure( startDir, entryCodeList, pythonScriptFileNameRoot )
logScriptFileNameRoot = 'log_'+pythonScriptFileNameRoot
job_list = []
for i, entry_code in enumerate(entryCodeList):
extraArgListStr = ''
if extraArgList:
extraArgListStr = ' '.join( extraArgList )
chain_code = chainCodeList[i]
entryCodeChar2and3 = entry_code[1:3]
entryDir = os.path.join( startDir, DATA_STR, entryCodeChar2and3, entry_code )
date_stamp = getDateTimeStampForFileName()
cmd = 'cd %s; python -u %s %s %s %s > %s/%s_%s%s.log 2>&1 ' % (
entryDir,
pythonScriptFileName,
entry_code,
chain_code,
extraArgListStr,
logScriptFileNameRoot,
entry_code,
chain_code,
date_stamp
)
if useAnyCmd:
cmdBase = pythonScriptFileName
cmdBase = cmdBase.replace("$x", entry_code)
cmdBase = cmdBase.replace("$c", chain_code)
cmd = 'cd %s; %s %s %s > %s/%s_%s%s.log 2>&1 ' % (
entryDir,
cmdBase,
entry_code,
chain_code,
logScriptFileNameRoot,
entry_code,
chain_code,
date_stamp
)
job = ( do_cmd, (cmd,) )
# nTdebug("Will schedule job cmd: %s" % cmd)
job_list.append( job )
f = ForkOff( processes_max = processes_max, max_time_to_wait = max_time_to_wait)
done_entry_list = f.forkoff_start( job_list, delay_between_submitting_jobs )
done_entry_list.sort()
not_done_entry_list = range(len(job_list))
for id in done_entry_list:
idx = not_done_entry_list.index(id)
if idx >= 0:
del(not_done_entry_list[idx])
nTmessage("In doScriptOnEntryList Finished list : %s" % done_entry_list)
nTmessage("In doScriptOnEntryList Unfinished list: %s" % not_done_entry_list)
for id in not_done_entry_list:
job = job_list[id]
_do_cmd, cmdTuple = job
cmd = cmdTuple[0]
nTerror("In doScriptOnEntryList failed forked: %s" % cmd)
def doFunctionOnEntryList(
f,
entryListFileName,
processes_max = 3, # default 3
max_time_to_wait = 600,
delay_between_submitting_jobs = 0,
extraArgList = None,
start_entry_id = start_entry_id,
max_entries_todo = max_entries_todo,
shuffleBeforeSelecting = False, # fails for chain ids when included.
entryList = None # as an alternative to a file.
):
"""Return True on error"""
if True: # DEFAULT: True
pid = os.getpid()
nTmessage("Use kill -2 %s (sending a INT (interrupt) to this Process ID) twice to kill all child processes." % pid)
nTmessage("entryListFileName : %s" % entryListFileName)
nTmessage("processes_max : %s" % processes_max )
nTmessage("max_time_to_wait : %s" % max_time_to_wait )
nTmessage("delay_between_submitting_jobs: %s" % delay_between_submitting_jobs)
nTmessage("extraArgList : %s" % str(extraArgList ))
nTmessage("start_entry_id : %s" % start_entry_id )
nTmessage("max_entries_todo : %s" % max_entries_todo )
nTmessage("shuffleBeforeSelecting : %s" % shuffleBeforeSelecting )
nTmessage("entryList : %s" % entryList)
# end if
# if os.chdir(cingDirTmp):
# raise SetupError("Failed to change to directory for temporary test files: "+cingDirTmp)
# Empty list means no filtering done.
entryCodeListFilter = []
# entryCodeListFilter = string.split("1n62")
if entryListFileName:
entryListFile = file(entryListFileName, 'r')
entryCodeList = []
entryCountTotal = 0
for line in entryListFile.readlines():
line = line.strip()
if line == '': # skip empty lines.
continue
entryCountTotal += 1
entryCode = line
if entryCode in entryCodeListFilter:
continue
entryCodeList.append( entryCode )
entryListFile.close()
else:
entryCodeList = entryList
entryCountTotal = len( entryList )
# end if
entryCountSelected = len( entryCodeList )
# lastEntryId is id of last entry excluding the entry itself.
lastEntryId = min(len(entryCodeList), start_entry_id+max_entries_todo)
if shuffleBeforeSelecting:
nTmessage("Shuffling entry list before selecting entries.")
entryCodeListCopy = entryCodeList[:]
shuffle(entryCodeListCopy)
entryCodeList = entryCodeListCopy[start_entry_id:lastEntryId] # no sense in starting at zero here; they're random.
else:
entryCodeList = entryCodeList[start_entry_id:lastEntryId] # no sense in starting at zero here; they're random.
entryCodeList.sort()
nTmessage('Read %05d entries ' % entryCountTotal)
nTmessage('Selected %05d entries ' % entryCountSelected)
nTmessage('Sliced %05d entries: %s' % (len(entryCodeList), entryCodeList ))
# nTmessage('Sliced %05d chains: %s' % (len(chainCodeList), chainCodeList ))
job_list = []
for _i, entry_code in enumerate(entryCodeList):
# extraArgListStr = ''
# if extraArgList:
# extraArgListStr = ' '.join( extraArgList )
argList = [entry_code]
if isinstance(extraArgList, list) or isinstance(extraArgList, tuple):
argList += extraArgList
# else:
# nTdebug("Skipping non-List/Tuple extraArgList: %s" % str(extraArgList))
# end if
job = ( f, tuple( argList ) )
job_list.append( job )
# end for
f = ForkOff( processes_max = processes_max, max_time_to_wait = max_time_to_wait)
done_entry_list = f.forkoff_start( job_list, delay_between_submitting_jobs )
done_entry_list.sort()
not_done_entry_list = range(len(job_list))
for id in done_entry_list:
idx = not_done_entry_list.index(id)
if idx >= 0:
del(not_done_entry_list[idx])
# end if
# end for
nTmessage("In doScriptOnEntryList Finished list : %s" % done_entry_list)
nTmessage("In doScriptOnEntryList Unfinished list: %s" % not_done_entry_list)
for id in not_done_entry_list:
job = job_list[id]
_do_cmd, cmdTuple = job
cmd = cmdTuple[0]
nTerror("In doFunctionOnEntryList failed forked: %s" % cmd)
def analyze(config, project, parameters, doPrint=0 ):
'''Analyze a run
Returns True on failure.
'''
nTmessage("\n-- analyze --")
inPath = config.directories.converted
modelList = asci2list(parameters.models)
if getDeepByKeysOrAttributes( parameters, USE_ANNEALED_STR):
inPath = config.directories.annealed
modelList = asci2list(parameters.modelsAnneal)
nTdebug( 'inPath: %s' % inPath)
nTdebug( 'modelList: %s' % modelList)
# first create the jobs, run later
analyzeJobs = []
for i in modelList:
job = Analyze(
config,
parameters,
project = project,
fileNum=i,
molecules=[
NTdict(
psfFile=parameters.psfFile,
pdbFile=parameters.baseName % i,
selectionCode='(not resn TIP3 and not resn ANI)'
),
],
inPath = inPath,
outPath = config.directories.analyzed,
jobName='analyze_%d' % i,
)
analyzeJobs.append(job)
#end for
job_list = []
for job in analyzeJobs:
if job.createScript():
nTerror("In refine#analyze failed to create at least one job's script.")
return True
if doPrint:
job.printScript()
#end if
# job.runScript()
job_list.append((job.runScript,) )
#end for
f = ForkOff(
processes_max=cing.ncpus,
max_time_to_wait=600,
verbosity=cing.verbosity
)
done_list = f.forkoff_start(job_list, 0) # delay 0 second between jobs.
nTmessage("Finished ids: %s", done_list)
if not done_list:
notDone = parameters.modelCountAnneal - len(done_list)
nTerror("Failed to analyze %s", notDone)
return True
def doScriptOnEntryList(
pythonScriptFileName,
entryListFileName,
startDir='.',
processes_max=3, # default 3
max_time_to_wait=600,
delay_between_submitting_jobs=1,
extraArgList=None,
start_entry_id=start_entry_id,
max_entries_todo=max_entries_todo,
expectPdbEntryList=True,
shuffleBeforeSelecting=False, # fails for chain ids when included.
entryList=() # as an alternative to a file.
):
"""Return True on error"""
if True: # DEFAULT: True
pid = os.getpid()
nTmessage(
"Use kill -2 %s (sending a INT (interrupt) to this Process ID) twice to kill all child processes."
% pid)
nTmessage("entryListFileName : %s" % entryListFileName)
nTmessage("startDir : %s" % startDir)
nTmessage("processes_max : %s" % processes_max)
nTmessage("max_time_to_wait : %s" % max_time_to_wait)
nTmessage("delay_between_submitting_jobs: %s" %
delay_between_submitting_jobs)
nTmessage("extraArgList : %s" % str(extraArgList))
nTmessage("start_entry_id : %s" % start_entry_id)
nTmessage("max_entries_todo : %s" % max_entries_todo)
nTmessage("expectPdbEntryList : %s" % expectPdbEntryList)
nTmessage("shuffleBeforeSelecting : %s" % shuffleBeforeSelecting)
# end if
# if os.chdir(cingDirTmp):
# raise SetupError("Failed to change to directory for temporary test files: "+cingDirTmp)
# Empty list means no filtering done.
entryCodeListFilter = []
# entryCodeListFilter = string.split("1n62")
if entryListFileName:
entryList = [line.strip() for line in open(entryListFileName)]
entryList = [x for x in entryList if x]
entryCountTotal = len(entryList)
if expectPdbEntryList:
chainCodeList = []
entryCodeList = []
for ss in entryList:
entryCode = ss[:4].lower()
if entryCode not in entryCodeListFilter:
if len(ss) > 4:
chainCode = ss[4].upper()
else:
chainCode = ''
entryCodeList.append(entryCode)
chainCodeList.append(chainCode)
entryCountSelected = len(entryCodeList)
else:
#Non-PDB
entryCodeList = [x for x in entryList if x not in entryCodeListFilter]
entryCountSelected = len(entryCodeList)
chainCodeList = [''] * entryCountSelected
# lastEntryId is id of last entry excluding the entry itself.
lastEntryId = min(len(entryCodeList), start_entry_id + max_entries_todo)
if shuffleBeforeSelecting:
nTmessage("Shuffling entry list before selecting entries.")
entryCodeListCopy = entryCodeList[:]
shuffle(entryCodeListCopy)
entryCodeList = entryCodeListCopy[
start_entry_id:
lastEntryId] # no sense in starting at zero here; they're random.
else:
entryCodeList = entryCodeList[
start_entry_id:
lastEntryId] # no sense in starting at zero here; they're random.
entryCodeList.sort()
chainCodeList = chainCodeList[start_entry_id:lastEntryId]
nTmessage('Read %05d entries ' % entryCountTotal)
nTmessage('Selected %05d entries ' % entryCountSelected)
nTmessage('Sliced %05d entries: %s' %
(len(entryCodeList), entryCodeList))
# nTmessage('Sliced %05d chains: %s' % (len(chainCodeList), chainCodeList ))
useAnyCmd = False
if pythonScriptFileName.endswith('.py'):
(_directory, pythonScriptFileNameRoot,
_ext) = nTpath(pythonScriptFileName)
else:
useAnyCmd = True
pythonScriptFileNameRoot = pythonScriptFileName.split()[
0] # get first word of eg.
if pythonScriptFileNameRoot.count('/'):
pythonScriptFileNameRoot = pythonScriptFileNameRoot.split('/')[-1]
if pythonScriptFileNameRoot.count('.'):
pythonScriptFileNameRoot = pythonScriptFileNameRoot.split('.')[0]
# 'cing -v 9 -n $x --initPDB $PDB/pdb$x.ent.gz --validateFastest --ranges cv')
mkSubDirStructure(startDir, entryCodeList, pythonScriptFileNameRoot)
logScriptFileNameRoot = 'log_' + pythonScriptFileNameRoot
job_list = []
for i, entry_code in enumerate(entryCodeList):
extraArgListStr = ''
if extraArgList:
extraArgListStr = ' '.join(extraArgList)
chain_code = chainCodeList[i]
entryCodeChar2and3 = entry_code[1:3]
entryDir = os.path.join(startDir, DATA_STR, entryCodeChar2and3,
entry_code)
date_stamp = getDateTimeStampForFileName()
cmd = 'cd %s; python -u %s %s %s %s > %s/%s_%s%s.log 2>&1 ' % (
entryDir, pythonScriptFileName, entry_code, chain_code,
extraArgListStr, logScriptFileNameRoot, entry_code, chain_code,
date_stamp)
if useAnyCmd:
cmdBase = pythonScriptFileName
cmdBase = cmdBase.replace("$x", entry_code)
cmdBase = cmdBase.replace("$c", chain_code)
cmd = 'cd %s; %s %s %s > %s/%s_%s%s.log 2>&1 ' % (
entryDir, cmdBase, entry_code, chain_code,
logScriptFileNameRoot, entry_code, chain_code, date_stamp)
job = (do_cmd, (cmd, ))
# nTdebug("Will schedule job cmd: %s" % cmd)
job_list.append(job)
f = ForkOff(processes_max=processes_max, max_time_to_wait=max_time_to_wait)
done_entry_list = f.forkoff_start(job_list, delay_between_submitting_jobs)
done_entry_list.sort()
not_done_entry_list = range(len(job_list))
for id in done_entry_list:
idx = not_done_entry_list.index(id)
if idx >= 0:
del (not_done_entry_list[idx])
nTmessage("In doScriptOnEntryList Finished list : %s" % done_entry_list)
nTmessage("In doScriptOnEntryList Unfinished list: %s" %
not_done_entry_list)
for id in not_done_entry_list:
job = job_list[id]
_do_cmd, cmdTuple = job
cmd = cmdTuple[0]
nTerror("In doScriptOnEntryList failed forked: %s" % cmd)
def doFunctionOnEntryList(
f,
entryListFileName,
processes_max=3, # default 3
max_time_to_wait=600,
delay_between_submitting_jobs=0,
extraArgList=None,
start_entry_id=start_entry_id,
max_entries_todo=max_entries_todo,
shuffleBeforeSelecting=False, # fails for chain ids when included.
entryList=None # as an alternative to a file.
):
"""Return True on error"""
if True: # DEFAULT: True
pid = os.getpid()
nTmessage(
"Use kill -2 %s (sending a INT (interrupt) to this Process ID) twice to kill all child processes."
% pid)
nTmessage("entryListFileName : %s" % entryListFileName)
nTmessage("processes_max : %s" % processes_max)
nTmessage("max_time_to_wait : %s" % max_time_to_wait)
nTmessage("delay_between_submitting_jobs: %s" %
delay_between_submitting_jobs)
nTmessage("extraArgList : %s" % str(extraArgList))
nTmessage("start_entry_id : %s" % start_entry_id)
nTmessage("max_entries_todo : %s" % max_entries_todo)
nTmessage("shuffleBeforeSelecting : %s" % shuffleBeforeSelecting)
nTmessage("entryList : %s" % entryList)
# end if
# if os.chdir(cingDirTmp):
# raise SetupError("Failed to change to directory for temporary test files: "+cingDirTmp)
# Empty list means no filtering done.
entryCodeListFilter = []
# entryCodeListFilter = string.split("1n62")
if entryListFileName:
entryListFile = file(entryListFileName, 'r')
entryCodeList = []
entryCountTotal = 0
for line in entryListFile.readlines():
line = line.strip()
if line == '': # skip empty lines.
continue
entryCountTotal += 1
entryCode = line
if entryCode in entryCodeListFilter:
continue
entryCodeList.append(entryCode)
entryListFile.close()
else:
entryCodeList = entryList
entryCountTotal = len(entryList)
# end if
entryCountSelected = len(entryCodeList)
# lastEntryId is id of last entry excluding the entry itself.
lastEntryId = min(len(entryCodeList), start_entry_id + max_entries_todo)
if shuffleBeforeSelecting:
nTmessage("Shuffling entry list before selecting entries.")
entryCodeListCopy = entryCodeList[:]
shuffle(entryCodeListCopy)
entryCodeList = entryCodeListCopy[
start_entry_id:
lastEntryId] # no sense in starting at zero here; they're random.
else:
entryCodeList = entryCodeList[
start_entry_id:
lastEntryId] # no sense in starting at zero here; they're random.
entryCodeList.sort()
nTmessage('Read %05d entries ' % entryCountTotal)
nTmessage('Selected %05d entries ' % entryCountSelected)
nTmessage('Sliced %05d entries: %s' %
(len(entryCodeList), entryCodeList))
# nTmessage('Sliced %05d chains: %s' % (len(chainCodeList), chainCodeList ))
job_list = []
for _i, entry_code in enumerate(entryCodeList):
# extraArgListStr = ''
# if extraArgList:
# extraArgListStr = ' '.join( extraArgList )
argList = [entry_code]
if isinstance(extraArgList, list) or isinstance(extraArgList, tuple):
argList += extraArgList
# else:
# nTdebug("Skipping non-List/Tuple extraArgList: %s" % str(extraArgList))
# end if
job = (f, tuple(argList))
job_list.append(job)
# end for
f = ForkOff(processes_max=processes_max, max_time_to_wait=max_time_to_wait)
done_entry_list = f.forkoff_start(job_list, delay_between_submitting_jobs)
done_entry_list.sort()
not_done_entry_list = range(len(job_list))
for id in done_entry_list:
idx = not_done_entry_list.index(id)
if idx >= 0:
del (not_done_entry_list[idx])
# end if
# end for
nTmessage("In doScriptOnEntryList Finished list : %s" % done_entry_list)
nTmessage("In doScriptOnEntryList Unfinished list: %s" %
not_done_entry_list)
for id in not_done_entry_list:
job = job_list[id]
_do_cmd, cmdTuple = job
cmd = cmdTuple[0]
nTerror("In doFunctionOnEntryList failed forked: %s" % cmd)