def run(self, timeout):
def target():
self.process = subprocess.Popen(self.cmd, shell=True)
self.process.communicate()
thread = threading.Thread(target=target)
thread.start()
thread.join(timeout)
if thread.is_alive():
print redStr("ERROR: timeout reached for this process")
self.process.terminate()
thread.join()
self.process = None
def submitProtocol(script, **params):
'''Launch a protocol, to a queue or executing directly.
If the queue options are found, it will be launched with
configuration (command and file template) found in project settings
This function should be called from ProjectDir
'''
#Load the config module
launch = loadLaunchModule()
launchFilename = script.replace('.py', '.job')
# This is for make a copy of nodes files
nodesFile = script.replace('.py', '.nodes')
params['nodesfileBackup'] = nodesFile
params['file'] = launchFilename
params['XMIPP_HOME'] = os.environ['XMIPP_HOME']
#create launch file
launchfile = open(launchFilename, 'w')
launchfile.write(launch.FileTemplate % params)
launchfile.close()
command = launch.Program + " " + launch.ArgsTemplate % params
from protlib_xmipp import greenStr, redStr
from protlib_sql import SqliteDb
print "** Submiting to queue: '%s'" % greenStr(command)
ps = Popen(command, shell=True, stdout=PIPE)
out = ps.communicate()[0]
import re
s = re.search('(\d+)', out)
if s:
return int(s.group(0))
else:
print "** Couldn't parse %s ouput: %s" % (greenStr(launch.Program), redStr(out))
return SqliteDb.UNKNOWN_JOBID
def qualifyModes(log,WorkingDir,NumberOfModes,StructureType,CollectivityThreshold):
currentDir=os.getcwd()
changeDir(log,WorkingDir)
fnVec=glob.glob("modes/vec.*")
if len(fnVec)<NumberOfModes:
fhWarning=open("warnings.xmd",'w')
fhWarning.write(redStr("There are only "+str(len(fnVec))+" modes instead of "+str(NumberOfModes)+". Check the number of modes you asked to compute and/or consider increasing cut-off distance. The maximum number of modes allowed by the method for atomic normal mode analysis is 6 times the number of RTB blocks and for pseudoatomic normal mode analysis 3 times the number of pseudoatoms. However, the protocol allows only up to 200 modes as 20-100 modes are usually enough. If the number of modes is below the minimum between these two numbers, consider increasing cut-off distance.")+"\n")
fhWarning.close()
fnDiag="diagrtb.eigenfacs"
if StructureType=="EM":
runJob(log,"nma_reformatForElNemo.sh","%d"%NumberOfModes)
fnDiag="diag_arpack.eigenfacs"
runJob(log,"echo","%s | nma_check_modes"%fnDiag)
deleteFile(log,fnDiag)
fh=open("Chkmod.res")
MDout=MetaData()
collectivityList=[]
for n in range(NumberOfModes):
line=fh.readline()
collectivity=float(line.split()[1])
collectivityList.append(collectivity)
id=MDout.addObject()
MDout.setValue(MDL_NMA_MODEFILE,os.path.join(WorkingDir,"modes/vec.%d"%(n+1)),id)
MDout.setValue(MDL_ORDER,long(n+1),id)
if n>=6:
MDout.setValue(MDL_ENABLED,1,id)
else:
MDout.setValue(MDL_ENABLED,-1,id)
MDout.setValue(MDL_NMA_COLLECTIVITY,collectivity,id)
if collectivity<CollectivityThreshold:
MDout.setValue(MDL_ENABLED,-1,id)
fh.close()
idxSorted=[i[0] for i in sorted(enumerate(collectivityList), key=lambda x:x[1])]
score=[0]*NumberOfModes
for i in range(NumberOfModes):
score[i]+=i+1
score[idxSorted[i]]+=NumberOfModes-i
i=0
for id in MDout:
score_i=float(score[i])/(2.0*NumberOfModes)
MDout.setValue(MDL_NMA_SCORE,score_i,id)
i+=1
MDout.write("modes.xmd")
deleteFile(log,"Chkmod.res")
changeDir(log,currentDir)
def computeModesPDB(log, WorkingDir, NumberOfModes, CutoffMode, Rc, RcPercentage, RTBblockSize, RTBForceConstant):
if CutoffMode=="Relative":
Rc=computeCutoff(WorkingDir,os.path.join(WorkingDir,"extra/atoms_distance.hist"),RcPercentage)
currentDir=os.getcwd()
changeDir(log,WorkingDir)
runJob(log,"nma_record_info_PDB.py","%d %d atoms.pdb %f %f"%(NumberOfModes,RTBblockSize,Rc,RTBForceConstant))
runJob(log,"nma_elnemo_pdbmat","")
runJob(log,"nma_diagrtb","")
if not os.path.exists("diagrtb.eigenfacs"):
fhWarning=open("warnings.xmd",'w')
fhWarning.write(redStr("Modes cannot be computed. Check the number of modes you asked to compute and/or consider increasing cut-off distance. The maximum number of modes allowed by the method for atomic normal mode analysis is 6 times the number of RTB blocks but the protocol allows only up to 200 modes as 20-100 modes are usually enough. If the number of modes is below the minimum between 200 and 6 times the number of RTB blocks, consider increasing cut-off distance.")+"\n")
fhWarning.close()
runJob(log,"rm","*.dat_run diagrtb.dat pdbmat.xyzm pdbmat.sdijf pdbmat.dat")
changeDir(log,currentDir)
def computeModes(log, WorkingDir, NumberOfModes, CutoffMode, Rc, RcPercentage):
if CutoffMode=="Relative":
Rc=computeCutoff(WorkingDir,os.path.join(WorkingDir,"extra/pseudoatoms_distance.hist"),RcPercentage)
currentDir=os.getcwd()
changeDir(log,WorkingDir)
runJob(log,"nma_record_info.py","%d pseudoatoms.pdb %d"%(NumberOfModes,int(Rc)))
runJob(log,"nma_pdbmat.pl","pdbmat.dat")
runJob(log,"nma_diag_arpack","")
if not os.path.exists("fort.11"):
fhWarning=open("warnings.xmd",'w')
fhWarning.write(redStr("Modes cannot be computed. Check the number of modes you asked to compute and/or consider increasing cut-off distance. The maximum number of modes allowed by the method for pseudoatomic normal mode analysis is 3 times the number of pseudoatoms but the protocol allows only up to 200 modes as 20-100 modes are usually enough. If the number of modes is below the minimum between 200 and 3 times the number of pseudoatoms, consider increasing cut-off distance.")+"\n")
fhWarning.close()
runJob(log,"rm","diag_arpack.in pdbmat.dat")
changeDir(log,currentDir)
def summary(self):
micrographs, particles = countParticles(self.ExtraDir)
if self.TiltPairs:
suffix = "tilt pairs"
items = "pairs"
micrographs /= 2
particles /= 2
else:
suffix = "micrographs"
items = "particles"
from protlib_xmipp import getMdSize
size = getMdSize(self.MicrographsMd)
summary = ["Input: [%s] with <%u> %s" % (self.importDir, size, suffix),
"Number of %(items)s manually picked: <%(particles)d> (from <%(micrographs)d> micrographs)" % locals()]
md=xmipp.MetaData(self.MicrographsMd)
if not md.containsLabel(xmipp.MDL_CTF_MODEL):
summary.append(redStr("There is no CTF information in the input micrographs: "))
summary.append("[%s]"%self.MicrographsMd)
return summary
def terminate(self):
if self.process != None:
self.process.terminate()
print redStr("Ctrl-c pressed, aborting this test")
def runProgramTests(self, program):
tests = self.progDict[program]
n = len(tests)
outPath = join(self.fnDir, program)
outDir = outPath
testName = ""
testNo = 1
md = MetaData(self._statistics)
for test, mpi, random, preruns, postruns, changeDirectory, testfiles, owner in tests:
# for test, mpi, random, preruns, postruns, changeDirectory, testfiles in tests:
objid = md.addObject()
if n > 1:
outDir = outPath + "_%02d" % testNo
testName = "(%d of %d)" % (testNo, n)
# test num
md.setValue(MDL_BLOCK_NUMBER, testNo, objid)
# program name
# print type(program)
md.setValue(MDL_PROGRAM, str(program), objid)
dtBegin = datetime.now()
timeStr = str(dtBegin)
# beginning time
md.setValue(MDL_DATE, str(timeStr), objid)
print "------------------------------------------------------------------------------------"
print warnStr(">>> Running test %(testName)s of %(program)s (%(timeStr)s)" % locals())
print " Output dir: "
print " ", outDir
print " Statistics file: "
print " ", self._statistics
print " Timeout: "
print " %d seconds" % self._timeout
if exists(outDir):
shutil.rmtree(outDir)
os.makedirs(outDir)
self.outDir = outDir
self.program = program
test = self.expandFormat(test)
self.runCommands(preruns, "prerun")
if mpi:
cmd = (
"mpirun -np 3 `which %s`" % program
) ##DO NOT REPLACE SO WE CAN TEST MPI EASILY.replace("xmipp_", "xmipp_mpi_")
else:
cmd = program
if changeDirectory:
cmd = "cd %s ; " % outDir + cmd + " %s > stdout.txt 2> stderr.txt" % test
else:
cmd += " %s > %s/stdout.txt 2> %s/stderr.txt" % (test, outDir, outDir)
print " Command: "
print " ", greenStr(cmd)
# run the test itself
command = Command(cmd)
self._command = command
try:
command.run(timeout=self._timeout)
except KeyboardInterrupt:
command.terminate()
# result = os.system(cmd)
# result = subprocess.call(cmd, shell=True)
self.runCommands(postruns, "postrun")
tdEnd = (datetime.now() - dtBegin).total_seconds()
# elapsed time
md.setValue(MDL_TIME, tdEnd, objid)
print " Elapsed time: %d seconds" % tdEnd
error = self.checkResult(testfiles, outDir, random)
if len(error):
self.error += error
self.errorFlag = True
print redStr("ERRORS:\n" + error)
md.setValue(MDL_ENABLED, -1, objid)
else:
md.setValue(MDL_ENABLED, 1, objid)
testNo += 1
md.setValue(MDL_USER, str(owner), objid)
md.write(self._statistics)
