def run(self, timeout): def target(): self.process = subprocess.Popen(self.cmd, shell=True) self.process.communicate() thread = threading.Thread(target=target) thread.start() thread.join(timeout) if thread.is_alive(): print redStr("ERROR: timeout reached for this process") self.process.terminate() thread.join() self.process = None
def submitProtocol(script, **params): '''Launch a protocol, to a queue or executing directly. If the queue options are found, it will be launched with configuration (command and file template) found in project settings This function should be called from ProjectDir ''' #Load the config module launch = loadLaunchModule() launchFilename = script.replace('.py', '.job') # This is for make a copy of nodes files nodesFile = script.replace('.py', '.nodes') params['nodesfileBackup'] = nodesFile params['file'] = launchFilename params['XMIPP_HOME'] = os.environ['XMIPP_HOME'] #create launch file launchfile = open(launchFilename, 'w') launchfile.write(launch.FileTemplate % params) launchfile.close() command = launch.Program + " " + launch.ArgsTemplate % params from protlib_xmipp import greenStr, redStr from protlib_sql import SqliteDb print "** Submiting to queue: '%s'" % greenStr(command) ps = Popen(command, shell=True, stdout=PIPE) out = ps.communicate()[0] import re s = re.search('(\d+)', out) if s: return int(s.group(0)) else: print "** Couldn't parse %s ouput: %s" % (greenStr(launch.Program), redStr(out)) return SqliteDb.UNKNOWN_JOBID
def qualifyModes(log,WorkingDir,NumberOfModes,StructureType,CollectivityThreshold): currentDir=os.getcwd() changeDir(log,WorkingDir) fnVec=glob.glob("modes/vec.*") if len(fnVec)<NumberOfModes: fhWarning=open("warnings.xmd",'w') fhWarning.write(redStr("There are only "+str(len(fnVec))+" modes instead of "+str(NumberOfModes)+". Check the number of modes you asked to compute and/or consider increasing cut-off distance. The maximum number of modes allowed by the method for atomic normal mode analysis is 6 times the number of RTB blocks and for pseudoatomic normal mode analysis 3 times the number of pseudoatoms. However, the protocol allows only up to 200 modes as 20-100 modes are usually enough. If the number of modes is below the minimum between these two numbers, consider increasing cut-off distance.")+"\n") fhWarning.close() fnDiag="diagrtb.eigenfacs" if StructureType=="EM": runJob(log,"nma_reformatForElNemo.sh","%d"%NumberOfModes) fnDiag="diag_arpack.eigenfacs" runJob(log,"echo","%s | nma_check_modes"%fnDiag) deleteFile(log,fnDiag) fh=open("Chkmod.res") MDout=MetaData() collectivityList=[] for n in range(NumberOfModes): line=fh.readline() collectivity=float(line.split()[1]) collectivityList.append(collectivity) id=MDout.addObject() MDout.setValue(MDL_NMA_MODEFILE,os.path.join(WorkingDir,"modes/vec.%d"%(n+1)),id) MDout.setValue(MDL_ORDER,long(n+1),id) if n>=6: MDout.setValue(MDL_ENABLED,1,id) else: MDout.setValue(MDL_ENABLED,-1,id) MDout.setValue(MDL_NMA_COLLECTIVITY,collectivity,id) if collectivity<CollectivityThreshold: MDout.setValue(MDL_ENABLED,-1,id) fh.close() idxSorted=[i[0] for i in sorted(enumerate(collectivityList), key=lambda x:x[1])] score=[0]*NumberOfModes for i in range(NumberOfModes): score[i]+=i+1 score[idxSorted[i]]+=NumberOfModes-i i=0 for id in MDout: score_i=float(score[i])/(2.0*NumberOfModes) MDout.setValue(MDL_NMA_SCORE,score_i,id) i+=1 MDout.write("modes.xmd") deleteFile(log,"Chkmod.res") changeDir(log,currentDir)
def computeModesPDB(log, WorkingDir, NumberOfModes, CutoffMode, Rc, RcPercentage, RTBblockSize, RTBForceConstant): if CutoffMode=="Relative": Rc=computeCutoff(WorkingDir,os.path.join(WorkingDir,"extra/atoms_distance.hist"),RcPercentage) currentDir=os.getcwd() changeDir(log,WorkingDir) runJob(log,"nma_record_info_PDB.py","%d %d atoms.pdb %f %f"%(NumberOfModes,RTBblockSize,Rc,RTBForceConstant)) runJob(log,"nma_elnemo_pdbmat","") runJob(log,"nma_diagrtb","") if not os.path.exists("diagrtb.eigenfacs"): fhWarning=open("warnings.xmd",'w') fhWarning.write(redStr("Modes cannot be computed. Check the number of modes you asked to compute and/or consider increasing cut-off distance. The maximum number of modes allowed by the method for atomic normal mode analysis is 6 times the number of RTB blocks but the protocol allows only up to 200 modes as 20-100 modes are usually enough. If the number of modes is below the minimum between 200 and 6 times the number of RTB blocks, consider increasing cut-off distance.")+"\n") fhWarning.close() runJob(log,"rm","*.dat_run diagrtb.dat pdbmat.xyzm pdbmat.sdijf pdbmat.dat") changeDir(log,currentDir)
def computeModes(log, WorkingDir, NumberOfModes, CutoffMode, Rc, RcPercentage): if CutoffMode=="Relative": Rc=computeCutoff(WorkingDir,os.path.join(WorkingDir,"extra/pseudoatoms_distance.hist"),RcPercentage) currentDir=os.getcwd() changeDir(log,WorkingDir) runJob(log,"nma_record_info.py","%d pseudoatoms.pdb %d"%(NumberOfModes,int(Rc))) runJob(log,"nma_pdbmat.pl","pdbmat.dat") runJob(log,"nma_diag_arpack","") if not os.path.exists("fort.11"): fhWarning=open("warnings.xmd",'w') fhWarning.write(redStr("Modes cannot be computed. Check the number of modes you asked to compute and/or consider increasing cut-off distance. The maximum number of modes allowed by the method for pseudoatomic normal mode analysis is 3 times the number of pseudoatoms but the protocol allows only up to 200 modes as 20-100 modes are usually enough. If the number of modes is below the minimum between 200 and 3 times the number of pseudoatoms, consider increasing cut-off distance.")+"\n") fhWarning.close() runJob(log,"rm","diag_arpack.in pdbmat.dat") changeDir(log,currentDir)
def summary(self): micrographs, particles = countParticles(self.ExtraDir) if self.TiltPairs: suffix = "tilt pairs" items = "pairs" micrographs /= 2 particles /= 2 else: suffix = "micrographs" items = "particles" from protlib_xmipp import getMdSize size = getMdSize(self.MicrographsMd) summary = ["Input: [%s] with <%u> %s" % (self.importDir, size, suffix), "Number of %(items)s manually picked: <%(particles)d> (from <%(micrographs)d> micrographs)" % locals()] md=xmipp.MetaData(self.MicrographsMd) if not md.containsLabel(xmipp.MDL_CTF_MODEL): summary.append(redStr("There is no CTF information in the input micrographs: ")) summary.append("[%s]"%self.MicrographsMd) return summary
def terminate(self): if self.process != None: self.process.terminate() print redStr("Ctrl-c pressed, aborting this test")
def runProgramTests(self, program): tests = self.progDict[program] n = len(tests) outPath = join(self.fnDir, program) outDir = outPath testName = "" testNo = 1 md = MetaData(self._statistics) for test, mpi, random, preruns, postruns, changeDirectory, testfiles, owner in tests: # for test, mpi, random, preruns, postruns, changeDirectory, testfiles in tests: objid = md.addObject() if n > 1: outDir = outPath + "_%02d" % testNo testName = "(%d of %d)" % (testNo, n) # test num md.setValue(MDL_BLOCK_NUMBER, testNo, objid) # program name # print type(program) md.setValue(MDL_PROGRAM, str(program), objid) dtBegin = datetime.now() timeStr = str(dtBegin) # beginning time md.setValue(MDL_DATE, str(timeStr), objid) print "------------------------------------------------------------------------------------" print warnStr(">>> Running test %(testName)s of %(program)s (%(timeStr)s)" % locals()) print " Output dir: " print " ", outDir print " Statistics file: " print " ", self._statistics print " Timeout: " print " %d seconds" % self._timeout if exists(outDir): shutil.rmtree(outDir) os.makedirs(outDir) self.outDir = outDir self.program = program test = self.expandFormat(test) self.runCommands(preruns, "prerun") if mpi: cmd = ( "mpirun -np 3 `which %s`" % program ) ##DO NOT REPLACE SO WE CAN TEST MPI EASILY.replace("xmipp_", "xmipp_mpi_") else: cmd = program if changeDirectory: cmd = "cd %s ; " % outDir + cmd + " %s > stdout.txt 2> stderr.txt" % test else: cmd += " %s > %s/stdout.txt 2> %s/stderr.txt" % (test, outDir, outDir) print " Command: " print " ", greenStr(cmd) # run the test itself command = Command(cmd) self._command = command try: command.run(timeout=self._timeout) except KeyboardInterrupt: command.terminate() # result = os.system(cmd) # result = subprocess.call(cmd, shell=True) self.runCommands(postruns, "postrun") tdEnd = (datetime.now() - dtBegin).total_seconds() # elapsed time md.setValue(MDL_TIME, tdEnd, objid) print " Elapsed time: %d seconds" % tdEnd error = self.checkResult(testfiles, outDir, random) if len(error): self.error += error self.errorFlag = True print redStr("ERRORS:\n" + error) md.setValue(MDL_ENABLED, -1, objid) else: md.setValue(MDL_ENABLED, 1, objid) testNo += 1 md.setValue(MDL_USER, str(owner), objid) md.write(self._statistics)