def start(self): # Path of the stack stackdata = apStack.getOnlyStackData(self.params['stackid']) fn_oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) # Convert the stack to Xmipp apXmipp.breakupStackIntoSingleFiles(fn_oldstack) # Run sort junk cmd = "xmipp_sort_by_statistics -i partlist.sel" apDisplay.printColor(cmd, "cyan") proc = subprocess.Popen(cmd, shell=True) proc.wait() # Create sorted stack apXmipp.gatherSingleFilesIntoStack("sort_junk.sel","sorted.hed") # Create average MRC apStack.averageStack("sorted.hed") # Remove intermediate stuff #os.unlink("partlist.sel") #shutil.rmtree("partfiles") # Upload results self.uploadResults() time.sleep(1) return
def start(self): # Path of the stack stackdata = apStack.getOnlyStackData(self.params['stackid']) fn_oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) # Convert the stack to Xmipp apXmipp.breakupStackIntoSingleFiles(fn_oldstack) # Run sort junk cmd = "xmipp_sort_by_statistics -i partlist.sel" apDisplay.printColor(cmd, "cyan") proc = subprocess.Popen(cmd, shell=True) proc.wait() # Create sorted stack apXmipp.gatherSingleFilesIntoStack("sort_junk.sel", "sorted.hed") # Create average MRC apStack.averageStack("sorted.hed") # Remove intermediate stuff #os.unlink("partlist.sel") #shutil.rmtree("partfiles") # Upload results self.uploadResults() time.sleep(1) return
def _boxParticlesFromImage(self, imgdata, parttree, imgstackfile): ''' Box Particles From the manipulated full size image file on disk ''' ### make corrected integrated frame image imgpath = self.getOriginalImagePath(imgdata) t0 = time.time() if self.params['phaseflipped'] is True: if self.params['fliptype'] == 'emanimage': ### ctf correct whole image using EMAN imgpath = self.phaseFlipWholeImage(imgpath, imgdata) elif self.params['fliptype'] == "spiderimage": imgpath = self.phaseFlipSpider(imgpath, imgdata) elif self.params['fliptype'][:9] == "ace2image": ### ctf correct whole image using Ace 2 imgpath = self.phaseFlipAceTwo(imgpath, imgdata) self.ctftimes.append(time.time()-t0) if imgpath is None: return None, None, None ### run apBoxer apDisplay.printMsg("boxing "+str(len(parttree))+" particles into temp file: "+imgstackfile) ### method to align helices t0 = time.time() if self.params['rotate'] is True: apBoxer.boxerRotate(imgpath, parttree, imgstackfile, self.boxsize) if self.params['finealign'] is True: from appionlib import apXmipp apXmipp.breakupStackIntoSingleFiles(imgstackfile, filetype="mrc") rotcmd = "s_finealign %s %i" %(self.params['rundir'], self.boxsize) apParam.runCmd(rotcmd, "HIP", verbose=True) # read in text file containing refined angles anglepath = os.path.join(self.params['rundir'], 'angles.out') f = open(anglepath, 'r') angles = f.readlines() # loop through parttree and add refined angle to rough angle for i in range(len(parttree)): partdict = parttree[i] fineangle = float(angles[i]) newangle = float(partdict['angle'])-fineangle partdict['angle'] = newangle # rerun apBoxer.boxerRotate with the new parttree containing final angles apBoxer.boxerRotate(imgpath, parttree, imgstackfile, self.boxsize, pixlimit=self.params['pixlimit']) else: apBoxer.boxer(imgpath, parttree, imgstackfile, self.boxsize, pixlimit=self.params['pixlimit']) self.batchboxertimes.append(time.time()-t0)
def runMaxlike(self): stackdata = apStack.getOnlyStackData(self.params["stackid"]) apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"]) ### process stack to local file self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed") proccmd = "proc2d " + stackfile + " " + self.params["localstack"] + " apix=" + str(apix) if self.params["highpass"] > 1: proccmd += " hp=" + str(self.params["highpass"]) if self.params["lowpass"] > 1: proccmd += " lp=" + str(self.params["lowpass"]) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"]) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params["rundir"], self.partlistdocfile) + " -nref 1 " + " -iter 10 " + " -o " + os.path.join(self.params["rundir"], "part" + self.timestamp) + " -fast -C 1e-18 " ) ### angle step if self.params["rotate"] is True: xmippopts += " -psi_step 90 " else: xmippopts += " -psi_step 360 " ### convergence criteria if self.params["converge"] == "fast": xmippopts += " -eps 5e-3 " elif self.params["converge"] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params["mirror"] is True: xmippopts += " -mirror " if self.params["maxshift"] is not None: xmippopts += " -max_shift %d " % (self.params["maxshift"]) ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe + " " + xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ### create a quick mrc emancmd = "proc2d part" + self.timestamp + "_ref000001.xmp average.mrc" apEMAN.executeEmanCmd(emancmd, verbose=True) apFile.removeStack(self.params["localstack"]) apFile.removeFilePattern("partfiles/*")
def runMaxlike(self): stackdata = apStack.getOnlyStackData(self.params['stackid']) apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) ### process stack to local file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp + ".hed") proccmd = "proc2d " + stackfile + " " + self.params[ 'localstack'] + " apix=" + str(apix) if self.params['highpass'] > 1: proccmd += " hp=" + str(self.params['highpass']) if self.params['lowpass'] > 1: proccmd += " lp=" + str(self.params['lowpass']) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles( self.params['localstack']) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params['rundir'], self.partlistdocfile) + " -nref 1 " + " -iter 10 " + " -o " + os.path.join(self.params['rundir'], "part" + self.timestamp) + " -fast -C 1e-18 ") ### angle step if self.params['rotate'] is True: xmippopts += " -psi_step 90 " else: xmippopts += " -psi_step 360 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " if self.params['maxshift'] is not None: xmippopts += " -max_shift %d " % (self.params['maxshift']) ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe + " " + xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ### create a quick mrc emancmd = "proc2d part" + self.timestamp + "_ref000001.xmp average.mrc" apEMAN.executeEmanCmd(emancmd, verbose=True) apFile.removeStack(self.params['localstack']) apFile.removeFilePattern("partfiles/*")
def xmippNormStack(self, inStackPath, outStackPath): ### convert stack into single spider files selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath) ### setup Xmipp command xmippexe = apParam.getExecPath("xmipp_normalize", die=True) apDisplay.printMsg("Using Xmipp to normalize particle stack") normtime = time.time() xmippopts = ( " " +" -i %s"%os.path.join(self.params['rundir'],selfile) +" -method Ramp " +" -background circle %i"%(self.stack['boxsize']/self.params['bin']*0.4) +" -remove_black_dust" +" -remove_white_dust" +" -thr_black_dust -%.2f"%(self.params['xmipp-norm']) +" -thr_white_dust %.2f"%(self.params['xmipp-norm']) ) xmippcmd = xmippexe+" "+xmippopts apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True) normtime = time.time() - normtime apDisplay.printMsg("Xmipp normalization time: "+apDisplay.timeString(normtime)) ### recombine particles to a single imagic stack tmpstack = "tmp.xmippStack.hed" apXmipp.gatherSingleFilesIntoStack(selfile,tmpstack) apFile.moveStack(tmpstack,outStackPath) ### clean up directory apFile.removeFile(selfile) apFile.removeDir("partfiles")
def setupParticles(self): self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed") self.stack = {} self.stack["data"] = apStack.getOnlyStackData(self.params["stackid"]) self.stack["apix"] = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) self.stack["part"] = apStack.getOneParticleFromStackId(self.params["stackid"]) self.stack["boxsize"] = apStack.getStackBoxsize(self.params["stackid"]) self.stack["file"] = os.path.join(self.stack["data"]["path"]["path"], self.stack["data"]["name"]) boxsize = int(math.floor(self.stack["boxsize"] / float(self.params["bin"] * 2))) * 2 apix = self.stack["apix"] * self.params["bin"] proccmd = "proc2d " + self.stack["file"] + " " + self.params["localstack"] + " apix=" + str(self.stack["apix"]) if self.params["bin"] > 1: clipsize = boxsize * self.params["bin"] proccmd += " shrink=%d clip=%d,%d " % (self.params["bin"], clipsize, clipsize) if self.params["highpass"] > 1: proccmd += " hp=" + str(self.params["highpass"]) if self.params["lowpass"] > 1: proccmd += " lp=" + str(self.params["lowpass"]) proccmd += " last=" + str(self.params["numpart"] - 1) apFile.removeStack(self.params["localstack"], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"]) return (boxsize, apix)
def makeStackInFormatFromDefault(stackdata, format='eman'): ''' Function to create the stack in the requested format Current default is Imagic format used in EMAN ''' if format not in formats: apDisplay.printError('Unknown stack format %s requested' % format) stackroot, instackdir, outstackdir = getFormattedStackInfo( stackdata, format) instackpath = os.path.join(instackdir, stackroot + '.hed') if format == 'eman': return stackdata['path']['path'] apParam.createDirectory(outstackdir) rundir = os.getcwd() os.chdir(outstackdir) if format == 'spider': outstackpath = os.path.join(outstackdir, stackroot + stack_exts[format][0]) emancmd = "proc2d %s %s spidersingle" % (instackpath, outstackpath) apEMAN.executeEmanCmd(emancmd, showcmd=True) if format == 'frealign': outstackpath = os.path.join(outstackdir, stackroot + stack_exts[format][0]) emancmd = "proc2d %s %s invert" % (instackpath, outstackpath) apEMAN.executeEmanCmd(emancmd, showcmd=True) if format == 'xmipp': ### convert stack into single spider files selfile = apXmipp.breakupStackIntoSingleFiles(instackpath) os.chdir(rundir) return outstackdir
def setupParticles(self): self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2 apix = self.stack['apix']*self.params['bin'] proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix']) if self.params['bin'] > 1: clipsize = boxsize*self.params['bin'] proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize) if self.params['highpass'] > 1: proccmd += " hp="+str(self.params['highpass']) if self.params['lowpass'] > 1: proccmd += " lp="+str(self.params['lowpass']) proccmd += " last="+str(self.params['numpart']-1) apFile.removeStack(self.params['localstack'], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) return (boxsize, apix)
def setupParticles(self): self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2 apix = self.stack['apix']*self.params['bin'] proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix']) if self.params['bin'] > 1: clipsize = boxsize*self.params['bin'] proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize) if self.params['highpass'] > 1: proccmd += " hp="+str(self.params['highpass']) if self.params['lowpass'] > 1: proccmd += " lp="+str(self.params['lowpass']) proccmd += " last="+str(self.params['numpart']-1) apFile.removeStack(self.params['localstack'], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) return (boxsize, apix)
def xmippNormStack(self, inStackPath, outStackPath): ### convert stack into single spider files selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath) ### setup Xmipp command xmippexe = apParam.getExecPath("xmipp_normalize", die=True) apDisplay.printMsg("Using Xmipp to normalize particle stack") normtime = time.time() xmippopts = (" " + " -i %s" % os.path.join(self.params['rundir'], selfile) + " -method Ramp " + " -background circle %i" % (self.stack['boxsize'] / self.params['bin'] * 0.4) + " -remove_black_dust" + " -remove_white_dust" + " -thr_black_dust -%.2f" % (self.params['xmipp-norm']) + " -thr_white_dust %.2f" % (self.params['xmipp-norm'])) xmippcmd = xmippexe + " " + xmippopts apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True) normtime = time.time() - normtime apDisplay.printMsg("Xmipp normalization time: " + apDisplay.timeString(normtime)) ### recombine particles to a single imagic stack tmpstack = "tmp.xmippStack.hed" apXmipp.gatherSingleFilesIntoStack(selfile, tmpstack) apFile.moveStack(tmpstack, outStackPath) ### clean up directory apFile.removeFile(selfile) apFile.removeDir("partfiles")
def makeStackInFormatFromDefault(stackdata, format="eman"): """ Function to create the stack in the requested format Current default is Imagic format used in EMAN """ if format not in formats: apDisplay.printError("Unknown stack format %s requested" % format) stackroot, instackdir, outstackdir = getFormattedStackInfo(stackdata, format) instackpath = os.path.join(instackdir, stackroot + ".hed") if format == "eman": return stackdata["path"]["path"] apParam.createDirectory(outstackdir) rundir = os.getcwd() os.chdir(outstackdir) if format == "spider": outstackpath = os.path.join(outstackdir, stackroot + stack_exts[format][0]) emancmd = "proc2d %s %s spidersingle" % (instackpath, outstackpath) apEMAN.executeEmanCmd(emancmd, showcmd=True) if format == "frealign": outstackpath = os.path.join(outstackdir, stackroot + stack_exts[format][0]) emancmd = "proc2d %s %s invert" % (instackpath, outstackpath) apEMAN.executeEmanCmd(emancmd, showcmd=True) if format == "xmipp": ### convert stack into single spider files selfile = apXmipp.breakupStackIntoSingleFiles(instackpath) os.chdir(rundir) return outstackdir
def convertToRefineStack(self): hedpath = self.stack['file'] selfile = apXmipp.breakupStackIntoSingleFiles(hedpath) self.stack['file'] = os.path.join(self.params['rundir'], selfile) self.stack['format'] = 'xmipp' # remove EMAN Imagic Stack Files apFile.removeFile(hedpath, warn=True) imgpath = hedpath[:-3] + 'img' apFile.removeFile(imgpath, warn=True)
def convertToRefineStack(self): hedpath = self.stack['file'] selfile = apXmipp.breakupStackIntoSingleFiles(hedpath) self.stack['file'] = os.path.join(self.params['rundir'],selfile) self.stack['format'] = 'xmipp' # remove EMAN Imagic Stack Files apFile.removeFile(hedpath, warn=True) imgpath = hedpath[:-3]+'img' apFile.removeFile(imgpath, warn=True)
def applyEnvelopeAndCTF(self, stack): ### get defocus lists numpart = self.params['projcount'] cut = int(numpart/80.0)+1 apDisplay.printMsg("%d particles per dot"%(cut)) if len(self.deflist1) == 0: self.getListOfDefoci(numpart) ### break up particles partlistdocfile = apXmipp.breakupStackIntoSingleFiles(stack, filetype="mrc") t0 = time.time() apDisplay.printMsg("Applying CTF and Envelop to particles") ### apply CTF using ACE2 ctfapplydocfile = self.applyCTFToDocFile(partlistdocfile) ### apply Envelop using ACE2 envelopdocfile = self.applyEnvelopToDocFile(ctfapplydocfile) ### correct CTF using ACE2 if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True: ctfcorrectdocfile = self.correctCTFToDocFile(envelopdocfile) else: ctfcorrectdocfile = envelopdocfile timeper = (time.time()-t0)/float(numpart) apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green") apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green") ### write corrected particle list to doc file ctfpartlist = [] ctfpartlistfile = os.path.join(self.params['rundir'], "ctfpartlist.lst") inf = open(ctfcorrectdocfile, 'r') outf = open(ctfpartlistfile, "w") for line in inf: ### get filename filename = line.strip().split()[0] if not os.path.isfile(filename): apDisplay.printError("CTF and envelop apply failed") ctfpartlist.append(filename) outf.write(filename+"\t1\n") inf.close() outf.close() ### merge individual files into a common stack ctfstack = os.path.join(self.params['rundir'], "ctfstack.hed") apXmipp.gatherSingleFilesIntoStack(ctfpartlistfile, ctfstack, filetype="mrc") if self.params['pad'] is True: emancmd = "proc2d %s %s.clip.hed clip=%d,%d" % (ctfstack, ctfstack[:-4], self.params['box'], self.params['box']) apParam.runCmd(emancmd, "EMAN") shutil.move("%s.clip.hed" % ctfstack[:-4], "%s.hed" % ctfstack[:-4]) shutil.move("%s.clip.img" % ctfstack[:-4], "%s.img" % ctfstack[:-4]) return ctfstack, ctfpartlist
def applyEnvelopeAndCTF(self, stack): ### get defocus lists numpart = self.params['projcount'] cut = int(numpart/80.0)+1 apDisplay.printMsg("%d particles per dot"%(cut)) if len(self.deflist1) == 0: self.getListOfDefoci(numpart) ### break up particles partlistdocfile = apXmipp.breakupStackIntoSingleFiles(stack, filetype="mrc") t0 = time.time() apDisplay.printMsg("Applying CTF and Envelop to particles") ### apply CTF using ACE2 ctfapplydocfile = self.applyCTFToDocFile(partlistdocfile) ### apply Envelop using ACE2 envelopdocfile = self.applyEnvelopToDocFile(ctfapplydocfile) ### correct CTF using ACE2 if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True: ctfcorrectdocfile = self.correctCTFToDocFile(envelopdocfile) else: ctfcorrectdocfile = envelopdocfile timeper = (time.time()-t0)/float(numpart) apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green") apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green") ### write corrected particle list to doc file ctfpartlist = [] ctfpartlistfile = os.path.join(self.params['rundir'], "ctfpartlist.lst") inf = open(ctfcorrectdocfile, 'r') outf = open(ctfpartlistfile, "w") for line in inf: ### get filename filename = line.strip().split()[0] if not os.path.isfile(filename): apDisplay.printError("CTF and envelop apply failed") ctfpartlist.append(filename) outf.write(filename+"\t1\n") inf.close() outf.close() ### merge individual files into a common stack ctfstack = os.path.join(self.params['rundir'], "ctfstack.hed") apXmipp.gatherSingleFilesIntoStack(ctfpartlistfile, ctfstack, filetype="mrc") if self.params['pad'] is True: emancmd = "proc2d %s %s.clip.hed clip=%d,%d" % (ctfstack, ctfstack[:-4], self.params['box'], self.params['box']) apParam.runCmd(emancmd, "EMAN") shutil.move("%s.clip.hed" % ctfstack[:-4], "%s.hed" % ctfstack[:-4]) shutil.move("%s.clip.img" % ctfstack[:-4], "%s.img" % ctfstack[:-4]) return ctfstack, ctfpartlist
def start(self): self.writeParams() stackdata = apStack.getOnlyStackData(self.params["stackid"], msg=False) stackpath = os.path.abspath(stackdata["path"]["path"]) stackfile = os.path.join(stackpath, "start.hed") self.params["localstack"] = os.path.join(stackpath, self.timestamp + ".hed") proccmd = "proc2d " + stackfile + " " + self.params["localstack"] + " apix=" + str(self.params["step"]) if self.params["bin"] > 1: proccmd += " shrink=%d" % int(self.params["bin"]) proccmd += " last=" + str(self.params["numpart"] - 1) apParam.runCmd(proccmd, "EMAN", verbose=True) if self.params["numpart"] != apFile.numImagesInStack(self.params["localstack"]): apDisplay.printError("Missing particles in stack") apXmipp.breakupStackIntoSingleFiles(self.params["localstack"], filetype="mrc") if self.params["prehip"] == "yes": if self.params["rise"] is not None: self.writeLLBO1() elif self.params["ll1"] is not None: self.writeLLBO2() elif self.params["rise"] is None and self.params["ll1"] is None: apDisplay.printError( "You must specify either rise and twist or (1,0) and (0,1) layer lines to run preHIP" ) apDisplay.printMsg("now running s_prehip to set up input files") apDisplay.printMsg( "Make sure the layer line/bessel order assignment is correct. If it is not, you may need to adjust some of the input variables." ) cmd = "s_prehip" proc = subprocess.Popen(cmd) proc.wait() else: apDisplay.printMsg("now running s_hip2: Phoelix programs") cmd = "s_hip2" proc = subprocess.Popen(cmd) proc.wait() # apParam.runCmd(cmd, package="Phoelix") self.putFilesInStack() self.insertHipRunData() self.insertHipParamsData() self.insertHipIterData() self.insertHipParticleData()
def start(self): self.writeParams() stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackpath = os.path.abspath(stackdata['path']['path']) stackfile = os.path.join(stackpath, "start.hed") self.params['localstack'] = os.path.join(stackpath, self.timestamp+".hed") proccmd = "proc2d "+stackfile+" "+self.params['localstack']+" apix="+str(self.params['step']) if self.params['bin'] > 1: proccmd += " shrink=%d"%int(self.params['bin']) proccmd += " last="+str(self.params['numpart']-1) apParam.runCmd(proccmd, "EMAN", verbose=True) if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']): apDisplay.printError("Missing particles in stack") apXmipp.breakupStackIntoSingleFiles(self.params['localstack'], filetype="mrc") if self.params['prehip'] == "yes": if self.params['rise'] is not None: self.writeLLBO1() elif self.params['ll1'] is not None: self.writeLLBO2() elif self.params['rise'] is None and self.params['ll1'] is None: apDisplay.printError("You must specify either rise and twist or (1,0) and (0,1) layer lines to run preHIP") apDisplay.printMsg("now running s_prehip to set up input files") apDisplay.printMsg("Make sure the layer line/bessel order assignment is correct. If it is not, you may need to adjust some of the input variables.") cmd = "s_prehip" proc = subprocess.Popen(cmd) proc.wait() else: apDisplay.printMsg("now running s_hip2: Phoelix programs") cmd = "s_hip2" proc = subprocess.Popen(cmd) proc.wait() #apParam.runCmd(cmd, package="Phoelix") self.putFilesInStack() self.insertHipRunData() self.insertHipParamsData() self.insertHipIterData() self.insertHipParticleData()
def runMakeSpectra(self, indata): """ From http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Makespectra This program generates a Fourier-Bessel decomposition of each image listed in a *.sel. Then creates a report ASCII file for each image, with the harmonic energy percentage as a function of the radius, with extension .SPT. """ spectratime = time.time() apDisplay.printMsg("Running make_spectra") #logging.debug('Inside make_spectra') #ROB tempFileNameforSpectra = "tempFileNameforSpectra.txt" tmpSelFile = apXmipp.breakupStackIntoSingleFiles( indata, numpart=self.params['numpart']) makespectracmd = ("xmipp_make_spectra -i %s -o %s \ -x0 %f -y0 %f -r1 %d -r2 %d -low %d -high %d " % ( tmpSelFile, tempFileNameforSpectra, self.xOffset, self.yOffset, self.params['spectrainnerradius'], self.params['spectraouterradius'], self.params['spectralowharmonic'], self.params['spectrahighharmonic'], )) #logging.debug(makespectracmd) apDisplay.printColor(makespectracmd, "cyan") proc = subprocess.Popen(makespectracmd, shell=True, stdout=subprocess.PIPE) proc.wait() apDisplay.printMsg("Spectra calculation complete in %s" % (apDisplay.timeString(time.time() - spectratime))) time.sleep(1) return tempFileNameforSpectra
def runMakeSpectra(self, indata): """ From http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Makespectra This program generates a Fourier-Bessel decomposition of each image listed in a *.sel. Then creates a report ASCII file for each image, with the harmonic energy percentage as a function of the radius, with extension .SPT. """ spectratime = time.time() apDisplay.printMsg("Running make_spectra") #logging.debug('Inside make_spectra') #ROB tempFileNameforSpectra="tempFileNameforSpectra.txt" tmpSelFile=apXmipp.breakupStackIntoSingleFiles(indata, numpart=self.params['numpart']); makespectracmd = ( "xmipp_make_spectra -i %s -o %s \ -x0 %f -y0 %f -r1 %d -r2 %d -low %d -high %d "% (tmpSelFile, tempFileNameforSpectra, self.xOffset, self.yOffset, self.params['spectrainnerradius'], self.params['spectraouterradius'], self.params['spectralowharmonic'], self.params['spectrahighharmonic'], ) ) #logging.debug(makespectracmd) apDisplay.printColor(makespectracmd, "cyan") proc = subprocess.Popen(makespectracmd, shell=True,stdout=subprocess.PIPE) proc.wait() apDisplay.printMsg("Spectra calculation complete in %s"%(apDisplay.timeString(time.time()-spectratime))) time.sleep(1) return tempFileNameforSpectra
def start(self): ''' OptiMod script ''' ############### define short-hand parameter names ############ refine = True rundir = self.params['rundir'] box = self.params['boxsize'] rbox = self.params['refineboxsize'] apix = self.params['apix'] rapix = self.params['refineapix'] npart = self.params['numpart'] nvol = self.params['num_volumes'] # sym = self.params['sym'] sym = "c1" psym = self.params['presumed_sym'] first = self.params['firstimage'] nws = self.params['non_weighted_sequence'] threes = self.params['threes'] lmask = self.params['linmask'] asqfilt = self.params['asqfilt'] anginc = self.params['ang_inc'] keep_ordered = self.params['keep_ordered_num'] hamwin = self.params['ham_win'] useEMAN1 = self.params['useEMAN1'] ipv = self.params['images_per_volume'] lp = self.params['3d_lpfilt'] nref = self.params['nref'] PCA = self.params['PCA'] neigens = self.params['numeigens'] recalc = self.params['recalculate'] preftype = self.params['preftype'] mrad = self.params['mask_radius'] irad = self.params['inner_radius'] orad = self.params['outer_radius'] tnproc = self.params['threadnproc'] nproc = self.params['nproc'] oldavgs = self.params['oldavgs'] avgs = os.path.join(self.params['rundir'], self.params['avgs']) start = self.params['start_at'] ############# copy to working directory ############ if not os.path.isfile(os.path.join(avgs)[:-4]+".hed"): shutil.copyfile(os.path.join(oldavgs[:-4]+".hed"), avgs[:-4]+".hed") if not os.path.isfile(os.path.join(avgs)[:-4]+".img"): shutil.copyfile(os.path.join(oldavgs[:-4]+".img"), avgs[:-4]+".img") if self.params['ravgs'] is not None: ravgs = os.path.join(self.params['rundir'], self.params['ravgs']) if not os.path.isfile(self.params['ravgs'][:-4]+".hed"): shutil.copy(self.params['oldravgs'][:-4]+".hed", os.path.join(self.params['ravgs'][:-4]+".hed")) if not os.path.isfile(self.params['ravgs'][:-4]+".img"): shutil.copy(self.params['oldravgs'][:-4]+".img", os.path.join(self.params['ravgs'][:-4]+".img")) ### Euler jumper assessment does not make sense when refine images are different ejassess = False else: ravgs = avgs ejassess = True ########### scale & prealign class averages, if specified ######### if self.params['scale'] is True: emancmd = "proc2d %s %s_scaled.img scale=%.3f clip=%i,%i edgenorm" \ % (avgs, avgs[:-4], self.scalefactor, 64, 64) avgs = avgs[:-4]+"_scaled.img" if start == "none": while os.path.isfile(avgs): apFile.removeStack(avgs) apParam.runCmd(emancmd, "EMAN") else: apDisplay.printColor("skipping stack scaling", "cyan") if self.imagicroot is not None and useEMAN1 is False: apIMAGIC.takeoverHeaders(avgs, npart, box) if self.imagicroot is not None: if self.params['prealign'] is True: if start == "none": avgs = apIMAGIC.prealignClassAverages(rundir, avgs) apIMAGIC.checkLogFileForErrors( os.path.join(rundir, "prealignClassAverages.log")) ### select between EMAN1 and IMAGIC for common lines if self.imagicroot is not None and useEMAN1 is False: commonlinesdir = os.path.join(rundir, "angular_reconstitution") else: commonlinesdir = os.path.join(rundir, "cross_common_lines") ### make links & calculate CCC matrix for 2D averages apDisplay.printColor("Calculating similarity matrix", "cyan") ccc_matrix = apCommonLines.calculate_ccc_matrix_2d(avgs) clsavgs = os.path.split(avgs)[1][:-4] if not os.path.isdir(commonlinesdir): os.mkdir(commonlinesdir) if os.path.islink(os.path.join(commonlinesdir, clsavgs+".hed")): os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".hed")) os.symlink(os.path.join(rundir, clsavgs+".hed"), os.path.join(commonlinesdir, clsavgs+".hed")) if os.path.islink(os.path.join(commonlinesdir, clsavgs+".img")): os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".img")) os.symlink(os.path.join(rundir, clsavgs+".img"), os.path.join(commonlinesdir, clsavgs+".img")) ################## create multiple 3d0s ##################### if start == "none": if self.imagicroot is not None and useEMAN1 is False: cmdlist = [] seqfile = open(os.path.join(rundir, "sequences_for_angular_reconstitution.dat"), "w") apDisplay.printColor("Running multiple raw volume calculations", "cyan") for i in range(nvol): sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart, ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws) apCommonLines.check_for_duplicates_in_sequence(sequence) seqfile.write(str(sequence)+"\n") ### create IMAGIC batch file for each model & append them to be threaded batchfile = apCommonLines.imagic_batch_file(sequence, i+1, avgs, sym, asqfilt, lmask, apix, box, anginc, keep_ordered, hamwin, lp, threes=threes, do_not_remove=False) proc = subprocess.Popen('chmod 755 '+batchfile, shell=True) proc.wait() cmdlist.append(batchfile) os.chdir(rundir) seqfile.close() apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10) ### check for errors after execution for i in range(nvol): apIMAGIC.checkLogFileForErrors( os.path.join(commonlinesdir, "3d"+str(i+1)+".log")) else: ### use EMAN1 cross-common lines cmdlist = [] seqfile = open(os.path.join(rundir, "sequences_for_cross_common_lines.dat"), "w") for i in range(nvol): sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart, ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws) apCommonLines.check_for_duplicates_in_sequence(sequence) seqfile.write(str(sequence)+"\n") vol_avgfile = os.path.join(commonlinesdir, "vol_%d_averages.hed" % (i+1)) vol_seqfile = os.path.join(commonlinesdir, "vol_%d_averages.txt" % (i+1)) vsf = open(vol_seqfile, "w") for j in range(ipv): vsf.write("%d\n" % sequence[j]) vsf.close() tmpdir = os.path.join(commonlinesdir, "tmp%d" % (i+1)) fullcmd = "proc2d %s %s list=%s ; " % (avgs, vol_avgfile, vol_seqfile) fullcmd+= "rm -rf %s ; mkdir %s ; cd %s ; " % (tmpdir, tmpdir, tmpdir) fullcmd+= "startAny %s sym=c1 proc=1 rounds=2 mask=%d ; " \ % (vol_avgfile, mrad/apix) fullcmd+= "mv threed.0a.mrc ../threed_%d.mrc ; " % (i+1) fullcmd+= "mv CCL.hed ../CCL_%d.hed ; " % (i+1) fullcmd+= "mv CCL.img ../CCL_%d.img ; " % (i+1) cmdlist.append(fullcmd) seqfile.close() apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10) for i in range(nvol): shutil.rmtree(os.path.join(commonlinesdir, "tmp%d" % (i+1))) else: apDisplay.printColor("skipping common lines volume calculations", "cyan") ############### move 3-D models volume directory ################ ### create volume directory volumedir = os.path.join(rundir, "volumes") if not os.path.isdir(volumedir): os.mkdir(volumedir) volumes = {} cmds = [] if start == "none": apDisplay.printColor("moving volumes for Xmipp 3-D Maximum Likelihood", "cyan") for i in range(nvol): if useEMAN1 is False: volume1 = os.path.join(commonlinesdir, "3d%d_ordered%d_filt.vol" % (i+1,i+1)) else: volume1 = os.path.join(commonlinesdir, "threed_%d.mrc" % (i+1)) volume2 = os.path.join(volumedir, "3d%d.vol" % (i+1)) if start == "none": if useEMAN1 is False: shutil.move(volume1, volume2) else: cmds.append("proc3d %s %s spidersingle" % (volume1, volume2)) volumes[(i+1)] = volume2 apThread.threadCommands(cmds, nproc=tnproc) #################### align 3-D models ################## if start == "3D_align" or start == "none": # if start != "3D_assess" and start != "3D_refine": ### run Maximum Likelihood 3-D alignment & align resulting volumes apDisplay.printColor("Running Xmipp maximum likelihood 3-D alignment", "cyan") vol_doc_file, alignref = apCommonLines.xmipp_max_like_3d_align( volumes, nproc, tnproc, nref) alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol) apDisplay.printColor("Aligning volumes based on 3-D ML parameters", "cyan") apCommonLines.align_volumes(alignparams, alignref, nvol, tnproc, apix) else: apDisplay.printColor("skipping 3D alignment", "cyan") vol_doc_file, alignref = apCommonLines.findAlignmentParams( os.path.join(rundir,"max_like_alignment"), nref) alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol) ##################### Principal Component Analysis ############## apDisplay.printColor("Calculating inter-volume similarity", "cyan") aligned_volumes = {} for i in range(nvol): aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.vol" % (i+1)) # aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.mrc" % (i+1)) if PCA is True: simfile, sim_matrix = apCommonLines.runPrincipalComponentAnalysis( aligned_volumes, nvol, neigens, box, apix, recalculate=recalc) else: simfile, sim_matrix = apCommonLines.calculate_ccc_matrix_3d( aligned_volumes, nvol) ################ 3-D affinity propagation ############# ### 3-D Affinity Propagation apDisplay.printColor("Averaging volumes with Affinity Propagation", "cyan") preffile = apCommonLines.set_preferences(sim_matrix, preftype, nvol) classes = apCommonLines.run_affinity_propagation(aligned_volumes, simfile, preffile, box, apix) ######### refine volumes using Xmipp projection matching ######### if start == "3D_align" or start == "3D_refine" or start == "none": # if start != "3D_assess": if not os.path.isdir("refinement"): os.mkdir("refinement") os.chdir("refinement") apXmipp.breakupStackIntoSingleFiles(ravgs) xmippcmd = "xmipp_normalize -i partlist.sel -method OldXmipp" apParam.runCmd(xmippcmd, "Xmipp") for i in classes.keys(): emancmd = "proc3d %s %s scale=%.3f clip=%d,%d,%d mask=%s spidersingle" \ % (os.path.join(rundir, "%d.mrc" % i), "%d.vol" % i, \ (1/self.scalefactor), rbox, rbox, rbox, (mrad / rapix)) apParam.runCmd(emancmd, "EMAN") apCommonLines.refine_volume(rundir, ravgs, i, mrad, irad, orad, rapix, nproc) emancmd = "proc3d %s %s apix=%.3f" \ % (os.path.join("refine_%d" % i, "3d%d_refined.vol" % i), \ os.path.join(rundir, "%d_r.mrc" % i), rapix) apParam.runCmd(emancmd, "EMAN") os.chdir(rundir) else: apDisplay.printColor("skipping 3D refinement", "cyan") ################# model evaluation ################ if start == "3D_align" or start == "3D_refine" or start == "3D_assess" or start == "none": # if start == "none" or start=="3D_assess": ### final model assessment try: euler_array = apCommonLines.getEulerValuesForModels(alignparams, rundir, nvol, npart, threes=threes) except: euler_array = None ejassess = False if refine is True: apCommonLines.assess_3Dclass_quality2(rundir, aligned_volumes, sim_matrix, classes, euler_array, box, rbox, rapix, ravgs, psym, npart, ejassess=ejassess) else: apCommonLines.assess_3Dclass_quality(rundir, aligned_volumes, sim_matrix, classes, euler_array, box, apix, avgs, psym, npart, ejassess=ejassess) apCommonLines.combineMetrics("final_model_stats.dat", "final_model_stats_sorted_by_Rcrit.dat", **{"CCPR":(1,1)}) ################## upload & cleanup ################## ### upload to database, if specified self.upload() ### make chimera snapshots if refine is True: for i in classes.keys(): if self.params['mass'] is not None: apChimera.filterAndChimera( os.path.join(self.params['rundir'], "%d_r.mrc" % i), res=self.params['3d_lpfilt'], apix=self.params['refineapix'], box=self.params['refineboxsize'], chimtype="snapshot", contour=2, zoom=1, sym="c1", color="gold", mass=self.params['mass'] ) ### cleanup snapshots = glob.glob("*.png") mtlfiles = glob.glob("*.mtl") objfiles = glob.glob("*.obj") pyfiles = glob.glob("*mrc.py") for file in mtlfiles: os.remove(file) for file in objfiles: os.remove(file) for file in pyfiles: os.remove(file) if not os.path.isdir("snapshots"): os.mkdir("snapshots") for s in snapshots: shutil.move(s, os.path.join("snapshots", s))
def start(self): self.insertMaxLikeJob() self.stack = {} self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) if self.params['virtualdata'] is not None: self.stack['file'] = self.params['virtualdata']['filename'] else: self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) self.estimateIterTime() self.dumpParameters() ### process stack to local file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") a = proc2dLib.RunProc2d() a.setValue('infile',self.stack['file']) a.setValue('outfile',self.params['localstack']) a.setValue('apix',self.stack['apix']) a.setValue('bin',self.params['bin']) a.setValue('last',self.params['numpart']-1) a.setValue('append',False) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: a.setValue('lowpass',self.params['lowpass']) if self.params['highpass'] is not None and self.params['highpass'] > 1: a.setValue('highpass',self.params['highpass']) if self.params['invert'] is True: a.setValue('invert') is True if self.params['virtualdata'] is not None: vparts = self.params['virtualdata']['particles'] plist = [int(p['particleNumber'])-1 for p in vparts] a.setValue('list',plist) # clip not yet implemented # self.params['clipsize'] is not None: # clipsize = int(self.clipsize)*self.params['bin'] # if clipsize % 2 == 1: # clipsize += 1 ### making sure that clipped boxsize is even # a.setValue('clip',clipsize) if self.params['virtualdata'] is not None: vparts = self.params['virtualdata']['particles'] plist = [int(p['particleNumber'])-1 for p in vparts] a.setValue('list',plist) #run proc2d a.run() if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']): apDisplay.printError("Missing particles in stack") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) ### stack is needed by uploadMaxlike.py #apFile.removeStack(self.params['localstack']) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " +" -i "+os.path.join(self.params['rundir'], self.partlistdocfile) +" -nref "+str(self.params['numrefs']) +" -iter "+str(self.params['maxiter']) +" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp) +" -psi_step "+str(self.params['psistep']) ) ### fast mode if self.params['fast'] is True: xmippopts += " -fast " if self.params['fastmode'] == "narrow": xmippopts += " -C 1e-10 " elif self.params['fastmode'] == "wide": xmippopts += " -C 1e-17 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " ### normalization if self.params['norm'] is True: xmippopts += " -norm " ### use student's T distribution if self.params['student'] is True: xmippopts += " -student " ### write cluster job file if self.params['cluster'] is True: self.writeClusterJobFile() ### find number of processors if self.params['nproc'] is None: nproc = nproc = apParam.getNumProcessors() else: nproc = self.params['nproc'] mpirun = self.checkMPI() self.estimateIterTime() if nproc > 2 and mpirun is not None: ### use multi-processor apDisplay.printColor("Using "+str(nproc)+" processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True) mpiruncmd = mpirun+" -np "+str(nproc)+" "+xmippexe+" "+xmippopts self.writeXmippLog(mpiruncmd) apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True) else: ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe+" "+xmippopts self.writeXmippLog(xmippcmd) apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ### minor post-processing self.createReferenceStack() self.readyUploadFlag() self.dumpParameters()
def start(self): # self.insertCL2DJob() self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join( self.params['rundir'], self.params['timestamp'] + ".hed") if os.path.isfile(self.params['localstack']): apFile.removeStack(self.params['localstack']) proccmd = "proc2d " + self.stack['file'] + " " + self.params[ 'localstack'] + " apix=" + str(self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] if clipsize % 2 == 1: clipsize += 1 ### making sure that clipped boxsize is even proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], clipsize, clipsize) proccmd += " last=" + str(self.params['numpart'] - 1) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp=" + str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp=" + str(self.params['lowpass']) apParam.runCmd(proccmd, "EMAN", verbose=True) if self.params['numpart'] != apFile.numImagesInStack( self.params['localstack']): apDisplay.printError("Missing particles in stack") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles( self.params['localstack']) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params['rundir'], self.partlistdocfile) + " -codes " + str(self.params['numrefs']) + " -iter " + str(self.params['maxiter']) + " -o " + os.path.join( self.params['rundir'], "part" + self.params['timestamp'])) if self.params['fast']: xmippopts += " -fast " if self.params['correlation']: xmippopts += " -useCorrelation " if self.params['classical']: xmippopts += " -classicalMultiref " if self.params['align']: xmippopts += " -alignImages " ### use multi-processor command apDisplay.printColor( "Using " + str(self.params['nproc']) + " processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True) mpiruncmd = self.mpirun + " -np " + str( self.params['nproc']) + " " + xmippexe + " " + xmippopts self.writeXmippLog(mpiruncmd) apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True, logfile="xmipp.std") self.params['runtime'] = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(self.params['runtime'])) ### minor post-processing self.createReferenceStack() self.parseOutput() self.clearIntermediateFiles() # self.readyUploadFlag() apParam.dumpParameters( self.params, "cl2d-" + self.params['timestamp'] + "-params.pickle") ### upload results ... this used to be two separate operations, I'm combining into one self.runparams = apParam.readRunParameters("cl2d-" + self.params['timestamp'] + "-params.pickle") self.apix = apStack.getStackPixelSizeFromStackId( self.runparams['stackid']) * self.runparams['bin'] self.Nlevels = len( glob.glob("part" + self.params['timestamp'] + "_level_??_.hed")) ### create average of aligned stacks & insert aligned stack info lastLevelStack = "part" + self.params[ 'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1) apStack.averageStack(lastLevelStack) self.boxsize = apFile.getBoxSize(lastLevelStack)[0] self.insertCL2DParamsIntoDatabase() if self.runparams['align'] is True: self.insertAlignStackRunIntoDatabase("alignedStack.hed") self.calcResolution(self.Nlevels - 1) self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1) ### loop over each class average stack & insert as clustering stacks self.insertClusterRunIntoDatabase() for level in range(self.Nlevels): ### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES if self.params['align'] is True: self.calcResolution(level) partdict = self.getClassificationAtLevel(level) for classnum in partdict: self.insertClusterStackIntoDatabase( "part" + self.params['timestamp'] + "_level_%02d_.hed" % level, classnum + 1, partdict[classnum], len(partdict))
def start(self): self.insertMaxLikeJob() self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.estimateIterTime() self.dumpParameters() ### process stack to local temp file proccmd = "proc2d " + self.stack['file'] + " temp.hed apix=" + str( self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], clipsize, clipsize) proccmd += " last=" + str(self.params['numpart'] - 1) apEMAN.executeEmanCmd(proccmd, verbose=True) ### process stack to final file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp + ".hed") proccmd = "proc2d temp.hed " + self.params[ 'localstack'] + " apix=" + str( self.stack['apix'] * self.params['bin']) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp=" + str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp=" + str(self.params['lowpass']) apEMAN.executeEmanCmd(proccmd, verbose=True) apFile.removeStack("temp.hed") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles( self.params['localstack']) ### convert stack into single spider files templateselfile = self.initializeTemplates() ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params['rundir'], self.partlistdocfile) + " -iter " + str(self.params['maxiter']) + " -ref " + templateselfile + " -o " + os.path.join(self.params['rundir'], "part" + self.timestamp) + " -psi_step " + str(self.params['psistep'])) ### fast mode if self.params['fast'] is True: xmippopts += " -fast " if self.params['fastmode'] == "narrow": xmippopts += " -C 1e-10 " elif self.params['fastmode'] == "wide": xmippopts += " -C 1e-18 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " ### normalization if self.params['norm'] is True: xmippopts += " -norm " ### find number of processors if self.params['nproc'] is None: nproc = nproc = apParam.getNumProcessors() else: nproc = self.params['nproc'] mpirun = self.checkMPI() if nproc > 2 and mpirun is not None: ### use multi-processor apDisplay.printColor("Using " + str(nproc - 1) + " processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True) mpiruncmd = mpirun + " -np " + str( nproc - 1) + " " + xmippexe + " " + xmippopts self.writeXmippLog(mpiruncmd) apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True) else: ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe + " " + xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ### minor post-processing self.createReferenceStack() self.readyUploadFlag() self.dumpParameters()
def start(self): # self.insertCL2DJob() self.stack = {} self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) if self.params['virtualdata'] is not None: self.stack['file'] = self.params['virtualdata']['filename'] else: self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) ### process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed") if os.path.isfile(self.params['localstack']): apFile.removeStack(self.params['localstack']) a = proc2dLib.RunProc2d() a.setValue('infile',self.stack['file']) a.setValue('outfile',self.params['localstack']) a.setValue('apix',self.stack['apix']) a.setValue('bin',self.params['bin']) a.setValue('last',self.params['numpart']-1) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: a.setValue('lowpass',self.params['lowpass']) if self.params['highpass'] is not None and self.params['highpass'] > 1: a.setValue('highpass',self.params['highpass']) if self.params['invert'] is True: a.setValue('invert',True) # clip not yet implemented # if self.params['clipsize'] is not None: # clipsize = int(self.clipsize)*self.params['bin'] # if clipsize % 2 == 1: # clipsize += 1 ### making sure that clipped boxsize is even # a.setValue('clip',clipsize) if self.params['virtualdata'] is not None: vparts = self.params['virtualdata']['particles'] plist = [int(p['particleNumber'])-1 for p in vparts] a.setValue('list',plist) #run proc2d a.run() if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']): apDisplay.printError("Missing particles in stack") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " +" -i "+os.path.join(self.params['rundir'], self.partlistdocfile) +" -codes "+str(self.params['numrefs']) +" -iter "+str(self.params['maxiter']) +" -o "+os.path.join(self.params['rundir'], "part"+self.params['timestamp']) ) if self.params['fast']: xmippopts += " -fast " if self.params['correlation']: xmippopts += " -useCorrelation " if self.params['classical']: xmippopts += " -classicalMultiref " if self.params['align']: xmippopts += " -alignImages " ### use multi-processor command apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True) mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts self.writeXmippLog(mpiruncmd) apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True, logfile="xmipp.std") self.params['runtime'] = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime'])) ### minor post-processing self.createReferenceStack() self.parseOutput() self.clearIntermediateFiles() # self.readyUploadFlag() apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle") ### upload results ... this used to be two separate operations, I'm combining into one self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle") self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin'] self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed")) ### create average of aligned stacks & insert aligned stack info lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1) apStack.averageStack(lastLevelStack) self.boxsize = apFile.getBoxSize(lastLevelStack)[0] self.insertCL2DParamsIntoDatabase() if self.runparams['align'] is True: self.insertAlignStackRunIntoDatabase("alignedStack.hed") self.calcResolution(self.Nlevels-1) self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1) ### loop over each class average stack & insert as clustering stacks self.insertClusterRunIntoDatabase() for level in range(self.Nlevels): ### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES if self.params['align'] is True: self.calcResolution(level) partdict = self.getClassificationAtLevel(level) for classnum in partdict: self.insertClusterStackIntoDatabase( "part"+self.params['timestamp']+"_level_%02d_.hed"%level, classnum+1, partdict[classnum], len(partdict))
def start(self): self.insertMaxLikeJob() self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.estimateIterTime() self.dumpParameters() ### process stack to local temp file proccmd = "proc2d "+self.stack['file']+" temp.hed apix="+str(self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize)*self.params['bin'] proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize) proccmd += " last="+str(self.params['numpart']-1) apEMAN.executeEmanCmd(proccmd, verbose=True) ### process stack to final file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") proccmd = "proc2d temp.hed "+self.params['localstack']+" apix="+str(self.stack['apix']*self.params['bin']) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp="+str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp="+str(self.params['lowpass']) apEMAN.executeEmanCmd(proccmd, verbose=True) apFile.removeStack("temp.hed") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) ### convert stack into single spider files templateselfile = self.initializeTemplates() ### setup Xmipp command aligntime = time.time() xmippopts = ( " " +" -i "+os.path.join(self.params['rundir'], self.partlistdocfile) +" -iter "+str(self.params['maxiter']) +" -ref "+templateselfile +" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp) +" -psi_step "+str(self.params['psistep']) ) ### fast mode if self.params['fast'] is True: xmippopts += " -fast " if self.params['fastmode'] == "narrow": xmippopts += " -C 1e-10 " elif self.params['fastmode'] == "wide": xmippopts += " -C 1e-18 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " ### normalization if self.params['norm'] is True: xmippopts += " -norm " ### find number of processors if self.params['nproc'] is None: nproc = nproc = apParam.getNumProcessors() else: nproc = self.params['nproc'] mpirun = self.checkMPI() if nproc > 2 and mpirun is not None: ### use multi-processor apDisplay.printColor("Using "+str(nproc-1)+" processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True) mpiruncmd = mpirun+" -np "+str(nproc-1)+" "+xmippexe+" "+xmippopts self.writeXmippLog(mpiruncmd) apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True) else: ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe+" "+xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ### minor post-processing self.createReferenceStack() self.readyUploadFlag() self.dumpParameters()