def takeoverHeaders(filename, numpart, boxsize, keepfiles=False): ### better workaround than copyFile ... still a workaround though imagicroot = checkImagicExecutablePath() basedir = os.path.split(filename)[0] basename = os.path.split(filename)[1] batchfile = os.path.join(basedir, "takeoverHeaders.batch") if basename[-4:] == ".img" or basename[-4:] == ".hed": stripped_file = basename[:-4] else: stripped_file = basename f = open(batchfile, 'w') f.write("#!/bin/csh -f\n") f.write("setenv IMAGIC_BATCH 1\n") f.write("cd %s\n" % (basedir)) f.write(str(imagicroot)+"/stand/testim.e <<EOF\n") f.write("test,1,%d\n" % (numpart)) f.write("%d,%d\n" % (boxsize, boxsize)) f.write("REAL\n") f.write("BLOBS\n") f.write("EOF\n") f.close() proc = subprocess.Popen('chmod 755 '+batchfile, shell=True) proc.wait() apParam.runCmd(batchfile, "IMAGIC") shutil.move(os.path.join(basedir, "test.hed"), os.path.join(basedir, stripped_file+".hed")) os.remove(os.path.join(basedir, "test.img")) if keepfiles is not True: os.remove(batchfile)
def scaleTemplates(self): reffile = os.path.join(self.params['rundir'], "references.hed") if self.params['apix'] != self.templatestack['apix']: scalefactor = float( self.templatestack['apix']) / self.params['apix'] templates = apImagicFile.readImagic(reffile) scaledtemplates = [] for templatearray in templates['images']: newarray = apTemplate.scaleTemplate(templatearray, scalefactor) scaledtemplates.append(newarray) apImagicFile.writeImagic(scaledtemplates, reffile) refbox = apFile.getBoxSize(reffile)[0] stbox = self.params['boxsize'] ### now clip the references to get identical boxsizes if stbox != refbox: while os.path.isfile(reffile + ".new.img"): apFile.removeStack(reffile + ".new.img") emancmd = "proc2d " + reffile + " " + reffile + ".new.hed clip=" + str( stbox) + " edgenorm" apParam.runCmd(emancmd, "EMAN") os.rename(reffile + ".new.hed", reffile) os.rename(reffile + ".new.img", reffile[:-4] + ".img") return
def takeoverHeaders(filename, numpart, boxsize, keepfiles=False): ### better workaround than copyFile ... still a workaround though imagicroot = checkImagicExecutablePath() basedir = os.path.split(filename)[0] basename = os.path.split(filename)[1] batchfile = os.path.join(basedir, "takeoverHeaders.batch") if basename[-4:] == ".img" or basename[-4:] == ".hed": stripped_file = basename[:-4] else: stripped_file = basename f = open(batchfile, 'w') f.write("#!/bin/csh -f\n") f.write("setenv IMAGIC_BATCH 1\n") f.write("cd %s\n" % (basedir)) f.write(str(imagicroot) + "/stand/testim.e <<EOF\n") f.write("test,1,%d\n" % (numpart)) f.write("%d,%d\n" % (boxsize, boxsize)) f.write("REAL\n") f.write("BLOBS\n") f.write("EOF\n") f.close() proc = subprocess.Popen('chmod 755 ' + batchfile, shell=True) proc.wait() apParam.runCmd(batchfile, "IMAGIC") shutil.move(os.path.join(basedir, "test.hed"), os.path.join(basedir, stripped_file + ".hed")) os.remove(os.path.join(basedir, "test.img")) if keepfiles is not True: os.remove(batchfile)
def start(self): ### database entry parameters package_table = 'ApXmippRefineIterData|xmippParams' ### set projection-matching path self.projmatchpath = os.path.abspath(os.path.join(self.params['rundir'], "recon", self.runparams['package_params']['WorkingDir'])) # self.projmatchpath = os.path.abspath(os.path.join(self.params['rundir'], self.runparams['package_params']['WorkingDir'])) ### check for variable root directories between file systems apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(self.params['rundir'], self.runparams['remoterundir'], self.runparams['rundir']) ### determine which iterations to upload lastiter = self.findLastCompletedIteration() uploadIterations = self.verifyUploadIterations(lastiter) ### upload each iteration for iteration in uploadIterations: apDisplay.printColor("uploading iteration %d" % iteration, "cyan") ### set package parameters, as they will appear in database entries package_database_object = self.instantiateProjMatchParamsData(iteration) ### move FSC file to results directory oldfscfile = os.path.join(self.projmatchpath, "Iter_%d" % iteration, "Iter_%d_resolution.fsc" % iteration) newfscfile = os.path.join(self.resultspath, "recon_%s_it%.3d_vol001.fsc" % (self.params['timestamp'],iteration)) if os.path.exists(oldfscfile): shutil.copyfile(oldfscfile, newfscfile) ### create a stack of class averages and reprojections (optional) self.compute_stack_of_class_averages_and_reprojections(iteration) ### create a text file with particle information self.createParticleDataFile(iteration) if not self.params['euleronly']: ### create mrc file of map for iteration and reference number oldvol = os.path.join(self.projmatchpath, "Iter_%d" % iteration, "Iter_%d_reconstruction.vol" % iteration) newvol = os.path.join(self.resultspath, "recon_%s_it%.3d_vol001.mrc" % (self.params['timestamp'], iteration)) mrccmd = "proc3d %s %s apix=%.3f" % (oldvol, newvol, self.runparams['apix']) apParam.runCmd(mrccmd, "EMAN") ### make chimera snapshot of volume self.createChimeraVolumeSnapshot(newvol, iteration) ### instantiate database objects self.insertRefinementRunData(iteration) self.insertRefinementIterationData(iteration, package_table, package_database_object) ### calculate Euler jumps if self.runparams['numiter'] > 1: self.calculateEulerJumpsAndGoodBadParticles(uploadIterations) ### query the database for the completed refinements BEFORE deleting any files ... returns a dictionary of lists ### e.g. {1: [5, 4, 3, 2, 1]} means 5 iters completed for refine 1 complete_refinements = self.verifyNumberOfCompletedRefinements(multiModelRefinementRun=False) if self.params['cleanup_files'] is True: self.cleanupFiles(complete_refinements)
def start(self): ### database entry parameters package_table = 'ApXmippML3DRefineIterData|xmippML3DParams' ### set ml3d path self.ml3dpath = os.path.abspath(os.path.join(self.params['rundir'], "recon", self.runparams['package_params']['WorkingDir'], "RunML3D")) ### check for variable root directories between file systems if os.path.split(self.runparams['remoterundir'])[1] == "recon": self.runparams['remoterundir'] = os.path.split(self.runparams['remoterundir'])[0] apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(self.params['rundir'], self.runparams['remoterundir'], self.params['rundir']) ### determine which iterations to upload lastiter = self.findLastCompletedIteration() uploadIterations = self.verifyUploadIterations(lastiter) ### create ml3d_lib.doc file somewhat of a workaround, but necessary to make projections total_num_2d_classes = self.createModifiedLibFile() ### upload each iteration for iteration in uploadIterations: ### set package parameters, as they will appear in database entries package_database_object = self.instantiateML3DParamsData(iteration) for j in range(self.runparams['NumberOfReferences']): ### calculate FSC for each iteration using split selfile (selfile requires root directory change) self.calculateFSCforIteration(iteration, j+1) ### create a stack of class averages and reprojections (optional) self.compute_stack_of_class_averages_and_reprojections(iteration, j+1) ### create a text file with particle information self.createParticleDataFile(iteration, j+1, total_num_2d_classes) ### create mrc file of map for iteration and reference number oldvol = os.path.join(self.ml3dpath, "ml3d_it%.6d_vol%.6d.vol" % (iteration, j+1)) newvol = os.path.join(self.resultspath, "recon_%s_it%.3d_vol%.3d.mrc" % (self.params['timestamp'], iteration, j+1)) mrccmd = "proc3d %s %s apix=%.3f" % (oldvol, newvol, self.runparams['apix']) apParam.runCmd(mrccmd, "EMAN") ### make chimera snapshot of volume self.createChimeraVolumeSnapshot(newvol, iteration, j+1) ### instantiate database objects self.insertRefinementRunData(iteration, j+1) self.insertRefinementIterationData(iteration, package_table, package_database_object, j+1) ### calculate Euler jumps if self.runparams['numiter'] > 1: self.calculateEulerJumpsAndGoodBadParticles(uploadIterations) ### query the database for the completed refinements BEFORE deleting any files ... returns a dictionary of lists ### e.g. {1: [5, 4, 3, 2, 1], 2: [6, 5, 4, 3, 2, 1]} means 5 iters completed for refine 1 & 6 iters completed for refine 2 complete_refinements = self.verifyNumberOfCompletedRefinements(multiModelRefinementRun=True) if self.params['cleanup_files'] is True: self.cleanupFiles(complete_refinements)
def applyEnvelopeAndCTF(self, stack): ### get defocus lists numpart = self.params['projcount'] cut = int(numpart/80.0)+1 apDisplay.printMsg("%d particles per dot"%(cut)) if len(self.deflist1) == 0: self.getListOfDefoci(numpart) ### break up particles partlistdocfile = apXmipp.breakupStackIntoSingleFiles(stack, filetype="mrc") t0 = time.time() apDisplay.printMsg("Applying CTF and Envelop to particles") ### apply CTF using ACE2 ctfapplydocfile = self.applyCTFToDocFile(partlistdocfile) ### apply Envelop using ACE2 envelopdocfile = self.applyEnvelopToDocFile(ctfapplydocfile) ### correct CTF using ACE2 if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True: ctfcorrectdocfile = self.correctCTFToDocFile(envelopdocfile) else: ctfcorrectdocfile = envelopdocfile timeper = (time.time()-t0)/float(numpart) apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green") apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green") ### write corrected particle list to doc file ctfpartlist = [] ctfpartlistfile = os.path.join(self.params['rundir'], "ctfpartlist.lst") inf = open(ctfcorrectdocfile, 'r') outf = open(ctfpartlistfile, "w") for line in inf: ### get filename filename = line.strip().split()[0] if not os.path.isfile(filename): apDisplay.printError("CTF and envelop apply failed") ctfpartlist.append(filename) outf.write(filename+"\t1\n") inf.close() outf.close() ### merge individual files into a common stack ctfstack = os.path.join(self.params['rundir'], "ctfstack.hed") apXmipp.gatherSingleFilesIntoStack(ctfpartlistfile, ctfstack, filetype="mrc") if self.params['pad'] is True: emancmd = "proc2d %s %s.clip.hed clip=%d,%d" % (ctfstack, ctfstack[:-4], self.params['box'], self.params['box']) apParam.runCmd(emancmd, "EMAN") shutil.move("%s.clip.hed" % ctfstack[:-4], "%s.hed" % ctfstack[:-4]) shutil.move("%s.clip.img" % ctfstack[:-4], "%s.img" % ctfstack[:-4]) return ctfstack, ctfpartlist
def addNoise(self, oldstack, noiselevel, SNR): ### create new image with modified SNR basename, extension = os.path.splitext(oldstack) formattedsnr = "%.3f" % (SNR) newstack = basename+"_snr"+formattedsnr+".hed" apFile.removeStack(newstack) emancmd = "proc2d "+oldstack+" "+newstack+" addnoise="+str(noiselevel) apParam.runCmd(emancmd, "EMAN") return newstack
def start(self): ### get analysis paramteres self.analysisdata = appiondata.ApAlignAnalysisRunData.direct_query(self.params['analysisId']) pixelsize = self.analysisdata['alignstack']['pixelsize'] boxsize = self.analysisdata['alignstack']['boxsize'] bin = self.analysisdata['imagicMSArun']['bin'] self.params['apix'] = float(pixelsize) * int(bin) self.params['boxsize'] = int(boxsize) / int(bin) self.params['num_particles'] = self.analysisdata['alignstack']['num_particles'] starttime=time.time() print self.params print "... stack pixel size: "+str(self.params['apix']) print "... stack box size: "+str(self.params['boxsize']) apDisplay.printColor("Running IMAGIC .batch file: See imagicMSAcluster log file(s) corresponding to # of classes for details", "cyan") ### insert run into database self.insertClusterRun() ### split the cluster numbers numclasslist = self.params['num_classes'].split(",") for item in numclasslist: numclusters = int(item) apDisplay.printColor("\n==========================\nprocessing class averages for " +str(numclusters)+" classes\n==========================\n", "green") ### create IMAGIC batch file batchfile = self.createImagicBatchFile(numclusters) ### execute IMAGIC batch file clustertime0 = time.time() proc = subprocess.Popen("chmod 775 "+str(batchfile), shell=True) proc.wait() apIMAGIC.executeImagicBatchFile(batchfile) logfile = open(os.path.join(self.params['rundir'], "imagicMSAcluster_classes_"+str(numclusters)+".log")) loglines = logfile.readlines() for line in loglines: if re.search("ERROR in program", line): apDisplay.printError("ERROR IN IMAGIC SUBROUTINE, please check the logfile: imagicMSAcluster_classes_"\ +str(numclusters)+".log") apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-clustertime0), "cyan") ### normalize classfile = os.path.join(self.params['rundir'], self.params['classumfile']) emancmd = "proc2d "+classfile+" "+classfile+".norm.hed norm" while os.path.isfile(classfile+".norm.img"): apStack.removeStack(alignstack+".norm.img") apParam.runCmd(emancmd, "EMAN") os.rename(classfile+".norm.hed", classfile) os.rename(classfile+".norm.img", classfile[:-4]+".img") ### connect to database self.insertClusterStack(numclusters)
def unpackResults(self): """ untar results, if this hasn't been done yet """ if os.path.exists(os.path.join(self.params["rundir"], "recon_results.tar.gz")): apParam.runCmd("tar -xvzf recon_results.tar.gz", "SHELL") apParam.runCmd("rm recon_results.tar.gz", "SHELL") if os.path.exists(os.path.join(self.params["rundir"], "volumes.tar.gz")): apParam.runCmd("tar -xvzf volumes.tar.gz", "SHELL") apParam.runCmd("rm volumes.tar.gz", "SHELL")
def _boxParticlesFromImage(self, imgdata, parttree, imgstackfile): ''' Box Particles From the manipulated full size image file on disk ''' ### make corrected integrated frame image imgpath = self.getOriginalImagePath(imgdata) t0 = time.time() if self.params['phaseflipped'] is True: if self.params['fliptype'] == 'emanimage': ### ctf correct whole image using EMAN imgpath = self.phaseFlipWholeImage(imgpath, imgdata) elif self.params['fliptype'] == "spiderimage": imgpath = self.phaseFlipSpider(imgpath, imgdata) elif self.params['fliptype'][:9] == "ace2image": ### ctf correct whole image using Ace 2 imgpath = self.phaseFlipAceTwo(imgpath, imgdata) self.ctftimes.append(time.time()-t0) if imgpath is None: return None, None, None ### run apBoxer apDisplay.printMsg("boxing "+str(len(parttree))+" particles into temp file: "+imgstackfile) ### method to align helices t0 = time.time() if self.params['rotate'] is True: apBoxer.boxerRotate(imgpath, parttree, imgstackfile, self.boxsize) if self.params['finealign'] is True: from appionlib import apXmipp apXmipp.breakupStackIntoSingleFiles(imgstackfile, filetype="mrc") rotcmd = "s_finealign %s %i" %(self.params['rundir'], self.boxsize) apParam.runCmd(rotcmd, "HIP", verbose=True) # read in text file containing refined angles anglepath = os.path.join(self.params['rundir'], 'angles.out') f = open(anglepath, 'r') angles = f.readlines() # loop through parttree and add refined angle to rough angle for i in range(len(parttree)): partdict = parttree[i] fineangle = float(angles[i]) newangle = float(partdict['angle'])-fineangle partdict['angle'] = newangle # rerun apBoxer.boxerRotate with the new parttree containing final angles apBoxer.boxerRotate(imgpath, parttree, imgstackfile, self.boxsize, pixlimit=self.params['pixlimit']) else: apBoxer.boxer(imgpath, parttree, imgstackfile, self.boxsize, pixlimit=self.params['pixlimit']) self.batchboxertimes.append(time.time()-t0)
def shift_images(self, filename): ### random shifts and rotations to a stack shiftstackname = filename[:-4]+"_rand.hed" apFile.removeStack(shiftstackname) # shiftfile = os.path.join(self.params['rundir'], "shift_rotate.lst") shiftfile = os.path.join(self.params['rundir'], "shift.lst") if os.path.isfile(shiftfile): apFile.removeFile(shiftfile) f = open(shiftfile, "a") # apDisplay.printMsg("Now randomly shifting and rotating particles") apDisplay.printMsg("Now randomly shifting particles") for i in range(self.params['projcount']): # if self.params['rotang'] != 0: # randrot = random.uniform(-1*self.params['rotang'], self.params['rotang']) randx = random.uniform(-1*self.params['shiftrad'], self.params['shiftrad']) randy = random.uniform(-1*self.params['shiftrad'], self.params['shiftrad']) if self.params['flip'] is True: flip = random.choice([0,1]) else: flip = 0 # if self.params['rotang'] != 0: # emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" rot="+str(randrot)+" trans="+str(randx)+","+str(randy) # else: # emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" trans="+str(randx)+","+str(randy) emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" trans="+str(randx)+","+str(randy) if flip == 0: if self.params['pad'] is False: emancmd = emancmd+" clip="+str(self.params['box'])+","+str(self.params['box'])+" edgenorm" else: emancmd = emancmd+" edgenorm" else: if self.params['pad'] is False: emancmd = emancmd+" flip clip="+str(self.params['box'])+","+str(self.params['box'])+" edgenorm" else: emancmd = emancmd+" flip edgenorm" apParam.runCmd(emancmd, "EMAN", showcmd=False) # if self.params['rotang'] != 0: # f.write("%.3f,"%(randrot)) # else: # f.write(",") f.write("%.3f,"%(randx)) f.write("%.3f,"%(randy)) f.write(str(flip)+"\n") f.close() return shiftstackname
def unpackResults(self): ''' untar results, if this hasn't been done yet ''' if os.path.exists( os.path.join(self.params['rundir'], "recon_results.tar.gz")): apParam.runCmd("tar -xvzf recon_results.tar.gz", "SHELL") apParam.runCmd("rm recon_results.tar.gz", "SHELL") if os.path.exists(os.path.join(self.params['rundir'], "volumes.tar.gz")): apParam.runCmd("tar -xvzf volumes.tar.gz", "SHELL") apParam.runCmd("rm volumes.tar.gz", "SHELL")
def post_process_stack(self, filename, pad, invert): ### pad out and/or invert projections if self.params['invert'] is True or pad is True: emancmd = "proc2d %s %s.process.hed " % (filename, filename) if self.params['invert'] is True: emancmd+="invert " if pad is True: emancmd+="clip=%d " % (self.params['box']*self.params['padF']) apParam.runCmd(emancmd, "EMAN") shutil.move("%s.process.hed" % filename, "%s.hed" % filename[:-4]) shutil.move("%s.process.img" % filename, "%s.img" % filename[:-4]) # self.params['projcount'] = int(split[-1][0]) ### last row, first value is last projection # self.params['projcount'] = self.numProj(ang=self.params['projinc'], sym='c1') ### for some reasons did not work for prop=20 # numpart = apFile.numImagesInStack(stackfile) return
def start(self): self.writeParams() stackdata = apStack.getOnlyStackData(self.params["stackid"], msg=False) stackpath = os.path.abspath(stackdata["path"]["path"]) stackfile = os.path.join(stackpath, "start.hed") self.params["localstack"] = os.path.join(stackpath, self.timestamp + ".hed") proccmd = "proc2d " + stackfile + " " + self.params["localstack"] + " apix=" + str(self.params["step"]) if self.params["bin"] > 1: proccmd += " shrink=%d" % int(self.params["bin"]) proccmd += " last=" + str(self.params["numpart"] - 1) apParam.runCmd(proccmd, "EMAN", verbose=True) if self.params["numpart"] != apFile.numImagesInStack(self.params["localstack"]): apDisplay.printError("Missing particles in stack") apXmipp.breakupStackIntoSingleFiles(self.params["localstack"], filetype="mrc") if self.params["prehip"] == "yes": if self.params["rise"] is not None: self.writeLLBO1() elif self.params["ll1"] is not None: self.writeLLBO2() elif self.params["rise"] is None and self.params["ll1"] is None: apDisplay.printError( "You must specify either rise and twist or (1,0) and (0,1) layer lines to run preHIP" ) apDisplay.printMsg("now running s_prehip to set up input files") apDisplay.printMsg( "Make sure the layer line/bessel order assignment is correct. If it is not, you may need to adjust some of the input variables." ) cmd = "s_prehip" proc = subprocess.Popen(cmd) proc.wait() else: apDisplay.printMsg("now running s_hip2: Phoelix programs") cmd = "s_hip2" proc = subprocess.Popen(cmd) proc.wait() # apParam.runCmd(cmd, package="Phoelix") self.putFilesInStack() self.insertHipRunData() self.insertHipParamsData() self.insertHipIterData() self.insertHipParticleData()
def prealignClassAverages(rundir, avgs): '''function to iteratively align class averages to each other prior to input into angular reconstitution (optional) ''' imagicroot = checkImagicExecutablePath() batchfile = os.path.join(rundir, "prealignClassAverages.batch") f = open(batchfile, 'w') f.write("#!/bin/csh -f\n") f.write("setenv IMAGIC_BATCH 1\n") f.write("cd " + rundir + "/\n") ### this is the actual alignment f.write( str(imagicroot) + "/align/alirefs.e <<EOF >> prealignClassAverages.log\n") f.write("ALL\n") f.write("CCF\n") f.write(str(os.path.basename(avgs)[:-4]) + "\n") f.write("NO\n") f.write("0.99\n") f.write(str(os.path.basename(avgs)[:-4]) + "_aligned\n") f.write("-999.\n") f.write("0.2\n") f.write("-180,180\n") f.write("NO\n") f.write("5\n") f.write("NO\n") f.write("EOF\n") f.close() avgs = avgs[:-4] + "_aligned.img" proc = subprocess.Popen('chmod 755 ' + batchfile, shell=True) proc.wait() apParam.runCmd(batchfile, "IMAGIC") return avgs
def applyBfactor(infile, fscfile, apix, mass=None, outfile=None): embfactorfile = "embfactor64.exe" embfactorexe = apParam.getExecPath("embfactor64.exe") if embfactorexe is None: apDisplay.printWarning("Could not find %s"%(embfactorfile)) return infile if outfile is None: outfile = os.path.splitext(infile)[0]+"-bfactor.mrc" cmd = embfactorexe cmd += " -FSC %s"%(fscfile) cmd += " -sampling %.3f"%(apix) ### this option always failed for me -neil #if mass is not None: # cmd += " -molweight %d"%(mass*1000) cmd += " %s"%(infile) cmd += " %s"%(outfile) apParam.runCmd(cmd, package="B-factor", verbose=True, showcmd=True) if not apVolume.isValidVolume(outfile): apDisplay.printWarning("B-factor correction failed %s"%(embfactorfile)) return infile return outfile
def start(self): self.writeParams() stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackpath = os.path.abspath(stackdata['path']['path']) stackfile = os.path.join(stackpath, "start.hed") self.params['localstack'] = os.path.join(stackpath, self.timestamp+".hed") proccmd = "proc2d "+stackfile+" "+self.params['localstack']+" apix="+str(self.params['step']) if self.params['bin'] > 1: proccmd += " shrink=%d"%int(self.params['bin']) proccmd += " last="+str(self.params['numpart']-1) apParam.runCmd(proccmd, "EMAN", verbose=True) if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']): apDisplay.printError("Missing particles in stack") apXmipp.breakupStackIntoSingleFiles(self.params['localstack'], filetype="mrc") if self.params['prehip'] == "yes": if self.params['rise'] is not None: self.writeLLBO1() elif self.params['ll1'] is not None: self.writeLLBO2() elif self.params['rise'] is None and self.params['ll1'] is None: apDisplay.printError("You must specify either rise and twist or (1,0) and (0,1) layer lines to run preHIP") apDisplay.printMsg("now running s_prehip to set up input files") apDisplay.printMsg("Make sure the layer line/bessel order assignment is correct. If it is not, you may need to adjust some of the input variables.") cmd = "s_prehip" proc = subprocess.Popen(cmd) proc.wait() else: apDisplay.printMsg("now running s_hip2: Phoelix programs") cmd = "s_hip2" proc = subprocess.Popen(cmd) proc.wait() #apParam.runCmd(cmd, package="Phoelix") self.putFilesInStack() self.insertHipRunData() self.insertHipParamsData() self.insertHipIterData() self.insertHipParticleData()
def scaleTemplates(self): reffile = os.path.join(self.params['rundir'], "references.hed") if self.params['apix'] != self.templatestack['apix']: scalefactor = float(self.templatestack['apix']) / self.params['apix'] templates = apImagicFile.readImagic(reffile) scaledtemplates = [] for templatearray in templates['images']: newarray = apTemplate.scaleTemplate(templatearray, scalefactor) scaledtemplates.append(newarray) apImagicFile.writeImagic(scaledtemplates, reffile) refbox = apFile.getBoxSize(reffile)[0] stbox = self.params['boxsize'] ### now clip the references to get identical boxsizes if stbox != refbox: while os.path.isfile(reffile+".new.img"): apFile.removeStack(reffile+".new.img") emancmd = "proc2d "+reffile+" "+reffile+".new.hed clip="+str(stbox)+" edgenorm" apParam.runCmd(emancmd, "EMAN") os.rename(reffile+".new.hed", reffile) os.rename(reffile+".new.img", reffile[:-4]+".img") return
def prealignClassAverages(rundir, avgs): '''function to iteratively align class averages to each other prior to input into angular reconstitution (optional) ''' imagicroot = checkImagicExecutablePath() batchfile = os.path.join(rundir, "prealignClassAverages.batch") f = open(batchfile, 'w') f.write("#!/bin/csh -f\n") f.write("setenv IMAGIC_BATCH 1\n") f.write("cd "+rundir+"/\n") ### this is the actual alignment f.write(str(imagicroot)+"/align/alirefs.e <<EOF >> prealignClassAverages.log\n") f.write("ALL\n") f.write("CCF\n") f.write(str(os.path.basename(avgs)[:-4])+"\n") f.write("NO\n") f.write("0.99\n") f.write(str(os.path.basename(avgs)[:-4])+"_aligned\n") f.write("-999.\n") f.write("0.2\n") f.write("-180,180\n") f.write("NO\n") f.write("5\n") f.write("NO\n") f.write("EOF\n") f.close() avgs = avgs[:-4]+"_aligned.img" proc = subprocess.Popen('chmod 755 '+batchfile, shell=True) proc.wait() apParam.runCmd(batchfile, "IMAGIC") return avgs
def applyBfactor(infile, fscfile, apix, mass=None, outfile=None): embfactorfile = "embfactor64.exe" embfactorexe = apParam.getExecPath("embfactor64.exe") if embfactorexe is None: apDisplay.printWarning("Could not find %s" % (embfactorfile)) return infile if outfile is None: outfile = os.path.splitext(infile)[0] + "-bfactor.mrc" cmd = embfactorexe cmd += " -FSC %s" % (fscfile) cmd += " -sampling %.3f" % (apix) ### this option always failed for me -neil #if mass is not None: # cmd += " -molweight %d"%(mass*1000) cmd += " %s" % (infile) cmd += " %s" % (outfile) apParam.runCmd(cmd, package="B-factor", verbose=True, showcmd=True) if not apVolume.isValidVolume(outfile): apDisplay.printWarning("B-factor correction failed %s" % (embfactorfile)) return infile return outfile
def runRelionPreprocess(self, newstackroot): ''' 1. Use stackIntoPicks.py to extract the particle locations from the selected stack. 2. Run makestack2.py without ctf correction or normalization using the stackIntoPicks result as the Particles run. 3. Run relion_preprocess with rescale and norm. Outputs .mrcs file. Neil: most of these steps could be done more generally ''' apDisplay.printWarning("Making a new stack from original images") # Build the stackIntoPicks command apDisplay.printMsg('Extracting the particle locations from your selected stack.') newstackrunname = self.params['runname']+"_particles" newstackrundir = self.params['rundir'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid sessionid = int(self.params['expid']) cmd = "stackIntoPicks.py " cmd += (" --stackid=%d --projectid=%d --runname=%s --rundir=%s "% (stackid,projectid,newstackrunname,newstackrundir)) cmd += (" --commit --expId=%d --jobtype=stackintopicks "% (sessionid)) # Run the command logfilepath = os.path.join(newstackrundir,'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd,logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Build the makestack2 command ''' This is the command we want to make a stack from a stackIntoPicks run makestack2.py --single=start.hed --selectionid=130 --invert --boxsize=320 --bin=2 --description="made from stackrun1 using stackintopicks" --runname=stack66 --rundir=/ami/data00/appion/zz07jul25b/stacks/stack66 --commit --preset=en --projectid=303 --session=zz07jul25b --no-rejects --no-wait --continue --expid=8556 --jobtype=makestack2 --ppn=1 --nodes=1 --walltime=240 --jobid=2016 ''' apDisplay.printMsg('Using selected stack particle locations to make a Relion ready stack....') # Get the ID of the stackIntoPicks run we just created #apParticle.getSelectionIdFromName(runname, sessionname) runq = appiondata.ApSelectionRunData() runq['name'] = newstackrunname runq['session'] = leginondata.SessionData.direct_query(self.params['expid']) rundatas = runq.query(results=1) print rundatas if rundatas: selectionid = rundatas[0].dbid else: apDisplay.printError("Error creating Relion ready stack. Could not find stackIntoPicks.py data in database.\n") # Gather all the makestack parameters totalpart = self.originalStackData.numpart numpart = totalpart if not self.params['last'] else min(self.params['last'],totalpart) stackpathname = os.path.basename( self.originalStackData.path ) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir,'start.hed') presetname = self.originalStackData.preset # binning is combination of the original binning of the stack and the preparation binnning bin = self.originalStackData.bin * self.params['bin'] unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin lowpasstext = setArgText( 'lowpass', ( self.params['lowpass'], self.originalStackData.lowpass ), False) highpasstext = setArgText( 'highpass', ( self.params['highpass'], self.originalStackData.highpass ), True) partlimittext = setArgText('partlimit',(numpart,),False) xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'],), True) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid reversetext = '--reverse' if self.originalStackData.reverse else '' defoctext = '--defocpair' if self.originalStackData.defocpair else '' inverttext = '--no-invert' if not self.invert else '' # Build the makestack2 command cmd = "makestack2.py " cmd += (" --single=%s --selectionid=%d %s --boxsize=%d --bin=%d "% (os.path.basename(newstackimagicfile),selectionid,inverttext, unbinnedboxsize,bin)) cmd += (" --description='Relion refinestack based on %s(id=%d)' --projectid=%d "% (stackpathname,stackid,projectid)) cmd += (" --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d "% (presetname,newstackrunname,newstackrundir,sessionname,sessionid)) cmd += " --no-wait --no-commit --no-continue --jobtype=makestack2 " # Run the command logfilepath = os.path.join(newstackrundir,'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd,logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Clean up boxfiles = glob.glob("*.box") for boxfile in boxfiles: apFile.removeFile(boxfile) # Make sure our new stack params reflects the changes made # Use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] self.stack['file'] = newstackroot+'.hed' # Run Relion pre-process command ''' Setup the follow relion preprocess command to normalize the new stack: relion_preprocess \ --o particles --norm --bg_radius 60 --white_dust -1 --black_dust -1 \ --operate_on /ami/data00/appion/zz07jul25b/stacks/stack63_no_xmipp_norm/start.hed ''' apDisplay.printMsg('Running Relion preprocessing to normalize your Relion Ready stack....') bg_radius = math.floor((self.stack['boxsize'] / 2) - 1) relioncmd = "relion_preprocess " relioncmd += " --o particles --norm --bg_radius %d "%(bg_radius) relioncmd += " --white_dust -1 --black_dust -1 --operate_on %s "%(newstackimagicfile) apParam.runCmd(relioncmd, package="Relion", verbose=True, showcmd=True) self.stack['file'] = 'particles.mrcs'
def runAlignmentGPU(self, stackfile, templatestack, origstackfile, xysearch, xystep, firstring=2, lastring=100, dataext=".spi", iternum=1, oldpartlist=None): """ inputs: stack template search params outputs: aligned stack rotation/shift params """ ### setup if dataext in templatestack: templatestack = templatestack[:-4] if dataext in stackfile: stackfile = stackfile[:-4] if dataext in origstackfile: origstackfile = origstackfile[:-4] t0 = time.time() rundir = "alignments" apParam.createDirectory(rundir) ### remove previous iterations apFile.removeFile(rundir+"/paramdoc%02d%s" % (iternum, dataext)) docfile = rundir+("/paramdoc%02d" % (iternum))+dataext ### required for apsh like commands #inputanglesfile = rundir+("/angles%02d" % (iternum))+dataext cmd = "/home/ryan/alignment_gpu/bin/alignment_gpu " cmd += "'%s' "%(stackfile+dataext) cmd += "'%s' "%(templatestack+dataext) cmd += "%d "%(xysearch) cmd += "%d "%(xystep) cmd += "%d "%(1) cmd += "%d "%(firstring) cmd += "%d "%(lastring) cmd += "'%s' "%(docfile) apParam.runCmd(cmd, package="Alignment_GPU", verbose=True, showcmd=True) ### convert spider rotation, shift data to python picklefile = rundir+("/paramdoc%02d" % (iternum))+".pickle" if oldpartlist is not None and iternum > 1: apDisplay.printMsg("updating particle doc info") partlist = alignment.updateRefBasedDocFile(oldpartlist, docfile, picklefile) elif iternum == 1: apDisplay.printMsg("reading initial particle doc info") partlist = self.readRefBasedDocFile(docfile, picklefile) else: apDisplay.printError("reading (not updating) particle doc info on iteration "+str(iternum)) ### write aligned stack -- with python loop alignedstack = rundir+("/alignedstack%02d" % (iternum)) alignment.alignStack(origstackfile, alignedstack, partlist, dataext) ### average stack emancmd = ( "proc2d "+alignedstack+dataext+" " +rundir+("/avgimg%02d" % (iternum))+".mrc " +" average") apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) td1 = time.time()-t0 apDisplay.printMsg("completed alignment of particles in "+apDisplay.timeString(td1)) if len(partlist) < 1: apDisplay.printError("Failed to find any particles") return alignedstack+dataext, partlist
def compute_stack_of_class_averages_and_reprojections(self, iteration, reference_number): ''' takes Xmipp single files, doc and sel files associated with ML3D, creates a stack of class averages in the results directory ''' ''' ### make projections, and put them back into resultspath selfile = "ml3d_it%.6d_vol%.6d.sel" % (iteration, reference_number) refvolume = "ml3d_it%.6d_vol%.6d.vol\n" % (iteration, reference_number) docfile = "ml3d_it%.6d_vol%.6d.doc" % (iteration, reference_number) apXmipp.compute_stack_of_class_averages_and_reprojections(self.ml3dpath, selfile, refvolume, docfile, \ self.runparams['boxsize'], self.resultspath, self.params['timestamp'], iteration, reference_number, extract=True) return ''' os.chdir(self.ml3dpath) ### remove "RunML3D/" from selfile (created by ML3D program), then extract header information to docfile selfile = "ml3d_it%.6d_vol%.6d.sel" % (iteration, reference_number) f = open(selfile, "r") lines = f.readlines() newlines = [re.sub("RunML3D/", "", line) for line in lines] f.close() f = open(selfile, "w") f.writelines(newlines) f.close() extractcmd = "xmipp_header_extract -i ml3d_it%.6d_vol%.6d.sel -o ml3d_it%.6d_vol%.6d.doc" \ % (iteration, reference_number, iteration, reference_number) apParam.runCmd(extractcmd, "Xmipp") ### create a projection params file and project the volume along identical Euler angles f = open("paramfile.descr", "w") f.write("ml3d_it%.6d_vol%.6d.vol\n" % (iteration, reference_number)) f.write("tmpproj 1 xmp\n") f.write("%d %d\n" % (self.runparams['boxsize'], self.runparams['boxsize'])) f.write("ml3d_it%.6d_vol%.6d.doc rot tilt psi\n" % (iteration, reference_number)) f.write("NULL\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.close() projectcmd = "xmipp_project -i paramfile.descr" apParam.runCmd(projectcmd, "Xmipp") ### get order of projections in docfile docfile = "ml3d_it%.6d_vol%.6d.doc" % (iteration, reference_number) d = open(docfile, "r") lines = d.readlines()[1:] d.close() projfile_sequence = [] for i, l in enumerate(lines): if i % 2 == 0: filename = os.path.basename(l.split()[1]) projfile_sequence.append(filename) else: pass ### create stack of projections and class averages projections = glob.glob("tmpproj**xmp") projections.sort() if len(projections) != len(projfile_sequence): apDisplay.printWarning("number of projections does not match number of classes for model %d, iteration %d") stackarray = [] stackname = os.path.join(self.resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (self.params['timestamp'], iteration, reference_number)) for i in range(len(projections)): stackarray.append(spider.read(projections[i])) stackarray.append(spider.read(projfile_sequence[i])) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### remove unnecessary files for file in glob.glob("tmpproj*"): apFile.removeFile(file) os.chdir(self.params['rundir']) return
def start(self): # self.insertCL2DJob() self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join( self.params['rundir'], self.params['timestamp'] + ".hed") if os.path.isfile(self.params['localstack']): apFile.removeStack(self.params['localstack']) proccmd = "proc2d " + self.stack['file'] + " " + self.params[ 'localstack'] + " apix=" + str(self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] if clipsize % 2 == 1: clipsize += 1 ### making sure that clipped boxsize is even proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], clipsize, clipsize) proccmd += " last=" + str(self.params['numpart'] - 1) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp=" + str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp=" + str(self.params['lowpass']) apParam.runCmd(proccmd, "EMAN", verbose=True) if self.params['numpart'] != apFile.numImagesInStack( self.params['localstack']): apDisplay.printError("Missing particles in stack") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles( self.params['localstack']) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params['rundir'], self.partlistdocfile) + " -codes " + str(self.params['numrefs']) + " -iter " + str(self.params['maxiter']) + " -o " + os.path.join( self.params['rundir'], "part" + self.params['timestamp'])) if self.params['fast']: xmippopts += " -fast " if self.params['correlation']: xmippopts += " -useCorrelation " if self.params['classical']: xmippopts += " -classicalMultiref " if self.params['align']: xmippopts += " -alignImages " ### use multi-processor command apDisplay.printColor( "Using " + str(self.params['nproc']) + " processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True) mpiruncmd = self.mpirun + " -np " + str( self.params['nproc']) + " " + xmippexe + " " + xmippopts self.writeXmippLog(mpiruncmd) apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True, logfile="xmipp.std") self.params['runtime'] = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(self.params['runtime'])) ### minor post-processing self.createReferenceStack() self.parseOutput() self.clearIntermediateFiles() # self.readyUploadFlag() apParam.dumpParameters( self.params, "cl2d-" + self.params['timestamp'] + "-params.pickle") ### upload results ... this used to be two separate operations, I'm combining into one self.runparams = apParam.readRunParameters("cl2d-" + self.params['timestamp'] + "-params.pickle") self.apix = apStack.getStackPixelSizeFromStackId( self.runparams['stackid']) * self.runparams['bin'] self.Nlevels = len( glob.glob("part" + self.params['timestamp'] + "_level_??_.hed")) ### create average of aligned stacks & insert aligned stack info lastLevelStack = "part" + self.params[ 'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1) apStack.averageStack(lastLevelStack) self.boxsize = apFile.getBoxSize(lastLevelStack)[0] self.insertCL2DParamsIntoDatabase() if self.runparams['align'] is True: self.insertAlignStackRunIntoDatabase("alignedStack.hed") self.calcResolution(self.Nlevels - 1) self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1) ### loop over each class average stack & insert as clustering stacks self.insertClusterRunIntoDatabase() for level in range(self.Nlevels): ### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES if self.params['align'] is True: self.calcResolution(level) partdict = self.getClassificationAtLevel(level) for classnum in partdict: self.insertClusterStackIntoDatabase( "part" + self.params['timestamp'] + "_level_%02d_.hed" % level, classnum + 1, partdict[classnum], len(partdict))
def start(self): # self.insertCL2DJob() self.stack = {} self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) if self.params['virtualdata'] is not None: self.stack['file'] = self.params['virtualdata']['filename'] else: self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) ### process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed") if os.path.isfile(self.params['localstack']): apFile.removeStack(self.params['localstack']) a = proc2dLib.RunProc2d() a.setValue('infile',self.stack['file']) a.setValue('outfile',self.params['localstack']) a.setValue('apix',self.stack['apix']) a.setValue('bin',self.params['bin']) a.setValue('last',self.params['numpart']-1) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: a.setValue('lowpass',self.params['lowpass']) if self.params['highpass'] is not None and self.params['highpass'] > 1: a.setValue('highpass',self.params['highpass']) if self.params['invert'] is True: a.setValue('invert',True) # clip not yet implemented # if self.params['clipsize'] is not None: # clipsize = int(self.clipsize)*self.params['bin'] # if clipsize % 2 == 1: # clipsize += 1 ### making sure that clipped boxsize is even # a.setValue('clip',clipsize) if self.params['virtualdata'] is not None: vparts = self.params['virtualdata']['particles'] plist = [int(p['particleNumber'])-1 for p in vparts] a.setValue('list',plist) #run proc2d a.run() if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']): apDisplay.printError("Missing particles in stack") ### setup Xmipp command aligntime = time.time() xmippopts = (" -i "+os.path.join(self.params['rundir'], self.params['localstack']) +" --nref "+str(self.params['numrefs']) +" --iter "+str(self.params['maxiter']) +" --odir "+str(self.params['rundir']) +" --oroot "+ "part"+str(self.params['timestamp']) +" --classifyAllImages" ) if self.params['correlation']: xmippopts += " --distance correlation" if self.params['classical']: xmippopts += " --classicalMultiref" ### use multi-processor command apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green") xmippexe = apParam.getExecPath(self.execFile, die=True) mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts self.writeXmippLog(mpiruncmd) apParam.runCmd(mpiruncmd, package="Xmipp 3", verbose=True, showcmd=True, logfile="xmipp.std") self.params['runtime'] = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime'])) ### post-processing # Create a stack for the class averages at each level Nlevels=glob.glob("level_*") for level in Nlevels: digits = level.split("_")[1] apParam.runCmd("xmipp_image_convert -i "+level+"/part"+self.params['timestamp']+"*xmd -o part" +self.params['timestamp']+"_level_"+digits+"_.hed", package="Xmipp 3", verbose=True) if self.params['align']: apParam.runCmd("xmipp_transform_geometry -i images.xmd -o %s_aligned.stk --apply_transform" % self.params['timestamp'], package="Xmipp 3", verbose=True) apParam.runCmd("xmipp_image_convert -i %s_aligned.xmd -o alignedStack.hed" % self.params['timestamp'], package="Xmipp 3", verbose=True) apFile.removeFile("%s_aligned.xmd" % self.params['timestamp']) apFile.removeFile("%s_aligned.stk" % self.params['timestamp']) self.parseOutput() apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle") ### upload results ... this used to be two separate operations, I'm combining into one self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle") self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin'] self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed")) ### create average of aligned stacks & insert aligned stack info lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1) apStack.averageStack(lastLevelStack) self.boxsize = apFile.getBoxSize(lastLevelStack)[0] self.insertCL2DParamsIntoDatabase() if self.runparams['align'] is True: self.insertAlignStackRunIntoDatabase("alignedStack.hed") self.calcResolution(self.Nlevels-1) self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1) ### loop over each class average stack & insert as clustering stacks self.insertClusterRunIntoDatabase() for level in range(self.Nlevels): ### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES if self.params['align'] is True: self.calcResolution(level) partdict = self.getClassificationAtLevel(level) for classnum in partdict: self.insertClusterStackIntoDatabase( "part"+self.params['timestamp']+"_level_%02d_.hed"%level, classnum+1, partdict[classnum], len(partdict)) self.clearIntermediateFiles()
def start(self): self.insertMaxLikeJob() self.stack = {} self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) if self.params['virtualdata'] is not None: self.stack['file'] = self.params['virtualdata']['filename'] else: self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) self.estimateIterTime() self.dumpParameters() ### process stack to local file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") a = proc2dLib.RunProc2d() a.setValue('infile',self.stack['file']) a.setValue('outfile',self.params['localstack']) a.setValue('apix',self.stack['apix']) a.setValue('bin',self.params['bin']) a.setValue('last',self.params['numpart']-1) a.setValue('append',False) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: a.setValue('lowpass',self.params['lowpass']) if self.params['highpass'] is not None and self.params['highpass'] > 1: a.setValue('highpass',self.params['highpass']) if self.params['invert'] is True: a.setValue('invert') is True if self.params['virtualdata'] is not None: vparts = self.params['virtualdata']['particles'] plist = [int(p['particleNumber'])-1 for p in vparts] a.setValue('list',plist) # clip not yet implemented # self.params['clipsize'] is not None: # clipsize = int(self.clipsize)*self.params['bin'] # if clipsize % 2 == 1: # clipsize += 1 ### making sure that clipped boxsize is even # a.setValue('clip',clipsize) if self.params['virtualdata'] is not None: vparts = self.params['virtualdata']['particles'] plist = [int(p['particleNumber'])-1 for p in vparts] a.setValue('list',plist) #run proc2d a.run() if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']): apDisplay.printError("Missing particles in stack") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) ### stack is needed by uploadMaxlike.py #apFile.removeStack(self.params['localstack']) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " +" -i "+os.path.join(self.params['rundir'], self.partlistdocfile) +" -nref "+str(self.params['numrefs']) +" -iter "+str(self.params['maxiter']) +" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp) +" -psi_step "+str(self.params['psistep']) ) ### fast mode if self.params['fast'] is True: xmippopts += " -fast " if self.params['fastmode'] == "narrow": xmippopts += " -C 1e-10 " elif self.params['fastmode'] == "wide": xmippopts += " -C 1e-17 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " ### normalization if self.params['norm'] is True: xmippopts += " -norm " ### use student's T distribution if self.params['student'] is True: xmippopts += " -student " ### write cluster job file if self.params['cluster'] is True: self.writeClusterJobFile() ### find number of processors if self.params['nproc'] is None: nproc = nproc = apParam.getNumProcessors() else: nproc = self.params['nproc'] mpirun = self.checkMPI() self.estimateIterTime() if nproc > 2 and mpirun is not None: ### use multi-processor apDisplay.printColor("Using "+str(nproc)+" processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True) mpiruncmd = mpirun+" -np "+str(nproc)+" "+xmippexe+" "+xmippopts self.writeXmippLog(mpiruncmd) apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True) else: ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe+" "+xmippopts self.writeXmippLog(xmippcmd) apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ### minor post-processing self.createReferenceStack() self.readyUploadFlag() self.dumpParameters()
def runAmpcor(): spidercmd = "spider bat/spi @enhance_edit" apParam.runCmd(spidercmd, package="SPIDER", verbose=False, showcmd=True) apFile.removeFile('pwsc.bat') apFile.removeFile('enhance_edit.bat') return
def start(self): ### database entry parameters package_table = 'ApXmippRefineIterData|xmippParams' ### set projection-matching path self.projmatchpath = os.path.abspath( os.path.join(self.params['rundir'], "recon", self.runparams['package_params']['WorkingDir'])) # self.projmatchpath = os.path.abspath(os.path.join(self.params['rundir'], self.runparams['package_params']['WorkingDir'])) ### check for variable root directories between file systems apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree( self.params['rundir'], self.runparams['remoterundir'], self.runparams['rundir']) ### determine which iterations to upload lastiter = self.findLastCompletedIteration() uploadIterations = self.verifyUploadIterations(lastiter) ### upload each iteration for iteration in uploadIterations: apDisplay.printColor("uploading iteration %d" % iteration, "cyan") ### set package parameters, as they will appear in database entries package_database_object = self.instantiateProjMatchParamsData( iteration) ### move FSC file to results directory oldfscfile = os.path.join(self.projmatchpath, "Iter_%d" % iteration, "Iter_%d_resolution.fsc" % iteration) newfscfile = os.path.join( self.resultspath, "recon_%s_it%.3d_vol001.fsc" % (self.params['timestamp'], iteration)) if os.path.exists(oldfscfile): shutil.copyfile(oldfscfile, newfscfile) ### create a stack of class averages and reprojections (optional) self.compute_stack_of_class_averages_and_reprojections(iteration) ### create a text file with particle information self.createParticleDataFile(iteration) if not self.params['euleronly']: ### create mrc file of map for iteration and reference number oldvol = os.path.join(self.projmatchpath, "Iter_%d" % iteration, "Iter_%d_reconstruction.vol" % iteration) newvol = os.path.join( self.resultspath, "recon_%s_it%.3d_vol001.mrc" % (self.params['timestamp'], iteration)) mrccmd = "proc3d %s %s apix=%.3f" % (oldvol, newvol, self.runparams['apix']) apParam.runCmd(mrccmd, "EMAN") ### make chimera snapshot of volume self.createChimeraVolumeSnapshot(newvol, iteration) ### instantiate database objects self.insertRefinementRunData(iteration) self.insertRefinementIterationData(iteration, package_table, package_database_object) ### calculate Euler jumps if self.runparams['numiter'] > 1: self.calculateEulerJumpsAndGoodBadParticles(uploadIterations) ### query the database for the completed refinements BEFORE deleting any files ... returns a dictionary of lists ### e.g. {1: [5, 4, 3, 2, 1]} means 5 iters completed for refine 1 complete_refinements = self.verifyNumberOfCompletedRefinements( multiModelRefinementRun=False) if self.params['cleanup_files'] is True: self.cleanupFiles(complete_refinements)
def compute_stack_of_class_averages_and_reprojections(self, iteration): ''' takes Xmipp single files, doc and sel files in projection-matching, creates a stack of class averages in the results directory ''' if bool(self.runparams['package_params']['CleanUpFiles']) is False: os.chdir( os.path.join(self.projmatchpath, "Iter_%d" % iteration, "ProjMatchClasses")) ### make projections, and put them back into resultspath selfile = "proj_match_classes.sel" refvolume = "../Iter_%d_reconstruction.vol" % iteration docfile = "proj_match_classes.doc" # apXmipp.compute_stack_of_class_averages_and_reprojections(d, selfile, refvolume, docfile, \ # self.runparams['boxsize'], self.resultspath, self.params['timestamp'], iteration) ### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile f = open(selfile, "r") lines = f.readlines() newlines = [ re.sub("ProjMatchClasses/", "", line) for line in lines ] f.close() f = open(selfile[:-4] + "_new.sel", "w") f.writelines(newlines) f.close() ### create a projection params file and project the volume along identical Euler angles f = open("paramfile.descr", "w") f.write("%s\n" % refvolume) f.write("tmpproj 1 xmp\n") f.write("%d %d\n" % (self.runparams['boxsize'], self.runparams['boxsize'])) f.write("%s rot tilt psi\n" % docfile) f.write("NULL\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.close() projectcmd = "xmipp_project -i paramfile.descr" apParam.runCmd(projectcmd, "Xmipp") ### get order of projections in docfile d = open(docfile, "r") lines = d.readlines()[1:] d.close() projfile_sequence = [] for i, l in enumerate(lines): if i % 2 == 0: filename = os.path.basename(l.split()[1]) projfile_sequence.append(filename) else: pass ### create stack of projections and class averages projections = glob.glob("tmpproj**xmp") projections.sort() if len(projections) != len(projfile_sequence): apDisplay.printWarning( "number of projections does not match number of classes") stackarray = [] stackname = os.path.join( self.resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (self.params['timestamp'], iteration, 1)) for i in range(len(projections)): stackarray.append(spider.read(projections[i])) stackarray.append(spider.read(projfile_sequence[i])) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### remove unnecessary files for file in glob.glob("tmpproj*"): apFile.removeFile(file) os.chdir(self.params['rundir']) else: apDisplay.printWarning( "all projection-matching files were cleaned up ... NOT creating class-average / re-projection stack" ) return
def start(self): # self.insertCL2DJob() self.stack = {} self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) if self.params['virtualdata'] is not None: self.stack['file'] = self.params['virtualdata']['filename'] else: self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) ### process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed") if os.path.isfile(self.params['localstack']): apFile.removeStack(self.params['localstack']) a = proc2dLib.RunProc2d() a.setValue('infile',self.stack['file']) a.setValue('outfile',self.params['localstack']) a.setValue('apix',self.stack['apix']) a.setValue('bin',self.params['bin']) a.setValue('last',self.params['numpart']-1) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: a.setValue('lowpass',self.params['lowpass']) if self.params['highpass'] is not None and self.params['highpass'] > 1: a.setValue('highpass',self.params['highpass']) if self.params['invert'] is True: a.setValue('invert',True) # clip not yet implemented # if self.params['clipsize'] is not None: # clipsize = int(self.clipsize)*self.params['bin'] # if clipsize % 2 == 1: # clipsize += 1 ### making sure that clipped boxsize is even # a.setValue('clip',clipsize) if self.params['virtualdata'] is not None: vparts = self.params['virtualdata']['particles'] plist = [int(p['particleNumber'])-1 for p in vparts] a.setValue('list',plist) #run proc2d a.run() if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']): apDisplay.printError("Missing particles in stack") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " +" -i "+os.path.join(self.params['rundir'], self.partlistdocfile) +" -codes "+str(self.params['numrefs']) +" -iter "+str(self.params['maxiter']) +" -o "+os.path.join(self.params['rundir'], "part"+self.params['timestamp']) ) if self.params['fast']: xmippopts += " -fast " if self.params['correlation']: xmippopts += " -useCorrelation " if self.params['classical']: xmippopts += " -classicalMultiref " if self.params['align']: xmippopts += " -alignImages " ### use multi-processor command apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True) mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts self.writeXmippLog(mpiruncmd) apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True, logfile="xmipp.std") self.params['runtime'] = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime'])) ### minor post-processing self.createReferenceStack() self.parseOutput() self.clearIntermediateFiles() # self.readyUploadFlag() apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle") ### upload results ... this used to be two separate operations, I'm combining into one self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle") self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin'] self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed")) ### create average of aligned stacks & insert aligned stack info lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1) apStack.averageStack(lastLevelStack) self.boxsize = apFile.getBoxSize(lastLevelStack)[0] self.insertCL2DParamsIntoDatabase() if self.runparams['align'] is True: self.insertAlignStackRunIntoDatabase("alignedStack.hed") self.calcResolution(self.Nlevels-1) self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1) ### loop over each class average stack & insert as clustering stacks self.insertClusterRunIntoDatabase() for level in range(self.Nlevels): ### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES if self.params['align'] is True: self.calcResolution(level) partdict = self.getClassificationAtLevel(level) for classnum in partdict: self.insertClusterStackIntoDatabase( "part"+self.params['timestamp']+"_level_%02d_.hed"%level, classnum+1, partdict[classnum], len(partdict))
def createProjectionsEmanProp(self, pad=False, invert=False): ### first get rid of projection artifacts from insufficient padding if self.params['threedfile'] is not None: origfile = self.params['threedfile'] elif self.params['modelid'] is not None: self.modelparams = appiondata.ApInitialModelData.direct_query(self.params['modelid']) origfile = os.path.join(self.modelparams['path']['path'], self.modelparams['name']) if self.params['apix'] is None: self.params['apix'] = self.modelparams['pixelsize'] if self.params['box'] is None: self.params['box'] = self.modelparams['boxsize'] clipped = os.path.join(self.params['rundir'], "clipped.mrc") newsize = self.params['box'] * 1.5 emancmd = "proc3d "+origfile+" "+clipped+" clip="+str(int(newsize))+","+str(int(newsize))+","+str(int(newsize))+" edgenorm" apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True) ### project resized file eulerfile = os.path.join(self.params['rundir'], "eulers.lst") tempeulerfile = os.path.join(self.params['rundir'], "tmpeulers.lst") filename = os.path.join(self.params['rundir'], "proj.img") temp = os.path.join(self.params['rundir'], "temp.img") emancmd = "project3d "+clipped+" out="+temp+" sym=c1 prop="+str(self.params['projinc'])+" > "+tempeulerfile apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True) ### update projection count & write eulerlist file f = open(tempeulerfile, "r") lines = f.readlines() f.close() apFile.removeStack(temp) apFile.removeFile(tempeulerfile) strip = [line.strip() for line in lines[2:]] ### first two lines are EMAN commands iters = int(math.ceil(float(self.params['projcount']) / len(strip))) self.params['projcount'] = iters * len(strip) while os.path.isfile(eulerfile): apFile.removeFile(eulerfile) f = open(eulerfile, "a") n = 1 for i in range(iters): for j in range(len(strip)): split = strip[j].split() f.write(str(n)+"\t") f.write(str(split[1])+"\t") f.write(str(split[2])+"\t") if self.params['rotang'] != 0: randrot = random.uniform(-1*self.params['rotang'], self.params['rotang']) f.write(str(randrot)+"\t\n") else: f.write(str(split[3])+"\t\n") n += 1 f.close() ### create actual projections apFile.removeStack(filename) t0 = time.time() emancmd = "project3d "+clipped+" out="+filename+" list="+eulerfile # emancmd = "project3d "+clipped+" out="+filename+" sym=c1 prop="+str(self.params['projinc'])+" > "+tempeulerfile apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True) apDisplay.printMsg("Finished project3d in %s, %.3f ms per iteration" %(apDisplay.timeString(time.time()-t0), 1.0e3 * (time.time()-t0)/float(self.params['projcount']))) ''' ### update projection count & write eulerlist file f = open(tempeulerfile, "r") lines = f.readlines() f.close() apFile.removeStack(temp) apFile.removeFile(tempeulerfile) strip = [line.strip() for line in lines[2:]] ### first two lines are EMAN commands iters = int(math.ceil(float(self.params['projcount']) / len(strip))) self.params['projcount'] = iters * len(strip) while os.path.isfile(eulerfile): apFile.removeFile(eulerfile) f = open(eulerfile, "a") n = 1 for i in range(iters): for j in range(len(strip)): split = strip[j].split() f.write(str(n)+"\t") f.write(str(split[1])+"\t") f.write(str(split[2])+"\t") f.write(str(split[3])+"\t\n") n += 1 f.close() ''' ### post-process stack self.post_process_stack(filename, pad, invert) return filename
def runRelionPreprocess(self, newstackroot): ''' 1. Use stackIntoPicks.py to extract the particle locations from the selected stack. 2. Run makestack2.py without ctf correction or normalization using the stackIntoPicks result as the Particles run. 3. Run relion_preprocess with rescale and norm. Outputs .mrcs file. Neil: most of these steps could be done more generally ''' apDisplay.printWarning("Making a new stack from original images") # Build the stackIntoPicks command apDisplay.printMsg( 'Extracting the particle locations from your selected stack.') newstackrunname = self.params['runname'] + "_particles" newstackrundir = self.params['rundir'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid sessionid = int(self.params['expid']) cmd = "stackIntoPicks.py " cmd += (" --stackid=%d --projectid=%d --runname=%s --rundir=%s " % (stackid, projectid, newstackrunname, newstackrundir)) cmd += (" --commit --expId=%d --jobtype=stackintopicks " % (sessionid)) # Run the command logfilepath = os.path.join(newstackrundir, 'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd, logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Build the makestack2 command ''' This is the command we want to make a stack from a stackIntoPicks run makestack2.py --single=start.hed --selectionid=130 --invert --boxsize=320 --bin=2 --description="made from stackrun1 using stackintopicks" --runname=stack66 --rundir=/ami/data00/appion/zz07jul25b/stacks/stack66 --commit --preset=en --projectid=303 --session=zz07jul25b --no-rejects --no-wait --continue --expid=8556 --jobtype=makestack2 --ppn=1 --nodes=1 --walltime=240 --jobid=2016 ''' apDisplay.printMsg( 'Using selected stack particle locations to make a Relion ready stack....' ) # Get the ID of the stackIntoPicks run we just created #apParticle.getSelectionIdFromName(runname, sessionname) runq = appiondata.ApSelectionRunData() runq['name'] = newstackrunname runq['session'] = leginondata.SessionData.direct_query( self.params['expid']) rundatas = runq.query(results=1) print rundatas if rundatas: selectionid = rundatas[0].dbid else: apDisplay.printError( "Error creating Relion ready stack. Could not find stackIntoPicks.py data in database.\n" ) # Gather all the makestack parameters totalpart = self.originalStackData.numpart numpart = totalpart if not self.params['last'] else min( self.params['last'], totalpart) stackpathname = os.path.basename(self.originalStackData.path) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir, 'start.hed') presetname = self.originalStackData.preset # binning is combination of the original binning of the stack and the preparation binnning bin = self.originalStackData.bin * self.params['bin'] unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin lowpasstext = setArgText( 'lowpass', (self.params['lowpass'], self.originalStackData.lowpass), False) highpasstext = setArgText( 'highpass', (self.params['highpass'], self.originalStackData.highpass), True) partlimittext = setArgText('partlimit', (numpart, ), False) xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'], ), True) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid reversetext = '--reverse' if self.originalStackData.reverse else '' defoctext = '--defocpair' if self.originalStackData.defocpair else '' inverttext = '--no-invert' if not self.invert else '' # Build the makestack2 command cmd = "makestack2.py " cmd += (" --single=%s --selectionid=%d %s --boxsize=%d --bin=%d " % (os.path.basename(newstackimagicfile), selectionid, inverttext, unbinnedboxsize, bin)) cmd += ( " --description='Relion refinestack based on %s(id=%d)' --projectid=%d " % (stackpathname, stackid, projectid)) cmd += ( " --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d " % (presetname, newstackrunname, newstackrundir, sessionname, sessionid)) cmd += " --no-wait --no-commit --no-continue --jobtype=makestack2 " # Run the command logfilepath = os.path.join(newstackrundir, 'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd, logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Clean up boxfiles = glob.glob("*.box") for boxfile in boxfiles: apFile.removeFile(boxfile) # Make sure our new stack params reflects the changes made # Use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] self.stack['file'] = newstackroot + '.hed' # Run Relion pre-process command ''' Setup the follow relion preprocess command to normalize the new stack: relion_preprocess \ --o particles --norm --bg_radius 60 --white_dust -1 --black_dust -1 \ --operate_on /ami/data00/appion/zz07jul25b/stacks/stack63_no_xmipp_norm/start.hed ''' apDisplay.printMsg( 'Running Relion preprocessing to normalize your Relion Ready stack....' ) bg_radius = math.floor((self.stack['boxsize'] / 2) - 1) relioncmd = "relion_preprocess " relioncmd += " --o particles --norm --bg_radius %d " % (bg_radius) relioncmd += " --white_dust -1 --black_dust -1 --operate_on %s " % ( newstackimagicfile) apParam.runCmd(relioncmd, package="Relion", verbose=True, showcmd=True) self.stack['file'] = 'particles.mrcs'
def compute_stack_of_class_averages_and_reprojections(dir, selfile, refvolume, docfile, boxsize, resultspath, timestamp, iteration, reference_number=1, extract=False): ''' takes Xmipp single files, doc and sel files in routine, creates a stack of class averages in the results directory ''' workingdir = os.getcwd() os.chdir(dir) if dir.endswith("/"): dir = dir[:-1] head, tail = os.path.split(dir) ### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile f = open(selfile, "r") lines = f.readlines() newlines = [re.sub(str(tail)+"/", "", line) for line in lines] f.close() f = open(selfile[:-4]+"_new.sel", "w") f.writelines(newlines) f.close() if extract is True: extractcmd = "xmipp_header_extract -i %s.sel -o %s.doc" % (selfile[:-4], docfile[:-4]) apParam.runCmd(extractcmd, "Xmipp") ### create a projection params file and project the volume along identical Euler angles f = open("paramfile.descr", "w") f.write("%s\n" % refvolume) f.write("tmpproj 1 xmp\n") f.write("%d %d\n" % (boxsize, boxsize)) f.write("%s rot tilt psi\n" % docfile) f.write("NULL\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.close() projectcmd = "xmipp_project -i paramfile.descr" apParam.runCmd(projectcmd, "Xmipp") ### get order of projections in docfile d = open(docfile, "r") lines = d.readlines()[1:] d.close() projfile_sequence = [] for i, l in enumerate(lines): if i % 2 == 0: filename = os.path.basename(l.split()[1]) projfile_sequence.append(filename) else: pass ### create stack of projections and class averages projections = glob.glob("tmpproj**xmp") projections.sort() if len(projections) != len(projfile_sequence): apDisplay.printWarning("number of projections does not match number of classes") stackarray = [] stackname = os.path.join(resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (timestamp, iteration, reference_number)) for i in range(len(projections)): stackarray.append(spider.read(projections[i])) stackarray.append(spider.read(projfile_sequence[i])) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### remove unnecessary files for file in glob.glob("tmpproj*"): apFile.removeFile(file) os.chdir(workingdir) return
def start(self): ''' OptiMod script ''' ############### define short-hand parameter names ############ refine = True rundir = self.params['rundir'] box = self.params['boxsize'] rbox = self.params['refineboxsize'] apix = self.params['apix'] rapix = self.params['refineapix'] npart = self.params['numpart'] nvol = self.params['num_volumes'] # sym = self.params['sym'] sym = "c1" psym = self.params['presumed_sym'] first = self.params['firstimage'] nws = self.params['non_weighted_sequence'] threes = self.params['threes'] lmask = self.params['linmask'] asqfilt = self.params['asqfilt'] anginc = self.params['ang_inc'] keep_ordered = self.params['keep_ordered_num'] hamwin = self.params['ham_win'] useEMAN1 = self.params['useEMAN1'] ipv = self.params['images_per_volume'] lp = self.params['3d_lpfilt'] nref = self.params['nref'] PCA = self.params['PCA'] neigens = self.params['numeigens'] recalc = self.params['recalculate'] preftype = self.params['preftype'] mrad = self.params['mask_radius'] irad = self.params['inner_radius'] orad = self.params['outer_radius'] tnproc = self.params['threadnproc'] nproc = self.params['nproc'] oldavgs = self.params['oldavgs'] avgs = os.path.join(self.params['rundir'], self.params['avgs']) start = self.params['start_at'] ############# copy to working directory ############ if not os.path.isfile(os.path.join(avgs)[:-4]+".hed"): shutil.copyfile(os.path.join(oldavgs[:-4]+".hed"), avgs[:-4]+".hed") if not os.path.isfile(os.path.join(avgs)[:-4]+".img"): shutil.copyfile(os.path.join(oldavgs[:-4]+".img"), avgs[:-4]+".img") if self.params['ravgs'] is not None: ravgs = os.path.join(self.params['rundir'], self.params['ravgs']) if not os.path.isfile(self.params['ravgs'][:-4]+".hed"): shutil.copy(self.params['oldravgs'][:-4]+".hed", os.path.join(self.params['ravgs'][:-4]+".hed")) if not os.path.isfile(self.params['ravgs'][:-4]+".img"): shutil.copy(self.params['oldravgs'][:-4]+".img", os.path.join(self.params['ravgs'][:-4]+".img")) ### Euler jumper assessment does not make sense when refine images are different ejassess = False else: ravgs = avgs ejassess = True ########### scale & prealign class averages, if specified ######### if self.params['scale'] is True: emancmd = "proc2d %s %s_scaled.img scale=%.3f clip=%i,%i edgenorm" \ % (avgs, avgs[:-4], self.scalefactor, 64, 64) avgs = avgs[:-4]+"_scaled.img" if start == "none": while os.path.isfile(avgs): apFile.removeStack(avgs) apParam.runCmd(emancmd, "EMAN") else: apDisplay.printColor("skipping stack scaling", "cyan") if self.imagicroot is not None and useEMAN1 is False: apIMAGIC.takeoverHeaders(avgs, npart, box) if self.imagicroot is not None: if self.params['prealign'] is True: if start == "none": avgs = apIMAGIC.prealignClassAverages(rundir, avgs) apIMAGIC.checkLogFileForErrors( os.path.join(rundir, "prealignClassAverages.log")) ### select between EMAN1 and IMAGIC for common lines if self.imagicroot is not None and useEMAN1 is False: commonlinesdir = os.path.join(rundir, "angular_reconstitution") else: commonlinesdir = os.path.join(rundir, "cross_common_lines") ### make links & calculate CCC matrix for 2D averages apDisplay.printColor("Calculating similarity matrix", "cyan") ccc_matrix = apCommonLines.calculate_ccc_matrix_2d(avgs) clsavgs = os.path.split(avgs)[1][:-4] if not os.path.isdir(commonlinesdir): os.mkdir(commonlinesdir) if os.path.islink(os.path.join(commonlinesdir, clsavgs+".hed")): os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".hed")) os.symlink(os.path.join(rundir, clsavgs+".hed"), os.path.join(commonlinesdir, clsavgs+".hed")) if os.path.islink(os.path.join(commonlinesdir, clsavgs+".img")): os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".img")) os.symlink(os.path.join(rundir, clsavgs+".img"), os.path.join(commonlinesdir, clsavgs+".img")) ################## create multiple 3d0s ##################### if start == "none": if self.imagicroot is not None and useEMAN1 is False: cmdlist = [] seqfile = open(os.path.join(rundir, "sequences_for_angular_reconstitution.dat"), "w") apDisplay.printColor("Running multiple raw volume calculations", "cyan") for i in range(nvol): sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart, ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws) apCommonLines.check_for_duplicates_in_sequence(sequence) seqfile.write(str(sequence)+"\n") ### create IMAGIC batch file for each model & append them to be threaded batchfile = apCommonLines.imagic_batch_file(sequence, i+1, avgs, sym, asqfilt, lmask, apix, box, anginc, keep_ordered, hamwin, lp, threes=threes, do_not_remove=False) proc = subprocess.Popen('chmod 755 '+batchfile, shell=True) proc.wait() cmdlist.append(batchfile) os.chdir(rundir) seqfile.close() apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10) ### check for errors after execution for i in range(nvol): apIMAGIC.checkLogFileForErrors( os.path.join(commonlinesdir, "3d"+str(i+1)+".log")) else: ### use EMAN1 cross-common lines cmdlist = [] seqfile = open(os.path.join(rundir, "sequences_for_cross_common_lines.dat"), "w") for i in range(nvol): sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart, ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws) apCommonLines.check_for_duplicates_in_sequence(sequence) seqfile.write(str(sequence)+"\n") vol_avgfile = os.path.join(commonlinesdir, "vol_%d_averages.hed" % (i+1)) vol_seqfile = os.path.join(commonlinesdir, "vol_%d_averages.txt" % (i+1)) vsf = open(vol_seqfile, "w") for j in range(ipv): vsf.write("%d\n" % sequence[j]) vsf.close() tmpdir = os.path.join(commonlinesdir, "tmp%d" % (i+1)) fullcmd = "proc2d %s %s list=%s ; " % (avgs, vol_avgfile, vol_seqfile) fullcmd+= "rm -rf %s ; mkdir %s ; cd %s ; " % (tmpdir, tmpdir, tmpdir) fullcmd+= "startAny %s sym=c1 proc=1 rounds=2 mask=%d ; " \ % (vol_avgfile, mrad/apix) fullcmd+= "mv threed.0a.mrc ../threed_%d.mrc ; " % (i+1) fullcmd+= "mv CCL.hed ../CCL_%d.hed ; " % (i+1) fullcmd+= "mv CCL.img ../CCL_%d.img ; " % (i+1) cmdlist.append(fullcmd) seqfile.close() apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10) for i in range(nvol): shutil.rmtree(os.path.join(commonlinesdir, "tmp%d" % (i+1))) else: apDisplay.printColor("skipping common lines volume calculations", "cyan") ############### move 3-D models volume directory ################ ### create volume directory volumedir = os.path.join(rundir, "volumes") if not os.path.isdir(volumedir): os.mkdir(volumedir) volumes = {} cmds = [] if start == "none": apDisplay.printColor("moving volumes for Xmipp 3-D Maximum Likelihood", "cyan") for i in range(nvol): if useEMAN1 is False: volume1 = os.path.join(commonlinesdir, "3d%d_ordered%d_filt.vol" % (i+1,i+1)) else: volume1 = os.path.join(commonlinesdir, "threed_%d.mrc" % (i+1)) volume2 = os.path.join(volumedir, "3d%d.vol" % (i+1)) if start == "none": if useEMAN1 is False: shutil.move(volume1, volume2) else: cmds.append("proc3d %s %s spidersingle" % (volume1, volume2)) volumes[(i+1)] = volume2 apThread.threadCommands(cmds, nproc=tnproc) #################### align 3-D models ################## if start == "3D_align" or start == "none": # if start != "3D_assess" and start != "3D_refine": ### run Maximum Likelihood 3-D alignment & align resulting volumes apDisplay.printColor("Running Xmipp maximum likelihood 3-D alignment", "cyan") vol_doc_file, alignref = apCommonLines.xmipp_max_like_3d_align( volumes, nproc, tnproc, nref) alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol) apDisplay.printColor("Aligning volumes based on 3-D ML parameters", "cyan") apCommonLines.align_volumes(alignparams, alignref, nvol, tnproc, apix) else: apDisplay.printColor("skipping 3D alignment", "cyan") vol_doc_file, alignref = apCommonLines.findAlignmentParams( os.path.join(rundir,"max_like_alignment"), nref) alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol) ##################### Principal Component Analysis ############## apDisplay.printColor("Calculating inter-volume similarity", "cyan") aligned_volumes = {} for i in range(nvol): aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.vol" % (i+1)) # aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.mrc" % (i+1)) if PCA is True: simfile, sim_matrix = apCommonLines.runPrincipalComponentAnalysis( aligned_volumes, nvol, neigens, box, apix, recalculate=recalc) else: simfile, sim_matrix = apCommonLines.calculate_ccc_matrix_3d( aligned_volumes, nvol) ################ 3-D affinity propagation ############# ### 3-D Affinity Propagation apDisplay.printColor("Averaging volumes with Affinity Propagation", "cyan") preffile = apCommonLines.set_preferences(sim_matrix, preftype, nvol) classes = apCommonLines.run_affinity_propagation(aligned_volumes, simfile, preffile, box, apix) ######### refine volumes using Xmipp projection matching ######### if start == "3D_align" or start == "3D_refine" or start == "none": # if start != "3D_assess": if not os.path.isdir("refinement"): os.mkdir("refinement") os.chdir("refinement") apXmipp.breakupStackIntoSingleFiles(ravgs) xmippcmd = "xmipp_normalize -i partlist.sel -method OldXmipp" apParam.runCmd(xmippcmd, "Xmipp") for i in classes.keys(): emancmd = "proc3d %s %s scale=%.3f clip=%d,%d,%d mask=%s spidersingle" \ % (os.path.join(rundir, "%d.mrc" % i), "%d.vol" % i, \ (1/self.scalefactor), rbox, rbox, rbox, (mrad / rapix)) apParam.runCmd(emancmd, "EMAN") apCommonLines.refine_volume(rundir, ravgs, i, mrad, irad, orad, rapix, nproc) emancmd = "proc3d %s %s apix=%.3f" \ % (os.path.join("refine_%d" % i, "3d%d_refined.vol" % i), \ os.path.join(rundir, "%d_r.mrc" % i), rapix) apParam.runCmd(emancmd, "EMAN") os.chdir(rundir) else: apDisplay.printColor("skipping 3D refinement", "cyan") ################# model evaluation ################ if start == "3D_align" or start == "3D_refine" or start == "3D_assess" or start == "none": # if start == "none" or start=="3D_assess": ### final model assessment try: euler_array = apCommonLines.getEulerValuesForModels(alignparams, rundir, nvol, npart, threes=threes) except: euler_array = None ejassess = False if refine is True: apCommonLines.assess_3Dclass_quality2(rundir, aligned_volumes, sim_matrix, classes, euler_array, box, rbox, rapix, ravgs, psym, npart, ejassess=ejassess) else: apCommonLines.assess_3Dclass_quality(rundir, aligned_volumes, sim_matrix, classes, euler_array, box, apix, avgs, psym, npart, ejassess=ejassess) apCommonLines.combineMetrics("final_model_stats.dat", "final_model_stats_sorted_by_Rcrit.dat", **{"CCPR":(1,1)}) ################## upload & cleanup ################## ### upload to database, if specified self.upload() ### make chimera snapshots if refine is True: for i in classes.keys(): if self.params['mass'] is not None: apChimera.filterAndChimera( os.path.join(self.params['rundir'], "%d_r.mrc" % i), res=self.params['3d_lpfilt'], apix=self.params['refineapix'], box=self.params['refineboxsize'], chimtype="snapshot", contour=2, zoom=1, sym="c1", color="gold", mass=self.params['mass'] ) ### cleanup snapshots = glob.glob("*.png") mtlfiles = glob.glob("*.mtl") objfiles = glob.glob("*.obj") pyfiles = glob.glob("*mrc.py") for file in mtlfiles: os.remove(file) for file in objfiles: os.remove(file) for file in pyfiles: os.remove(file) if not os.path.isdir("snapshots"): os.mkdir("snapshots") for s in snapshots: shutil.move(s, os.path.join("snapshots", s))
def start(self): ### get stack parameteres self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackId']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackId']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackId']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### copy stack into working directory if os.path.isfile(self.stack['file']): apDisplay.printColor("copying stack into running directoroy", "cyan") if self.stack['file'][-4:] == ".img" or self.stack['file'][ -4:] == ".hed": strippedfile = self.stack['file'][:-4] else: strippedfile = self.stack['file'] while os.path.isfile( os.path.join(self.params['rundir'], "start.img")): apFile.removeStack( os.path.join(self.params['rundir'], "start.img")) emancmd = "proc2d "+strippedfile+".hed "+os.path.join(self.params['rundir'], "start.hed ")+\ "first=0 last="+str(self.params['numpart']-1) apParam.runCmd(emancmd, "EMAN") else: apDisplay.printError("stack not found in database") ### get template stack parameters self.templatestack = {} self.templatestack[ 'data'] = appiondata.ApTemplateStackData.direct_query( self.params['templateStackId']) self.templatestack['apix'] = self.templatestack['data']['apix'] self.templatestack['boxsize'] = self.templatestack['data']['boxsize'] self.templatestack['file'] = os.path.join( self.templatestack['data']['path']['path'], self.templatestack['data']['templatename']) self.templatestack['numimages'] = self.templatestack['data'][ 'numimages'] ### copy templates into working directory if os.path.isfile(self.templatestack['file']): apDisplay.printColor("copying templates into running directoroy", "cyan") ts = os.path.join(self.params['rundir'], "references.img") while os.path.isfile(ts): apFile.removeStack(ts) if self.templatestack['file'][-4:] == ".img" or self.templatestack[ 'file'][-4:] == ".hed": strippedfile = self.templatestack['file'][:-4] else: strippedfile = self.templatestack['file'] emancmd = "proc2d " + strippedfile + ".img " + ts apParam.runCmd(emancmd, "EMAN") else: apDisplay.printError("template stack not found in database") ### set new pixelsize if self.params['bin'] is not None and self.params['bin'] != 0: self.params['apix'] = float(self.stack['apix']) * int( self.params['bin']) else: self.params['apix'] = self.stack['apix'] ### scale, low-pass, and high-pass filter stack ... do this with imagic, because it determines the appropriate boxsizes scalingbatchfile = self.createImagicBatchFileScaling() preptime = time.time() proc = subprocess.Popen("chmod 775 " + str(scalingbatchfile), shell=True) proc.wait() os.chdir(self.params['rundir']) apParam.runCmd(scalingbatchfile, "IMAGIC") apIMAGIC.checkLogFileForErrors( os.path.join(self.params['rundir'], "prepareStack.log")) apDisplay.printColor( "finished IMAGIC in " + apDisplay.timeString(time.time() - preptime), "cyan") ### set new boxsize, done only after scaling is complete if self.params['bin'] is not None: self.params['boxsize'] = apFile.getBoxSize( os.path.join(self.params['rundir'], "start.hed"))[0] else: self.params['boxsize'] = self.stack['boxsize'] ### make sure template stack boxsize matches that of the input stack if self.params['apix'] != self.templatestack['apix'] or self.params[ 'boxsize'] != self.templatestack['boxsize']: self.scaleTemplates() starttime = time.time() print self.params print "... stack pixel size: " + str(self.params['apix']) print "... stack box size: " + str(self.params['boxsize']) apDisplay.printColor( "Running IMAGIC .batch file: See multiReferenceAlignment.log file for details", "cyan") ### create IMAGIC batch file batchfile = self.createImagicBatchFileMRA() ### execute IMAGIC batch file aligntime0 = time.time() proc = subprocess.Popen("chmod 775 " + str(batchfile), shell=True) proc.wait() os.chdir(self.params['rundir']) apParam.runCmd(batchfile, "IMAGIC") apIMAGIC.checkLogFileForErrors( os.path.join(self.params['rundir'], "multiReferenceAlignment.log")) apDisplay.printColor( "finished IMAGIC in " + apDisplay.timeString(time.time() - aligntime0), "cyan") ### get particle parameters (shift, rotate, refnum, mirror, ccc) partparams = self.getParticleParams() ### average stack alignstack = os.path.join(self.params['rundir'], "alignstack.hed") apStack.averageStack(alignstack) ### normalize particles (otherwise found problems in viewing with stackviewer) emancmd = "proc2d " + alignstack + " " + alignstack + ".norm.hed norm" while os.path.isfile(alignstack + ".norm.img"): apFile.removeStack(alignstack + ".norm.img") apParam.runCmd(emancmd, "EMAN") os.rename(alignstack + ".norm.hed", alignstack) os.rename(alignstack + ".norm.img", alignstack[:-4] + ".img") ### normalize references emancmd = "proc2d " + ts + " " + ts + ".norm.hed norm" while os.path.isfile(ts + ".norm.img"): apFile.removeStack(ts + ".norm.img") apParam.runCmd(emancmd, "EMAN") os.rename(ts + ".norm.hed", ts) os.rename(ts + ".norm.img", ts[:-4] + ".img") ### remove copied stack while os.path.isfile(os.path.join(self.params['rundir'], "start.img")): apFile.removeStack(os.path.join(self.params['rundir'], "start.img")) ### insert run into database self.insertAlignmentRun(insert=True) self.insertParticlesIntoDatabase(partparams, insert=True)
def start(self): ### get stack parameteres self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackId']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackId']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackId']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### copy stack into working directory if os.path.isfile(self.stack['file']): apDisplay.printColor("copying stack into running directoroy", "cyan") if self.stack['file'][-4:] == ".img" or self.stack['file'][-4:] == ".hed": strippedfile = self.stack['file'][:-4] else: strippedfile = self.stack['file'] while os.path.isfile(os.path.join(self.params['rundir'], "start.img")): apFile.removeStack(os.path.join(self.params['rundir'], "start.img")) emancmd = "proc2d "+strippedfile+".hed "+os.path.join(self.params['rundir'], "start.hed ")+\ "first=0 last="+str(self.params['numpart']-1) apParam.runCmd(emancmd, "EMAN") else: apDisplay.printError("stack not found in database") ### get template stack parameters self.templatestack = {} self.templatestack['data'] = appiondata.ApTemplateStackData.direct_query(self.params['templateStackId']) self.templatestack['apix'] = self.templatestack['data']['apix'] self.templatestack['boxsize'] = self.templatestack['data']['boxsize'] self.templatestack['file'] = os.path.join(self.templatestack['data']['path']['path'], self.templatestack['data']['templatename']) self.templatestack['numimages'] = self.templatestack['data']['numimages'] ### copy templates into working directory if os.path.isfile(self.templatestack['file']): apDisplay.printColor("copying templates into running directoroy", "cyan") ts = os.path.join(self.params['rundir'], "references.img") while os.path.isfile(ts): apFile.removeStack(ts) if self.templatestack['file'][-4:] == ".img" or self.templatestack['file'][-4:] == ".hed": strippedfile = self.templatestack['file'][:-4] else: strippedfile = self.templatestack['file'] emancmd = "proc2d "+strippedfile+".img "+ts apParam.runCmd(emancmd, "EMAN") else: apDisplay.printError("template stack not found in database") ### set new pixelsize if self.params['bin'] is not None and self.params['bin'] != 0: self.params['apix'] = float(self.stack['apix']) * int(self.params['bin']) else: self.params['apix'] = self.stack['apix'] ### scale, low-pass, and high-pass filter stack ... do this with imagic, because it determines the appropriate boxsizes scalingbatchfile = self.createImagicBatchFileScaling() preptime = time.time() proc = subprocess.Popen("chmod 775 "+str(scalingbatchfile), shell=True) proc.wait() os.chdir(self.params['rundir']) apParam.runCmd(scalingbatchfile, "IMAGIC") apIMAGIC.checkLogFileForErrors(os.path.join(self.params['rundir'], "prepareStack.log")) apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-preptime), "cyan") ### set new boxsize, done only after scaling is complete if self.params['bin'] is not None: self.params['boxsize'] = apFile.getBoxSize(os.path.join(self.params['rundir'], "start.hed"))[0] else: self.params['boxsize'] = self.stack['boxsize'] ### make sure template stack boxsize matches that of the input stack if self.params['apix'] != self.templatestack['apix'] or self.params['boxsize'] != self.templatestack['boxsize']: self.scaleTemplates() starttime=time.time() print self.params print "... stack pixel size: "+str(self.params['apix']) print "... stack box size: "+str(self.params['boxsize']) apDisplay.printColor("Running IMAGIC .batch file: See multiReferenceAlignment.log file for details", "cyan") ### create IMAGIC batch file batchfile = self.createImagicBatchFileMRA() ### execute IMAGIC batch file aligntime0 = time.time() proc = subprocess.Popen("chmod 775 "+str(batchfile), shell=True) proc.wait() os.chdir(self.params['rundir']) apParam.runCmd(batchfile, "IMAGIC") apIMAGIC.checkLogFileForErrors(os.path.join(self.params['rundir'], "multiReferenceAlignment.log")) apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-aligntime0), "cyan") ### get particle parameters (shift, rotate, refnum, mirror, ccc) partparams = self.getParticleParams() ### average stack alignstack = os.path.join(self.params['rundir'], "alignstack.hed") apStack.averageStack(alignstack) ### normalize particles (otherwise found problems in viewing with stackviewer) emancmd = "proc2d "+alignstack+" "+alignstack+".norm.hed norm" while os.path.isfile(alignstack+".norm.img"): apFile.removeStack(alignstack+".norm.img") apParam.runCmd(emancmd, "EMAN") os.rename(alignstack+".norm.hed", alignstack) os.rename(alignstack+".norm.img", alignstack[:-4]+".img") ### normalize references emancmd = "proc2d "+ts+" "+ts+".norm.hed norm" while os.path.isfile(ts+".norm.img"): apFile.removeStack(ts+".norm.img") apParam.runCmd(emancmd, "EMAN") os.rename(ts+".norm.hed", ts) os.rename(ts+".norm.img", ts[:-4]+".img") ### remove copied stack while os.path.isfile(os.path.join(self.params['rundir'], "start.img")): apFile.removeStack(os.path.join(self.params['rundir'], "start.img")) ### insert run into database self.insertAlignmentRun(insert=True) self.insertParticlesIntoDatabase(partparams, insert=True)
def compute_stack_of_class_averages_and_reprojections(dir, selfile, refvolume, docfile, boxsize, resultspath, timestamp, iteration, reference_number=1, extract=False): ''' takes Xmipp single files, doc and sel files in routine, creates a stack of class averages in the results directory ''' workingdir = os.getcwd() os.chdir(dir) if dir.endswith("/"): dir = dir[:-1] head, tail = os.path.split(dir) ### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile f = open(selfile, "r") lines = f.readlines() newlines = [re.sub(str(tail) + "/", "", line) for line in lines] f.close() f = open(selfile[:-4] + "_new.sel", "w") f.writelines(newlines) f.close() if extract is True: extractcmd = "xmipp_header_extract -i %s.sel -o %s.doc" % ( selfile[:-4], docfile[:-4]) apParam.runCmd(extractcmd, "Xmipp") ### create a projection params file and project the volume along identical Euler angles f = open("paramfile.descr", "w") f.write("%s\n" % refvolume) f.write("tmpproj 1 xmp\n") f.write("%d %d\n" % (boxsize, boxsize)) f.write("%s rot tilt psi\n" % docfile) f.write("NULL\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.close() projectcmd = "xmipp_project -i paramfile.descr" apParam.runCmd(projectcmd, "Xmipp") ### get order of projections in docfile d = open(docfile, "r") lines = d.readlines()[1:] d.close() projfile_sequence = [] for i, l in enumerate(lines): if i % 2 == 0: filename = os.path.basename(l.split()[1]) projfile_sequence.append(filename) else: pass ### create stack of projections and class averages projections = glob.glob("tmpproj**xmp") projections.sort() if len(projections) != len(projfile_sequence): apDisplay.printWarning( "number of projections does not match number of classes") stackarray = [] stackname = os.path.join( resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (timestamp, iteration, reference_number)) for i in range(len(projections)): stackarray.append(spider.read(projections[i])) stackarray.append(spider.read(projfile_sequence[i])) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### remove unnecessary files for file in glob.glob("tmpproj*"): apFile.removeFile(file) os.chdir(workingdir) return
def start(self): t0 = time.time() self.checkAnalysisRun() # get stack parameters if self.params['alignid'] is not None: self.alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignid']) stackpixelsize = self.alignstackdata['pixelsize'] stack_box_size = self.alignstackdata['boxsize'] self.params['boxsize'] = stack_box_size / int(self.params['bin']) self.params['apix'] = stackpixelsize * int(self.params['bin']) orig_path = self.alignstackdata['path']['path'] orig_file = self.alignstackdata['imagicfile'] linkingfile = orig_path+"/"+orig_file linkingfile = linkingfile.replace(".hed", "") else: apDisplay.printError("stack not in the database") # link stack file to working directory if not os.path.isfile(linkingfile+".hed"): apDisplay.printError("stackfile does not exist: "+linkingfile+".img") else: if not os.path.isfile(os.path.join(str(self.params['rundir']), "start.img")): apDisplay.printMsg("copying aligned stack into working directory for operations with IMAGIC") # shutil.copyfile(linkingfile+".img", str(self.params['rundir'])+"/start.img") # shutil.copyfile(linkingfile+".hed", str(self.params['rundir'])+"/start.hed") lnkcmd1 = "ln -s "+linkingfile+".img "+os.path.join(self.params['rundir'], "start.img") lnkcmd2 = "ln -s "+linkingfile+".hed "+os.path.join(self.params['rundir'], "start.hed") proc = subprocess.Popen(lnkcmd1, shell=True) proc.wait() proc = subprocess.Popen(lnkcmd2, shell=True) proc.wait() ### header bug apImagicFile.setImagic4DHeader(os.path.join(self.params['rundir'], "start.hed")) else: apDisplay.printColor("aligned stack already exists in working directory", "green") ### NEED TO CONVERT FILTERING PARAMETERS TO IMAGIC FORMAT BETWEEN 0-1 if self.params['lpfilt'] is not None: self.params['lpfilt_imagic'] = 2 * float(self.params['apix']) / int(self.params['lpfilt']) else: self.params['lpfilt_imagic'] = False if float(self.params['lpfilt_imagic']) > 1: self.params['lpfilt_imagic'] = 1 # imagic cannot perform job when lowpass > 1 if self.params['hpfilt'] is not None: self.params['hpfilt_imagic'] = 2 * float(self.params['apix']) / int(self.params['hpfilt']) else: self.params['hpfilt_imagic'] = False print self.params print "... aligned stack pixel size: "+str(self.params['apix']) print "... aligned stack box size: "+str(self.params['boxsize']) apDisplay.printColor("Running IMAGIC .batch file: See imagicMultivariateStatisticalAnalysis.log for details", "cyan") ### create imagic batch file filename = self.createImagicBatchFile() ### execute batch file that was created aligntime = time.time() proc = subprocess.Popen('chmod 775 '+filename, shell=True) proc.wait() apParam.runCmd(filename, package="IMAGIC") logfile = open(os.path.join(self.params['rundir'], "imagicMultivariateStatisticalAnalysis.log")) apIMAGIC.checkLogFileForErrors(os.path.join(self.params['rundir'], "imagicMultivariateStatisticalAnalysis.log")) if not os.path.isfile(os.path.join(self.params['rundir'], "eigenimages.hed")): apDisplay.printError("IMAGIC did not run and did not create eigenimages") aligntime = time.time() - aligntime ### remove copied stack # while os.path.isfile(os.path.join(self.params['rundir'], "start.img")): # apFile.removeStack(os.path.join(self.params['rundir'], "start.img")) ### normalize eigenimages eigenimages = os.path.join(self.params['rundir'], "eigenimages.img") emancmd = "proc2d "+str(eigenimages)+" "+str(eigenimages)+" inplace" apParam.runCmd(emancmd, package="EMAN") ### upload alignment imagicstack = os.path.join(self.params['rundir'], "start.hed") inserttime = time.time() if self.params['commit'] is True: self.insertAnalysis(imagicstack, runtime=aligntime, insert=True) else: apDisplay.printWarning("not committing results to DB") inserttime = time.time() - inserttime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def start(self): ### simple is written in Fortran, which cannot take inputs of certain length, therefore one needs ### to change to the directory to minimize the filename length, in particular for the stack os.chdir(self.params['rundir']) ### stack needs to be centered if self.params['no_center'] is False: if os.path.isfile(os.path.join(self.params['rundir'], "ali.hed")): apFile.removeStack(os.path.join(self.params['rundir'], "ali.hed")) centstack = os.path.join(self.params['rundir'], "ali.hed") centcmd = "cenalignint %s > cenalignint.log" % (self.stack['file']) apParam.runCmd(centcmd, "EMAN") ### process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".spi") if os.path.isfile(self.params['localstack']): apFile.removeFile(self.params['localstack']) if self.params['no_center'] is False: proccmd = "proc2d "+centstack+" "+self.params['localstack']+" apix="+str(self.stack['apix']) else: proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], self.boxsize, self.boxsize) proccmd += " last="+str(self.params['numpart']-1) proccmd += " spiderswap" # if self.params['highpass'] is not None and self.params['highpass'] > 1: # proccmd += " hp="+str(self.params['highpass']) # if self.params['lowpass'] is not None and self.params['lowpass'] > 1: # proccmd += " lp="+str(self.params['lowpass']) apParam.runCmd(proccmd, "EMAN", verbose=True) # if self.params['numpart'] != int(spider.getSpiderHeader(self.params['localstack'])[-2]): # apDisplay.printError("Missing particles in stack") ### setup Simple command aligntime = time.time() simpleopts = ("" +" stk=%s" % os.path.basename(self.params['localstack']) +" box=%d" % self.boxsize +" nptcls=%d" % self.params['numpart'] +" smpd=%.3f" % self.apix +" ring2=%d" % self.params['ring2'] +" ncls=%d" % self.params['ncls'] +" minp=%d" % self.params['minp'] +" nvars=%d" % self.params['nvars'] +" nthr=%d" % self.params['nproc'] ) if self.params['no_kmeans'] is True: simpleopts += " kmeans=off" if self.params['nran'] is not None: simpleopts += "nran=%d" % self.params['nran'] ### SIMPLE 2D clustering apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green") simpleexe = apParam.getExecPath("cluster", die=True) simplecmd = "%s %s" % (simpleexe, simpleopts) self.writeSimpleLog(simplecmd) apParam.runCmd(simplecmd, package="SIMPLE", verbose=True, showcmd=True, logfile="cluster.std") self.params['runtime'] = time.time() - aligntime apDisplay.printMsg("Alignment & Classification time: "+apDisplay.timeString(self.params['runtime'])) ### SIMPLE spider to Fourier format clsavgs = "cavgstk.spi" if not os.path.isfile(os.path.join(self.params['rundir'], clsavgs)): apDisplay.printError("class averages were not created! try rerunning with centering, more particles, or less ppc") try: nptcls = spider.getSpiderHeader(clsavgs)[-2] except: nptcls = self.params['ncls'] apDisplay.printWarning("class average file may not have been created! Please check existence of file cavgstk.spi") projfile = "projs" projext = ".fim" simpleexe = apParam.getExecPath("spi_to_fim", die=True) simpleopts = ("" +" stk=%s" % clsavgs +" box=%d" % self.boxsize +" nptcls=%d" % nptcls +" smpd=%.3f" % self.apix +" outbdy=%s" % projfile +" msk=%d" % self.params['mask'] ) simplecmd = "%s %s" % (simpleexe, simpleopts) self.writeSimpleLog(simplecmd) apParam.runCmd(simplecmd, package="SIMPLE", verbose=True, showcmd=True, logfile="spi_to_fim.std") ### SIMPLE origami, ab initio 3D reconstruction refinetime = time.time() simpleexe = apParam.getExecPath("origami", die=True) simpleopts = ("" +" fstk=%s" % projfile+projext +" froms=%d" % self.params['froms'] +" tos=%d" % self.params['tos'] +" lp=%d" % self.params['lp'] +" hp=%d" % self.params['hp'] +" maxits=%d" % self.params['maxits'] +" msk=%d" % self.params['mask'] +" mw=%d" % self.params['mw'] +" frac=%.3f" % self.params['frac'] +" amsklp=%d" % self.params['amsklp'] +" edge=%d" % self.params['edge'] +" trs=%d" % self.params['trs'] +" nthr=%d" % self.params['nproc'] ) simplecmd = "%s %s" % (simpleexe, simpleopts) self.writeSimpleLog(simplecmd) apParam.runCmd(simplecmd, package="SIMPLE", verbose=True, showcmd=True, logfile="origami.std") refinetime = time.time() - refinetime apDisplay.printMsg("Origami reconstruction time: "+apDisplay.timeString(refinetime)) # ''' ### minor post-processing self.clearIntermediateFiles() apParam.dumpParameters(self.params, "simple-"+self.params['timestamp']+"-params.pickle") ### upload results self.runparams = apParam.readRunParameters("simple-"+self.params['timestamp']+"-params.pickle") ### create average of aligned and clustered stacks, convert to IMAGIC alignedStackSpi = "inplalgnstk.spi" alignedStack = "inplalgnstk.hed" if os.path.isfile(alignedStack): apFile.removeStack(alignedStack) emancmd = "proc2d %s %s flip" % (alignedStackSpi, alignedStack) apParam.runCmd(emancmd, "EMAN") clusterStackSpi = "cavgstk.spi" clusterStack = "cavgstk.hed" if os.path.isfile(clusterStack): apFile.removeStack(clusterStack) emancmd = "proc2d %s %s flip" % (clusterStackSpi, clusterStack) apParam.runCmd(emancmd, "EMAN") # apStack.averageStack(alignedStack) ### parse alignment and classification results if self.params['no_center'] is False: self.alignD = self.getAlignParameters(centparams="cenalignint.log") else: self.alignD = self.getAlignParameters() if self.params['no_kmeans'] is False: self.classD = self.getClassification("kmeans.spi", clusterStack) else: self.classD = self.getClassification("hcl.spi", clusterStack) ### upload to database self.insertSIMPLEAlignParamsIntoDatabase() self.insertAlignStackRunIntoDatabase(alignedStack, clusterStack) self.calcResolution(alignedStack) self.insertAlignParticlesIntoDatabase() self.insertClusterRunIntoDatabase() self.insertClusterStackIntoDatabase(clusterStack, len(self.classD)) self.insertSIMPLEOrigamiParamsIntoDatabase()
def calculateFSCforIteration(self, iteration, reference_number): ''' requires manipulation of data using Xmipp programs, benefits from MPI parallelization ''' os.chdir(self.ml3dpath) ### split selfile selfile = "ml3d_it%.6d_class_vol%.6d.sel" % (iteration,reference_number) xmipp_split_selfile_cmd = "xmipp_selfile_split -i %s" % selfile apParam.runCmd(xmipp_split_selfile_cmd, "Xmipp") ### make even and odd docfiles from split selfile xmipp_mk_docfile_cmd1 = "xmipp_docfile_select_subset -i ml3d_it%.6d.doc -sel ml3d_it%.6d_class_vol%.6d_1.sel -o tmpdocfile_1.doc" \ % (iteration,iteration,reference_number) apParam.runCmd(xmipp_mk_docfile_cmd1, "Xmipp") xmipp_mk_docfile_cmd2 = "xmipp_docfile_select_subset -i ml3d_it%.6d.doc -sel ml3d_it%.6d_class_vol%.6d_2.sel -o tmpdocfile_2.doc" \ % (iteration,iteration,reference_number) apParam.runCmd(xmipp_mk_docfile_cmd2, "Xmipp") ### create angular class averages from split docfiles and selfiles if self.params['nproc'] > 1: xmipp_average_cmd1 = "mpirun -np %d xmipp_mpi_angular_class_average -i tmpdocfile_1.doc -lib ml3d_lib_new.doc -o tmpclass_1" \ % (self.params['nproc']) xmipp_average_cmd2 = "mpirun -np %d xmipp_mpi_angular_class_average -i tmpdocfile_2.doc -lib ml3d_lib_new.doc -o tmpclass_2" \ % (self.params['nproc']) else: xmipp_average_cmd1 = "xmipp_angular_class_average -i tmpdocfile_1.doc -lib ml3d_lib_new.doc -o tmpclass_1" xmipp_average_cmd2 = "xmipp_angular_class_average -i tmpdocfile_2.doc -lib ml3d_lib_new.doc -o tmpclass_2" apParam.runCmd(xmipp_average_cmd1, "Xmipp") apParam.runCmd(xmipp_average_cmd2, "Xmipp") ### reconstruct even volume if self.params['nproc'] > 1: xmipp_reconstruct_cmd = "mpirun -np %d xmipp_mpi_reconstruct_fourier -i tmpclass_1_classes.sel -o %s -weight" \ % (self.params['nproc'], "even.vol") else: xmipp_reconstruct_cmd = "xmipp_reconstruct_fourier -i tmpclass_1_classes.sel -o %s -weight" \ % ("even.vol") apParam.runCmd(xmipp_reconstruct_cmd, "Xmipp") ### reconstruct odd volume if self.params['nproc'] > 1: xmipp_reconstruct_cmd = "mpirun -np %d xmipp_mpi_reconstruct_fourier -i tmpclass_2_classes.sel -o %s -weight" \ % (self.params['nproc'], "odd.vol") else: xmipp_reconstruct_cmd = "xmipp_reconstruct_fourier -i tmpclass_2_classes.sel -o %s -weight" \ % ("odd.vol") apParam.runCmd(xmipp_reconstruct_cmd, "Xmipp") ### calculate FSC apDisplay.printMsg("calculating FSC resolution for iteration %d, volume %d using split selfile" % (iteration,reference_number)) xmipp_resolution_cmd = "xmipp_resolution_fsc -ref even.vol -i odd.vol -sam %.3f" % (self.runparams['apix']) apParam.runCmd(xmipp_resolution_cmd, "Xmipp") ### rename FSC file, get resolution value, and remove unwanted files oldfscfile = os.path.join(self.ml3dpath, "odd.vol.frc") newfscfile = os.path.join(self.resultspath, "recon_%s_it%.3d_vol%.3d.fsc" % (self.params['timestamp'],iteration,reference_number)) if os.path.exists(oldfscfile): shutil.move(oldfscfile, newfscfile) apFile.removeFile("ml3d_it%.6d_class_vol%.6d_1.sel" % (iteration,reference_number)) apFile.removeFile("ml3d_it%.6d_class_vol%.6d_2.sel" % (iteration,reference_number)) list = [] list += [f for f in glob.glob("odd*")] list += [f for f in glob.glob("even*")] list += [f for f in glob.glob("tmp*")] for file in list: apFile.removeFile(file) os.chdir(self.params['rundir']) return
def createParticleDataFile(self, iteration, reference_number, total_num_2d_classes): ''' puts all relevant particle information into a single text file that can be read by the uploader ''' os.chdir(self.ml3dpath) ### create docfile for the iteration and reference number docfile = os.path.join(self.ml3dpath, "ml3d_it%.6d.doc" % (iteration)) selfile = "ml3d_it%.6d_class_vol%.6d.sel" % (iteration,reference_number) makedocfilecmd = "xmipp_docfile_select_subset -i %s -sel %s -o ml3d_it%.6d_class_vol%.6d.doc" \ % (docfile, selfile, iteration,reference_number) apParam.runCmd(makedocfilecmd, "Xmipp") classes_per_volume = total_num_2d_classes / len(self.modeldata) ### read output from ml3d ml3ddocfile = os.path.join(self.ml3dpath, "ml3d_it%.6d_class_vol%.6d.doc" % (iteration, reference_number)) ml3df = open(ml3ddocfile, "r") ml3dlines = ml3df.readlines()[1:] ml3dsplitlines = [l.strip().split() for l in ml3dlines] ml3df.close() ### write data in appion format to input file for uploading to the database particledataf = open(os.path.join(self.resultspath, "particle_data_%s_it%.3d_vol%.3d.txt" % (self.params['timestamp'], iteration, reference_number)), "w") # particledataf.write("### column info: ") particledataf.write("#%8s" % "partnum") particledataf.write("%10s" % "phi") particledataf.write("%10s" % "theta") particledataf.write("%10s" % "omega") particledataf.write("%10s" % "shiftx") particledataf.write("%10s" % "shifty") # particledataf.write("(7) mirror") particledataf.write("%8s" % "3D_ref#") particledataf.write("%8s" % "2D_cls#") particledataf.write("%10s" % "qfact") particledataf.write("%8s" % "keptp") particledataf.write("%8s\n" % "p_keptp") for i in range(len(ml3dsplitlines)/2): if int(float(ml3dsplitlines[i*2+1][7])) % classes_per_volume == 0: n = classes_per_volume else: n = int(float(ml3dsplitlines[i*2+1][7])) % classes_per_volume phi = float(ml3dsplitlines[i*2+1][2])) theta = float(ml3dsplitlines[i*2+1][3])) psi = float(ml3dsplitlines[i*2+1][4])) mirror = bool(float(ml3dsplitlines[i*2+1][8]))) if mirror is True: phi, theta, psi = apXmipp.calculate_equivalent_Eulers_without_flip(phi, theta, psi) particledataf.write("%9d" % (int(ml3dsplitlines[i*2][1][-10:-4])+1)) ### NOTE: IT IS IMPORTANT TO START WITH 1, OTHERWISE STACKMAPPING IS WRONG!!! particledataf.write("%10.4f" % phi) particledataf.write("%10.4f" % theta) particledataf.write("%10.4f" % psi) particledataf.write("%10.4f" % float(ml3dsplitlines[i*2+1][5])) particledataf.write("%10.4f" % float(ml3dsplitlines[i*2+1][6])) # particledataf.write("%8d" % int(float(ml3dsplitlines[i*2+1][8]))) # deprecated: mirror is flipped already. Set to False particledataf.write("%8d" % int(reference_number)) particledataf.write("%8d" % n) particledataf.write("%10.4f" % 0) particledataf.write("%8d" % 1) particledataf.write("%8d\n" % 1) particledataf.close() os.chdir(self.params['rundir']) return
def start(self): self.ace2correct = self.getACE2Path() ### determine amount of memory needed for entire stack memorylimit = 0.3 bytelimit = memorylimit*(1024**3) writeiters = 1 partbytes = 4*self.params['box']*self.params['box'] if partbytes*self.params['projcount'] > bytelimit: writeiters = int(math.ceil(float(partbytes)*self.params['projcount'] / bytelimit)) partsperiter = int(float(self.params['projcount']) / writeiters) ### number of particles read each time ### some defaults, and workarounds for now self.params['projpergraph'] = 100 self.params['filesperdir'] = partsperiter if self.params['filesperdir'] > 2048: self.params['filesperdir'] = 2048 ### first create projections if self.params['preforient'] is True: filename = self.createProjections(pad=self.params['pad'], invert=self.params['invert']) else: filename = self.createProjectionsEmanProp(pad=self.params['pad'], invert=self.params['invert']) ### shift & rotate randomly if self.params['rotang']!=0 or self.params['shiftrad']!=0: shiftstackname = self.shift_images(filename) else: shiftstackname = filename ### read MRC stats to figure out noise level addition mean1, stdev1 = self.readFileStats(shiftstackname) ### determine noise multiplication factor to ensure that appropriate amount of noise gets added to particles inside circular mask multfactor = 1.0/((float(self.params['radius'])/self.params['box'])*(float(self.params['radius'])/self.params['box'])*math.pi) ### calculate noiselevel additions and add noise to an initial ratio of 1.8, simulating beam and structural damage ### NOTE: THERE ARE DIFFERENT DEFINITIONS OF SNR, see below noiselevel1 = math.sqrt((float(stdev1)*float(stdev1)) / float(self.params['snr1'])) # noiselevel1 = float(stdev1) / float(self.params['snr1']) noiselevel1 = noiselevel1 * multfactor noisystack = self.addNoise(shiftstackname, noiselevel1, SNR=self.params['snr1']) ### get list of defocus values self.getListOfDefoci(self.params['projcount']) ### apply envelope and ctf to each .mrc file, then correct based on how well ace2 works on raw micrographs ctfstack, ctfpartlist = self.applyEnvelopeAndCTF(noisystack) #recoverlists = self.recoverLists() ### read IMAGIC stats to figure out noise level addition mean2, stdev2 = self.readFileStats(ctfstack) ### cascading of noise processes according to Frank and Al-Ali (1975) & Baxter (2009) # snr2 = 1 / ((1+1/float(self.params['snrtot'])) / (1/float(self.params['snr1']) + 1) - 1) snr2 = (1+1/self.params['snr1'])/(1/self.params['snrtot']-1/self.params['snr1']) ### NOTE: THERE ARE DIFFERENT DEFINITIONS OF SNR, see below noiselevel2 = math.sqrt((float(stdev2)*float(stdev2)) / float(snr2)) # noiselevel2 = float(stdev2) / float(snr2) noiselevel2 = noiselevel2 * multfactor ### add a last layer of noise noisystack2 = self.addNoise(ctfstack, noiselevel2, SNR=self.params['snrtot']) ### low-pass / high-pass filter resulting stack, if specified if self.params['hpfilt'] is not None or self.params['lpfilt'] is not None or self.params['norm'] is True: filtstack = noisystack2[:-4] if self.params['norm'] is True: filtstack = filtstack+"_norm.hed" else: filtstack = filtstack+"_filt.hed" apFile.removeStack(filtstack) emancmd = "proc2d "+noisystack2+" "+filtstack+" apix="+str(self.params['apix'])+" " if self.params['hpfilt'] is not None: emancmd = emancmd+"hp="+str(self.params['hpfilt'])+" " if self.params['lpfilt'] is not None: emancmd = emancmd+"lp="+str(self.params['lpfilt'])+" " if self.params['norm'] is True: emancmd = emancmd+"norm="+str(self.params['norm'])+" " apParam.runCmd(emancmd, "EMAN") self.params['finalstack'] = os.path.basename(filtstack) finalstack = filtstack else: self.params['finalstack'] = os.path.basename(noisystack2) finalstack = noisystack2 ### post-processing: create average file for viewing on webpages apStack.averageStack(finalstack) ### upload if commit is checked self.uploadData(ctfpartlist) ### post-processing: Create Stack Mean Plot if self.params['commit'] is True: stackid = apStack.getStackIdFromPath(finalstack) if stackid is not None: apStackMeanPlot.makeStackMeanPlot(stackid, gridpoints=8)
def createProjections(self, pad=False, invert=False): timestamp = apParam.makeTimestamp() eulerlist = self.setEulers() eulerfile = os.path.join(self.params['rundir'], "eulers.lst") f = open(eulerfile, "w") projcount = numpy.zeros((len(eulerlist)), dtype=numpy.uint16) angsum = numpy.zeros((len(eulerlist),3), dtype=numpy.float32) t0 = time.time() for i in range(self.params['projcount']): projnum = int(random.random()*len(eulerlist)) alt = random.gauss(eulerlist[projnum][0], self.params['projstdev']) az = random.gauss(eulerlist[projnum][1], self.params['projstdev']) #phi = random.random()*360.0-180.0 #phi = random.random()*360.0 if self.params['rotang'] != 0: phi = random.uniform(-1*self.params['rotang'], self.params['rotang']) else: phi = 0.0 f.write("%.8f\t%.8f\t%.8f\n"%(alt,az,phi)) ### stats projcount[projnum] += 1 angsum[projnum,0] += alt angsum[projnum,1] += az angsum[projnum,2] += phi apDisplay.printMsg("Finished random in %s, %.3f ns per iteration" %(apDisplay.timeString(time.time()-t0), 1.0e6 * (time.time()-t0)/float(self.params['projcount']))) f.close() print "projection count", projcount for i in range(len(eulerlist)): angavg = angsum[i,:]/projcount[i] print "angle average %d: %03.3f, %03.3f, %03.3f"%(i, angavg[0], angavg[1], angavg[2]) ### first get rid of projection artifacts from insufficient padding if self.params['threedfile'] is not None: origfile = self.params['threedfile'] elif self.params['modelid'] is not None: self.modelparams = appiondata.ApInitialModelData.direct_query(self.params['modelid']) origfile = os.path.join(self.modelparams['path']['path'], self.modelparams['name']) if self.params['apix'] is None: self.params['apix'] = self.modelparams['pixelsize'] if self.params['box'] is None: self.params['box'] = self.modelparams['boxsize'] clipped = os.path.join(self.params['rundir'], "clipped.mrc") newsize = self.params['box'] * 1.5 emancmd = "proc3d "+origfile+" "+clipped+" clip="+str(int(newsize))+","+str(int(newsize))+","+str(int(newsize))+" edgenorm" apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True) ### project resized file filename = os.path.join(self.params['rundir'], 'proj.img') apFile.removeStack(filename) emancmd = "project3d "+clipped+" out="+filename+" list="+eulerfile t0 = time.time() apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True) apDisplay.printMsg("Finished project3d in %s, %.3f ms per iteration" %(apDisplay.timeString(time.time()-t0), 1.0e3 * (time.time()-t0)/float(self.params['projcount']))) ### post-process stack self.post_process_stack(filename, pad, invert) return filename
def start(self): ### simple is written in Fortran, which cannot take inputs of certain length, therefore one needs ### to change to the directory to minimize the filename length, in particular for the stack os.chdir(self.params['rundir']) ### stack needs to be centered if self.params['no_center'] is False: if os.path.isfile(os.path.join(self.params['rundir'], "ali.hed")): apFile.removeStack( os.path.join(self.params['rundir'], "ali.hed")) centstack = os.path.join(self.params['rundir'], "ali.hed") centcmd = "cenalignint %s > cenalignint.log" % (self.stack['file']) apParam.runCmd(centcmd, "EMAN") ### process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join( self.params['rundir'], self.params['timestamp'] + ".spi") if os.path.isfile(self.params['localstack']): apFile.removeFile(self.params['localstack']) if self.params['no_center'] is False: proccmd = "proc2d " + centstack + " " + self.params[ 'localstack'] + " apix=" + str(self.stack['apix']) else: proccmd = "proc2d " + self.stack['file'] + " " + self.params[ 'localstack'] + " apix=" + str(self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], self.boxsize, self.boxsize) proccmd += " last=" + str(self.params['numpart'] - 1) proccmd += " spiderswap" # if self.params['highpass'] is not None and self.params['highpass'] > 1: # proccmd += " hp="+str(self.params['highpass']) # if self.params['lowpass'] is not None and self.params['lowpass'] > 1: # proccmd += " lp="+str(self.params['lowpass']) apParam.runCmd(proccmd, "EMAN", verbose=True) # if self.params['numpart'] != int(spider.getSpiderHeader(self.params['localstack'])[-2]): # apDisplay.printError("Missing particles in stack") ### setup Simple command aligntime = time.time() simpleopts = ( "" + " stk=%s" % os.path.basename(self.params['localstack']) + " box=%d" % self.boxsize + " nptcls=%d" % self.params['numpart'] + " smpd=%.3f" % self.apix + " ring2=%d" % self.params['ring2'] + " ncls=%d" % self.params['ncls'] + " minp=%d" % self.params['minp'] + " nvars=%d" % self.params['nvars'] + " nthr=%d" % self.params['nproc']) if self.params['no_kmeans'] is True: simpleopts += " kmeans=off" if self.params['nran'] is not None: simpleopts += "nran=%d" % self.params['nran'] ### SIMPLE 2D clustering apDisplay.printColor( "Using " + str(self.params['nproc']) + " processors!", "green") simpleexe = apParam.getExecPath("cluster", die=True) simplecmd = "%s %s" % (simpleexe, simpleopts) self.writeSimpleLog(simplecmd) apParam.runCmd(simplecmd, package="SIMPLE", verbose=True, showcmd=True, logfile="cluster.std") self.params['runtime'] = time.time() - aligntime apDisplay.printMsg("Alignment & Classification time: " + apDisplay.timeString(self.params['runtime'])) ### SIMPLE spider to Fourier format clsavgs = "cavgstk.spi" if not os.path.isfile(os.path.join(self.params['rundir'], clsavgs)): apDisplay.printError( "class averages were not created! try rerunning with centering, more particles, or less ppc" ) try: nptcls = spider.getSpiderHeader(clsavgs)[-2] except: nptcls = self.params['ncls'] apDisplay.printWarning( "class average file may not have been created! Please check existence of file cavgstk.spi" ) projfile = "projs" projext = ".fim" simpleexe = apParam.getExecPath("spi_to_fim", die=True) simpleopts = ("" + " stk=%s" % clsavgs + " box=%d" % self.boxsize + " nptcls=%d" % nptcls + " smpd=%.3f" % self.apix + " outbdy=%s" % projfile + " msk=%d" % self.params['mask']) simplecmd = "%s %s" % (simpleexe, simpleopts) self.writeSimpleLog(simplecmd) apParam.runCmd(simplecmd, package="SIMPLE", verbose=True, showcmd=True, logfile="spi_to_fim.std") ### SIMPLE origami, ab initio 3D reconstruction refinetime = time.time() simpleexe = apParam.getExecPath("origami", die=True) simpleopts = ( "" + " fstk=%s" % projfile + projext + " froms=%d" % self.params['froms'] + " tos=%d" % self.params['tos'] + " lp=%d" % self.params['lp'] + " hp=%d" % self.params['hp'] + " maxits=%d" % self.params['maxits'] + " msk=%d" % self.params['mask'] + " mw=%d" % self.params['mw'] + " frac=%.3f" % self.params['frac'] + " amsklp=%d" % self.params['amsklp'] + " edge=%d" % self.params['edge'] + " trs=%d" % self.params['trs'] + " nthr=%d" % self.params['nproc']) simplecmd = "%s %s" % (simpleexe, simpleopts) self.writeSimpleLog(simplecmd) apParam.runCmd(simplecmd, package="SIMPLE", verbose=True, showcmd=True, logfile="origami.std") refinetime = time.time() - refinetime apDisplay.printMsg("Origami reconstruction time: " + apDisplay.timeString(refinetime)) # ''' ### minor post-processing self.clearIntermediateFiles() apParam.dumpParameters( self.params, "simple-" + self.params['timestamp'] + "-params.pickle") ### upload results self.runparams = apParam.readRunParameters("simple-" + self.params['timestamp'] + "-params.pickle") ### create average of aligned and clustered stacks, convert to IMAGIC alignedStackSpi = "inplalgnstk.spi" alignedStack = "inplalgnstk.hed" if os.path.isfile(alignedStack): apFile.removeStack(alignedStack) emancmd = "proc2d %s %s flip" % (alignedStackSpi, alignedStack) apParam.runCmd(emancmd, "EMAN") clusterStackSpi = "cavgstk.spi" clusterStack = "cavgstk.hed" if os.path.isfile(clusterStack): apFile.removeStack(clusterStack) emancmd = "proc2d %s %s flip" % (clusterStackSpi, clusterStack) apParam.runCmd(emancmd, "EMAN") # apStack.averageStack(alignedStack) ### parse alignment and classification results if self.params['no_center'] is False: self.alignD = self.getAlignParameters(centparams="cenalignint.log") else: self.alignD = self.getAlignParameters() if self.params['no_kmeans'] is False: self.classD = self.getClassification("kmeans.spi", clusterStack) else: self.classD = self.getClassification("hcl.spi", clusterStack) ### upload to database self.insertSIMPLEAlignParamsIntoDatabase() self.insertAlignStackRunIntoDatabase(alignedStack, clusterStack) self.calcResolution(alignedStack) self.insertAlignParticlesIntoDatabase() self.insertClusterRunIntoDatabase() self.insertClusterStackIntoDatabase(clusterStack, len(self.classD)) self.insertSIMPLEOrigamiParamsIntoDatabase()