def setupParticles(self): self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed") self.stack = {} self.stack["data"] = apStack.getOnlyStackData(self.params["stackid"]) self.stack["apix"] = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) self.stack["part"] = apStack.getOneParticleFromStackId(self.params["stackid"]) self.stack["boxsize"] = apStack.getStackBoxsize(self.params["stackid"]) self.stack["file"] = os.path.join(self.stack["data"]["path"]["path"], self.stack["data"]["name"]) boxsize = int(math.floor(self.stack["boxsize"] / float(self.params["bin"] * 2))) * 2 apix = self.stack["apix"] * self.params["bin"] proccmd = "proc2d " + self.stack["file"] + " " + self.params["localstack"] + " apix=" + str(self.stack["apix"]) if self.params["bin"] > 1: clipsize = boxsize * self.params["bin"] proccmd += " shrink=%d clip=%d,%d " % (self.params["bin"], clipsize, clipsize) if self.params["highpass"] > 1: proccmd += " hp=" + str(self.params["highpass"]) if self.params["lowpass"] > 1: proccmd += " lp=" + str(self.params["lowpass"]) proccmd += " last=" + str(self.params["numpart"] - 1) apFile.removeStack(self.params["localstack"], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"]) return (boxsize, apix)
def stackToSpiderStack(stack,spiderstack,apix,boxsize,lp=0,hp=0,bin=1,numpart=0): """ convert an input stack (i.e. imagic) and write it out to a spider-formatted stack """ emancmd = "proc2d " if not os.path.isfile(stack): apDisplay.printError("stackfile does not exist: "+stack) emancmd += stack+" " emancmd += "apix="+str(apix)+" " if lp > 0: emancmd += "lp="+str(lp)+" " if hp > 0: emancmd += "hp="+str(hp)+" " if bin > 1: clipboxsize = boxsize*bin emancmd += "shrink="+str(bin)+" " emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" " if numpart > 0: emancmd += "last="+str(numpart-1)+" " emancmd += "spiderswap edgenorm" starttime = time.time() apDisplay.printColor("Running spider stack conversion this can take a while", "cyan") from appionlib import apEMAN apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def start(self): ### copy stack to rundir newstack = os.path.join(self.params['rundir'], "start.hed") if os.path.isfile(newstack): apDisplay.printError("Stack already exists") emancmd = "proc2d %s %s" % (self.params['stackfile'], newstack) if self.params['normalize'] is True: emancmd += " edgenorm" apEMAN.executeEmanCmd(emancmd) ### set final parameters boxsize = apFile.getBoxSize(newstack) apDisplay.printMsg("Boxsize: %i" % boxsize[0]) if not boxsize or boxsize <= 0: apDisplay.printError("Could not determine stack size") else: self.boxsize = boxsize[0] self.numpart = apFile.numImagesInStack(newstack) apDisplay.printMsg("Num part: %i" % self.numpart) if not self.numpart or self.numpart <= 0: apDisplay.printError("Could not determine number of particles") if self.numpart <= 1: apDisplay.printError("Not enough particles to upload") apStack.averageStack(newstack) #self.params['commit'] = False self.createStackData()
def prepareStack(self): ### setup box and mask sizes self.alignstackdata = self.getAlignedStack() maskpixrad = self.params['maskrad'] / self.alignstackdata[ 'pixelsize'] / self.params['bin'] boxpixdiam = int(math.ceil(maskpixrad) + 1) * 2 if boxpixdiam * self.params['bin'] > self.alignstackdata['boxsize']: boxpixdiam = math.floor(self.alignstackdata['boxsize'] / self.params['bin']) clippixdiam = boxpixdiam * self.params['bin'] apDisplay.printMsg("Pixel mask radius=" + str(maskpixrad)) ### convert aligned stack to local dir oldalignedstack = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['imagicfile']) alignedstackname = "alignedstack.hed" alignedstack = os.path.join(self.params['rundir'], alignedstackname) apFile.removeFile(alignedstack) emancmd = ("proc2d %s %s shrink=%d clip=%d,%d" % (oldalignedstack, alignedstack, self.params['bin'], clippixdiam, clippixdiam)) if self.params['numpart'] is not None and self.params[ 'numpart'] < self.numpart: emancmd += " last=%d" % (self.params['numpart'] - 1) self.numpart = self.params['numpart'] apEMAN.executeEmanCmd(emancmd, verbose=True) return alignedstack
def createTemplateStack(self): """ takes the spider file and creates an average template of all particles """ templatestack = os.path.join(self.params['rundir'], "templatestack00.spi") apFile.removeFile(templatestack, warn=True) ### hack to use standard filtering library templateparams = {} templateparams['apix'] = self.stack['apix'] templateparams['rundir'] = os.path.join(self.params['rundir'], "templates") templateparams['templateIds'] = self.templatelist templateparams['bin'] = self.params['bin'] templateparams['lowpass'] = self.params['lowpass'] templateparams['median'] = None templateparams['pixlimit'] = None print templateparams apParam.createDirectory(os.path.join(self.params['rundir'], "templates")) filelist = apTemplate.getTemplates(templateparams) for mrcfile in filelist: emancmd = ("proc2d templates/"+mrcfile+" "+templatestack +" clip="+str(self.boxsize)+","+str(self.boxsize) +" spider ") if self.params['inverttemplates'] is True: emancmd += " invert " apEMAN.executeEmanCmd(emancmd, showcmd=False) return templatestack
def boxMaskStack(bmstackf, partdatas, box, xmask, ymask, falloff, imask=None, norotate=False): from appionlib.apSpider import operations from appionlib import apEMAN import os # create blank image for mask using SPIDER maskfile = "boxmask.spi" operations.createBoxMask(maskfile,box,xmask,ymask,falloff,imask) # convert mask to MRC apEMAN.executeEmanCmd("proc2d boxmask.spi boxmask.mrc",verbose=False,showcmd=False) os.remove("boxmask.spi") maskarray = mrc.read("boxmask.mrc") # box particles maskedparts = [] for i in range(len(partdatas)): if norotate is True: rotatemask = maskarray else: angle = (-partdatas[i]['angle'])-90 rotatemask = ndimage.rotate(maskarray, angle=angle, reshape=False, order=1) maskedparts.append(rotatemask) # write to stack apImagicFile.writeImagic(maskedparts, bmstackf) os.remove("boxmask.mrc") return bmstackf
def phaseFlipWholeImage(self, inimgpath, imgdata): outimgpath = os.path.join(self.params['rundir'], self.shortname+"-ctfcorrect.dwn.mrc") ### High tension on CM is given in kv instead of v so do not divide by 1000 in that case if imgdata['scope']['tem']['name'] == "CM": voltage = imgdata['scope']['high tension'] else: voltage = (imgdata['scope']['high tension'])/1000 apix = apDatabase.getPixelSize(imgdata) defocus, ampconst = self.getDefocusAmpConstForImage(imgdata, True) defocus *= 1.0e6 self.checkDefocus(defocus, self.shortname) ### get all CTF parameters, we also need to get the CS value from the database ctfdata = self.getBestCtfValue(imgdata, False) #ampconst = ctfdata['amplitude_contrast'] ### we could use this too # find cs cs = self.getCS(ctfdata) parmstr = ("parm=%f,200,1,%.3f,0,17.4,9,1.53,%i,%.1f,%f" %(defocus, ampconst, voltage, cs, apix)) emancmd = ("applyctf %s %s %s setparm flipphase" % (inimgpath, outimgpath, parmstr)) apDisplay.printMsg("phaseflipping entire micrograph with defocus "+str(round(defocus,3))+" microns") apEMAN.executeEmanCmd(emancmd, showcmd = True) return outimgpath
def processFrealignVolume(self, iternum, symname): volumeImagicFile = os.path.join(self.params['rundir'], "threed.%03da.hed" % (iternum)) if not os.path.isfile(volumeImagicFile): apDisplay.printError("Failed to find volume: %s" % (volumeImagicFile)) volumeMrcFile = os.path.join(self.params['rundir'], "threed.%03da.mrc" % (iternum)) if not os.path.isfile(volumeMrcFile): emancmd = "proc3d %s %s apix=%.3f origin=0,0,0 norm=0,1" % ( volumeImagicFile, volumeMrcFile, self.apix) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) if not os.path.isfile(volumeMrcFile): apDisplay.printError("Failed to convert volume") apChimera.filterAndChimera(volumeMrcFile, res=10, apix=self.apix, chimtype='snapshot', contour=1.0, zoom=self.params['zoom'], sym=symname, silhouette=True, mass=self.params['mass']) return volumeMrcFile
def convertSpiderStack(self, spiderstack): """ takes the stack file and creates a spider file ready for processing """ emancmd = "proc2d " if not os.path.isfile(spiderstack): apDisplay.printError("stackfile does not exist: " + spiderstack) emancmd += spiderstack + " " imagicstack = os.path.join(self.params['rundir'], "alignstack.hed") apFile.removeFile(imagicstack, warn=True) emancmd += imagicstack + " " starttime = time.time() apDisplay.printColor( "Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor( "finished eman in " + apDisplay.timeString(time.time() - starttime), "cyan") numpart = apFile.numImagesInStack(imagicstack) if numpart != self.params['numpart']: apDisplay.printError( "The number of particles returned is different from the number requested" ) return imagicstack
def setupVolumes(self, boxsize, apix): voldocfile = "volumelist"+self.timestamp+".doc" f = open(voldocfile, "w") if self.params['modelstr'] is not None: for i, modelid in enumerate(self.params['modelids']): ### Scale volume mrcvolfile = os.path.join(self.params['rundir'], "volume%s_%02d_%05d.mrc"%(self.timestamp, i+1, modelid)) apModel.rescaleModel(modelid, mrcvolfile, boxsize, apix) ### Convert volume to spider spivolfile = os.path.join(self.params['rundir'], "volume%s_%02d_%05d.spi"%(self.timestamp, i+1, modelid)) emancmd = "proc3d %s %s spidersingle"%(mrcvolfile, spivolfile) apEMAN.executeEmanCmd(emancmd, verbose=False) ### Normalize volume normalvolfile = self.normalizeVolume(spivolfile) ### Write to selection file f.write(normalvolfile+" 1\n") else: for i in range(self.params['nvol']): ### Create Gaussian sphere spivolfile = os.path.join(self.params['rundir'], "volume%s_%02d.spi"%(self.timestamp, i+1)) self.createGaussianSphere(spivolfile, boxsize) ### Normalize volume normalvolfile = self.normalizeVolume(spivolfile) ### Write to selection file f.write(normalvolfile+" 1\n") f.close() return voldocfile
def genProj(self, euler, density, tilt=None): """ Rotate model and then project at two tilt angles This could go faster if we could add Euler angles... 90,-90 puts cameras where the would be in the microscope """ (alt, az, phi) = euler rotcmd = ( "proc3d "+density+" rotated.mrc rot=" +str(alt)+","+str(az)+","+str(phi) ) apEMAN.executeEmanCmd(rotcmd, verbose=False, showcmd=False) if abs(tilt - self.tiltangle) < 5.0: bigtilt = self.tiltangle - self.notangle euler = ("%.3f,%.3f,%.3f"%( bigtilt + random.gauss(0, self.params['slopcam']), 90+random.gauss(0, self.params['slopcam']), -90+random.gauss(0, self.params['slopcam']))) cmd = "project3d rotated.mrc out="+self.params['tiltstack']+" euler="+euler elif abs(tilt - self.notangle) < 5.0: #euler = "%.3f,90,-90"%(0.0) euler = ( "%.3f,%.3f,%.3f"%( random.gauss(0, self.params['slopcam']), 90+random.gauss(0, self.params['slopcam']), -90+random.gauss(0, self.params['slopcam']))) cmd = "project3d rotated.mrc out="+self.params['notstack']+" euler="+euler else: apDisplay.printError("unknown tilt angle: %.3f"%(tilt)) apEMAN.executeEmanCmd(cmd, verbose=False, showcmd=False)
def genProj(self, euler, density, tilt=None): """ Rotate model and then project at two tilt angles This could go faster if we could add Euler angles... 90,-90 puts cameras where the would be in the microscope """ (alt, az, phi) = euler rotcmd = ("proc3d " + density + " rotated.mrc rot=" + str(alt) + "," + str(az) + "," + str(phi)) apEMAN.executeEmanCmd(rotcmd, verbose=False, showcmd=False) if abs(tilt - self.tiltangle) < 5.0: bigtilt = self.tiltangle - self.notangle euler = ("%.3f,%.3f,%.3f" % (bigtilt + random.gauss(0, self.params['slopcam']), 90 + random.gauss(0, self.params['slopcam']), -90 + random.gauss(0, self.params['slopcam']))) cmd = "project3d rotated.mrc out=" + self.params[ 'tiltstack'] + " euler=" + euler elif abs(tilt - self.notangle) < 5.0: #euler = "%.3f,90,-90"%(0.0) euler = ("%.3f,%.3f,%.3f" % (random.gauss(0, self.params['slopcam']), 90 + random.gauss(0, self.params['slopcam']), -90 + random.gauss(0, self.params['slopcam']))) cmd = "project3d rotated.mrc out=" + self.params[ 'notstack'] + " euler=" + euler else: apDisplay.printError("unknown tilt angle: %.3f" % (tilt)) apEMAN.executeEmanCmd(cmd, verbose=False, showcmd=False)
def start(self): ### load parameters runparams = self.readRunParameters() ### align references self.alignReferences(runparams) ### create an aligned stack self.createAlignedReferenceStack() ### read particles self.lastiter = self.findLastIterNumber() if self.params['sort'] is True: self.sortFolder() reflist = self.readRefDocFile() partlist = self.readPartDocFile(reflist) self.writePartDocFile(partlist) ### create aligned stacks alignimagicfile = self.createAlignedStacks(partlist, runparams['localstack']) apStack.averageStack(alignimagicfile) ### calculate resolution for each reference apix = apStack.getStackPixelSizeFromStackId(runparams['stackid'])*runparams['bin'] self.calcResolution(partlist, alignimagicfile, apix) ### insert into databse self.insertRunIntoDatabase(alignimagicfile, runparams) self.insertParticlesIntoDatabase(runparams['stackid'], partlist) apFile.removeStack(runparams['localstack'], warn=False) rmcmd = "/bin/rm -fr partfiles/*" apEMAN.executeEmanCmd(rmcmd, verbose=False, showcmd=False) apFile.removeFilePattern("partfiles/*")
def createSpiderFile(self): """ takes the stack file and creates a spider file ready for processing """ emancmd = "proc2d " if not os.path.isfile(self.stack['file']): apDisplay.printError("stackfile does not exist: "+self.stack['file']) emancmd += self.stack['file']+" " spiderstack = os.path.join(self.params['rundir'], "start.spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack+" " emancmd += "apix="+str(self.stack['apix'])+" " if self.params['lowpass'] > 0: emancmd += "lp="+str(self.params['lowpass'])+" " if self.params['highpass'] > 0: emancmd += "hp="+str(self.params['highpass'])+" " if self.params['bin'] > 1: clipboxsize = self.boxsize*self.params['bin'] emancmd += "shrink="+str(self.params['bin'])+" " emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" " emancmd += "last="+str(self.params['numpart']-1)+" " emancmd += "spider edgenorm" starttime = time.time() apDisplay.printColor("Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan") return spiderstack
def createTemplateStack(self): """ takes the spider file and creates an average template of all particles """ templatestack = os.path.join(self.params['rundir'], "templatestack00.spi") apFile.removeFile(templatestack, warn=True) ### hack to use standard filtering library templateparams = {} templateparams['apix'] = self.stack['apix'] templateparams['rundir'] = os.path.join(self.params['rundir'], "templates") templateparams['templateIds'] = self.templatelist templateparams['bin'] = self.params['bin'] templateparams['lowpass'] = self.params['lowpass'] templateparams['median'] = None templateparams['pixlimit'] = None print templateparams apParam.createDirectory( os.path.join(self.params['rundir'], "templates")) filelist = apTemplate.getTemplates(templateparams) for mrcfile in filelist: emancmd = ("proc2d templates/" + mrcfile + " " + templatestack + " clip=" + str(self.boxsize) + "," + str(self.boxsize) + " spiderswap ") if self.params['inverttemplates'] is True: emancmd += " invert " apEMAN.executeEmanCmd(emancmd, showcmd=False) return templatestack
def runEMANmra(self): # set up cls files emancmd = "proc2d %s.hed cls mraprep" %self.params['currentcls'] apEMAN.executeEmanCmd(emancmd, verbose=False) # run EMAN projection matching emancmd = "classesbymra %s %s.hed mask=%i split imask=-1 logit=1 sep=1 phase" %( self.params['localstack'], self.params['currentcls'], self.workingmask ) if self.params['nproc'] > 1: apEMAN.executeRunpar(emancmd, self.params['nproc']) else: apEMAN.executeEmanCmd(emancmd, verbose=True) # create stack of aligned particles # first get list of cls files clslist=glob.glob('cls*.lst') clslist.sort() emantar = tarfile.open("cls.eman.tar","w") clsarray = [[]for i in range(self.params['numpart'])] for cls in clslist: f = open(cls) lines = f.readlines() f.close() for l in range(1,len(lines)): d=lines[l].strip().split() if len(d) < 4: continue part = int(d[0]) stack = d[1] cc = float(d[2][:-1]) (rot,x,y,mirror) = d[3].split(',') clsarray[part]=[part,stack,cc,float(rot),float(x),float(y),int(mirror)] # for tarring cls files emantar.add(cls) emantar.close() # remove eman cls####.lst files for cls in clslist: os.remove(cls) # create a new cls file with all particles f = open("cls_all.lst","w") f.write("#LST\n") for p in clsarray: f.write("%i\t%s\t%.2f, %.6f,%.3f,%.3f,%i\n" % (p[0],p[1],p[2],p[3],p[4],p[5],p[6])) f.close() # create aligned particles apEMAN.alignParticlesInLST("cls_all.lst","mrastack.hed") if self.params['currentiter'] > 1: apFile.removeStack(self.params['alignedstack']) self.params['alignedstack']=os.path.abspath("mrastack") self.params['alignedstack'] = string.replace(self.params['alignedstack'],"/jetstor/APPION","") # remove cls file used for alignment os.remove("cls_all.lst")
def processTemplate(self, mrcfile): ### shrink apDisplay.printMsg("Binning template by a factor of " + str(self.params['bin'])) clipsize = int( math.floor(self.stack['boxsize'] / self.params['bin']) * self.params['bin']) emancmd = ("proc2d " + mrcfile + " " + mrcfile + " shrink=" + str(self.params['bin']) + " spiderswap ") emancmd += "clip=" + str(clipsize) + "," + str(clipsize) + " " apEMAN.executeEmanCmd(emancmd) ### normalize and center #apDisplay.printMsg("Normalize and centering template") #emancmd = "proc2d "+mrcfile+" "+mrcfile+" edgenorm center" #apEMAN.executeEmanCmd(emancmd) ### convert to SPIDER apDisplay.printMsg("Converting template to SPIDER") templatefile = "template.spi" if os.path.isfile(templatefile): apFile.removeFile(templatefile, warn=True) emancmd = "proc2d template.mrc " + templatefile + " spiderswap " apEMAN.executeEmanCmd(emancmd) return templatefile
def selectRandomParticles(self): """ takes the spider file and creates an average template of all particles """ ### create random keep list numrandpart = int(self.params['numpart'] / 100) + 2 apDisplay.printMsg("Selecting 1% of particles (" + str(numrandpart) + ") to average") # create random list keepdict = {} randlist = [] for i in range(numrandpart): rand = int(random.random() * self.params['numpart']) while rand in keepdict: rand = int(random.random() * self.params['numpart']) keepdict[rand] = 1 randlist.append(rand) # sort and write to file randlist.sort() f = open("randkeep.lst", "w") for rand in randlist: f.write(str(rand) + "\n") f.close() emancmd = "proc2d " + self.stack[ 'file'] + " template.mrc list=randkeep.lst average edgenorm" apEMAN.executeEmanCmd(emancmd) templatefile = self.processTemplate("template.mrc") return templatefile
def convertStackToSpider(self, emanstackfile, classnum): """ takes the stack file and creates a spider file ready for processing """ if not os.path.isfile(emanstackfile): apDisplay.printError("stackfile does not exist: "+emanstackfile) ### first high pass filter particles apDisplay.printMsg("pre-filtering particles") apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid']) emancmd = ("proc2d "+emanstackfile+" "+emanstackfile +" apix="+str(apix)+" hp="+str(self.params['highpasspart']) +" inplace") apEMAN.executeEmanCmd(emancmd, verbose=True) ### convert imagic stack to spider emancmd = "proc2d " emancmd += emanstackfile+" " spiderstack = os.path.join(self.params['rundir'], str(classnum), "otrstack"+self.timestamp+".spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack+" " emancmd += "spiderswap edgenorm" starttime = time.time() apDisplay.printColor("Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan") return spiderstack
def useClusterForTemplate(self): apDisplay.printMsg("Using clustering stack to make templates") clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid']) self.params['apix'] = clusterstackdata['clusterrun']['alignstack']['pixelsize'] stackfile = os.path.join(clusterstackdata['path']['path'], clusterstackdata['avg_imagicfile']) self.params['templatelist'] = [] stackid = clusterstackdata['clusterrun']['alignstack']['stack'].dbid sessiondata = apStack.getSessionDataFromStackId(stackid) self.params['sessionname'] = sessiondata['name'] ### check to see if stackimagenum is within the boundary of the stack numpart = apFile.numImagesInStack(stackfile) for i in self.params['imgnums'].split(","): partnum = int(i) if partnum > numpart or partnum < 0: apDisplay.printError("'imgnums' is NOT within the boundary of the stack: %d > %d"%(partnum,numpart)) apDisplay.printMsg("Extracting image %d from cluster stack" % partnum) templatename = "cluster%d-average%d.mrc"%(self.params['clusterid'],partnum) abstemplatepath= os.path.join(clusterstackdata['path']['path'], templatename) ### run proc2d with params['stackimgnum'] emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, abstemplatepath, partnum, partnum) ### create template apDisplay.printMsg("creating "+templatename) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'].append(abstemplatepath)
def setupParticles(self): self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2 apix = self.stack['apix']*self.params['bin'] proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix']) if self.params['bin'] > 1: clipsize = boxsize*self.params['bin'] proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize) if self.params['highpass'] > 1: proccmd += " hp="+str(self.params['highpass']) if self.params['lowpass'] > 1: proccmd += " lp="+str(self.params['lowpass']) proccmd += " last="+str(self.params['numpart']-1) apFile.removeStack(self.params['localstack'], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) return (boxsize, apix)
def rescaleVolume(infile, outfile, inapix, outapix=None, newbox=None, spider=False): """ scale an existing model - provide an input model & output (strings) an input a/pix & output a/pix (floats) and the final box size (after scaling) currently uses EMAN's proc3d to do this """ origbox = getModelDimensions(infile) if newbox is None: newbox = origbox if outapix is None: outapix = inapix apDisplay.printMsg( ("rescaling %s with boxsize %d by a factor of %.3f\n" +"\tand saving to %s with a boxsize %d") %(infile, origbox, scalefactor, outfile, newbox)) emancmd = "proc3d %s %s " % (infile, outfile) scalefactor = float(inapix/outapix) # only do it if it is bigger than 10% if abs(scalefactor-1.0) > 0.1: emancmd += "scale=%.3f " % scalefactor emancmd += "clip=%i,%i,%i norm=0,1 " % (newbox, newbox, newbox) apEMAN.executeEmanCmd(emancmd, verbose=True) if spider is True: emancmd += "spidersingle " return
def createSpiderFile(self): """ takes the stack file and creates a spider file ready for processing """ emancmd = "proc2d " if not os.path.isfile(self.stack["file"]): apDisplay.printError("stackfile does not exist: " + self.stack["file"]) emancmd += self.stack["file"] + " " spiderstack = os.path.join(self.params["rundir"], "start.spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack + " " emancmd += "apix=" + str(self.stack["apix"]) + " " if self.params["lowpass"] > 0: emancmd += "lp=" + str(self.params["lowpass"]) + " " emancmd += "last=" + str(self.params["numpart"] - 1) + " " emancmd += "shrink=" + str(self.params["bin"]) + " " clipsize = int(math.floor(self.stack["boxsize"] / self.params["bin"] / 2.0) * self.params["bin"] * 2) emancmd += "clip=" + str(clipsize) + "," + str(clipsize) + " " emancmd += "spiderswap edgenorm" starttime = time.time() apDisplay.printColor("Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor("finished eman in " + apDisplay.timeString(time.time() - starttime), "cyan") return spiderstack
def MRCtoSPI(infile, rundir): # convert file to spider file tmpspifile = apParam.randomString(8)+".spi" tmpspifile = os.path.join(rundir,tmpspifile) emancmd = "proc3d %s %s spidersingle" %(infile,tmpspifile) apEMAN.executeEmanCmd(emancmd, verbose=True) return tmpspifile
def createOrientationReference(self): """ convert spider orientation reference """ orientref = os.path.join(self.params['rundir'], "orient.spi") apFile.removeFile(orientref, warn=True) #templatedata = apTemplate.getTemplateFromId(self.params['orientref']) #templatefile = os.path.join(templatedata['path']['path'], templatedata['templatename']) ### hack to use standard filtering library templateparams = {} templateparams['apix'] = self.stack['apix'] templateparams['rundir'] = os.path.join(self.params['rundir'], "templates") templateparams['templateIds'] = [self.params['orientref'],] templateparams['bin'] = self.params['bin'] templateparams['lowpass'] = self.params['lowpass'] templateparams['median'] = None templateparams['pixlimit'] = None print 'Converting orientation reference:\n', templateparams apParam.createDirectory(os.path.join(self.params['rundir'], "templates")) filelist = apTemplate.getTemplates(templateparams) mrcfile = filelist[0] localclip = self.clipsize/self.params['bin'] emancmd = ("proc2d templates/"+mrcfile+" "+orientref +" clip="+str(localclip)+","+str(localclip) +" edgenorm spiderswap-single ") if self.params['inverttemplates'] is True: emancmd += " invert " apEMAN.executeEmanCmd(emancmd, showcmd=False) return orientref
def createSpiderFile(self): """ takes the stack file and creates a spider file ready for processing """ emancmd = "proc2d " if not os.path.isfile(self.stack['file']): apDisplay.printError("stackfile does not exist: " + self.stack['file']) emancmd += self.stack['file'] + " " spiderstack = os.path.join(self.params['rundir'], "start.spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack + " " emancmd += "apix=" + str(self.stack['apix']) + " " if self.params['lowpass'] > 0: emancmd += "lp=" + str(self.params['lowpass']) + " " emancmd += "last=" + str(self.params['numpart'] - 1) + " " emancmd += "shrink=" + str(self.params['bin']) + " " emancmd += "clip=" + str(self.clipsize) + "," + str( self.clipsize) + " " emancmd += "spiderswap" starttime = time.time() apDisplay.printColor("Converting stack to spider format", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor( "finished eman in " + apDisplay.timeString(time.time() - starttime), "cyan") return spiderstack
def createTemplateStack(self): """ convert spider template stack """ templatestack = os.path.join(self.params['rundir'], "templatestack.spi") apFile.removeFile(templatestack, warn=True) ### hack to use standard filtering library templateparams = {} templateparams['apix'] = self.stack['apix'] templateparams['rundir'] = os.path.join(self.params['rundir'], "templates") templateparams['templateIds'] = self.templatelist templateparams['bin'] = self.params['bin'] templateparams['lowpass'] = self.params['lowpass'] templateparams['median'] = None templateparams['pixlimit'] = None print 'Converting reference templates:\n', templateparams apParam.createDirectory( os.path.join(self.params['rundir'], "templates")) filelist = apTemplate.getTemplates(templateparams) localclip = self.clipsize / self.params['bin'] for mrcfile in filelist: emancmd = ("proc2d templates/" + mrcfile + " " + templatestack + " clip=" + str(localclip) + "," + str(localclip) + " edgenorm spiderswap ") if self.params['inverttemplates'] is True: emancmd += " invert " apEMAN.executeEmanCmd(emancmd, showcmd=False) return templatestack
def stackToSpiderStack(stack,spiderstack,apix,boxsize,lp=0,hp=0,bin=1,numpart=0): """ convert an input stack (i.e. imagic) and write it out to a spider-formatted stack """ emancmd = "proc2d " if not os.path.isfile(stack): apDisplay.printError("stackfile does not exist: "+stack) emancmd += stack+" " apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack+" " emancmd += "apix="+str(apix)+" " if lp > 0: emancmd += "lp="+str(lp)+" " if hp > 0: emancmd += "hp="+str(hp)+" " if bin > 1: clipboxsize = boxsize*bin emancmd += "shrink="+str(bin)+" " emancmd += "clip="+str(clipboxsize)+","+str(clipboxsize)+" " if numpart > 0: emancmd += "last="+str(numpart-1)+" " emancmd += "spiderswap edgenorm" starttime = time.time() apDisplay.printColor("Running spider stack conversion this can take a while", "cyan") from appionlib import apEMAN apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def start(self): ### copy stack to rundir newstack = os.path.join(self.params['rundir'], "start.hed") if os.path.isfile(newstack): apDisplay.printError("Stack already exists") emancmd = "proc2d %s %s"%(self.params['stackfile'], newstack) if self.params['normalize'] is True: emancmd += " edgenorm" apEMAN.executeEmanCmd(emancmd) ### set final parameters boxsize = apFile.getBoxSize(newstack) apDisplay.printMsg("Boxsize: %i"%boxsize[0]) if not boxsize or boxsize <= 0: apDisplay.printError("Could not determine stack size") else: self.boxsize = boxsize[0] self.numpart = apFile.numImagesInStack(newstack) apDisplay.printMsg("Num part: %i"%self.numpart) if not self.numpart or self.numpart <= 0: apDisplay.printError("Could not determine number of particles") if self.numpart <= 1: apDisplay.printError("Not enough particles to upload") apStack.averageStack(newstack) #self.params['commit'] = False self.createStackData()
def centerParticles(stack, mask=None, maxshift=None): apDisplay.printMsg("Centering stack: "+stack) stacksize = apFile.stackSize(stack) freemem = mem.free()*1024 #convert memory to bytes apDisplay.printMsg("file is %s, mem is %s" %(apDisplay.bytes(stacksize), apDisplay.bytes(freemem))) ### from EMAN FAQ: need to have at least 3x as much ram as the size of the file memsize = freemem/3.0 numfrac = int(math.ceil(stacksize/memsize)) apDisplay.printMsg("file is %s, will be split into %d fractions" %(apDisplay.bytes(stacksize), numfrac)) for i in range(numfrac): emancmd = "cenalignint "+stack if numfrac > 1: emancmd += " frac="+str(i)+"/"+str(numfrac) if mask is not None: emancmd += " mask="+str(mask) if maxshift is not None: emancmd += " maxshift="+str(maxshift) apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) return
def start(self): """ This is the core of your function. """ stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) original_stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) filtered_stackfile = os.path.join(self.params['rundir'], self.timestamp+".hed") apFile.removeStack(filtered_stackfile, warn=False) apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) boxsize = apStack.getStackBoxsize(self.params['stackid']) emancmd = "proc2d %s %s apix=%.3f "%(original_stackfile, filtered_stackfile, apix) if self.params['lowpass'] is not None: emancmd += " lp=%.3f "%(self.params['lowpass']) if self.params['highpass'] is not None: emancmd += " hp=%.3f "%(self.params['highpass']) if self.params['bin'] is not None and self.params['bin'] > 1: ## determine a multiple of the bin that is divisible by 2 and less than the boxsize clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2*self.params['bin'] emancmd += " shrink=%d clip=%d,%d "%(self.params['bin'], clipsize, clipsize) emancmd += " last=%d "%(self.params['numpart']-1) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) ### confirm that it worked if self.params['numpart'] != apFile.numImagesInStack(filtered_stackfile): apDisplay.printError("Missing particles in stack") ### run the radon transform code self.radonAlign(filtered_stackfile) ### insert info into database self.commitToDatabase()
def makeNewStack(oldstack, newstack, listfile=None, remove=False, bad=False): if not os.path.isfile(oldstack): apDisplay.printWarning("could not find old stack: "+oldstack) if os.path.isfile(newstack): if remove is True: apDisplay.printWarning("removing old stack: "+newstack) time.sleep(2) apFile.removeStack(newstack) else: apDisplay.printError("new stack already exists: "+newstack) apDisplay.printMsg("creating a new stack\n\t"+newstack+ "\nfrom the oldstack\n\t"+oldstack+"\nusing list file\n\t"+str(listfile)) emancmd = "proc2d "+oldstack+" "+newstack if listfile is not None: emancmd += " list="+listfile apEMAN.executeEmanCmd(emancmd, verbose=True) if bad is True and listfile is not None: ### run only if num bad particles < num good particles newstacknumpart = apFile.numImagesInStack(newstack) oldstacknumpart = apFile.numImagesInStack(oldstack) if newstacknumpart > oldstacknumpart/2: ### create bad.hed stack with all bad particles badstack = os.path.join(os.path.dirname(newstack), "bad.hed") emancmd = "proc2d %s %s exclude=%s"%(oldstack, badstack, listfile) apEMAN.executeEmanCmd(emancmd, verbose=True) else: apDisplay.printMsg("Rejecting more particles than keeping, not creating a bad stack") return
def runRmeasure(self, volfile): emancmd = "proc3d %s %s"%(volfile, "rmeasure.mrc") apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc") #apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan") apFile.removeFile("rmeasure.mrc")
def assemblePngs(keys, tag, montagestack): apDisplay.printMsg("assembling pngs into montage") # get path from montagestack stackpath = os.path.dirname(os.path.abspath(montagestack)) montagecmd = "montage -geometry +4+4 " montagestackdata = apImagicFile.readImagic(montagestack) for i,key in enumerate(keys): if i % 20 == 0: sys.stderr.write(".") pngfile = key+".png" array = montagestackdata['images'][i] apImage.arrayToPng(array, pngfile, normalize=True, msg=False) #proccmd = "proc2d "+montagestack+" "+pngfile+" first="+str(i)+" last="+str(i) #apEMAN.executeEmanCmd(proccmd, verbose=False, showcmd=False) montagecmd += pngfile+" " apDisplay.printMsg("montaging") montagefile = os.path.join(stackpath,"montage"+tag+".png") montagecmd += montagefile apEMAN.executeEmanCmd(montagecmd, verbose=True) #rotatecmd = "mogrify -rotate 180 -flop "+montagefile #apEMAN.executeEmanCmd(rotatecmd, verbose=False) for key in keys: apFile.removeFile(key+".png") if not os.path.isfile(montagefile): apDisplay.printWarning("failed to create montage file")
def convertStackToSpider(self, emanstackfile, spiderstack=None): """ takes the stack file and creates a spider file ready for processing """ if not os.path.isfile(emanstackfile): apDisplay.printError("stackfile does not exist: "+emanstackfile) ### first high pass filter particles #apDisplay.printMsg("pre-filtering particles") #apix = apStack.getStackPixelSizeFromStackId(self.params['partstackid']) #emancmd = ("proc2d "+emanstackfile+" "+emanstackfile # +" apix="+str(apix)+" hp="+str(self.params['highpasspart']) # +" inplace") #apEMAN.executeEmanCmd(emancmd, verbose=True) ### convert imagic stack to spider emancmd = "proc2d " emancmd += emanstackfile+" " if spiderstack is None: spiderstack = os.path.join(self.params['rundir'], "start.spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack+" " emancmd += "spiderswap edgenorm" starttime = time.time() apDisplay.printColor("Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan") return spiderstack
def runRmeasure(self, volfile): emancmd = "proc3d %s %s"%(volfile, "rmeasure.mrc") apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['bin'] self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc") #apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan") apFile.removeFile("rmeasure.mrc")
def makeStackInFormatFromDefault(stackdata, format="eman"): """ Function to create the stack in the requested format Current default is Imagic format used in EMAN """ if format not in formats: apDisplay.printError("Unknown stack format %s requested" % format) stackroot, instackdir, outstackdir = getFormattedStackInfo(stackdata, format) instackpath = os.path.join(instackdir, stackroot + ".hed") if format == "eman": return stackdata["path"]["path"] apParam.createDirectory(outstackdir) rundir = os.getcwd() os.chdir(outstackdir) if format == "spider": outstackpath = os.path.join(outstackdir, stackroot + stack_exts[format][0]) emancmd = "proc2d %s %s spidersingle" % (instackpath, outstackpath) apEMAN.executeEmanCmd(emancmd, showcmd=True) if format == "frealign": outstackpath = os.path.join(outstackdir, stackroot + stack_exts[format][0]) emancmd = "proc2d %s %s invert" % (instackpath, outstackpath) apEMAN.executeEmanCmd(emancmd, showcmd=True) if format == "xmipp": ### convert stack into single spider files selfile = apXmipp.breakupStackIntoSingleFiles(instackpath) os.chdir(rundir) return outstackdir
def createTemplateStack(self): """ convert spider template stack """ templatestack = os.path.join(self.params['rundir'], "templatestack.spi") apFile.removeFile(templatestack, warn=True) ### hack to use standard filtering library templateparams = {} templateparams['apix'] = self.stack['apix'] templateparams['rundir'] = os.path.join(self.params['rundir'], "templates") templateparams['templateIds'] = self.templatelist templateparams['bin'] = self.params['bin'] templateparams['lowpass'] = self.params['lowpass'] templateparams['median'] = None templateparams['pixlimit'] = None print 'Converting reference templates:\n', templateparams apParam.createDirectory(os.path.join(self.params['rundir'], "templates")) filelist = apTemplate.getTemplates(templateparams) localclip = self.clipsize/self.params['bin'] for mrcfile in filelist: emancmd = ("proc2d templates/"+mrcfile+" "+templatestack +" clip="+str(localclip)+","+str(localclip) +" edgenorm spiderswap ") if self.params['inverttemplates'] is True: emancmd += " invert " apEMAN.executeEmanCmd(emancmd, showcmd=False) return templatestack
def selectRandomParticles(self): """ takes the spider file and creates an average template of all particles """ ### create random keep list numrandpart = int(self.params["numpart"] / 100) + 2 apDisplay.printMsg("Selecting 1% of particles (" + str(numrandpart) + ") to average") # create random list keepdict = {} randlist = [] for i in range(numrandpart): rand = int(random.random() * self.params["numpart"]) while rand in keepdict: rand = int(random.random() * self.params["numpart"]) keepdict[rand] = 1 randlist.append(rand) # sort and write to file randlist.sort() f = open("randkeep.lst", "w") for rand in randlist: f.write(str(rand) + "\n") f.close() emancmd = "proc2d " + self.stack["file"] + " template.mrc list=randkeep.lst average edgenorm" apEMAN.executeEmanCmd(emancmd) templatefile = self.processTemplate("template.mrc") return templatefile
def start(self): self.runtime = 0 self.checkCoranRun() ### convert stack to spider self.alignstackdata = self.getAlignedStack() maskpixrad = self.params['maskrad']/self.alignstackdata['pixelsize']/self.params['bin'] boxpixdiam = int(math.ceil(maskpixrad)+1)*2 if boxpixdiam*self.params['bin'] > self.alignstackdata['boxsize']: boxpixdiam = math.floor(self.alignstackdata['boxsize']/self.params['bin']) clippixdiam = boxpixdiam*self.params['bin'] apDisplay.printMsg("Pixel mask radius="+str(maskpixrad)) oldalignedstack = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['imagicfile']) alignedstackname = re.sub("\.", "_", self.alignstackdata['imagicfile'])+".spi" alignedstack = os.path.join(self.params['rundir'], alignedstackname) apFile.removeFile(alignedstack) emancmd = ("proc2d %s %s spiderswap shrink=%d clip=%d,%d edgenorm" %(oldalignedstack,alignedstack,self.params['bin'],clippixdiam,clippixdiam)) if self.params['numpart'] is not None: emancmd += " last=%d"%(self.params['numpart']-1) numpart = self.params['numpart'] else: numpart = self.getNumAlignedParticles() apEMAN.executeEmanCmd(emancmd, verbose=True) ### get number of processors nproc = apParam.getNumProcessors() esttime = classification.estimateTime(numpart, maskpixrad) apDisplay.printColor("Running spider this can take awhile, estimated time: "+\ apDisplay.timeString(esttime),"cyan") ### do correspondence analysis corantime = time.time() self.contriblist = classification.correspondenceAnalysis( alignedstack, boxsize=boxpixdiam, maskpixrad=maskpixrad, numpart=numpart, numfactors=self.params['numfactors'], nproc=nproc) corantime = time.time() - corantime ### make dendrogram dendrotime = time.time() classification.makeDendrogram(numfactors=min(3,self.params['numfactors']),nproc=nproc) dendrotime = time.time() - dendrotime inserttime = time.time() if self.params['commit'] is True: self.runtime = corantime self.insertCoranRun(insert=True) else: apDisplay.printWarning("not committing results to DB") inserttime = time.time() - inserttime apFile.removeFile(alignedstack, warn=True) apDisplay.printMsg("Correspondence Analysis time: "+apDisplay.timeString(corantime)) apDisplay.printMsg("Make Dendrogram time: "+apDisplay.timeString(dendrotime)) apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"): """ inputs: aligned particle stack number of classes timestamp output directory output: class averages class variances dendrogram.png """ if timestamp is None: timestamp = apParam.makeTimestamp() classavg = rundir+"/"+("classavgstack_%s_%03d" % (timestamp, numclasses)) classvar = rundir+"/"+("classvarstack_%s_%03d" % (timestamp, numclasses)) thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext) ### create class doc files mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True) mySpider.toSpider( "CL HE", thresh, spyder.fileFilter(dendrogramfile), # dendrogram doc file rundir+"/classdoc_"+timestamp+"_****", # class doc file ) ### delete existing files sys.stderr.write("delete existing files") for dext in (".hed", ".img", dataext): apFile.removeFile(classavg+dext) apFile.removeFile(classvar+dext) print "" ### create class averages sys.stderr.write("create class averages") for i in range(classes): sys.stderr.write(".") classnum = i+1 mySpider.toSpiderQuiet( "AS R", spyder.fileFilter(alignedstack)+"@******", rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)), "A", (classavg+"@%04d" % (classnum)), (classvar+"@%04d" % (classnum)), ) mySpider.close() print "" ### convert to IMAGIC emancmd = "proc2d "+classavg+".spi "+classavg+".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) emancmd = "proc2d "+classvar+".spi "+classvar+".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) return classavg,classvar
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"): """ inputs: aligned particle stack number of classes timestamp output directory output: class averages class variances dendrogram.png """ if timestamp is None: timestamp = apParam.makeTimestamp() classavg = rundir+"/"+("classavgstack_%s_%03d" % (timestamp, numclasses)) classvar = rundir+"/"+("classvarstack_%s_%03d" % (timestamp, numclasses)) thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext) ### create class doc files mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True) mySpider.toSpider( "CL HE", thresh, spyder.fileFilter(dendrogramfile), # dendrogram doc file rundir+"/classdoc_"+timestamp+"_****", # class doc file ) ### delete existing files sys.stderr.write("delete existing files") for dext in (".hed", ".img", dataext): apFile.removeFile(classavg+dext) apFile.removeFile(classvar+dext) print "" ### create class averages sys.stderr.write("create %i class averages"%classes) for i in range(classes): sys.stderr.write(".") classnum = i+1 mySpider.toSpiderQuiet( "AS R", spyder.fileFilter(alignedstack)+"@******", rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)), "A", (classavg+"@%04d" % (classnum)), (classvar+"@%04d" % (classnum)), ) mySpider.close() print "" ### convert to IMAGIC emancmd = "proc2d "+classavg+".spi "+classavg+".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) emancmd = "proc2d "+classvar+".spi "+classvar+".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) return classavg,classvar
def emanSpiderToStack(partlist): apFile.removeStack("emanspi.hed", warn=False) for part in partlist: spider.write(part, "temp.spi") emancmd = "proc2d temp.spi emanspi.hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False) apFile.removeFile("temp.spi") return
def runRmeasure(self): finalrawvolfile = os.path.join(self.params['rundir'], "rawvolume%s-%03d.spi"%(self.timestamp, self.params['numiters'])) emancmd = "proc3d %s %s"%(finalrawvolfile, "rmeasure.mrc") apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc") #apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan") apFile.removeFile("rmeasure.mrc")
def emanMrcToStack(partlist): apFile.removeStack("emanmrc.hed", warn=False) for part in partlist: mrc.write(part, "temp.mrc") emancmd = "proc2d temp.mrc emanmrc.hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False) apFile.removeFile("temp.mrc") return
def start(self): self.stackdata = appiondata.ApStackData.direct_query( self.params['stackid']) if self.params['vertical'] is not True: self.alignstackdata = appiondata.ApAlignStackData.direct_query( self.params['alignstackid']) # Path of the stack self.stackdata = apStack.getOnlyStackData(self.params['stackid']) fn_oldstack = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) rotfile = None if self.params['vertical'] is not True: # get averaged image: self.alignstackdata = appiondata.ApAlignStackData.direct_query( self.params['alignstackid']) avgimg = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['avgmrcfile']) # Convert averaged aligned mrcfile to spider spiavg = os.path.join(self.params['rundir'], "avg.spi") emancmd = "proc2d %s %s spiderswap edgenorm" % (avgimg, spiavg) apEMAN.executeEmanCmd(emancmd, verbose=True) # find rotation for vertical alignment rot = self.findRotation(spiavg) apDisplay.printMsg("found average rotation: %.2f" % rot) rotlist = self.getInplaneRotations() rotfile = self.createRotationSpiList(rotlist, rot) # Convert the original stack to spider spistack = self.convertStackToSpider(fn_oldstack) # boxmask the particles spimaskfile = "masked" + self.timestamp + ".spi" self.boxMask(spistack, spimaskfile, rotfile) # Convert the spider stack to imagic imgstack = self.convertStackToImagic(spimaskfile) # Create average MRC apStack.averageStack(imgstack) # Clean up apDisplay.printMsg("deleting temporary processing files") os.remove(spistack) os.remove(spimaskfile) # Upload results if self.params['commit'] is True: oldstackparts = apStack.getStackParticlesFromId( self.params['stackid']) apStack.commitMaskedStack(self.params, oldstackparts, newname='start.hed') time.sleep(1) return
def insertParticlesIntoDatabase(self, stackid, partlist): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in partlist: count += 1 if count % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = partdict['refnum'] refq['iteration'] = self.lastiter refsearch = "part" + self.params['timestamp'] + "_ref*" + str( partdict['refnum']) + "*" refbase = os.path.splitext(glob.glob(refsearch)[0])[0] refq['mrcfile'] = refbase + ".mrc" refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] if partdict['refnum'] in self.resdict: refq['ssnr_resolution'] = self.resdict[partdict['refnum']] reffile = os.path.join(self.params['rundir'], refq['mrcfile']) if not os.path.isfile(reffile): emancmd = "proc2d " + refbase + ".xmp " + refbase + ".mrc" apEMAN.executeEmanCmd(emancmd, verbose=False) if not os.path.isfile(reffile): apDisplay.printError("could not find reference file: " + reffile) ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partdict['partnum'] alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle(stackid, partdict['partnum']) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['rotation'] = partdict['inplane'] alignpartq['mirror'] = partdict['mirror'] alignpartq['ref'] = refq alignpartq['spread'] = partdict['spread'] ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(count) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan") return
def genProj(euler, tilt=False): (alt, az, phi) = euler rotcmd = ( "proc3d groel.mrc rotated.mrc rot=" +str(alt)+","+str(az)+","+str(phi) ) apEMAN.executeEmanCmd(rotcmd, verbose=False, showcmd=False) if tilt is True: cmd = "project3d rotated.mrc euler=15,90,-90" else: cmd = "project3d rotated.mrc euler=0,0,0" apEMAN.executeEmanCmd(cmd, verbose=False, showcmd=False)
def runIMAGICclassify(self): bfile = "msaclassify.job" outfile = "classes" apFile.removeStack(outfile) numIters = int(self.params['numpart']*self.params['itermult']) decrement = self.params['start']-self.params['end'] if self.params['iter']>0: decrement /= float(self.params['iter']) numClasses = self.params['start']-(decrement*self.params['currentiter']) stackfile = self.params['alignedstack'] self.params['currentnumclasses'] = numClasses f = open(bfile,'w') f.write("#!/bin/csh -f\n") f.write("setenv IMAGIC_BATCH 1\n") f.write("%s/msa/classify.e <<EOF\n" % self.imagicroot) f.write("IMAGES\n") f.write("%s\n"%stackfile) f.write("0\n") f.write("%i\n"%self.params['activeeigen']) f.write("YES\n") f.write("%i\n"%numClasses) f.write("classes_start\n") f.write("EOF\n") f.write("%s/msa/classum.e <<EOF\n" % self.imagicroot) f.write("%s\n"%stackfile) f.write("classes_start\n") f.write("%s\n"%outfile) f.write("YES\n") f.write("NONE\n") f.write("0\n") if int(self.imagicversion) >= 120619: f.write("NONE\n") # Mode of summing statistics f.write("EOF\n") ## make eigenimage stack appion-compatible f.write("proc2d eigenim.hed eigenim.hed inplace\n") f.write("touch msaclassify_done.txt\n") f.write("exit\n") f.close() ## execute the batch file aligntime0 = time.time() apEMAN.executeEmanCmd("chmod 755 "+bfile) apDisplay.printColor("Running IMAGIC .batch file: %s"%(os.path.abspath(bfile)), "cyan") apIMAGIC.executeImagicBatchFile(os.path.abspath(bfile)) os.remove("msaclassify_done.txt") if not os.path.exists(outfile+".hed"): apDisplay.printError("ERROR IN IMAGIC SUBROUTINE") apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-aligntime0), "cyan")
def processVolume(self, spivolfile, iternum=0): ### set values apix = apStack.getStackPixelSizeFromStackId( self.params['tiltstackid']) * self.params['tiltbin'] boxsize = self.getBoxSize() rawspifile = os.path.join( self.params['rundir'], "rawvolume%s-%03d.spi" % (self.timestamp, iternum)) mrcvolfile = os.path.join( self.params['rundir'], "volume%s-%03d.mrc" % (self.timestamp, iternum)) lowpass = self.params['lowpassvol'] ### copy original to raw file shutil.copy(spivolfile, rawspifile) ### convert to mrc emancmd = ("proc3d " + spivolfile + " " + mrcvolfile + " norm=0,1 apix=" + str(apix)) apEMAN.executeEmanCmd(emancmd, verbose=False) ### median filter rawvol = mrc.read(mrcvolfile) medvol = ndimage.median_filter(rawvol, size=self.params['median']) mrc.write(medvol, mrcvolfile) ### low pass filter emancmd = ("proc3d " + mrcvolfile + " " + mrcvolfile + " center norm=0,1 apix=" + str(apix) + " lp=" + str(lowpass)) apEMAN.executeEmanCmd(emancmd, verbose=False) ### set origin emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " origin=0,0,0 " apEMAN.executeEmanCmd(emancmd, verbose=False) ### mask volume emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " mask=" + str( self.params['radius']) apEMAN.executeEmanCmd(emancmd, verbose=False) ### convert to spider apFile.removeFile(spivolfile) emancmd = "proc3d " + mrcvolfile + " " + spivolfile + " spidersingle" apEMAN.executeEmanCmd(emancmd, verbose=False) ### image with chimera if self.params['skipchimera'] is False: if self.params['mass'] is not None: apDisplay.printMsg("Using scale by mass method") apChimera.setVolumeMass(mrcvolfile, apix=apix, mass=self.params['mass']) apChimera.renderSnapshots(mrcvolfile, self.params['contour'], self.params['zoom'], 'c1') return mrcvolfile