def getResolutionData(self, avgpath, iternum): evenvol = os.path.join(avgpath, "avglist3_%dp1.mrc" % (self.params["rescut"])) oddvol = os.path.join(avgpath, "avglist3_%dp2.mrc" % (self.params["rescut"])) evenhed = os.path.join(avgpath, "avglist3_%dp1.hed" % (self.params["rescut"])) oddhed = os.path.join(avgpath, "avglist3_%dp2.hed" % (self.params["rescut"])) emancmd1 = "proc3d %s %s" % (evenvol, evenhed) apEMAN.executeEmanCmd(emancmd1, verbose=True) emancmd2 = "proc3d %s %s" % (oddvol, oddhed) apEMAN.executeEmanCmd(emancmd2, verbose=True) fscfile = "fsc.eotest.%d" % (iternum) fscpath = os.path.join(avgpath, fscfile) emancmd3 = "proc3d %s %s fsc=%s" % (evenhed, oddhed, fscpath) apEMAN.executeEmanCmd(emancmd3, verbose=True) if not os.path.isfile(fscpath): apDisplay.printWarning("Could not find FSC file: " + fscpath) return None f = open(fscpath, "r") xy = f.readlines() lines = len(xy) boxsize = lines * 2.0 f.close() # calculate the resolution: halfres = apRecon.calcRes(fscpath, boxsize, self.params["step"]) apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms" % (halfres), "cyan") # save to database resq = appiondata.ApResolutionData() resq["half"] = halfres resq["fscfile"] = fscpath return resq
def getResolutionData(self, avgpath, iternum): evenvol = os.path.join(avgpath, "avglist3_%dp1.mrc"%(self.params['rescut'])) oddvol = os.path.join(avgpath, "avglist3_%dp2.mrc"%(self.params['rescut'])) evenhed = os.path.join(avgpath, "avglist3_%dp1.hed"%(self.params['rescut'])) oddhed = os.path.join(avgpath, "avglist3_%dp2.hed"%(self.params['rescut'])) emancmd1 = 'proc3d %s %s' % (evenvol, evenhed) apEMAN.executeEmanCmd(emancmd1, verbose=True) emancmd2 = 'proc3d %s %s' % (oddvol, oddhed) apEMAN.executeEmanCmd(emancmd2, verbose=True) fscfile = 'fsc.eotest.%d'%(iternum) fscpath = os.path.join(avgpath, fscfile) emancmd3 = 'proc3d %s %s fsc=%s' % (evenhed, oddhed, fscpath) apEMAN.executeEmanCmd(emancmd3, verbose=True) if not os.path.isfile(fscpath): apDisplay.printWarning("Could not find FSC file: "+fscpath) return None f = open(fscpath, 'r') xy = f.readlines() lines = len(xy) boxsize = (lines * 2.0) f.close() # calculate the resolution: halfres = apRecon.calcRes(fscpath, boxsize, self.params['step']) apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms"%(halfres), "cyan") # save to database resq=appiondata.ApResolutionData() resq['half'] = halfres resq['fscfile'] = fscpath return resq
def getResolutionData(self, iternum): fscfile = 'fsc.eotest.%d'%(iternum) fscpath = os.path.join(self.params['rundir'], "iter%03d"%(iternum), fscfile) if not os.path.isfile(fscpath): apDisplay.printWarning("Could not find FSC file: "+fscpath) return None # calculate the resolution: halfres = apRecon.calcRes(fscpath, self.boxsize, self.apix) apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms"%(halfres), "cyan") # save to database resq=appiondata.ApResolutionData() resq['half'] = halfres resq['fscfile'] = fscfile return resq
def getResolutionData(self, iteration): fsc = 'fsc.eotest.'+iteration['num'] fscfile = os.path.join(self.params['rundir'],fsc) if not os.path.isfile(fscfile): apDisplay.printWarning("Could not find FSC file: "+fscfile) iteration['fscfile'] = fscfile if not fsc in self.params['fscs']: return None # calculate the resolution: halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix']) # save to database resq=appiondata.ApResolutionData() resq['half'] = halfres resq['fscfile'] = fsc return resq
def start(self): self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) # symmetry info if self.params['sym']=='Icos': self.params['symtype']='I' self.params['symfold']=None else: self.params['symtype']=self.params['sym'][0] self.params['symfold']=int(self.params['sym'][1:]) # eman "d" symmetry is spider "ci" if self.params['symtype'].upper()=="D": self.params['symtype'] = "CI" # create symmetry doc file sydoc="sym.spi" refine.symmetryDoc(self.params['symtype'],self.params['symfold'],sydoc) # convert incr to array of increments ang_inc=self.params['incr'].split(',') self.params['numiter'] = len(ang_inc) self.params['increments']=[] for i in range(0,self.params['numiter']): self.params['increments'].append(int(ang_inc[i])) # convert stack to spider stack spiderstack=os.path.join(self.params['rundir'],'start.spi') operations.stackToSpiderStack( self.stack['file'], spiderstack, apix=self.stack['apix'], boxsize=self.stack['boxsize'], numpart=self.params['numpart'] ) # create filtered stack spiderstackfilt=spiderstack if (self.params['lowpass']+self.params['highpass']) > 0: spiderstackfilt=os.path.join(self.params['rundir'],'start_filt.spi') operations.stackToSpiderStack( self.stack['file'], spiderstackfilt, apix=self.stack['apix'], boxsize=self.stack['boxsize'], lp=self.params['lowpass'], hp=self.params['highpass'], numpart=self.params['numpart'] ) # rescale initial model if necessary outvol = os.path.join(self.params['rundir'],"vol000.spi") apModel.rescaleModel(self.params['modelid'], outvol, self.stack['boxsize'], self.stack['apix'], spider=True) self.params['itervol']=outvol for iter in range(1,self.params['numiter']+1): # create projections for projection matching apDisplay.printMsg("creating reference projections of volume: %s" % self.params['itervol']) projs,numprojs,ang,sel = refine.createProjections( incr=self.params['increments'][iter-1], boxsz=self.stack['boxsize'], symfold=self.params['symfold'], invol=self.params['itervol'], rad=self.params['rad'], ) # run reference-based alignment apDisplay.printMsg("running reference-based alignment (AP MQ)") apmqfile = "apmq%03d.spi" % iter outang = "angular%03d.spi" % iter shf = "shifts%03d.spi" % iter shiftedStack="parts_shifted.spi" apFile.removeFile(shf) refine.spiderAPMQ( projs=projs, numprojs=numprojs, tsearch=self.params['xysearch'], tstep=self.params['xystep'], firstRing=self.params['firstring'], lastRing=self.params['lastring'], stackfile=spiderstackfilt, nump=self.params['numpart'], ang=ang, apmqfile=apmqfile, outang=outang, nproc=self.params['proc'], ) # use cross-correlation to find the sub-pixel alignment # of the particles, # results will be saved in "peakfile.spi" apFile.removeFile(shiftedStack) # don't use MPI here - for some reason slower? mySpi=spyder.SpiderSession(dataext=".spi", logo=False, log=False) apmqlist = refine.readDocFile(apmqfile) avgccrot = 0 apDisplay.printMsg("creating shifted stack") for p in range(0,self.params['numpart']): ref=int(float(apmqlist[p][2])) ccrot=float(apmqlist[p][3]) inplane=float(apmqlist[p][4]) avgccrot+=ccrot # invert the sign - ref projs will be rotated inplane*=-1 # get corresponding projection if (ref <= 0): # mirror projection if necessary ref*=-1 refimg=spyder.fileFilter(projs)+"@"+str(ref) refine.mirrorImg(refimg,"_3",inMySpi=mySpi) img="_3" else: img=spyder.fileFilter(projs)+"@"+str(ref) refine.rotAndShiftImg(img,"_2",inplane,inMySpi=mySpi) refine.maskImg("_2","_3",self.params['rad'],"D","E", center=int((self.stack['boxsize']/2)+1), inMySpi=mySpi) # pad ref image & stack image to twice the size refine.padImg("_3","_2",2*self.stack['boxsize'],"N",1,1,0,inMySpi=mySpi) stackimg=spyder.fileFilter(spiderstack)+"@"+str(p+1) refine.padImg(stackimg,"_1",2*self.stack['boxsize'],"B",1,1,inMySpi=mySpi) # calculate cross-correlation refine.getCC("_1","_2","_1",inMySpi=mySpi) # crop the correllation image to allowable shift amount shift=int(self.params['allowedShift']*self.stack['boxsize']) dim=2*shift+1 topleftx=self.stack['boxsize']-shift+1 refine.windowImg("_1","_2",dim,topleftx,topleftx,inMySpi=mySpi) # find the sub-pixel location of cc peak mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17","_2","0") # create new stack of shifted particles shpos=spyder.fileFilter(shiftedStack)+"@"+str(p+1) mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN") mySpi.toSpiderQuiet("GP x17","_2",str(shift+1)+","+str(shift+1)) refine.copyImg(stackimg,shpos,inMySpi=mySpi) mySpi.toSpiderQuiet("ELSE") #mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16") mySpi.toSpiderQuiet("SH F",stackimg,shpos,"-x15,-x16") mySpi.toSpiderQuiet("ENDIF") # save shifts to file mySpi.toSpiderQuiet("SD "+str(p+1)+",x15,x16,x17",spyder.fileFilter(shf)) mySpi.toSpiderQuiet("SD E",spyder.fileFilter(shf)) mySpi.close() # create class average images refine.createClassAverages( shiftedStack, projs, apmqfile, numprojs, self.stack['boxsize'], shifted=True, ) # rename class averages & variacnes for iteration cmd="/bin/mv classes.hed classes.%d.hed;" % iter cmd+="/bin/mv classes.img classes.%d.img;" % iter cmd+="/bin/mv variances.hed variances.%d.hed;" % iter cmd+="/bin/mv variances.img variances.%d.img;" % iter proc = subprocess.Popen(cmd, shell=True) proc.wait() # calculate the stddev for the apmq cc's for throwing out particles avgccrot/=self.params['numpart'] stdccrot = 0 for p in range(0,self.params['numpart']): stdccrot+=abs(float(apmqlist[p][3])-avgccrot) stdccrot/=self.params['numpart'] cccutoff=avgccrot+(stdccrot*self.params['keepsig']) apDisplay.printMsg("average cc: %f" %avgccrot) apDisplay.printMsg("setting cutoff to: %f" %cccutoff) # create new selection file that only has particles with good cc's selectfile="select%03d.spi" % iter apFile.removeFile(selectfile) mySpi = spyder.SpiderSession(nproc=self.params['proc'],dataext=".spi", logo=False, log=False) i=1 for p in range(0,self.params['numpart']): ccrot=float(apmqlist[p][3]) if ccrot>=cccutoff: mySpi.toSpiderQuiet("x11=%d" % (p+1)) mySpi.toSpiderQuiet("SD %d,x11" % i,spyder.fileFilter(selectfile)) i+=1 mySpi.close() # calculate the new 3d structure using centered projections # and the corrected angles from the angular doc file apDisplay.printMsg("creating 3d volume") out_rawvol="vol_raw%03d.spi" % iter if self.params['voliter'] is not None: refine.iterativeBackProjection( shiftedStack, selectfile, rad=self.params['rad'], ang=outang, out=out_rawvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['bpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc'], ) else: refine.backProjection( shiftedStack, selectfile, ang=outang, out=out_rawvol, sym=sydoc, nproc=self.params['proc'] ) # create even & odd select files apDisplay.printMsg("creating even/odd volumes") oddfile="selectodd%03d.spi" % iter evenfile="selecteven%03d.spi" % iter refine.docSplit(selectfile,oddfile,evenfile) # get the even & odd volumesa oddvol="vol1%03d.spi" % iter evenvol="vol2%03d.spi" % iter if self.params['voliter'] is not None: refine.iterativeBackProjection( shiftedStack, oddfile, rad=self.params['rad'], ang=outang, out=oddvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['eobpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc'] ) refine.iterativeBackProjection( shiftedStack, evenfile, rad=self.params['rad'], ang=outang, out=evenvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['eobpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc'] ) else: refine.backProjection( shiftedStack, oddfile, ang=outang, out=oddvol, sym=sydoc, nproc=self.params['proc'], ) refine.backProjection( shiftedStack, evenfile, ang=outang, out=evenvol, sym=sydoc, nproc=self.params['proc'], ) # calculate the FSC apDisplay.printMsg("calculating FSC") fscfile="fsc%03d.spi" % iter emanfsc="fsc.eotest.%d" % iter refine.calcFSC(oddvol,evenvol,fscfile) # convert to eman-style fscfile refine.spiderFSCtoEMAN(fscfile,emanfsc) # calculate the resolution at 0.5 FSC & write to file res=apRecon.calcRes(emanfsc,self.stack['boxsize'],self.stack['apix']) restxt="resolution.txt" if iter==1 and os.path.isfile(restxt): os.remove(restxt) resfile=open(restxt,"a") resfile.write("iter %d:\t%.3f\n" % (iter,res)) resfile.close() # filter & normalize the volume to be used as a reference in the next round outvol="vol%03d.spi" % iter emancmd="proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % (out_rawvol,outvol,self.stack['apix'],res,self.params['rad']) if self.params['imask'] is not None: emancmd+=" imask=%d" % self.params['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) # create mrc files of volumes emancmd="proc3d %s %s apix=%.3f mask=%d norm" % (out_rawvol,"threed.%da.mrc" % iter, self.stack['apix'],self.params['rad']) if self.params['imask'] is not None: emancmd+=" imask=%d" % self.params['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) emancmd="proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % (out_rawvol,"threed.%da.lp.mrc" % iter, self.stack['apix'],res, self.params['rad']) if self.params['imask'] is not None: emancmd+=" imask=%d" % self.param['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) # set this model as start for next iteration, remove previous os.remove(self.params['itervol']) os.remove(out_rawvol) self.params['itervol']=outvol # clean up directory apDisplay.printMsg("cleaning up directory") if os.path.isfile(oddvol): os.remove(oddvol) if os.path.isfile(evenvol): os.remove(evenvol) if os.path.isfile(ang): os.remove(ang) if os.path.isfile(sel): os.remove(sel) if os.path.isfile(projs): os.remove(projs) if os.path.isfile(oddfile): os.remove(oddfile) if os.path.isfile(evenfile): os.remove(evenfile) if os.path.isfile(emanfsc) and os.path.isfile(fscfile): os.remove(fscfile) if os.path.isfile(shiftedStack): os.remove(shiftedStack) os.remove(self.params['itervol'])
def insertIteration(self, iteration): refineparamsq = appiondata.ApEmanRefineIterData() refineparamsq['ang'] = iteration['ang'] refineparamsq['lpfilter'] = iteration['lpfilter'] refineparamsq['hpfilter'] = iteration['hpfilter'] refineparamsq['pad'] = iteration['pad'] refineparamsq['EMAN_maxshift'] = iteration['maxshift'] refineparamsq['EMAN_hard'] = iteration['hard'] refineparamsq['EMAN_classkeep'] = iteration['classkeep'] refineparamsq['EMAN_classiter'] = iteration['classiter'] refineparamsq['EMAN_filt3d'] = iteration['filt3d'] refineparamsq['EMAN_shrink'] = iteration['shrink'] refineparamsq['EMAN_euler2'] = iteration['euler2'] refineparamsq['EMAN_xfiles'] = iteration['xfiles'] refineparamsq['EMAN_median'] = iteration['median'] refineparamsq['EMAN_phasecls'] = iteration['phasecls'] refineparamsq['EMAN_fscls'] = iteration['fscls'] refineparamsq['EMAN_refine'] = iteration['refine'] refineparamsq['EMAN_goodbad'] = iteration['goodbad'] refineparamsq['EMAN_perturb'] = iteration['perturb'] refineparamsq['MsgP_cckeep'] = iteration['msgpasskeep'] refineparamsq['MsgP_minptls'] = iteration['msgpassminp'] #create Chimera snapshots fscfile = os.path.join(self.params['rundir'], "fsc.eotest." + iteration['num']) halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix']) if self.params['snapfilter']: halfres = self.params['snapfilter'] volumeDensity = 'threed.' + iteration['num'] + 'a.mrc' volDensPath = os.path.join(self.params['rundir'], volumeDensity) apChimera.filterAndChimera(volDensPath, halfres, self.params['apix'], self.params['boxsize'], 'snapshot', self.params['contour'], self.params['zoom'], sym=iteration['sym']['eman_name'], mass=self.params['mass']) ## uncommment this for chimera image only runs... if self.params['chimeraonly'] is True: return # insert resolution data if halfres != True: resData = self.getResolutionData(iteration) else: apDisplay.printWarning( "resolution reported as nan, not committing results to database" ) return if self.params['package'] == 'EMAN': refineclassavg = 'classes_eman.' + iteration['num'] + '.img' postrefineclassavg = None elif self.params['package'] == 'EMAN/SpiCoran': refineclassavg = 'classes_eman.' + iteration['num'] + '.img' postrefineclassavg = 'classes_coran.' + iteration['num'] + '.img' elif self.params['package'] == 'EMAN/MsgP': refineclassavg = 'classes_eman.' + iteration['num'] + '.img' postrefineclassavg = 'classes_msgp.' + iteration['num'] + '.img' else: apDisplay.printError("Refinement Package Not Valid") # insert refinement results refineq = appiondata.ApRefineIterData() refineq['refineRun'] = self.params['refineRun'] refineq['emanParams'] = refineparamsq refineq['iteration'] = iteration['num'] refineq['resolution'] = resData refineq['rMeasure'] = self.getRMeasureData(iteration) refineq['mask'] = iteration['mask'] refineq['imask'] = iteration['imask'] refineq['symmetry'] = iteration['sym'] refineq['exemplar'] = False classvar = 'classes.' + iteration['num'] + '.var.img' refineq['refineClassAverages'] = refineclassavg refineq['postRefineClassAverages'] = postrefineclassavg if classvar in self.params['classvars']: refineq['classVariance'] = classvar if volumeDensity in self.params['volumes']: refineq['volumeDensity'] = volumeDensity apDisplay.printMsg("inserting Refinement Data into database") if self.params['commit'] is True: refineq.insert() else: apDisplay.printWarning("not committing results to database") #insert FSC data fscfile = os.path.join(self.params['rundir'], "fsc.eotest." + iteration['num']) self.insertFSC(fscfile, refineq, self.params['commit']) halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix']) apDisplay.printColor("FSC 0.5 Resolution: " + str(halfres), "cyan") # get projections eulers for iteration: eulers = self.getEulersFromProj(iteration['num']) # get list of bad particles for this iteration badprtls = self.readParticleLog(self.params['rundir'], iteration['num']) # expand cls.*.tar into temp file clsf = os.path.join(self.params['rundir'], "cls." + iteration['num'] + ".tar") #print "reading",clsf clstar = tarfile.open(clsf) clslist = clstar.getmembers() clsnames = clstar.getnames() #print "extracting",clsf,"into temp directory" for clsfile in clslist: clstar.extract(clsfile, self.params['tmpdir']) clstar.close() # for each class, insert particle alignment info into database apDisplay.printColor( "Inserting Particle Classification Data for " + str(len(clsnames)) + " classes", "magenta") t0 = time.time() for cls in clsnames: self.insertRefineParticleData(cls, iteration, eulers, badprtls, refineq, len(clsnames)) apDisplay.printColor( "\nFinished in " + apDisplay.timeString(time.time() - t0), "magenta") # remove temp directory for file in os.listdir(self.params['tmpdir']): os.remove(os.path.join(self.params['tmpdir'], file)) os.rmdir(self.params['tmpdir']) #create euler freq map apDisplay.printMsg("creating euler frequency map") refrunid = int(self.params['refineRun'].dbid) iternum = int(iteration['num']) if self.params['package'] != 'EMAN': postrefine = True else: postrefine = False apEulerDraw.createEulerImages(refrunid, iternum, path=self.params['rundir'], postrefine=postrefine) return
def start(self): self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) # symmetry info if self.params['sym'] == 'Icos': self.params['symtype'] = 'I' self.params['symfold'] = None else: self.params['symtype'] = self.params['sym'][0] self.params['symfold'] = int(self.params['sym'][1:]) # eman "d" symmetry is spider "ci" if self.params['symtype'].upper() == "D": self.params['symtype'] = "CI" # create symmetry doc file sydoc = "sym.spi" refine.symmetryDoc(self.params['symtype'], self.params['symfold'], sydoc) # convert incr to array of increments ang_inc = self.params['incr'].split(',') self.params['numiter'] = len(ang_inc) self.params['increments'] = [] for i in range(0, self.params['numiter']): self.params['increments'].append(int(ang_inc[i])) # convert stack to spider stack spiderstack = os.path.join(self.params['rundir'], 'start.spi') operations.stackToSpiderStack(self.stack['file'], spiderstack, apix=self.stack['apix'], boxsize=self.stack['boxsize'], numpart=self.params['numpart']) # create filtered stack spiderstackfilt = spiderstack if (self.params['lowpass'] + self.params['highpass']) > 0: spiderstackfilt = os.path.join(self.params['rundir'], 'start_filt.spi') operations.stackToSpiderStack(self.stack['file'], spiderstackfilt, apix=self.stack['apix'], boxsize=self.stack['boxsize'], lp=self.params['lowpass'], hp=self.params['highpass'], numpart=self.params['numpart']) # rescale initial model if necessary outvol = os.path.join(self.params['rundir'], "vol000.spi") apModel.rescaleModel(self.params['modelid'], outvol, self.stack['boxsize'], self.stack['apix'], spider=True) self.params['itervol'] = outvol for iter in range(1, self.params['numiter'] + 1): # create projections for projection matching apDisplay.printMsg("creating reference projections of volume: %s" % self.params['itervol']) projs, numprojs, ang, sel = refine.createProjections( incr=self.params['increments'][iter - 1], boxsz=self.stack['boxsize'], symfold=self.params['symfold'], invol=self.params['itervol'], rad=self.params['rad'], ) # run reference-based alignment apDisplay.printMsg("running reference-based alignment (AP MQ)") apmqfile = "apmq%03d.spi" % iter outang = "angular%03d.spi" % iter shf = "shifts%03d.spi" % iter shiftedStack = "parts_shifted.spi" apFile.removeFile(shf) refine.spiderAPMQ( projs=projs, numprojs=numprojs, tsearch=self.params['xysearch'], tstep=self.params['xystep'], firstRing=self.params['firstring'], lastRing=self.params['lastring'], stackfile=spiderstackfilt, nump=self.params['numpart'], ang=ang, apmqfile=apmqfile, outang=outang, nproc=self.params['proc'], ) # use cross-correlation to find the sub-pixel alignment # of the particles, # results will be saved in "peakfile.spi" apFile.removeFile(shiftedStack) # don't use MPI here - for some reason slower? mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False) apmqlist = refine.readDocFile(apmqfile) avgccrot = 0 apDisplay.printMsg("creating shifted stack") for p in range(0, self.params['numpart']): ref = int(float(apmqlist[p][2])) ccrot = float(apmqlist[p][3]) inplane = float(apmqlist[p][4]) avgccrot += ccrot # invert the sign - ref projs will be rotated inplane *= -1 # get corresponding projection if (ref <= 0): # mirror projection if necessary ref *= -1 refimg = spyder.fileFilter(projs) + "@" + str(ref) refine.mirrorImg(refimg, "_3", inMySpi=mySpi) img = "_3" else: img = spyder.fileFilter(projs) + "@" + str(ref) refine.rotAndShiftImg(img, "_2", inplane, inMySpi=mySpi) refine.maskImg("_2", "_3", self.params['rad'], "D", "E", center=int((self.stack['boxsize'] / 2) + 1), inMySpi=mySpi) # pad ref image & stack image to twice the size refine.padImg("_3", "_2", 2 * self.stack['boxsize'], "N", 1, 1, 0, inMySpi=mySpi) stackimg = spyder.fileFilter(spiderstack) + "@" + str(p + 1) refine.padImg(stackimg, "_1", 2 * self.stack['boxsize'], "B", 1, 1, inMySpi=mySpi) # calculate cross-correlation refine.getCC("_1", "_2", "_1", inMySpi=mySpi) # crop the correllation image to allowable shift amount shift = int(self.params['allowedShift'] * self.stack['boxsize']) dim = 2 * shift + 1 topleftx = self.stack['boxsize'] - shift + 1 refine.windowImg("_1", "_2", dim, topleftx, topleftx, inMySpi=mySpi) # find the sub-pixel location of cc peak mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17", "_2", "0") # create new stack of shifted particles shpos = spyder.fileFilter(shiftedStack) + "@" + str(p + 1) mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN") mySpi.toSpiderQuiet("GP x17", "_2", str(shift + 1) + "," + str(shift + 1)) refine.copyImg(stackimg, shpos, inMySpi=mySpi) mySpi.toSpiderQuiet("ELSE") #mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16") mySpi.toSpiderQuiet("SH F", stackimg, shpos, "-x15,-x16") mySpi.toSpiderQuiet("ENDIF") # save shifts to file mySpi.toSpiderQuiet("SD " + str(p + 1) + ",x15,x16,x17", spyder.fileFilter(shf)) mySpi.toSpiderQuiet("SD E", spyder.fileFilter(shf)) mySpi.close() # create class average images refine.createClassAverages( shiftedStack, projs, apmqfile, numprojs, self.stack['boxsize'], shifted=True, ) # rename class averages & variacnes for iteration cmd = "/bin/mv classes.hed classes.%d.hed;" % iter cmd += "/bin/mv classes.img classes.%d.img;" % iter cmd += "/bin/mv variances.hed variances.%d.hed;" % iter cmd += "/bin/mv variances.img variances.%d.img;" % iter proc = subprocess.Popen(cmd, shell=True) proc.wait() # calculate the stddev for the apmq cc's for throwing out particles avgccrot /= self.params['numpart'] stdccrot = 0 for p in range(0, self.params['numpart']): stdccrot += abs(float(apmqlist[p][3]) - avgccrot) stdccrot /= self.params['numpart'] cccutoff = avgccrot + (stdccrot * self.params['keepsig']) apDisplay.printMsg("average cc: %f" % avgccrot) apDisplay.printMsg("setting cutoff to: %f" % cccutoff) # create new selection file that only has particles with good cc's selectfile = "select%03d.spi" % iter apFile.removeFile(selectfile) mySpi = spyder.SpiderSession(nproc=self.params['proc'], dataext=".spi", logo=False, log=False) i = 1 for p in range(0, self.params['numpart']): ccrot = float(apmqlist[p][3]) if ccrot >= cccutoff: mySpi.toSpiderQuiet("x11=%d" % (p + 1)) mySpi.toSpiderQuiet("SD %d,x11" % i, spyder.fileFilter(selectfile)) i += 1 mySpi.close() # calculate the new 3d structure using centered projections # and the corrected angles from the angular doc file apDisplay.printMsg("creating 3d volume") out_rawvol = "vol_raw%03d.spi" % iter if self.params['voliter'] is not None: refine.iterativeBackProjection( shiftedStack, selectfile, rad=self.params['rad'], ang=outang, out=out_rawvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['bpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc'], ) else: refine.backProjection(shiftedStack, selectfile, ang=outang, out=out_rawvol, sym=sydoc, nproc=self.params['proc']) # create even & odd select files apDisplay.printMsg("creating even/odd volumes") oddfile = "selectodd%03d.spi" % iter evenfile = "selecteven%03d.spi" % iter refine.docSplit(selectfile, oddfile, evenfile) # get the even & odd volumesa oddvol = "vol1%03d.spi" % iter evenvol = "vol2%03d.spi" % iter if self.params['voliter'] is not None: refine.iterativeBackProjection( shiftedStack, oddfile, rad=self.params['rad'], ang=outang, out=oddvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['eobpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc']) refine.iterativeBackProjection( shiftedStack, evenfile, rad=self.params['rad'], ang=outang, out=evenvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['eobpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc']) else: refine.backProjection( shiftedStack, oddfile, ang=outang, out=oddvol, sym=sydoc, nproc=self.params['proc'], ) refine.backProjection( shiftedStack, evenfile, ang=outang, out=evenvol, sym=sydoc, nproc=self.params['proc'], ) # calculate the FSC apDisplay.printMsg("calculating FSC") fscfile = "fsc%03d.spi" % iter emanfsc = "fsc.eotest.%d" % iter refine.calcFSC(oddvol, evenvol, fscfile) # convert to eman-style fscfile refine.spiderFSCtoEMAN(fscfile, emanfsc) # calculate the resolution at 0.5 FSC & write to file res = apRecon.calcRes(emanfsc, self.stack['boxsize'], self.stack['apix']) restxt = "resolution.txt" if iter == 1 and os.path.isfile(restxt): os.remove(restxt) resfile = open(restxt, "a") resfile.write("iter %d:\t%.3f\n" % (iter, res)) resfile.close() # filter & normalize the volume to be used as a reference in the next round outvol = "vol%03d.spi" % iter emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % ( out_rawvol, outvol, self.stack['apix'], res, self.params['rad']) if self.params['imask'] is not None: emancmd += " imask=%d" % self.params['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) # create mrc files of volumes emancmd = "proc3d %s %s apix=%.3f mask=%d norm" % ( out_rawvol, "threed.%da.mrc" % iter, self.stack['apix'], self.params['rad']) if self.params['imask'] is not None: emancmd += " imask=%d" % self.params['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % ( out_rawvol, "threed.%da.lp.mrc" % iter, self.stack['apix'], res, self.params['rad']) if self.params['imask'] is not None: emancmd += " imask=%d" % self.param['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) # set this model as start for next iteration, remove previous os.remove(self.params['itervol']) os.remove(out_rawvol) self.params['itervol'] = outvol # clean up directory apDisplay.printMsg("cleaning up directory") if os.path.isfile(oddvol): os.remove(oddvol) if os.path.isfile(evenvol): os.remove(evenvol) if os.path.isfile(ang): os.remove(ang) if os.path.isfile(sel): os.remove(sel) if os.path.isfile(projs): os.remove(projs) if os.path.isfile(oddfile): os.remove(oddfile) if os.path.isfile(evenfile): os.remove(evenfile) if os.path.isfile(emanfsc) and os.path.isfile(fscfile): os.remove(fscfile) if os.path.isfile(shiftedStack): os.remove(shiftedStack) os.remove(self.params['itervol'])
def insertIteration(self, iteration): refineparamsq=appiondata.ApEmanRefineIterData() refineparamsq['ang']=iteration['ang'] refineparamsq['lpfilter']=iteration['lpfilter'] refineparamsq['hpfilter']=iteration['hpfilter'] refineparamsq['pad']=iteration['pad'] refineparamsq['EMAN_maxshift']=iteration['maxshift'] refineparamsq['EMAN_hard']=iteration['hard'] refineparamsq['EMAN_classkeep']=iteration['classkeep'] refineparamsq['EMAN_classiter']=iteration['classiter'] refineparamsq['EMAN_filt3d']=iteration['filt3d'] refineparamsq['EMAN_shrink']=iteration['shrink'] refineparamsq['EMAN_euler2']=iteration['euler2'] refineparamsq['EMAN_xfiles']=iteration['xfiles'] refineparamsq['EMAN_median']=iteration['median'] refineparamsq['EMAN_phasecls']=iteration['phasecls'] refineparamsq['EMAN_fscls']=iteration['fscls'] refineparamsq['EMAN_refine']=iteration['refine'] refineparamsq['EMAN_goodbad']=iteration['goodbad'] refineparamsq['EMAN_perturb']=iteration['perturb'] refineparamsq['MsgP_cckeep']=iteration['msgpasskeep'] refineparamsq['MsgP_minptls']=iteration['msgpassminp'] #create Chimera snapshots fscfile = os.path.join(self.params['rundir'], "fsc.eotest."+iteration['num']) halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix']) if self.params['snapfilter']: halfres = self.params['snapfilter'] volumeDensity = 'threed.'+iteration['num']+'a.mrc' volDensPath = os.path.join(self.params['rundir'], volumeDensity) apChimera.filterAndChimera(volDensPath, halfres, self.params['apix'], self.params['boxsize'], 'snapshot', self.params['contour'], self.params['zoom'], sym=iteration['sym']['eman_name'], mass=self.params['mass']) ## uncommment this for chimera image only runs... if self.params['chimeraonly'] is True: return # insert resolution data if halfres != True: resData = self.getResolutionData(iteration) else: apDisplay.printWarning("resolution reported as nan, not committing results to database") return if self.params['package']== 'EMAN': refineclassavg='classes_eman.'+iteration['num']+'.img' postrefineclassavg=None elif self.params['package']== 'EMAN/SpiCoran': refineclassavg='classes_eman.'+iteration['num']+'.img' postrefineclassavg='classes_coran.'+iteration['num']+'.img' elif self.params['package']== 'EMAN/MsgP': refineclassavg='classes_eman.'+iteration['num']+'.img' postrefineclassavg='classes_msgp.'+iteration['num']+'.img' else: apDisplay.printError("Refinement Package Not Valid") # insert refinement results refineq = appiondata.ApRefineIterData() refineq['refineRun'] = self.params['refineRun'] refineq['emanParams'] = refineparamsq refineq['iteration'] = iteration['num'] refineq['resolution'] = resData refineq['rMeasure'] = self.getRMeasureData(iteration) refineq['mask'] = iteration['mask'] refineq['imask'] = iteration['imask'] refineq['symmetry']=iteration['sym'] refineq['exemplar'] = False classvar = 'classes.'+iteration['num']+'.var.img' refineq['refineClassAverages'] = refineclassavg refineq['postRefineClassAverages'] = postrefineclassavg if classvar in self.params['classvars']: refineq['classVariance'] = classvar if volumeDensity in self.params['volumes']: refineq['volumeDensity'] = volumeDensity apDisplay.printMsg("inserting Refinement Data into database") if self.params['commit'] is True: refineq.insert() else: apDisplay.printWarning("not committing results to database") #insert FSC data fscfile = os.path.join(self.params['rundir'], "fsc.eotest."+iteration['num']) self.insertFSC(fscfile, refineq, self.params['commit']) halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix']) apDisplay.printColor("FSC 0.5 Resolution: "+str(halfres), "cyan") # get projections eulers for iteration: eulers = self.getEulersFromProj(iteration['num']) # get list of bad particles for this iteration badprtls = self.readParticleLog(self.params['rundir'], iteration['num']) # expand cls.*.tar into temp file clsf=os.path.join(self.params['rundir'], "cls."+iteration['num']+".tar") #print "reading",clsf clstar=tarfile.open(clsf) clslist=clstar.getmembers() clsnames=clstar.getnames() #print "extracting",clsf,"into temp directory" for clsfile in clslist: clstar.extract(clsfile,self.params['tmpdir']) clstar.close() # for each class, insert particle alignment info into database apDisplay.printColor("Inserting Particle Classification Data for " +str(len(clsnames))+" classes", "magenta") t0 = time.time() for cls in clsnames: self.insertRefineParticleData(cls, iteration, eulers, badprtls, refineq, len(clsnames)) apDisplay.printColor("\nFinished in "+apDisplay.timeString(time.time()-t0), "magenta") # remove temp directory for file in os.listdir(self.params['tmpdir']): os.remove(os.path.join(self.params['tmpdir'],file)) os.rmdir(self.params['tmpdir']) #create euler freq map apDisplay.printMsg("creating euler frequency map") refrunid = int(self.params['refineRun'].dbid) iternum = int(iteration['num']) if self.params['package'] != 'EMAN': postrefine = True else: postrefine = False apEulerDraw.createEulerImages(refrunid, iternum, path=self.params['rundir'], postrefine=postrefine) return