def checkConflicts(self): if self.params["stackid"] is None: apDisplay.printError("stack id was not defined") if self.params["modelid"] is None: apDisplay.printError("model id was not defined") if self.params["runname"] is None: apDisplay.printError("missing a reconstruction name") if self.params["last"] is None: self.params["last"] = apStack.getNumberStackParticlesFromId(self.params["stackid"]) self.boxsize = apStack.getStackBoxsize(self.params["stackid"], msg=False) self.apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) if self.params["reconstackid"] is not None: reconboxsize = apStack.getStackBoxsize(self.params["reconstackid"], msg=False) reconapix = apStack.getStackPixelSizeFromStackId(self.params["reconstackid"]) refinenumpart = apStack.getNumberStackParticlesFromId(self.params["stackid"]) reconnumpart = apStack.getNumberStackParticlesFromId(self.params["reconstackid"]) if reconboxsize != self.boxsize: apDisplay.printError("Boxsize do not match for stacks") if reconapix != self.apix: apDisplay.printError("Pixelsize do not match for stacks") if refinenumpart != reconnumpart: apDisplay.printError("Number of particles do not match for stacks") maxmask = math.floor(self.apix * (self.boxsize - 10) / 2.0) if self.params["mask"] is None: apDisplay.printWarning("mask was not defined, setting to boxsize: %d" % (maxmask)) self.params["mask"] = maxmask if self.params["mask"] > maxmask: apDisplay.printWarning("mask was too big, setting to boxsize: %d" % (maxmask)) self.params["mask"] = maxmask if self.params["noctf"] is True: apDisplay.printWarning("Using no CTF method") self.params["wgh"] = -1.0 if self.params["nodes"] > 1 and self.params["cluster"] is False: apDisplay.printError("cluster mode must be enabled to run on more than 1 node") if self.params["ppn"] is None: if self.params["cluster"] is True: apDisplay.printError("you must define ppn for cluster mode") self.params["ppn"] = apParam.getNumProcessors() apDisplay.printMsg("Setting number of processors to %d" % (self.params["ppn"])) if self.params["rpn"] is None: self.params["rpn"] = self.params["ppn"] self.params["nproc"] = self.params["nodes"] * self.params["rpn"] ### get the symmetry data if self.params["sym"] is None: apDisplay.printError("Symmetry was not defined") else: self.symmdata = apSymmetry.findSymmetry(self.params["sym"]) self.params["symm_id"] = self.symmdata.dbid self.params["symm_name"] = self.symmdata["eman_name"] apDisplay.printMsg("Selected symmetry %s with id %s" % (self.symmdata["eman_name"], self.symmdata.dbid)) ### set cs value self.params["cs"] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['modelid'] is None: apDisplay.printError("model id was not defined") if self.params['runname'] is None: apDisplay.printError("missing a reconstruction name") if self.params['last'] is None: self.params['last'] = apStack.getNumberStackParticlesFromId(self.params['stackid']) self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False) self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) if self.params['reconstackid'] is not None: reconboxsize = apStack.getStackBoxsize(self.params['reconstackid'], msg=False) reconapix = apStack.getStackPixelSizeFromStackId(self.params['reconstackid']) refinenumpart = apStack.getNumberStackParticlesFromId(self.params['stackid']) reconnumpart = apStack.getNumberStackParticlesFromId(self.params['reconstackid']) if reconboxsize != self.boxsize: apDisplay.printError("Boxsize do not match for stacks") if reconapix != self.apix: apDisplay.printError("Pixelsize do not match for stacks") if refinenumpart != reconnumpart: apDisplay.printError("Number of particles do not match for stacks") maxmask = math.floor(self.apix*(self.boxsize-10)/2.0) if self.params['mask'] is None: apDisplay.printWarning("mask was not defined, setting to boxsize: %d"%(maxmask)) self.params['mask'] = maxmask if self.params['mask'] > maxmask: apDisplay.printWarning("mask was too big, setting to boxsize: %d"%(maxmask)) self.params['mask'] = maxmask if self.params['noctf'] is True: apDisplay.printWarning("Using no CTF method") self.params['wgh'] = -1.0 if self.params['nodes'] > 1 and self.params['cluster'] is False: apDisplay.printError("cluster mode must be enabled to run on more than 1 node") if self.params['ppn'] is None: if self.params['cluster'] is True: apDisplay.printError("you must define ppn for cluster mode") self.params['ppn'] = apParam.getNumProcessors() apDisplay.printMsg("Setting number of processors to %d"%(self.params['ppn'])) if self.params['rpn'] is None: self.params['rpn'] = self.params['ppn'] self.params['nproc'] = self.params['nodes']*self.params['rpn'] ### get the symmetry data if self.params['sym'] is None: apDisplay.printError("Symmetry was not defined") else: self.symmdata = apSymmetry.findSymmetry(self.params['sym']) self.params['symm_id'] = self.symmdata.dbid self.params['symm_name'] = self.symmdata['eman_name'] apDisplay.printMsg("Selected symmetry %s with id %s"%(self.symmdata['eman_name'], self.symmdata.dbid)) ### set cs value self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
def medianVolume(self): volpath = os.path.join(self.params['rundir'], "volumes/*a.mrc") mrcfiles = glob.glob(volpath) volumes = [] for filename in mrcfiles: if os.path.isfile(filename): vol = mrc.read(filename) print filename, vol.shape volumes.append(vol) volarray = numpy.asarray(volumes, dtype=numpy.float32) try: medarray = numpy.median(volarray, axis=0) except: medarray = numpy.median(volarray) medfile = os.path.join(self.params['rundir'], "volumes/medianVolume.mrc") print medfile, medarray.shape mrc.write(medarray, medfile) apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) uploadcmd = ( ("uploadModel.py --projectid=%d --session=%s --file=%s " +"--apix=%.3f --sym=%s --name=satmedian-recon%d.mrc --res=30 --description='%s %d'") %(self.params['projectid'], sessiondata['name'], medfile, apix, self.params['symmname'], self.params['reconid'], "SAT selected median volume for recon", self.params['reconid'], ) ) apDisplay.printColor(uploadcmd, "purple") f = open("upload.sh", "w") f.write(uploadcmd+"\n") f.close()
def setupParticles(self): self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed") self.stack = {} self.stack["data"] = apStack.getOnlyStackData(self.params["stackid"]) self.stack["apix"] = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) self.stack["part"] = apStack.getOneParticleFromStackId(self.params["stackid"]) self.stack["boxsize"] = apStack.getStackBoxsize(self.params["stackid"]) self.stack["file"] = os.path.join(self.stack["data"]["path"]["path"], self.stack["data"]["name"]) boxsize = int(math.floor(self.stack["boxsize"] / float(self.params["bin"] * 2))) * 2 apix = self.stack["apix"] * self.params["bin"] proccmd = "proc2d " + self.stack["file"] + " " + self.params["localstack"] + " apix=" + str(self.stack["apix"]) if self.params["bin"] > 1: clipsize = boxsize * self.params["bin"] proccmd += " shrink=%d clip=%d,%d " % (self.params["bin"], clipsize, clipsize) if self.params["highpass"] > 1: proccmd += " hp=" + str(self.params["highpass"]) if self.params["lowpass"] > 1: proccmd += " lp=" + str(self.params["lowpass"]) proccmd += " last=" + str(self.params["numpart"] - 1) apFile.removeStack(self.params["localstack"], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"]) return (boxsize, apix)
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("Please provide a user id, e.g. --stackid=15") if not 'radonshift' in dir(radermacher): apDisplay.printError("Radermacher python module is too old, please update") boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False) pixelsize = apStack.getStackPixelSizeFromStackId(self.params['stackid'], msg=False) numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'], msg=False) apDisplay.printMsg("\n\n") ### get info about the stack apDisplay.printMsg("Information about stack id %d"%(self.params['stackid'])) apDisplay.printMsg("\tboxsize %d pixels"%(boxsize)) apDisplay.printMsg("\tpixelsize %.3f Angstroms"%(pixelsize)) apDisplay.printMsg("\tsize %d particles"%(numpart)) if self.params['maskrad'] is None: self.params['pixelmaskrad'] = boxsize/2 self.params['maskrad'] = self.params['pixelmaskrad']*pixelsize else: self.params['pixelmaskrad'] = self.params['maskrad']/pixelsize if self.params['pixelmaskrad'] > boxsize/2: apDisplay.printError("Mask radius larger than boxsize") if self.params['numpart'] is None: self.params['numpart'] = numpart if self.params['numpart'] > numpart: apDisplay.printError("Requested more particles than available in stack") if self.params['numrefs'] is None: ### num refs should go approx. as the sqrt of the number of particles self.params['numrefs'] = int(math.ceil(math.sqrt(self.params['numpart'])))+1
def start(self): """ this is the main component of the script where all the processing is done """ ### initialize some variables self.runq = None self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) apDisplay.printMsg("Pixel size: %.5f"%(self.apix)) self.boxsize = apStack.getStackBoxsize(self.params['stackid']) apDisplay.printMsg("Box size: %d"%(self.boxsize)) self.checkResults() self.stackmapping = apRecon.partnum2defid(self.params['stackid']) self.numiter = self.getNumberOfIterations() for i in range(self.numiter): iternum = i+1 apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green") self.uploadIteration(iternum) reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir']) if reconrunid: apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid)) eulerjump = apEulerJump.ApEulerJump() eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid']) apRecon.setGoodBadParticlesFromReconId(reconrunid) else: apDisplay.printWarning("Could not find recon run id")
def runRmeasure(self, volfile): emancmd = "proc3d %s %s"%(volfile, "rmeasure.mrc") apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc") #apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan") apFile.removeFile("rmeasure.mrc")
def medianVolume(self): volpath = os.path.join(self.params['rundir'], "volumes/*a.mrc") mrcfiles = glob.glob(volpath) volumes = [] for filename in mrcfiles: if os.path.isfile(filename): vol = mrc.read(filename) print filename, vol.shape volumes.append(vol) volarray = numpy.asarray(volumes, dtype=numpy.float32) try: medarray = numpy.median(volarray, axis=0) except: medarray = numpy.median(volarray) medfile = os.path.join(self.params['rundir'], "volumes/medianVolume.mrc") print medfile, medarray.shape mrc.write(medarray, medfile) apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) uploadcmd = ( ("uploadModel.py --projectid=%d --session=%s --file=%s " +"--apix=%.3f --sym=%s --name=satmedian-recon%d.mrc --res=30 --description='%s %d'") %(self.params['projectid'], sessiondata['name'], medfile, apix, self.params['symmname'], self.params['reconid'], "SAT selected median volume for recon", self.params['reconid'], ) ) apDisplay.printColor(uploadcmd, "purple") f = open("upload.sh", "w") f.write(uploadcmd+"\n") f.close()
def start(self): self.runtime = 0 self.partlist = [] self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.checkNoRefRun() ### convert stack to spider spiderstack = self.createSpiderFile() ### create initialization template if self.params['initmethod'] == 'allaverage': templatefile = self.averageTemplate() elif self.params['initmethod'] == 'selectrand': templatefile = self.selectRandomParticles() elif self.params['initmethod'] == 'randpart': templatefile = self.pickRandomParticle() elif self.params['initmethod'] == 'template': templatefile = self.getTemplate() else: apDisplay.printError("unknown initialization method defined: " +str(self.params['initmethod'])+" not in "+str(self.initmethods)) apDisplay.printColor("Running spider this can take awhile","cyan") ### run the alignment aligntime = time.time() pixrad = int(round(self.params['partrad']/self.stack['apix']/self.params['bin'])) alignedstack, self.partlist = alignment.refFreeAlignParticles( spiderstack, templatefile, self.params['numpart'], pixrad, self.params['firstring'], self.params['lastring'], rundir = ".") aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ### remove large, worthless stack spiderstack = os.path.join(self.params['rundir'], "start.spi") apDisplay.printMsg("Removing un-aligned stack: "+spiderstack) apFile.removeFile(spiderstack, warn=False) ### convert stack to imagic imagicstack = self.convertSpiderStack(alignedstack) apFile.removeFile(alignedstack) inserttime = time.time() if self.params['commit'] is True: self.runtime = aligntime self.insertNoRefRun(imagicstack, insert=True) else: apDisplay.printWarning("not committing results to DB") inserttime = time.time() - inserttime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def setupParticles(self): self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2 apix = self.stack['apix']*self.params['bin'] proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix']) if self.params['bin'] > 1: clipsize = boxsize*self.params['bin'] proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize) if self.params['highpass'] > 1: proccmd += " hp="+str(self.params['highpass']) if self.params['lowpass'] > 1: proccmd += " lp="+str(self.params['lowpass']) proccmd += " last="+str(self.params['numpart']-1) apFile.removeStack(self.params['localstack'], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) return (boxsize, apix)
def runMaxlike(self): stackdata = apStack.getOnlyStackData(self.params['stackid']) apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) ### process stack to local file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp + ".hed") proccmd = "proc2d " + stackfile + " " + self.params[ 'localstack'] + " apix=" + str(apix) if self.params['highpass'] > 1: proccmd += " hp=" + str(self.params['highpass']) if self.params['lowpass'] > 1: proccmd += " lp=" + str(self.params['lowpass']) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles( self.params['localstack']) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params['rundir'], self.partlistdocfile) + " -nref 1 " + " -iter 10 " + " -o " + os.path.join(self.params['rundir'], "part" + self.timestamp) + " -fast -C 1e-18 ") ### angle step if self.params['rotate'] is True: xmippopts += " -psi_step 90 " else: xmippopts += " -psi_step 360 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " if self.params['maxshift'] is not None: xmippopts += " -max_shift %d " % (self.params['maxshift']) ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe + " " + xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ### create a quick mrc emancmd = "proc2d part" + self.timestamp + "_ref000001.xmp average.mrc" apEMAN.executeEmanCmd(emancmd, verbose=True) apFile.removeStack(self.params['localstack']) apFile.removeFilePattern("partfiles/*")
def insertAlignStackRunIntoDatabase(self, alignimagicfile): apDisplay.printMsg("Inserting CL2D Run into DB") ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.runparams['runname'] alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) # if uniquerun: # apDisplay.printError("Run name '"+self.runparams['runname']+"' and path already exist in database") ### setup cl2d run cl2dq = appiondata.ApCL2DRunData() cl2dq['runname'] = self.runparams['runname'] cl2dq['run_seconds'] = self.runparams['runtime'] cl2dq['fast'] = self.runparams['fast'] self.cl2dqdata=cl2dq ### finish alignment run alignrunq['cl2drun'] = cl2dq alignrunq['hidden'] = False alignrunq['runname'] = self.runparams['runname'] alignrunq['description'] = self.runparams['description'] alignrunq['lp_filt'] = self.runparams['lowpass'] alignrunq['hp_filt'] = self.runparams['highpass'] alignrunq['bin'] = self.runparams['bin'] ### setup alignment stack alignstackq = appiondata.ApAlignStackData() if self.runparams['align'] is True: ### option to create aligned stack alignstackq['imagicfile'] = alignimagicfile alignstackq['avgmrcfile'] = "average.mrc" alignstackq['refstackfile'] = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1) alignstackq['iteration'] = self.runparams['maxiter'] alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) alignstackq['alignrun'] = alignrunq ### check to make sure files exist if self.runparams['align'] is True: ### option to create aligned stack alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(alignimagicfilepath): apDisplay.printError("could not find stack file: "+alignimagicfilepath) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: "+avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: "+refstackfile) alignstackq['stack'] = apStack.getOnlyStackData(self.runparams['stackid']) alignstackq['boxsize'] = self.boxsize alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin'] alignstackq['description'] = self.runparams['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.runparams['numpart'] ### insert if self.params['commit'] is True: alignstackq.insert() self.alignstackdata = alignstackq return
def getBeamTiltFromStackParticleNumber(self): stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum+1, msg=True) pdata = stackparticledata['particle'] ### step 1: get axial beam tilt on to image coordinate direction imageshift = pdata['image']['scope']['image shift'] zerobeamtilt = {'x':0.0e-3, 'y':0.0e-3} axialbeamtilt = self.transformImageShiftToBeamTilt(imageshift, self.tem, self.cam, self.ht, zerobeamtilt, self.mag) beamtilt_rotationtransform = self.retrieveBeamTiltMatrix(self.tem, self.cam, self.ht, self.mag) axialbtvector = numpy.array((axialbeamtilt['y'],axialbeamtilt['x'])) axialbtvector_on_image = numpy.dot(beamtilt_rotationtransform, axialbtvector) apDisplay.printMsg( "Axial Beam Tilt on Image Coordinate ( %5.2f, %5.2f)" % (axialbtvector_on_image[1]*1e3,axialbtvector_on_image[0]*1e3)) ### step 2: get off-axis beam tilt pcoord = {'x':pdata['xcoord']*self.imgpix,'y':pdata['ycoord']*self.imgpix} apix = apStack.getStackPixelSizeFromStackId(self.stackid) camdata = pdata['image']['camera'] cambin = camdata['binning'] camdim = camdata['dimension'] camoff = camdata['offset'] camcenter = {'x':self.camsize['x']/2,'y':self.camsize['y']/2} apDisplay.printMsg( 'Particle coord (%5d,%5d)' % (pdata['xcoord'],pdata['ycoord'])) mcoord = {'y':pdata['ycoord']*cambin['y']+camoff['y']-camcenter['y'], 'x':pdata['xcoord']*cambin['x']+camoff['x']-camcenter['x']} apDisplay.printMsg( 'Particle from center in camera pixel (%5d,%5d)' % (mcoord['x'],mcoord['y'])) coord = {'x':mcoord['x']*self.imgpix*cambin['x'], 'y':mcoord['y']*self.imgpix*cambin['y']} apDisplay.printMsg( 'Particle from center (A) (%5.2f,%5.2f)' % (coord['x']*1e10,coord['y']*1e10)) offaxisbeamtilt = self.getCombinedOffAxisBeamTilt(self.c2diameter,self.beamdiameter,coord) #offaxisbeamtilt = {'x':0.0,'y':0.0} ### step 3: combine beam tilts totalbeamtilt = {'x':axialbtvector_on_image[1]+offaxisbeamtilt['x'], 'y':axialbtvector_on_image[0]-offaxisbeamtilt['y'] } apDisplay.printMsg( "Total Beam Tilt on Image Coordinate (mrad) ( %5.2f, %5.2f)" % (totalbeamtilt['x']*1e3,totalbeamtilt['y']*1e3)) return totalbeamtilt
def start(self): ### load parameters runparams = self.readRunParameters() ### align references self.alignReferences(runparams) ### create an aligned stack self.createAlignedReferenceStack() ### read particles self.lastiter = self.findLastIterNumber() if self.params['sort'] is True: self.sortFolder() reflist = self.readRefDocFile() partlist = self.readPartDocFile(reflist) self.writePartDocFile(partlist) ### create aligned stacks alignimagicfile = self.createAlignedStacks(partlist, runparams['localstack']) apStack.averageStack(alignimagicfile) ### calculate resolution for each reference apix = apStack.getStackPixelSizeFromStackId(runparams['stackid'])*runparams['bin'] self.calcResolution(partlist, alignimagicfile, apix) ### insert into databse self.insertRunIntoDatabase(alignimagicfile, runparams) self.insertParticlesIntoDatabase(runparams['stackid'], partlist) apFile.removeStack(runparams['localstack'], warn=False) rmcmd = "/bin/rm -fr partfiles/*" apEMAN.executeEmanCmd(rmcmd, verbose=False, showcmd=False) apFile.removeFilePattern("partfiles/*")
def setupParticles(self): self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2 apix = self.stack['apix']*self.params['bin'] proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix']) if self.params['bin'] > 1: clipsize = boxsize*self.params['bin'] proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize) if self.params['highpass'] > 1: proccmd += " hp="+str(self.params['highpass']) if self.params['lowpass'] > 1: proccmd += " lp="+str(self.params['lowpass']) proccmd += " last="+str(self.params['numpart']-1) apFile.removeStack(self.params['localstack'], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) return (boxsize, apix)
def insertAlignStackRunIntoDatabase(self, alignimagicfile): apDisplay.printMsg("Inserting CL2D Run into DB") ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.runparams['runname'] alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) # if uniquerun: # apDisplay.printError("Run name '"+self.runparams['runname']+"' and path already exist in database") ### setup cl2d run cl2dq = appiondata.ApCL2DRunData() cl2dq['runname'] = self.runparams['runname'] cl2dq['run_seconds'] = self.runparams['runtime'] cl2dq['fast'] = self.runparams['fast'] self.cl2dqdata=cl2dq ### finish alignment run alignrunq['cl2drun'] = cl2dq alignrunq['hidden'] = False alignrunq['runname'] = self.runparams['runname'] alignrunq['description'] = self.runparams['description'] alignrunq['lp_filt'] = self.runparams['lowpass'] alignrunq['hp_filt'] = self.runparams['highpass'] alignrunq['bin'] = self.runparams['bin'] ### setup alignment stack alignstackq = appiondata.ApAlignStackData() if self.runparams['align'] is True: ### option to create aligned stack alignstackq['imagicfile'] = alignimagicfile alignstackq['avgmrcfile'] = "average.mrc" alignstackq['refstackfile'] = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1) alignstackq['iteration'] = self.runparams['maxiter'] alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) alignstackq['alignrun'] = alignrunq ### check to make sure files exist if self.runparams['align'] is True: ### option to create aligned stack alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(alignimagicfilepath): apDisplay.printError("could not find stack file: "+alignimagicfilepath) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: "+avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: "+refstackfile) alignstackq['stack'] = apStack.getOnlyStackData(self.runparams['stackid']) alignstackq['boxsize'] = self.boxsize alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin'] alignstackq['description'] = self.runparams['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.runparams['numpart'] ### insert if self.params['commit'] is True: alignstackq.insert() self.alignstackdata = alignstackq return
def convertStackToSpider(self, emanstackfile, classnum): """ takes the stack file and creates a spider file ready for processing """ if not os.path.isfile(emanstackfile): apDisplay.printError("stackfile does not exist: "+emanstackfile) ### first high pass filter particles apDisplay.printMsg("pre-filtering particles") apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid']) emancmd = ("proc2d "+emanstackfile+" "+emanstackfile +" apix="+str(apix)+" hp="+str(self.params['highpasspart']) +" inplace") apEMAN.executeEmanCmd(emancmd, verbose=True) ### convert imagic stack to spider emancmd = "proc2d " emancmd += emanstackfile+" " spiderstack = os.path.join(self.params['rundir'], str(classnum), "otrstack"+self.timestamp+".spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack+" " emancmd += "spiderswap edgenorm" starttime = time.time() apDisplay.printColor("Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan") return spiderstack
def start(self): """ this is the main component of the script where all the processing is done """ ### initialize some variables self.runq = None self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) apDisplay.printMsg("Pixel size: %.5f"%(self.apix)) self.boxsize = apStack.getStackBoxsize(self.params['stackid']) apDisplay.printMsg("Box size: %d"%(self.boxsize)) self.checkResults() self.stackmapping = apRecon.partnum2defid(self.params['stackid']) self.numiter = self.getNumberOfIterations() for i in range(self.numiter): iternum = i+1 apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green") self.uploadIteration(iternum) reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir']) if reconrunid: apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid)) eulerjump = apEulerJump.ApEulerJump() eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid']) apRecon.setGoodBadParticlesFromReconId(reconrunid) else: apDisplay.printWarning("Could not find recon run id")
def runEoTest(self, corrSelectOdd, corrSelectEven, cnum, apshstack, apsheuler, iternum): apshOddVolfile = os.path.join(self.params['rundir'], str(cnum), "apshVolume_Odd-%03d.spi"%(iternum)) apshEvenVolfile = os.path.join(self.params['rundir'], str(cnum), "apshVolume_Even-%03d.spi"%(iternum)) self.APSHbackProject(apshstack, apsheuler, apshOddVolfile, cnum, corrSelectOdd) self.APSHbackProject(apshstack, apsheuler, apshEvenVolfile, cnum, corrSelectEven) fscout = os.path.join(self.params['rundir'], str(cnum), "FSCout-%03d.spi"%(iternum)) backproject.calcFSC(apshOddVolfile, apshEvenVolfile, fscout) ### Calculate FSC - taken from Neil's RCT script apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] emancmd = "proc3d %s %s"%(apshEvenVolfile, apshEvenVolfile+".mrc") apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) emancmd = "proc3d %s %s"%(apshOddVolfile, apshOddVolfile+".mrc") apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) fscfile = os.path.join(self.params['rundir'], "fscdata%s.fsc"%(self.timestamp)) emancmd = "proc3d %s %s fsc=%s"%(apshEvenVolfile+".mrc", apshOddVolfile+".mrc", fscfile) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) if not os.path.isfile(fscfile): apDisplay.printError("Even-Odd fsc calculation failed") boxsize = self.getBoxSize() self.fscresolution = apRecon.getResolutionFromFSCFile(fscfile, boxsize, apix, msg=True) apDisplay.printColor( ("Final FSC resolution: %.5f" % (self.fscresolution)), "cyan") return fscout
def checkConflicts(self): if self.params['runname'] is None: apDisplay.printError("enter a stack run name, e.g. combinestack1") if self.params['description'] is None: apDisplay.printError("enter a stack description") if self.params['stacks'] and ',' in self.params['stacks']: #remember stackids are a list of strings stackids = self.params['stacks'].split(',') self.params['stackids'] = stackids else: apDisplay.printError("enter a list of stack ids to combine, e.g. --stackids=11,14,7") ### check to make sure all pixel and box size are the same self.newboxsize = None self.newpixelsize = None for stackidstr in self.params['stackids']: if not re.match("^[0-9]+$", stackidstr): apDisplay.printError("Stack id '%s' is not an integer"%(stackidstr)) stackid = int(stackidstr) boxsize = apStack.getStackBoxsize(stackid, msg=False) pixelsize = apStack.getStackPixelSizeFromStackId(stackid, msg=False) apDisplay.printMsg("Stack id: %d\tBoxsize: %d\tPixelsize: %.3f"%(stackid, boxsize, pixelsize)) if self.newboxsize is None: self.newboxsize = boxsize if self.newpixelsize is None: self.newpixelsize = pixelsize if boxsize != self.newboxsize: apDisplay.printError("Trying to combine stacks with different box sizes") if abs(pixelsize - self.newpixelsize) > 0.01: apDisplay.printError("Trying to combine stacks with different pixel sizes")
def runMaxlike(self): stackdata = apStack.getOnlyStackData(self.params["stackid"]) apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"]) ### process stack to local file self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed") proccmd = "proc2d " + stackfile + " " + self.params["localstack"] + " apix=" + str(apix) if self.params["highpass"] > 1: proccmd += " hp=" + str(self.params["highpass"]) if self.params["lowpass"] > 1: proccmd += " lp=" + str(self.params["lowpass"]) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"]) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params["rundir"], self.partlistdocfile) + " -nref 1 " + " -iter 10 " + " -o " + os.path.join(self.params["rundir"], "part" + self.timestamp) + " -fast -C 1e-18 " ) ### angle step if self.params["rotate"] is True: xmippopts += " -psi_step 90 " else: xmippopts += " -psi_step 360 " ### convergence criteria if self.params["converge"] == "fast": xmippopts += " -eps 5e-3 " elif self.params["converge"] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params["mirror"] is True: xmippopts += " -mirror " if self.params["maxshift"] is not None: xmippopts += " -max_shift %d " % (self.params["maxshift"]) ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe + " " + xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ### create a quick mrc emancmd = "proc2d part" + self.timestamp + "_ref000001.xmp average.mrc" apEMAN.executeEmanCmd(emancmd, verbose=True) apFile.removeStack(self.params["localstack"]) apFile.removeFilePattern("partfiles/*")
def start(self): ### load parameters self.runparams = self.readRunParameters() self.apix = apStack.getStackPixelSizeFromStackId( self.runparams['stackid']) * self.runparams['bin'] self.Nlevels = len( glob.glob("part" + self.params['timestamp'] + "_level_??_.hed")) ### create average of aligned stacks & insert aligned stack info lastLevelStack = "part" + self.params[ 'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1) self.boxsize = apFile.getBoxSize(lastLevelStack)[0] if self.runparams['align']: self.insertAlignStackRunIntoDatabase("alignedStack.hed") self.calcResolution(self.Nlevels - 1) self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1) ### loop over each class average stack & insert as clustering stacks self.insertClusterRunIntoDatabase() for level in range(self.Nlevels): if self.runparams['align']: self.calcResolution(level) partdict = self.getClassificationAtLevel(level) for classnum in partdict: self.insertClusterStackIntoDatabase( "part" + self.params['timestamp'] + "_level_%02d_.hed" % level, classnum + 1, partdict[classnum], len(partdict))
def start(self): """ This is the core of your function. """ stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) original_stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) filtered_stackfile = os.path.join(self.params['rundir'], self.timestamp+".hed") apFile.removeStack(filtered_stackfile, warn=False) apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) boxsize = apStack.getStackBoxsize(self.params['stackid']) emancmd = "proc2d %s %s apix=%.3f "%(original_stackfile, filtered_stackfile, apix) if self.params['lowpass'] is not None: emancmd += " lp=%.3f "%(self.params['lowpass']) if self.params['highpass'] is not None: emancmd += " hp=%.3f "%(self.params['highpass']) if self.params['bin'] is not None and self.params['bin'] > 1: ## determine a multiple of the bin that is divisible by 2 and less than the boxsize clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2*self.params['bin'] emancmd += " shrink=%d clip=%d,%d "%(self.params['bin'], clipsize, clipsize) emancmd += " last=%d "%(self.params['numpart']-1) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) ### confirm that it worked if self.params['numpart'] != apFile.numImagesInStack(filtered_stackfile): apDisplay.printError("Missing particles in stack") ### run the radon transform code self.radonAlign(filtered_stackfile) ### insert info into database self.commitToDatabase()
def runRmeasure(self): finalrawvolfile = os.path.join(self.params['rundir'], "rawvolume%s-%03d.spi"%(self.timestamp, self.params['numiters'])) emancmd = "proc3d %s %s"%(finalrawvolfile, "rmeasure.mrc") apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc") #apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan") apFile.removeFile("rmeasure.mrc")
def insertOtrRun(self, volfile): ### setup resolutions fscresq = appiondata.ApResolutionData() fscresq["type"] = "fsc" fscresq["half"] = self.fscresolution fscresq["fscfile"] = "fscdata" + self.timestamp + ".fsc" rmeasureq = appiondata.ApResolutionData() rmeasureq["type"] = "rmeasure" rmeasureq["half"] = self.rmeasureresolution rmeasureq["fscfile"] = None ### insert rct run data otrrunq = appiondata.ApOtrRunData() otrrunq["runname"] = self.params["runname"] tempstr = "" for cnum in self.classlist: tempstr += str(cnum) + "," classliststr = tempstr[:-1] otrrunq["classnums"] = classliststr otrrunq["numiter"] = self.params["numiters"] otrrunq["euleriter"] = self.params["refineiters"] otrrunq["maskrad"] = self.params["radius"] otrrunq["lowpassvol"] = self.params["lowpassvol"] otrrunq["highpasspart"] = self.params["highpasspart"] otrrunq["median"] = self.params["median"] otrrunq["description"] = self.params["description"] otrrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"])) otrrunq["alignstack"] = self.alignstackdata otrrunq["tiltstack"] = apStack.getOnlyStackData(self.params["tiltstackid"]) otrrunq["numpart"] = self.numpart otrrunq["fsc_resolution"] = fscresq otrrunq["rmeasure_resolution"] = rmeasureq if self.params["commit"] is True: otrrunq.insert() ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq["otrrun"] = otrrunq densq["path"] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq["name"] = os.path.basename(volfile) densq["hidden"] = False densq["norm"] = True densq["symmetry"] = appiondata.ApSymmetryData.direct_query(25) densq["pixelsize"] = apStack.getStackPixelSizeFromStackId(self.params["partstackid"]) * self.params["bin"] densq["boxsize"] = self.getBoxSize() densq["lowpass"] = self.params["lowpassvol"] densq["highpass"] = self.params["highpasspart"] densq["mask"] = self.params["radius"] # densq['iterid'] = self.params['numiters'] densq["description"] = self.params["description"] densq["resolution"] = self.fscresolution densq["rmeasure"] = self.rmeasureresolution densq["session"] = apStack.getSessionDataFromStackId(self.params["tiltstackid"]) densq["md5sum"] = apFile.md5sumfile(volfile) if self.params["commit"] is True: densq.insert() return
def insertOtrRun(self, volfile): ### setup resolutions fscresq = appiondata.ApResolutionData() fscresq['type'] = "fsc" fscresq['half'] = self.fscresolution fscresq['fscfile'] = "fscdata"+self.timestamp+".fsc" rmeasureq = appiondata.ApResolutionData() rmeasureq['type'] = "rmeasure" rmeasureq['half'] = self.rmeasureresolution rmeasureq['fscfile'] = None ### insert rct run data otrrunq = appiondata.ApOtrRunData() otrrunq['runname'] = self.params['runname'] tempstr = "" for cnum in self.classlist: tempstr += str(cnum)+"," classliststr = tempstr[:-1] otrrunq['classnums'] = classliststr otrrunq['numiter'] = self.params['numiters'] otrrunq['euleriter'] = self.params['refineiters'] otrrunq['maskrad'] = self.params['radius'] otrrunq['lowpassvol'] = self.params['lowpassvol'] otrrunq['highpasspart'] = self.params['highpasspart'] otrrunq['median'] = self.params['median'] otrrunq['description'] = self.params['description'] otrrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) otrrunq['alignstack'] = self.alignstackdata otrrunq['tiltstack'] = apStack.getOnlyStackData(self.params['tiltstackid']) otrrunq['numpart'] = self.numpart otrrunq['fsc_resolution'] = fscresq otrrunq['rmeasure_resolution'] = rmeasureq if self.params['commit'] is True: otrrunq.insert() ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['otrrun'] = otrrunq densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25) densq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] densq['boxsize'] = self.getBoxSize() densq['lowpass'] = self.params['lowpassvol'] densq['highpass'] = self.params['highpasspart'] densq['mask'] = self.params['radius'] #densq['iterid'] = self.params['numiters'] densq['description'] = self.params['description'] densq['resolution'] = self.fscresolution densq['rmeasure'] = self.rmeasureresolution densq['session'] = apStack.getSessionDataFromStackId(self.params['tiltstackid']) densq['md5sum'] = apFile.md5sumfile(volfile) if self.params['commit'] is True: densq.insert() return
def insertRunIntoDatabase(self, alignedPartStack, alignedClassStack, runparams): apDisplay.printMsg("Inserting ISAC Run into DB") ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = runparams['runname'] alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database") ### setup ISAC like run isacq = appiondata.ApSparxISACRunData() isacq['runname'] = runparams['runname'] isacq['job'] = self.getISACJobData(runparams) ### finish alignment run alignrunq['isacrun'] = isacq alignrunq['hidden'] = False alignrunq['runname'] = runparams['runname'] alignrunq['description'] = runparams['description'] alignrunq['lp_filt'] = runparams['lowpass'] alignrunq['hp_filt'] = runparams['highpass'] alignrunq['bin'] = runparams['bin'] ### setup alignment stack alignstackq = appiondata.ApAlignStackData() alignstackq['imagicfile'] = alignedPartStack alignstackq['avgmrcfile'] = "average.mrc" alignstackq['refstackfile'] = alignedClassStack alignstackq['iteration'] = self.lastiter alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) alignstackq['alignrun'] = alignrunq ### check to make sure files exist alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(alignimagicfilepath): apDisplay.printError("could not find stack file: "+alignimagicfilepath) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: "+avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: "+refstackfile) alignstackq['stack'] = apStack.getOnlyStackData(runparams['stackid']) alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0] alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])*self.params['bin'] alignstackq['description'] = runparams['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.numPart ### insert if self.params['commit'] is True: alignstackq.insert() self.alignstackdata = alignstackq return
def getBeamTiltFromStackParticleNumber(self): stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum + 1, msg=True) pdata = stackparticledata['particle'] ### step 1: get axial beam tilt on to image coordinate direction imageshift = pdata['image']['scope']['image shift'] zerobeamtilt = {'x': 0.0e-3, 'y': 0.0e-3} axialbeamtilt = self.transformImageShiftToBeamTilt( imageshift, self.tem, self.cam, self.ht, zerobeamtilt, self.mag) beamtilt_rotationtransform = self.retrieveBeamTiltMatrix( self.tem, self.cam, self.ht, self.mag) axialbtvector = numpy.array((axialbeamtilt['y'], axialbeamtilt['x'])) axialbtvector_on_image = numpy.dot(beamtilt_rotationtransform, axialbtvector) apDisplay.printMsg( "Axial Beam Tilt on Image Coordinate ( %5.2f, %5.2f)" % (axialbtvector_on_image[1] * 1e3, axialbtvector_on_image[0] * 1e3)) ### step 2: get off-axis beam tilt pcoord = { 'x': pdata['xcoord'] * self.imgpix, 'y': pdata['ycoord'] * self.imgpix } apix = apStack.getStackPixelSizeFromStackId(self.stackid) camdata = pdata['image']['camera'] cambin = camdata['binning'] camdim = camdata['dimension'] camoff = camdata['offset'] camcenter = {'x': self.camsize['x'] / 2, 'y': self.camsize['y'] / 2} apDisplay.printMsg('Particle coord (%5d,%5d)' % (pdata['xcoord'], pdata['ycoord'])) mcoord = { 'y': pdata['ycoord'] * cambin['y'] + camoff['y'] - camcenter['y'], 'x': pdata['xcoord'] * cambin['x'] + camoff['x'] - camcenter['x'] } apDisplay.printMsg('Particle from center in camera pixel (%5d,%5d)' % (mcoord['x'], mcoord['y'])) coord = { 'x': mcoord['x'] * self.imgpix * cambin['x'], 'y': mcoord['y'] * self.imgpix * cambin['y'] } apDisplay.printMsg('Particle from center (A) (%5.2f,%5.2f)' % (coord['x'] * 1e10, coord['y'] * 1e10)) offaxisbeamtilt = self.getCombinedOffAxisBeamTilt( self.c2diameter, self.beamdiameter, coord) #offaxisbeamtilt = {'x':0.0,'y':0.0} ### step 3: combine beam tilts totalbeamtilt = { 'x': axialbtvector_on_image[1] + offaxisbeamtilt['x'], 'y': axialbtvector_on_image[0] - offaxisbeamtilt['y'] } apDisplay.printMsg( "Total Beam Tilt on Image Coordinate (mrad) ( %5.2f, %5.2f)" % (totalbeamtilt['x'] * 1e3, totalbeamtilt['y'] * 1e3)) return totalbeamtilt
def checkConflicts(self): if self.params["stackid"] is None: apDisplay.printError("Stack id was not defined") if self.params["mandir"] is None: apDisplay.printError("Directory containing mandatory files was not defined") if self.params["description"] is None: apDisplay.printError("Run description was not defined") stackdata = apStack.getOnlyStackData(self.params["stackid"], msg=False) stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"]) if self.params["numpart"] is None: self.params["numpart"] = apFile.numImagesInStack(stackfile) apDisplay.printWarning("Number of particles not defined, processing full stack") if self.params["numpart"] > apFile.numImagesInStack(stackfile): apDisplay.printError( "trying to use more particles " + str(self.params["numpart"]) + " than available " + str(apFile.numImagesInStack(stackfile)) ) if self.params["replen"] is None: apDisplay.printError("Helical repeat was not defined") if self.params["subunits"] is None: apDisplay.printError("Number of subunits was not defined") if self.params["diameter"] is None: apDisplay.printError("Filament diameter was not defined") if self.params["diaminner"] is None: apDisplay.printError("Inner filament diameter was not defined") if self.params["rescut"] is None: apDisplay.printError("Phase residual cutoff was not defined") if self.params["filval"] is None: apDisplay.printError("Filter value was not defined") if self.params["bin"] is None: self.params["bin"] = 1 apDisplay.printWarning("Binning factor was not defined, binning by one") if self.params["cont"] is None: apDisplay.printError("Must specify if a contrast change is needed") if self.params["prehip"] is None: apDisplay.printError("Must specify if prehip needs to be executed or not") step = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) self.params["step"] = step onerep = math.floor(self.params["replen"] / step) calcyht2 = math.floor(self.params["diameter"] / step) boxsize = apStack.getStackBoxsize(self.params["stackid"]) self.params["boxsize"] = boxsize if self.params["xlngth"] < onerep: apDisplay.printError("Filament segment length can not be less than one helical repeat") if self.params["xlngth"] > boxsize: apDisplay.printError("Filament segment length can not be greater than stack box size") if self.params["yht2"] < calcyht2: apDisplay.printError("Filament box height can not be less than filament diameter") if self.params["padval"] < self.params["xlngth"]: apDisplay.printError("Pad value can not be less than the filament segment length") if not self.checkval(self.params["yht2"]): apDisplay.printError("Filament box height must be a power of two") if not self.checkval(self.params["padval"]): apDisplay.printError("Pad value must be a power of two")
def __initializeStack(self): # This sets parameters of our new stack based on the params of the original stack # TODO: use originalStackData to populate these self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.stack['format'] = 'imagic' self.stack['phaseflipped'] = self.originalStackData.phaseFlipped
def processVolume(self, spivolfile, iternum=0): ### set values apix = apStack.getStackPixelSizeFromStackId( self.params['tiltstackid']) * self.params['tiltbin'] boxsize = self.getBoxSize() rawspifile = os.path.join( self.params['rundir'], "rawvolume%s-%03d.spi" % (self.timestamp, iternum)) mrcvolfile = os.path.join( self.params['rundir'], "volume%s-%03d.mrc" % (self.timestamp, iternum)) lowpass = self.params['lowpassvol'] ### copy original to raw file shutil.copy(spivolfile, rawspifile) ### convert to mrc emancmd = ("proc3d " + spivolfile + " " + mrcvolfile + " norm=0,1 apix=" + str(apix)) apEMAN.executeEmanCmd(emancmd, verbose=False) ### median filter rawvol = mrc.read(mrcvolfile) medvol = ndimage.median_filter(rawvol, size=self.params['median']) mrc.write(medvol, mrcvolfile) ### low pass filter emancmd = ("proc3d " + mrcvolfile + " " + mrcvolfile + " center norm=0,1 apix=" + str(apix) + " lp=" + str(lowpass)) apEMAN.executeEmanCmd(emancmd, verbose=False) ### set origin emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " origin=0,0,0 " apEMAN.executeEmanCmd(emancmd, verbose=False) ### mask volume emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " mask=" + str( self.params['radius']) apEMAN.executeEmanCmd(emancmd, verbose=False) ### convert to spider apFile.removeFile(spivolfile) emancmd = "proc3d " + mrcvolfile + " " + spivolfile + " spidersingle" apEMAN.executeEmanCmd(emancmd, verbose=False) ### image with chimera if self.params['skipchimera'] is False: if self.params['mass'] is not None: apDisplay.printMsg("Using scale by mass method") apChimera.setVolumeMass(mrcvolfile, apix=apix, mass=self.params['mass']) apChimera.renderSnapshots(mrcvolfile, self.params['contour'], self.params['zoom'], 'c1') return mrcvolfile
def processVolume(self, spivolfile, cnum, iternum=0): ### set values apix = apStack.getStackPixelSizeFromStackId( self.params['tiltstackid']) * self.params['bin'] boxsize = self.getBoxSize() volfilename = os.path.splitext(spivolfile)[0] rawspifile = volfilename + "-raw.spi" mrcvolfile = volfilename + ".mrc" lowpass = self.params['lowpassvol'] ### copy original to raw file shutil.copy(spivolfile, rawspifile) ### convert to mrc emancmd = ("proc3d " + spivolfile + " " + mrcvolfile + " norm=0,1 apix=" + str(apix)) apEMAN.executeEmanCmd(emancmd, verbose=False) ### median filter rawvol = mrc.read(mrcvolfile) medvol = ndimage.median_filter(rawvol, size=self.params['median']) mrc.write(medvol, mrcvolfile) ### low pass filter emancmd = ("proc3d " + mrcvolfile + " " + mrcvolfile + " center norm=0,1 apix=" + str(apix) + " lp=" + str(lowpass)) apEMAN.executeEmanCmd(emancmd, verbose=False) ### set origin emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " origin=0,0,0 " apEMAN.executeEmanCmd(emancmd, verbose=False) ### mask volume emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " mask=" + str( self.params['radius']) apEMAN.executeEmanCmd(emancmd, verbose=False) ### convert to spider apFile.removeFile(spivolfile) emancmd = "proc3d " + mrcvolfile + " " + spivolfile + " spidersingle" apEMAN.executeEmanCmd(emancmd, verbose=False) ### image with chimera if self.params['skipchimera'] is False: apChimera.renderSnapshots(mrcvolfile, self.params['contour'], self.params['zoom'], 'c1') animationthread = threading.Thread( target=apChimera.renderAnimation, args=(mrcvolfile, self.params['contour'], self.params['zoom'], 'c1')) animationthread.setDaemon(1) animationthread.start() return mrcvolfile
def runRmeasure(self): finalrawvolfile = os.path.join( self.params['rundir'], "rawvolume%s-%03d.spi" % (self.timestamp, self.params['numiters'])) emancmd = "proc3d %s %s" % (finalrawvolfile, "rmeasure.mrc") apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) apix = apStack.getStackPixelSizeFromStackId( self.params['tiltstackid']) * self.params['tiltbin'] self.rmeasureresolution = apRecon.runRMeasure(apix, "rmeasure.mrc") #apDisplay.printColor("Final Rmeasure resolution: "+str(self.rmeasureresolution), "cyan") apFile.removeFile("rmeasure.mrc")
def __initializeStack(self): # This sets parameters of our new stack based on the params of the original stack # TODO: use originalStackData to populate these self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.stack['format'] = 'imagic' self.stack['phaseflipped'] = self.originalStackData.phaseFlipped
def convertStackToSpider(self, emanstackfile): """ takes the stack file and creates a spider file ready for processing """ if not os.path.isfile(emanstackfile): apDisplay.printError("stackfile does not exist: " + emanstackfile) tempstack = os.path.join(self.params['rundir'], "filter" + self.timestamp + ".hed") ### first high pass filter particles apDisplay.printMsg("pre-filtering particles") apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid']) boxsize = self.getBoxSize() emancmd = ("proc2d " + emanstackfile + " " + tempstack + " apix=" + str(apix) + " ") if self.params['highpasspart'] is not None and self.params[ 'highpasspart'] > 0: emancmd += "hp=" + str(self.params['highpasspart']) + " " if self.params[ 'lowpasspart'] is not None and self.params['lowpasspart'] > 0: emancmd += "lp=" + str(self.params['lowpasspart']) + " " if self.params['tiltbin'] > 1: clipsize = boxsize * self.params['tiltbin'] emancmd += " shrink=%d clip=%d,%d " % (self.params['tiltbin'], clipsize, clipsize) apEMAN.executeEmanCmd(emancmd, verbose=True) ### convert imagic stack to spider emancmd = "proc2d " emancmd += tempstack + " " spiderstack = os.path.join(self.params['rundir'], "rctstack" + self.timestamp + ".spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack + " " emancmd += "spiderswap edgenorm" starttime = time.time() apDisplay.printColor( "Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) time.sleep(1) # wait a sec, for things to finish apDisplay.printColor( "finished eman in " + apDisplay.timeString(time.time() - starttime), "cyan") apFile.removeStack(tempstack, warn=False) apFile.removeStack(emanstackfile, warn=False) if not os.path.isfile(spiderstack): apDisplay.printError("Failed to create a spider stack") return spiderstack
def convertVolToSpider(self, mrcvolfile=None, modelid=None, apix=None, spivolfile=None): """ takes the mrc volume and creates a spider file ready for processing """ if modelid is not None: initModelData = appiondata.ApInitialModelData.direct_query(modelid) mrcvolfile = initModelData['path']['path'] + "/" + initModelData[ 'name'] apix = initModelData['pixelsize'] stackapix = apStack.getStackPixelSizeFromStackId( self.params['partstackid']) * self.params['bin'] stackboxsize = apStack.getStackBoxsize( self.params['partstackid']) / self.params['bin'] print apix print mrcvolfile if not os.path.isfile(mrcvolfile): apDisplay.printError("volfile does not exist: " + mrcvolfile) ### first high pass filter particles #apDisplay.printMsg("pre-filtering particles") #apix = apStack.getStackPixelSizeFromStackId(self.params['partstackid']) #emancmd = ("proc3d "+mrcvolfile+" "+emanstackfile # +" apix="+str(apix)+" hp="+str(self.params['highpasspart']) # +" inplace") #apEMAN.executeEmanCmd(emancmd, verbose=True) ### convert imagic stack to spider emancmd = "proc3d " emancmd += mrcvolfile + " " if spivolfile is None: spivolfile = os.path.join(self.params['rundir'], "threed-0a.spi") apFile.removeFile(spivolfile, warn=True) emancmd += spivolfile + " " emancmd += "scale=" + str(apix / stackapix) + " " emancmd += "clip=" + str(stackboxsize) + "," + str( stackboxsize) + "," + str(stackboxsize) + " " emancmd += "spidersingle" starttime = time.time() apDisplay.printColor("Running spider volume conversion", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor( "finished conversion in " + apDisplay.timeString(time.time() - starttime), "cyan") return spivolfile
def checkConflicts(self): ### setup correct database after we have read the project id if 'projectid' in self.params and self.params['projectid'] is not None: apDisplay.printMsg("Using split database") # use a project database newdbname = apProject.getAppionDBFromProjectId(self.params['projectid']) sinedon.setConfig('appiondata', db=newdbname) apDisplay.printColor("Connected to database: '"+newdbname+"'", "green") ### get stack data self.stack = {} if self.params['stackid'] is not None: self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) else: self.stack['data'] = appiondata.ApAlignStackData.direct_query(self.params['alignstackid']) self.stack['apix'] = self.stack['pixelsize'] self.stack['boxsize'] = self.stack['boxsize'] self.stack['file'] = self.stack['imagicfile'] ### check conflicts if self.params['stackid'] is None and self.params['alignstackid'] is None: apDisplay.printError("stack id OR alignstack id was not defined") if self.params['stackid'] is not None and self.params['alignstackid'] is not None: apDisplay.printError("either specify stack id OR alignstack id, not both") if self.params['generations'] is None: apDisplay.printError("number of generations was not provided") maxparticles = 500000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) if self.params['numpart'] > apFile.numImagesInStack(self.stack['file']): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(self.stack['file']))) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(self.stack['file']) boxsize = apStack.getStackBoxsize(self.params['stackid']) if self.params['ou'] is None: self.params['ou'] = (boxsize / 2.0) - 2 self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" %(self.params['clipsize'],self.clipsize)) self.clipsize = self.params['clipsize'] self.mpirun = self.checkMPI() if self.mpirun is None: apDisplay.printError("There is no MPI installed")
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("Stack id was not defined") if self.params['mandir'] is None: apDisplay.printError("Directory containing mandatory files was not defined") if self.params['description'] is None: apDisplay.printError("Run description was not defined") stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) apDisplay.printWarning("Number of particles not defined, processing full stack") if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart'])+" than available "+str(apFile.numImagesInStack(stackfile))) if self.params['replen'] is None: apDisplay.printError("Helical repeat was not defined") if self.params['subunits'] is None: apDisplay.printError("Number of subunits was not defined") if self.params['diameter'] is None: apDisplay.printError("Filament diameter was not defined") if self.params['diaminner'] is None: apDisplay.printError("Inner filament diameter was not defined") if self.params['rescut'] is None: apDisplay.printError("Phase residual cutoff was not defined") if self.params['filval'] is None: apDisplay.printError("Filter value was not defined") if self.params['bin'] is None: self.params['bin'] = 1 apDisplay.printWarning("Binning factor was not defined, binning by one") if self.params['cont'] is None: apDisplay.printError("Must specify if a contrast change is needed") if self.params['prehip'] is None: apDisplay.printError("Must specify if prehip needs to be executed or not") step = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.params['step'] = step onerep = math.floor(self.params['replen'] / step) calcyht2 = math.floor(self.params['diameter'] / step) boxsize = apStack.getStackBoxsize(self.params['stackid']) self.params['boxsize'] = boxsize if self.params['xlngth'] < onerep: apDisplay.printError("Filament segment length can not be less than one helical repeat") if self.params['xlngth'] > boxsize: apDisplay.printError("Filament segment length can not be greater than stack box size") if self.params['yht2'] < calcyht2: apDisplay.printError("Filament box height can not be less than filament diameter") if self.params['padval'] < self.params['xlngth']: apDisplay.printError("Pad value can not be less than the filament segment length") if not self.checkval(self.params['yht2']): apDisplay.printError("Filament box height must be a power of two") if not self.checkval(self.params['padval']): apDisplay.printError("Pad value must be a power of two")
def useStackForTemplate(self): apDisplay.printMsg("Using stack to make templates") sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) self.params['sessionname'] = sessiondata['name'] self.params['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) stackdata = apStack.getOnlyStackData(self.params['stackid']) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) self.params['templatelist'] = [] if self.params['avgstack'] is True: ### averaging all particles together apDisplay.printMsg("Averaging all images in stack") templatename = "stack%d-average.mrc" % (self.params['stackid']) oldtemplatepath = os.path.join(stackdata['path']['path'], "average.mrc") abstemplatepath = os.path.join(stackdata['path']['path'], templatename) if os.path.isfile(oldtemplatepath): shutil.copy(oldtemplatepath, abstemplatepath) else: ### average all images using proc2d emancmd = "proc2d %s %s average" % (stackfile, abstemplatepath) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'] = [abstemplatepath] elif self.params['imgnums'] is not None: ### check to see if stackimagenum is within the boundary of the stack numpart = apFile.numImagesInStack(stackfile) for i in self.params['imgnums'].split(","): partnum = int(i) if partnum > numpart or partnum < 0: apDisplay.printError( "'imgnums' is NOT within the boundary of the stack: %d > %d" % (partnum, numpart)) apDisplay.printMsg("Extracting image %d from stack" % partnum) templatename = "stack%d-particle%d.mrc" % ( self.params['stackid'], partnum) abstemplatepath = os.path.join(self.params['rundir'], templatename) ### run proc2d with params['stackimgnum'] emancmd = "proc2d %s %s first=%d last=%d" % ( stackfile, abstemplatepath, partnum, partnum) ### create template apDisplay.printMsg("creating " + templatename) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'].append(abstemplatepath)
def processVolume(self, spivolfile, cnum, iternum=0): ### set values apix = apStack.getStackPixelSizeFromStackId(self.params["tiltstackid"]) * self.params["bin"] boxsize = self.getBoxSize() volfilename = os.path.splitext(spivolfile)[0] rawspifile = volfilename + "-raw.spi" mrcvolfile = volfilename + ".mrc" lowpass = self.params["lowpassvol"] ### copy original to raw file shutil.copy(spivolfile, rawspifile) ### convert to mrc emancmd = "proc3d " + spivolfile + " " + mrcvolfile + " norm=0,1 apix=" + str(apix) apEMAN.executeEmanCmd(emancmd, verbose=False) ### median filter rawvol = mrc.read(mrcvolfile) medvol = ndimage.median_filter(rawvol, size=self.params["median"]) mrc.write(medvol, mrcvolfile) ### low pass filter emancmd = ( "proc3d " + mrcvolfile + " " + mrcvolfile + " center norm=0,1 apix=" + str(apix) + " lp=" + str(lowpass) ) apEMAN.executeEmanCmd(emancmd, verbose=False) ### set origin emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " origin=0,0,0 " apEMAN.executeEmanCmd(emancmd, verbose=False) ### mask volume emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " mask=" + str(self.params["radius"]) apEMAN.executeEmanCmd(emancmd, verbose=False) ### convert to spider apFile.removeFile(spivolfile) emancmd = "proc3d " + mrcvolfile + " " + spivolfile + " spidersingle" apEMAN.executeEmanCmd(emancmd, verbose=False) ### image with chimera if self.params["skipchimera"] is False: apChimera.renderSnapshots(mrcvolfile, self.params["contour"], self.params["zoom"], "c1") animationthread = threading.Thread( target=apChimera.renderAnimation, args=(mrcvolfile, self.params["contour"], self.params["zoom"], "c1") ) animationthread.setDaemon(1) animationthread.start() return mrcvolfile
def getInstrumentParams(self): self.stackpix = apStack.getStackPixelSizeFromStackId(self.stackid, True) * 1e-10 stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum+1, msg=True) pdata = stackparticledata['particle'] self.imgpix = apDatabase.getPixelSize(pdata['image'])*1e-10 scopedata = pdata['image']['scope'] self.tem = scopedata['tem'] self.cam = pdata['image']['camera']['ccdcamera'] self.ht = scopedata['high tension'] self.Cs = 2e-3 self.mag = scopedata['magnification'] self.camsize = {'x':4096,'y':4096} self.beamdiameter = 1e-6 self.c2diameter = 1.0e-4 self.wavelength = fftfun.getElectronWavelength(self.ht)
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stackid was not defined") if self.params['modelid'] is None: apDisplay.printError("model id was not defined") if self.params['runname'] is None: apDisplay.printError("new runname was not defined") self.params['totalpart'] = apStack.getNumberStackParticlesFromId(self.params['stackid']) if 'last' not in self.params.keys() or self.params['last'] is None: self.params['last'] = self.params['totalpart'] self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False) self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.refineboxsize = self.boxsize * self.params['bin'] ### set cs value self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData()) self.modelids = map((lambda x: int(x)),self.params['modelid'].split(',')) self.checkPackageConflicts()
def getInstrumentParams(self): self.stackpix = apStack.getStackPixelSizeFromStackId( self.stackid, True) * 1e-10 stackparticledata = apStack.getOneParticleFromStackId(self.stackid, self.imgnum + 1, msg=True) pdata = stackparticledata['particle'] self.imgpix = apDatabase.getPixelSize(pdata['image']) * 1e-10 scopedata = pdata['image']['scope'] self.tem = scopedata['tem'] self.cam = pdata['image']['camera']['ccdcamera'] self.ht = scopedata['high tension'] self.Cs = 2e-3 self.mag = scopedata['magnification'] self.camsize = {'x': 4096, 'y': 4096} self.beamdiameter = 1e-6 self.c2diameter = 1.0e-4 self.wavelength = fftfun.getElectronWavelength(self.ht)
def processVolume(self, spivolfile, iternum=0): ### set values apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] boxsize = self.getBoxSize() rawspifile = os.path.join(self.params['rundir'], "rawvolume%s-%03d.spi"%(self.timestamp, iternum)) mrcvolfile = os.path.join(self.params['rundir'], "volume%s-%03d.mrc"%(self.timestamp, iternum)) lowpass = self.params['lowpassvol'] ### copy original to raw file shutil.copy(spivolfile, rawspifile) ### convert to mrc emancmd = ("proc3d "+spivolfile+" "+mrcvolfile+" norm=0,1 apix="+str(apix)) apEMAN.executeEmanCmd(emancmd, verbose=False) ### median filter rawvol = mrc.read(mrcvolfile) medvol = ndimage.median_filter(rawvol, size=self.params['median']) mrc.write(medvol, mrcvolfile) ### low pass filter emancmd = ("proc3d "+mrcvolfile+" "+mrcvolfile+" center norm=0,1 apix=" +str(apix)+" lp="+str(lowpass)) apEMAN.executeEmanCmd(emancmd, verbose=False) ### set origin emancmd = "proc3d "+mrcvolfile+" "+mrcvolfile+" origin=0,0,0 " apEMAN.executeEmanCmd(emancmd, verbose=False) ### mask volume emancmd = "proc3d "+mrcvolfile+" "+mrcvolfile+" mask="+str(self.params['radius']) apEMAN.executeEmanCmd(emancmd, verbose=False) ### convert to spider apFile.removeFile(spivolfile) emancmd = "proc3d "+mrcvolfile+" "+spivolfile+" spidersingle" apEMAN.executeEmanCmd(emancmd, verbose=False) ### image with chimera if self.params['skipchimera'] is False: if self.params['mass'] is not None: apDisplay.printMsg("Using scale by mass method") apChimera.setVolumeMass(mrcvolfile, apix=apix, mass=self.params['mass']) apChimera.renderSnapshots(mrcvolfile, self.params['contour'], self.params['zoom'], 'c1') return mrcvolfile
def convertStackToSpider(self, emanstackfile): """ takes the stack file and creates a spider file ready for processing """ if not os.path.isfile(emanstackfile): apDisplay.printError("stackfile does not exist: "+emanstackfile) tempstack = os.path.join(self.params['rundir'], "filter"+self.timestamp+".hed") ### first high pass filter particles apDisplay.printMsg("pre-filtering particles") apix = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid']) boxsize = self.getBoxSize() emancmd = ("proc2d "+emanstackfile+" "+tempstack +" apix="+str(apix)+" ") if self.params['highpasspart'] is not None and self.params['highpasspart'] > 0: emancmd += "hp="+str(self.params['highpasspart'])+" " if self.params['lowpasspart'] is not None and self.params['lowpasspart'] > 0: emancmd += "lp="+str(self.params['lowpasspart'])+" " if self.params['tiltbin'] > 1: clipsize = boxsize*self.params['tiltbin'] emancmd += " shrink=%d clip=%d,%d "%(self.params['tiltbin'], clipsize, clipsize) apEMAN.executeEmanCmd(emancmd, verbose=True) ### convert imagic stack to spider emancmd = "proc2d " emancmd += tempstack+" " spiderstack = os.path.join(self.params['rundir'], "rctstack"+self.timestamp+".spi") apFile.removeFile(spiderstack, warn=True) emancmd += spiderstack+" " emancmd += "spiderswap edgenorm" starttime = time.time() apDisplay.printColor("Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) time.sleep(1) # wait a sec, for things to finish apDisplay.printColor("finished eman in "+apDisplay.timeString(time.time()-starttime), "cyan") apFile.removeStack(tempstack, warn=False) apFile.removeStack(emanstackfile, warn=False) if not os.path.isfile(spiderstack): apDisplay.printError("Failed to create a spider stack") return spiderstack
def convertVolToSpider(self, mrcvolfile=None, modelid=None, apix=None, spivolfile=None): """ takes the mrc volume and creates a spider file ready for processing """ if modelid is not None: initModelData = appiondata.ApInitialModelData.direct_query(modelid) mrcvolfile = initModelData["path"]["path"] + "/" + initModelData["name"] apix = initModelData["pixelsize"] stackapix = apStack.getStackPixelSizeFromStackId(self.params["partstackid"]) * self.params["bin"] stackboxsize = apStack.getStackBoxsize(self.params["partstackid"]) / self.params["bin"] print apix print mrcvolfile if not os.path.isfile(mrcvolfile): apDisplay.printError("volfile does not exist: " + mrcvolfile) ### first high pass filter particles # apDisplay.printMsg("pre-filtering particles") # apix = apStack.getStackPixelSizeFromStackId(self.params['partstackid']) # emancmd = ("proc3d "+mrcvolfile+" "+emanstackfile # +" apix="+str(apix)+" hp="+str(self.params['highpasspart']) # +" inplace") # apEMAN.executeEmanCmd(emancmd, verbose=True) ### convert imagic stack to spider emancmd = "proc3d " emancmd += mrcvolfile + " " if spivolfile is None: spivolfile = os.path.join(self.params["rundir"], "threed-0a.spi") apFile.removeFile(spivolfile, warn=True) emancmd += spivolfile + " " emancmd += "scale=" + str(apix / stackapix) + " " emancmd += "clip=" + str(stackboxsize) + "," + str(stackboxsize) + "," + str(stackboxsize) + " " emancmd += "spidersingle" starttime = time.time() apDisplay.printColor("Running spider volume conversion", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor("finished conversion in " + apDisplay.timeString(time.time() - starttime), "cyan") return spivolfile
def useStackForTemplate(self): apDisplay.printMsg("Using stack to make templates") sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) self.params['sessionname'] = sessiondata['name'] self.params['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) stackdata = apStack.getOnlyStackData(self.params['stackid']) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) self.params['templatelist'] = [] if self.params['avgstack'] is True: ### averaging all particles together apDisplay.printMsg("Averaging all images in stack") templatename = "stack%d-average.mrc"%(self.params['stackid']) oldtemplatepath = os.path.join(stackdata['path']['path'], "average.mrc") abstemplatepath = os.path.join(stackdata['path']['path'], templatename) if os.path.isfile(oldtemplatepath): shutil.copy(oldtemplatepath, abstemplatepath) else: ### average all images using proc2d emancmd = "proc2d %s %s average" % (stackfile, abstemplatepath) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'] = [abstemplatepath] elif self.params['imgnums'] is not None: ### check to see if stackimagenum is within the boundary of the stack numpart = apFile.numImagesInStack(stackfile) for i in self.params['imgnums'].split(","): partnum = int(i) if partnum > numpart or partnum < 0: apDisplay.printError("'imgnums' is NOT within the boundary of the stack: %d > %d"%(partnum,numpart)) apDisplay.printMsg("Extracting image %d from stack" % partnum) templatename = "stack%d-particle%d.mrc"%(self.params['stackid'],partnum) abstemplatepath= os.path.join(self.params['rundir'], templatename) ### run proc2d with params['stackimgnum'] emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, abstemplatepath, partnum, partnum) ### create template apDisplay.printMsg("creating "+templatename) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'].append(abstemplatepath)
def start(self): ### load parameters runparams = self.readRunParameters() runparams['localstack'] = "start1.hdf" self.params.update(runparams) alignedClassStackHDF = "alignedClasses.hdf" alignedPartStackHDF = "alignedParticles.hdf" apFile.removeStack(alignedClassStackHDF, warn=False) apFile.removeStack(alignedPartStackHDF, warn=False) ISACParser = parseISAC() ISACParser.setAlignParams(self.params) ISACParser.trackParticlesInISAC() ### align classes ISACParser.alignClassAverages(alignedClassStackHDF) alignedClassStack = apEMAN2.stackHDFToIMAGIC(alignedClassStackHDF) apStack.averageStack(alignedClassStack) self.lastiter = ISACParser.numGenerations ### align particles to classes AND create aligned stacks ISACParser.alignParticlesToClasses(self.params['localstack'], alignedClassStackHDF, alignedPartStackHDF) alignedPartStack = apEMAN2.stackHDFToIMAGIC(alignedPartStackHDF) self.numPart = sparx.EMUtil.get_image_count(alignedPartStackHDF) partList = ISACParser.createPartList() ### calculate resolution for each reference apix = apStack.getStackPixelSizeFromStackId( self.params['stackid']) * self.params['bin'] self.calcResolution(partList, alignedPartStack, apix) ### insert into database self.insertRunIntoDatabase(alignedPartStack, alignedClassStack, self.params) self.insertParticlesIntoDatabase(self.params['stackid'], partList)
def createFastFreeHandInputLineTemplate(self): self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False) self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.apix = 6.02 one_particledata = apStack.getOneParticleFromStackId(self.params['stackid'], particlenumber=1, msg=True) scopedata = one_particledata['particle']['image']['scope'] ht_kv = scopedata['high tension'] / 1000.0 cs_mm = scopedata['tem']['cs'] * 1000 angSearch = self.params['angSearch'] constant_inputlines = [ '%.3f,%.3f,%2f,%.f' %(self.apix,self.params['snr'],cs_mm,ht_kv) ,'1,0' ,'../%s' %(self.stackfile) ,'../%s' %(self.modelfile) ,'../%s' %(self.frealign_paramfile) ,'out.mrc' ,'-%d,%d,-%d,%d' %(angSearch,angSearch,angSearch,angSearch) ,'%.1f,%.2f,%.2f' %(self.params['minres'],self.params['maxres'],self.params['radius']) ,'%s' %(self.params['scoringtype'].upper()) ] return constant_inputlines
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stackid was not defined") if self.params['modelid'] is None: apDisplay.printError("model id was not defined") if self.params['runname'] is None: apDisplay.printError("new runname was not defined") self.params['totalpart'] = apStack.getNumberStackParticlesFromId( self.params['stackid']) if 'last' not in self.params.keys() or self.params['last'] is None: self.params['last'] = self.params['totalpart'] self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False) self.apix = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.refineboxsize = self.boxsize * self.params['bin'] ### set cs value self.params['cs'] = apInstrument.getCsValueFromSession( self.getSessionData()) self.modelids = map((lambda x: int(x)), self.params['modelid'].split(',')) self.checkPackageConflicts()
def checkConflicts(self): if self.params['runname'] is None: apDisplay.printError("enter a stack run name, e.g. combinestack1") if self.params['description'] is None: apDisplay.printError("enter a stack description") if self.params['stacks'] and ',' in self.params['stacks']: #remember stackids are a list of strings stackids = self.params['stacks'].split(',') self.params['stackids'] = stackids else: apDisplay.printError( "enter a list of stack ids to combine, e.g. --stackids=11,14,7" ) ### check to make sure all pixel and box size are the same self.newboxsize = None self.newpixelsize = None for stackidstr in self.params['stackids']: if not re.match("^[0-9]+$", stackidstr): apDisplay.printError("Stack id '%s' is not an integer" % (stackidstr)) stackid = int(stackidstr) boxsize = apStack.getStackBoxsize(stackid, msg=False) pixelsize = apStack.getStackPixelSizeFromStackId(stackid, msg=False) apDisplay.printMsg("Stack id: %d\tBoxsize: %d\tPixelsize: %.3f" % (stackid, boxsize, pixelsize)) if self.newboxsize is None: self.newboxsize = boxsize if self.newpixelsize is None: self.newpixelsize = pixelsize if boxsize != self.newboxsize: apDisplay.printError( "Trying to combine stacks with different box sizes") if abs(pixelsize - self.newpixelsize) > 0.01: apDisplay.printError( "Trying to combine stacks with different pixel sizes")
def checkConflicts(self): ### first get all stack data self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### modify boxsize and pixelsize according to binning self.boxsize = self.stack['boxsize'] self.clipsize = int( math.floor(self.boxsize / float(self.params['bin'] * 2))) * 2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" % (self.params['clipsize'], self.clipsize)) self.clipsize = self.params['clipsize'] self.apix = self.stack['apix'] if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] if clipsize % 2 == 1: clipsize += 1 ### making sure that clipped boxsize is even self.boxsize = clipsize self.apix = self.apix * self.params['bin'] ### basic error checking if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['ncls'] is None: apDisplay.printError("a number of classes was not provided") maxparticles = 1000000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles " + str(self.params['numpart']) + " than available " + str(apFile.numImagesInStack(stackfile))) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) if self.params['numpart'] > 5000: apDisplay.printWarning( "initial model calculation may not work with less than 5000 particles" ) self.mpirun = self.checkMPI() # if self.mpirun is None: # apDisplay.printError("There is no MPI installed") if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() ### SIMPLE defaults and error checking if self.params['ring2'] is None: self.params['ring2'] = (self.boxsize / 2) - 2 if self.params['ncls'] > 2000: apDisplay.printError( "number of classes should be less than 2000 for subsequent ORIGAMI run to work" ) if self.params['ncls'] > self.params['numpart']: self.params['ncls'] = self.params['numpart'] / self.params['minp'] if self.params['mask'] is None: self.params['mask'] = (self.boxsize / 2) - 2 if self.params['mw'] is None: apDisplay.printError( "please specify the molecular weight (in kDa)")
def runEoTest(self, alignstack, eulerfile): evenvolfile = os.path.join(self.params['rundir'], "evenvolume%s.spi" % (self.timestamp)) oddvolfile = os.path.join(self.params['rundir'], "oddvolume%s.spi" % (self.timestamp)) eveneulerfile = os.path.join(self.params['rundir'], "eveneulers%s.spi" % (self.timestamp)) oddeulerfile = os.path.join(self.params['rundir'], "oddeulers%s.spi" % (self.timestamp)) evenpartlist = os.path.join(self.params['rundir'], "evenparts%s.lst" % (self.timestamp)) oddpartlist = os.path.join(self.params['rundir'], "oddparts%s.lst" % (self.timestamp)) ### Create New Doc Files of = open(oddeulerfile, "w") ef = open(eveneulerfile, "w") op = open(oddpartlist, "w") ep = open(evenpartlist, "w") inf = open(eulerfile, "r") evenpart = 0 oddpart = 0 for line in inf: spidict = operations.spiderInLine(line) if spidict: partnum = spidict['row'] if partnum % 2 == 0: ep.write("%d\n" % (partnum - 1)) evenpart += 1 outline = operations.spiderOutLine(evenpart, spidict['floatlist']) ef.write(outline) elif partnum % 2 == 1: op.write("%d\n" % (partnum - 1)) oddpart += 1 outline = operations.spiderOutLine(oddpart, spidict['floatlist']) of.write(outline) inf.close() of.close() ef.close() op.close() ep.close() ### Create stacks evenstack = os.path.join(self.params['rundir'], "evenstack%s.spi" % (self.timestamp)) emancmd = "proc2d %s %s list=%s spiderswap" % (alignstack, evenstack, evenpartlist) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) oddstack = os.path.join(self.params['rundir'], "oddstack%s.spi" % (self.timestamp)) emancmd = "proc2d %s %s list=%s spiderswap" % (alignstack, oddstack, oddpartlist) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) ### Create Volumes backproject.backproject3F(evenstack, eveneulerfile, evenvolfile, evenpart) backproject.backproject3F(oddstack, oddeulerfile, oddvolfile, oddpart) if not os.path.isfile(evenvolfile) or not os.path.isfile(oddvolfile): apDisplay.printError("Even-Odd volume creation failed") ### Calculate FSC apix = apStack.getStackPixelSizeFromStackId( self.params['tiltstackid']) * self.params['tiltbin'] emancmd = "proc3d %s %s" % (evenvolfile, evenvolfile + ".mrc") apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) emancmd = "proc3d %s %s" % (oddvolfile, oddvolfile + ".mrc") apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) fscfile = os.path.join(self.params['rundir'], "fscdata%s.fsc" % (self.timestamp)) emancmd = "proc3d %s %s fsc=%s" % (evenvolfile + ".mrc", oddvolfile + ".mrc", fscfile) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) if not os.path.isfile(fscfile): apDisplay.printError("Even-Odd fsc calculation failed") boxsize = self.getBoxSize() self.fscresolution = apRecon.getResolutionFromFSCFile(fscfile, boxsize, apix, msg=True) apDisplay.printColor( ("Final FSC resolution: %.5f" % (self.fscresolution)), "cyan") for fname in (evenvolfile, oddvolfile, evenstack, oddstack, eveneulerfile, oddeulerfile, evenpartlist, oddpartlist): apFile.removeFile(fname)