def start(self): knownstackdata = apStack.getOnlyStackData(self.params['knownstackid']) fullstackdata = apStack.getOnlyStackData(self.params['fullstackid']) ### get good particle numbers includeParticle, tiltParticlesData = self.getGoodParticles() self.numpart = len(includeParticle) ### write kept particles to file self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile"+self.timestamp+".lst") apDisplay.printMsg("writing to keepfile "+self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum)+"\n") kf.close() ### make new stack of tilted particle from that run fullstackfile = os.path.join(fullstackdata['path']['path'], fullstackdata['name']) sb = os.path.splitext(fullstackdata['name']) newname = "tiltpairsub%d" % self.params['knownstackid']+sb[-1] newstackfile = os.path.join(self.params['rundir'], newname) apFile.removeStack(newstackfile, warn=False) apStack.makeNewStack(fullstackfile, newstackfile, self.params['keepfile']) if not os.path.isfile(newstackfile): apDisplay.printError("No stack was created") self.params['stackid'] = self.params['fullstackid'] apStack.commitSubStack(self.params, newname, sorted=False) apStack.averageStack(stack=newstackfile) newstackid = apStack.getStackIdFromPath(newstackfile) if self.params['meanplot'] is True: apDisplay.printMsg("creating Stack Mean Plot montage for stackid") apStackMeanPlot.makeStackMeanPlot(newstackid)
def start(self): self.iflag = 1 self.setIBLOW() ### get stack info self.stackdata = apStack.getOnlyStackData(self.params["stackid"]) self.refinestackfile = os.path.join(self.stackdata["path"]["path"], self.stackdata["name"]) apImagicFile.setImagic4DHeader(self.refinestackfile) ### copy stack or start job file if self.params["cluster"] is False: # create alias to stack data pass if self.params["cluster"] is True: self.setupMultiNode() if self.params["reconstackid"] is not None: self.reconstackdata = apStack.getOnlyStackData(self.params["stackid"]) self.reconstackfile = os.path.join(self.stackdata["path"]["path"], self.stackdata["name"]) else: self.reconstackfile = self.refinestackfile ### create initial model file self.currentvol = os.path.basename(self.setupInitialModel()) ### create parameter file self.currentparam = os.path.basename(self.setupParticleParams()) apDisplay.printColor( "Initial files:\n Stack: %s\n Volume: %s\n Params: %s\n" % (os.path.basename(self.refinestackfile), self.currentvol, self.currentparam), "violet", ) ## run frealign for number for refinement cycles for i in range(self.params["numiter"]): iternum = i + 1 if self.params["cluster"] is True: self.multiNodeRun(iternum) else: self.singleNodeRun(iternum) time.sleep(2) ### calculate FSC # emancmd = 'proc3d %s %s fsc=fsc.eotest.%d' % (evenvol, oddvol, self.params['iter']) # apEMAN.executeEmanCmd(emancmd, verbose=True) if self.params["cluster"] is True: self.prepareForCluster() print "Done!"
def setRunDir(self): """ this function only runs if no rundir is defined at the command line """ if self.params['rundir'] is None: if ('sessionname' in self.params and self.params['sessionname'] is not None ): # command line users may use sessionname rather than expId sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.params['rundir'] = self.getDefaultBaseAppionDir(sessiondata,[self.processdirname,self.params['runname']]) else: if ('expId' in self.params and self.params['expId']): # expId should always be included from appionwrapper derived appionscript sessiondata = apDatabase.getSessionDataFromSessionId(self.params['expId']) self.params['rundir'] = self.getDefaultBaseAppionDir(sessiondata,[self.processdirname,self.params['runname']]) # The rest should not be needed with appionwrapper format from appionlib import apStack if ( self.params['rundir'] is None and 'reconid' in self.params and self.params['reconid'] is not None ): self.params['stackid'] = apStack.getStackIdFromRecon(self.params['reconid'], msg=False) if ( self.params['rundir'] is None and 'stackid' in self.params and self.params['stackid'] is not None ): #auto set the run directory stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) path = os.path.abspath(stackdata['path']['path']) path = os.path.dirname(path) path = os.path.dirname(path) self.params['rundir'] = os.path.join(path, self.processdirname, self.params['runname']) self.params['outdir'] = self.params['rundir']
def setupParticles(self): self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed") self.stack = {} self.stack["data"] = apStack.getOnlyStackData(self.params["stackid"]) self.stack["apix"] = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) self.stack["part"] = apStack.getOneParticleFromStackId(self.params["stackid"]) self.stack["boxsize"] = apStack.getStackBoxsize(self.params["stackid"]) self.stack["file"] = os.path.join(self.stack["data"]["path"]["path"], self.stack["data"]["name"]) boxsize = int(math.floor(self.stack["boxsize"] / float(self.params["bin"] * 2))) * 2 apix = self.stack["apix"] * self.params["bin"] proccmd = "proc2d " + self.stack["file"] + " " + self.params["localstack"] + " apix=" + str(self.stack["apix"]) if self.params["bin"] > 1: clipsize = boxsize * self.params["bin"] proccmd += " shrink=%d clip=%d,%d " % (self.params["bin"], clipsize, clipsize) if self.params["highpass"] > 1: proccmd += " hp=" + str(self.params["highpass"]) if self.params["lowpass"] > 1: proccmd += " lp=" + str(self.params["lowpass"]) proccmd += " last=" + str(self.params["numpart"] - 1) apFile.removeStack(self.params["localstack"], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"]) return (boxsize, apix)
def start(self): """ This is the core of your function. """ stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) original_stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) filtered_stackfile = os.path.join(self.params['rundir'], self.timestamp+".hed") apFile.removeStack(filtered_stackfile, warn=False) apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) boxsize = apStack.getStackBoxsize(self.params['stackid']) emancmd = "proc2d %s %s apix=%.3f "%(original_stackfile, filtered_stackfile, apix) if self.params['lowpass'] is not None: emancmd += " lp=%.3f "%(self.params['lowpass']) if self.params['highpass'] is not None: emancmd += " hp=%.3f "%(self.params['highpass']) if self.params['bin'] is not None and self.params['bin'] > 1: ## determine a multiple of the bin that is divisible by 2 and less than the boxsize clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2*self.params['bin'] emancmd += " shrink=%d clip=%d,%d "%(self.params['bin'], clipsize, clipsize) emancmd += " last=%d "%(self.params['numpart']-1) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) ### confirm that it worked if self.params['numpart'] != apFile.numImagesInStack(filtered_stackfile): apDisplay.printError("Missing particles in stack") ### run the radon transform code self.radonAlign(filtered_stackfile) ### insert info into database self.commitToDatabase()
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['description'] is None: apDisplay.printError("run description was not defined") if self.params['templatelist'] is None: apDisplay.printError("template list was not provided") if self.params['lastring'] is None: apDisplay.printError("a last ring radius was not provided") if self.params['runname'] is None: apDisplay.printError("run name was not defined") stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize(self.params['stackid'])/self.params['bin'] if self.params['lastring'] > boxsize/2-2: apDisplay.printError("last ring radius is too big for boxsize " +str(self.params['lastring'])+" > "+str(boxsize/2-2)) if self.params['lastring']+self.params['xysearch'] > boxsize/2-2: apDisplay.printError("last ring plus xysearch radius is too big for boxsize " +str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2)) ### convert / check template data self.templatelist = self.params['templatelist'].strip().split(",") if not self.templatelist or type(self.templatelist) != type([]): apDisplay.printError("could not parse template list="+self.params['templatelist']) self.params['numtemplate'] = len(self.templatelist) apDisplay.printMsg("Found "+str(self.params['numtemplate'])+" templates")
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['numrefs'] is None: apDisplay.printError("a number of classes was not provided") maxparticles = 500000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize(self.params['stackid']) self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" %(self.params['clipsize'],self.clipsize)) self.clipsize = self.params['clipsize'] if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) self.mpirun = self.checkMPI() if self.mpirun is None: apDisplay.printError("There is no MPI installed") if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() if self.params['nproc'] < 2: apDisplay.printError("Only the MPI version of CL2D is currently supported, must run with > 1 CPU")
def start(self): ### new stack path stackdata = apStack.getOnlyStackData(self.params['stackid']) oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) ### make sure that old stack is numbered alignedstack = os.path.join(self.params['rundir'], 'alignstack.hed') apStack.checkForPreviousStack(alignedstack) ### run centering algorithm self.runMaxlike() ### create aligned stacks partlist = self.readPartDocFile() stackfile = self.createAlignedStacks(partlist) if not os.path.isfile(alignedstack): apDisplay.printError("No stack was created") ### get number of particles numpart = apStack.getNumberStackParticlesFromId(self.params['stackid']) self.writeFakeKeepFile(numpart) self.params['description'] += ( " ... %d maxlike centered substack id %d" % (numpart, self.params['stackid'])) apStack.commitSubStack(self.params, newname='alignstack.hed', centered=True) apStack.averageStack(stack=alignedstack)
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['start'] is None: apDisplay.printError("a number of starting classes was not provided") if self.params['end'] is None: apDisplay.printError("a number of ending classes was not provided") if self.params['runname'] is None: apDisplay.printError("run name was not defined") self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) # check for virtual stack self.params['virtualdata'] = None if not os.path.isfile(stackfile): vstackdata = apStack.getVirtualStackParticlesFromId(self.params['stackid']) npart = len(vstackdata['particles']) self.params['virtualdata'] = vstackdata else: npart = apFile.numImagesInStack(stackfile) if self.params['numpart'] is None: self.params['numpart'] = npart elif self.params['numpart'] > npart: apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) self.boxsize = apStack.getStackBoxsize(self.params['stackid']) self.workingboxsize = math.floor(self.boxsize/self.params['bin']) if not self.params['mask']: self.params['mask'] = (self.boxsize/2)-2 self.workingmask = math.floor(self.params['mask']/self.params['bin']) if self.params['mramethod'] == 'imagic': self.imagicroot = apIMAGIC.checkImagicExecutablePath() self.imagicversion = apIMAGIC.getImagicVersion(self.imagicroot)
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['rad'] is None: apDisplay.printError("no particle radius set") if self.params['modelid'] is None: apDisplay.printError("model id was not defined") if self.params['lastring'] is None: apDisplay.printError("a last ring radius was not provided") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if self.params['incr'] is None: apDisplay.printError("angular increments are not specified") stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) numstackp = apFile.numImagesInStack(stackfile) if self.params['numpart'] > numstackp: apDisplay.printError("trying to use more particles "+str(numstackp) +" than available "+str(numstackp)) elif self.params['numpart'] == 0: apDisplay.printWarning("using all "+str(numstackp)+" particles") self.params['numpart'] = numstackp boxsize = apStack.getStackBoxsize(self.params['stackid']) if self.params['lastring'] > boxsize/2-2: apDisplay.printError("last ring radius is too big for boxsize " +str(self.params['lastring'])+" > "+str(boxsize/2-2)) if self.params['lastring']+self.params['xysearch'] > boxsize/2-2: apDisplay.printError("last ring plus xysearch radius is too big for boxsize " +str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2)) if (self.params['xysearch'] % self.params['xystep']) > 0: apDisplay.printError("translational search (xy-search) must be divisible by search step (xy-step)")
def start(self): ### new stack path stackdata = apStack.getOnlyStackData(self.params['stackid']) oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) ### checks centerstack = os.path.join(self.params['rundir'], 'align.img') badstack = os.path.join(self.params['rundir'], 'bad.img') apStack.checkForPreviousStack(centerstack) ### run centering algorithm keeplist = self.centerParticles(oldstack, centerstack, badstack) if not os.path.isfile(centerstack): apDisplay.printError("No stack was created") self.params['keepfile'] = os.path.join(self.params['rundir'], 'keepfile.txt') ### get number of particles self.params['description'] += ( (" ... %d eman centered substack id %d" % (numparticles, self.params['stackid'])) ) apStack.commitSubStack(self.params, newname=os.path.basename(centerstack), centered=True) apStack.averageStack(stack=centerstack) if os.path.isfile(badstack): apStack.averageStack(stack=badstack, outfile='badaverage.mrc')
def runMaxlike(self): stackdata = apStack.getOnlyStackData(self.params["stackid"]) apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"]) ### process stack to local file self.params["localstack"] = os.path.join(self.params["rundir"], self.timestamp + ".hed") proccmd = "proc2d " + stackfile + " " + self.params["localstack"] + " apix=" + str(apix) if self.params["highpass"] > 1: proccmd += " hp=" + str(self.params["highpass"]) if self.params["lowpass"] > 1: proccmd += " lp=" + str(self.params["lowpass"]) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params["localstack"]) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params["rundir"], self.partlistdocfile) + " -nref 1 " + " -iter 10 " + " -o " + os.path.join(self.params["rundir"], "part" + self.timestamp) + " -fast -C 1e-18 " ) ### angle step if self.params["rotate"] is True: xmippopts += " -psi_step 90 " else: xmippopts += " -psi_step 360 " ### convergence criteria if self.params["converge"] == "fast": xmippopts += " -eps 5e-3 " elif self.params["converge"] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params["mirror"] is True: xmippopts += " -mirror " if self.params["maxshift"] is not None: xmippopts += " -max_shift %d " % (self.params["maxshift"]) ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe + " " + xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ### create a quick mrc emancmd = "proc2d part" + self.timestamp + "_ref000001.xmp average.mrc" apEMAN.executeEmanCmd(emancmd, verbose=True) apFile.removeStack(self.params["localstack"]) apFile.removeFilePattern("partfiles/*")
def initializeRefinementUploadVariables(self): """ untars results, reads required parameters, establishes necessary objects for upload script """ apDisplay.printColor("uploading refinement results for %s reconstruction routine" % (self.package), "cyan") ### establish directories and make an appion results directory self.basepath = os.path.abspath(self.params["rundir"]) self.resultspath = os.path.abspath(os.path.join(self.params["rundir"], str(self.package) + "_results")) if not os.path.isdir(self.resultspath): os.mkdir(self.resultspath) self.reconpath = os.path.abspath( os.path.join(self.params["rundir"], self.runparams["reconstruction_working_dir"]) ) ### get all stack parameters, map particles in reconstruction to particles in stack, get all model data self.stackdata = apStack.getOnlyStackData(self.runparams["stackid"]) self.stackmapping = apRecon.partnum2defid(self.runparams["stackid"]) self.modeldata = [] if len(self.runparams["modelid"].split(",")) > 1: models = self.runparams["modelid"].split(",") for i in range(len(models)): self.modeldata.append(appiondata.ApInitialModelData.direct_query(int(models[i]))) else: self.modeldata.append(appiondata.ApInitialModelData.direct_query(self.runparams["modelid"])) return
def start(self): self.runtime = 0 self.partlist = [] self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.checkNoRefRun() ### convert stack to spider spiderstack = self.createSpiderFile() ### create initialization template if self.params['initmethod'] == 'allaverage': templatefile = self.averageTemplate() elif self.params['initmethod'] == 'selectrand': templatefile = self.selectRandomParticles() elif self.params['initmethod'] == 'randpart': templatefile = self.pickRandomParticle() elif self.params['initmethod'] == 'template': templatefile = self.getTemplate() else: apDisplay.printError("unknown initialization method defined: " +str(self.params['initmethod'])+" not in "+str(self.initmethods)) apDisplay.printColor("Running spider this can take awhile","cyan") ### run the alignment aligntime = time.time() pixrad = int(round(self.params['partrad']/self.stack['apix']/self.params['bin'])) alignedstack, self.partlist = alignment.refFreeAlignParticles( spiderstack, templatefile, self.params['numpart'], pixrad, self.params['firstring'], self.params['lastring'], rundir = ".") aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) ### remove large, worthless stack spiderstack = os.path.join(self.params['rundir'], "start.spi") apDisplay.printMsg("Removing un-aligned stack: "+spiderstack) apFile.removeFile(spiderstack, warn=False) ### convert stack to imagic imagicstack = self.convertSpiderStack(alignedstack) apFile.removeFile(alignedstack) inserttime = time.time() if self.params['commit'] is True: self.runtime = aligntime self.insertNoRefRun(imagicstack, insert=True) else: apDisplay.printWarning("not committing results to DB") inserttime = time.time() - inserttime apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime)) apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def start(self): # Path of the stack stackdata = apStack.getOnlyStackData(self.params['stackid']) fn_oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) # Convert the stack to Xmipp apXmipp.breakupStackIntoSingleFiles(fn_oldstack) # Run sort junk cmd = "xmipp_sort_by_statistics -i partlist.sel" apDisplay.printColor(cmd, "cyan") proc = subprocess.Popen(cmd, shell=True) proc.wait() # Create sorted stack apXmipp.gatherSingleFilesIntoStack("sort_junk.sel","sorted.hed") # Create average MRC apStack.averageStack("sorted.hed") # Remove intermediate stuff #os.unlink("partlist.sel") #shutil.rmtree("partfiles") # Upload results self.uploadResults() time.sleep(1) return
def checkConflicts(self): if self.params["stackid"] is None: apDisplay.printError("stack id was not defined") if self.params["description"] is None: apDisplay.printError("run description was not defined") if self.params["runname"] is None: apDisplay.printError("run name was not defined") maxparticles = 150000 if self.params["numpart"] > maxparticles: apDisplay.printError( "too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params["numpart"]) ) if self.params["initmethod"] not in self.initmethods: apDisplay.printError( "unknown initialization method defined: " + str(self.params["initmethod"]) + " not in " + str(self.initmethods) ) stackdata = apStack.getOnlyStackData(self.params["stackid"], msg=False) stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"]) if self.params["numpart"] > apFile.numImagesInStack(stackfile): apDisplay.printError( "trying to use more particles " + str(self.params["numpart"]) + " than available " + str(apFile.numImagesInStack(stackfile)) )
def start(self): ### universal particle counter self.partnum = 1 ### final stack file self.combinefile = os.path.join( self.params['rundir'], self.params['stackfilename'] ) if os.path.isfile(self.combinefile): apDisplay.printError("A stack with name "+self.params['stackfilename']+" and path " +self.params['rundir']+" already exists.") ### loop through stacks for stackstr in self.params['stackids']: stackid = int(stackstr) ### get stack data stackdata = apStack.getOnlyStackData(stackid) ### append particle to stack file self.appendToStack(stackdata) if self.params['commit'] is True: ### insert stack data apDisplay.printColor("inserting new stack particles from stackid="+str(stackid), "cyan") self.commitStack(stackid) else: apDisplay.printWarning("not committing data to database") apStack.averageStack(stack=self.combinefile)
def setupParticles(self): self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed") self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) boxsize = int(math.floor(self.stack['boxsize']/float(self.params['bin']*2)))*2 apix = self.stack['apix']*self.params['bin'] proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix']) if self.params['bin'] > 1: clipsize = boxsize*self.params['bin'] proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize) if self.params['highpass'] > 1: proccmd += " hp="+str(self.params['highpass']) if self.params['lowpass'] > 1: proccmd += " lp="+str(self.params['lowpass']) proccmd += " last="+str(self.params['numpart']-1) apFile.removeStack(self.params['localstack'], warn=False) apEMAN.executeEmanCmd(proccmd, verbose=True) ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack']) return (boxsize, apix)
def insertAlignStackRunIntoDatabase(self, alignimagicfile): apDisplay.printMsg("Inserting CL2D Run into DB") ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.runparams['runname'] alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) # if uniquerun: # apDisplay.printError("Run name '"+self.runparams['runname']+"' and path already exist in database") ### setup cl2d run cl2dq = appiondata.ApCL2DRunData() cl2dq['runname'] = self.runparams['runname'] cl2dq['run_seconds'] = self.runparams['runtime'] cl2dq['fast'] = self.runparams['fast'] self.cl2dqdata=cl2dq ### finish alignment run alignrunq['cl2drun'] = cl2dq alignrunq['hidden'] = False alignrunq['runname'] = self.runparams['runname'] alignrunq['description'] = self.runparams['description'] alignrunq['lp_filt'] = self.runparams['lowpass'] alignrunq['hp_filt'] = self.runparams['highpass'] alignrunq['bin'] = self.runparams['bin'] ### setup alignment stack alignstackq = appiondata.ApAlignStackData() if self.runparams['align'] is True: ### option to create aligned stack alignstackq['imagicfile'] = alignimagicfile alignstackq['avgmrcfile'] = "average.mrc" alignstackq['refstackfile'] = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1) alignstackq['iteration'] = self.runparams['maxiter'] alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) alignstackq['alignrun'] = alignrunq ### check to make sure files exist if self.runparams['align'] is True: ### option to create aligned stack alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(alignimagicfilepath): apDisplay.printError("could not find stack file: "+alignimagicfilepath) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: "+avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: "+refstackfile) alignstackq['stack'] = apStack.getOnlyStackData(self.runparams['stackid']) alignstackq['boxsize'] = self.boxsize alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin'] alignstackq['description'] = self.runparams['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.runparams['numpart'] ### insert if self.params['commit'] is True: alignstackq.insert() self.alignstackdata = alignstackq return
def createAlignedStacks(self, partlist): stackid = self.params['stackid'] stackdata = apStack.getOnlyStackData(stackid) origstackfile = os.path.join(stackdata['path']['path'], stackdata['name']) imagesdict = apImagicFile.readImagic(origstackfile) i = 0 t0 = time.time() apDisplay.printMsg("rotating and shifting particles at "+time.asctime()) alignstack = [] while i < len(partlist): partimg = imagesdict['images'][i] partdict = partlist[i] partnum = i+1 #print partnum, partdict, partimg.shape if partdict['partnum'] != partnum: apDisplay.printError("particle shifting "+str(partnum)+" != "+str(partdict)) xyshift = (partdict['xshift'], partdict['yshift']) alignpartimg = apImage.xmippTransform(partimg, rot=partdict['inplane'], shift=xyshift, mirror=partdict['mirror']) alignstack.append(alignpartimg) i += 1 apDisplay.printMsg("rotate then shift %d particles in %s"%(i,apDisplay.timeString(time.time()-t0))) alignstackarray = numpy.asarray(alignstack) self.alignimagicfile = "alignstack.hed" apImagicFile.writeImagic(alignstackarray, self.alignimagicfile)
def start(self): #new stack path stackdata = apStack.getOnlyStackData(self.params['stackid']) oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) #make sure that old stack is numbered apEMAN.checkStackNumbering(oldstack) alignedstack = os.path.join(self.params['rundir'], 'ali.img') badstack = os.path.join(self.params['rundir'], 'bad.img') apStack.checkForPreviousStack(alignedstack) #run centering algorithm apStack.centerParticles(oldstack, self.params['mask'], self.params['maxshift']) self.params['keepfile'] = os.path.join(self.params['rundir'],'keepfile.txt') apEMAN.writeStackParticlesToFile(alignedstack, self.params['keepfile']) if not os.path.isfile(alignedstack, ): apDisplay.printError("No stack was created") #get number of particles f = open(self.params['keepfile'], "r") numparticles = len(f.readlines()) f.close() self.params['description'] += ( (" ... %d eman centered substack id %d" % (numparticles, self.params['stackid'])) ) apStack.commitSubStack(self.params, newname='ali.hed', centered=True) apStack.averageStack(stack=alignedstack) if (os.path.exists(badstack)): apStack.averageStack(stack=badstack, outfile='badaverage.mrc')
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") #if self.params['description'] is None: # apDisplay.printError("run description was not defined") if self.params['numrefs'] is None: apDisplay.printError("a number of classes was not provided") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if not self.params['fastmode'] in self.fastmodes: apDisplay.printError("fast mode must be on of: "+str(self.fastmodes)) if not self.params['converge'] in self.convergemodes: apDisplay.printError("converge mode must be on of: "+str(self.convergemodes)) maxparticles = 500000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize(self.params['stackid']) self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" %(self.params['clipsize'],self.clipsize)) self.clipsize = self.params['clipsize'] if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile)
def checksizes(self): newnots = apFile.numImagesInStack(self.params['notstack']) newtilts = apFile.numImagesInStack(self.params['tiltstack']) stackdata1 = apStack.getOnlyStackData(self.params['stack1'], msg=False) stackpath1 = os.path.join(stackdata1['path']['path'], stackdata1['name']) oldnots = apFile.numImagesInStack(stackpath1) stackdata2 = apStack.getOnlyStackData(self.params['stack2'], msg=False) stackpath2 = os.path.join(stackdata2['path']['path'], stackdata2['name']) oldtilts = apFile.numImagesInStack(stackpath2) if newtilts != oldtilts: apDisplay.printWarning("tilted stack are different sizes: %d vs. %d part"%(newtilts,oldtilts)) else: apDisplay.printMsg("tilted stack are the sames sizes: %d part"%(newtilts)) if newnots != oldnots: apDisplay.printWarning("untilted stack are different sizes: %d vs. %d part"%(newnots,oldnots)) else: apDisplay.printMsg("untilted stack are the sames sizes: %d part"%(newnots))
def setRunDir(self): stackdata = apStack.getOnlyStackData(self.params['tiltstackid'], msg=False) path = stackdata['path']['path'] uppath = os.path.dirname(os.path.dirname(os.path.abspath(path))) classliststr = operations.intListToString(self.classlist) self.params['rundir'] = os.path.join(uppath, "rctvolume", self.params['runname'] )
def setRunDir(self): stackdata = apStack.getOnlyStackData(self.params["partstackid"], msg=False) path = stackdata["path"]["path"] uppath = os.path.dirname(os.path.dirname(os.path.abspath(path))) self.params["rundir"] = os.path.join(uppath, "SpiderRecon", self.params["runname"]) print self.params["rundir"]
def checkConflicts(self): if self.params['tiltstackid'] is None: apDisplay.printError("Enter a tilted stack ID") if self.params['notstackid'] is None: apDisplay.printError("Enter a untilted stack ID") if self.params['runname'] is None: apDisplay.printError("Enter run name") self.tiltstackdata = apStack.getOnlyStackData(self.params['tiltstackid'])
def setRunDir(self): """ This funcion is only run when the user does not specifiy 'rundir' """ self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) path = self.stackdata['path']['path'] uppath = os.path.abspath(os.path.join(path, "../..")) self.params['rundir'] = os.path.join(uppath, "align", self.params['runname'])
def insertOtrRun(self, volfile): ### setup resolutions fscresq = appiondata.ApResolutionData() fscresq['type'] = "fsc" fscresq['half'] = self.fscresolution fscresq['fscfile'] = "fscdata"+self.timestamp+".fsc" rmeasureq = appiondata.ApResolutionData() rmeasureq['type'] = "rmeasure" rmeasureq['half'] = self.rmeasureresolution rmeasureq['fscfile'] = None ### insert rct run data otrrunq = appiondata.ApOtrRunData() otrrunq['runname'] = self.params['runname'] tempstr = "" for cnum in self.classlist: tempstr += str(cnum)+"," classliststr = tempstr[:-1] otrrunq['classnums'] = classliststr otrrunq['numiter'] = self.params['numiters'] otrrunq['euleriter'] = self.params['refineiters'] otrrunq['maskrad'] = self.params['radius'] otrrunq['lowpassvol'] = self.params['lowpassvol'] otrrunq['highpasspart'] = self.params['highpasspart'] otrrunq['median'] = self.params['median'] otrrunq['description'] = self.params['description'] otrrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) otrrunq['alignstack'] = self.alignstackdata otrrunq['tiltstack'] = apStack.getOnlyStackData(self.params['tiltstackid']) otrrunq['numpart'] = self.numpart otrrunq['fsc_resolution'] = fscresq otrrunq['rmeasure_resolution'] = rmeasureq if self.params['commit'] is True: otrrunq.insert() ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['otrrun'] = otrrunq densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25) densq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] densq['boxsize'] = self.getBoxSize() densq['lowpass'] = self.params['lowpassvol'] densq['highpass'] = self.params['highpasspart'] densq['mask'] = self.params['radius'] #densq['iterid'] = self.params['numiters'] densq['description'] = self.params['description'] densq['resolution'] = self.fscresolution densq['rmeasure'] = self.rmeasureresolution densq['session'] = apStack.getSessionDataFromStackId(self.params['tiltstackid']) densq['md5sum'] = apFile.md5sumfile(volfile) if self.params['commit'] is True: densq.insert() return
def insertOtrRun(self, volfile): ### setup resolutions fscresq = appiondata.ApResolutionData() fscresq["type"] = "fsc" fscresq["half"] = self.fscresolution fscresq["fscfile"] = "fscdata" + self.timestamp + ".fsc" rmeasureq = appiondata.ApResolutionData() rmeasureq["type"] = "rmeasure" rmeasureq["half"] = self.rmeasureresolution rmeasureq["fscfile"] = None ### insert rct run data otrrunq = appiondata.ApOtrRunData() otrrunq["runname"] = self.params["runname"] tempstr = "" for cnum in self.classlist: tempstr += str(cnum) + "," classliststr = tempstr[:-1] otrrunq["classnums"] = classliststr otrrunq["numiter"] = self.params["numiters"] otrrunq["euleriter"] = self.params["refineiters"] otrrunq["maskrad"] = self.params["radius"] otrrunq["lowpassvol"] = self.params["lowpassvol"] otrrunq["highpasspart"] = self.params["highpasspart"] otrrunq["median"] = self.params["median"] otrrunq["description"] = self.params["description"] otrrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"])) otrrunq["alignstack"] = self.alignstackdata otrrunq["tiltstack"] = apStack.getOnlyStackData(self.params["tiltstackid"]) otrrunq["numpart"] = self.numpart otrrunq["fsc_resolution"] = fscresq otrrunq["rmeasure_resolution"] = rmeasureq if self.params["commit"] is True: otrrunq.insert() ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq["otrrun"] = otrrunq densq["path"] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq["name"] = os.path.basename(volfile) densq["hidden"] = False densq["norm"] = True densq["symmetry"] = appiondata.ApSymmetryData.direct_query(25) densq["pixelsize"] = apStack.getStackPixelSizeFromStackId(self.params["partstackid"]) * self.params["bin"] densq["boxsize"] = self.getBoxSize() densq["lowpass"] = self.params["lowpassvol"] densq["highpass"] = self.params["highpasspart"] densq["mask"] = self.params["radius"] # densq['iterid'] = self.params['numiters'] densq["description"] = self.params["description"] densq["resolution"] = self.fscresolution densq["rmeasure"] = self.rmeasureresolution densq["session"] = apStack.getSessionDataFromStackId(self.params["tiltstackid"]) densq["md5sum"] = apFile.md5sumfile(volfile) if self.params["commit"] is True: densq.insert() return
def setRunDir(self): stackid = int(self.params['stackid']) #stackdata = apStack.getOnlyStackData(stackid, msg=False) stackdata = apStack.getOnlyStackData(stackid) print "stackdata: ", stackdata path = stackdata['path']['path'] #substitude the stacks with dbstacks folder uppath = os.path.dirname(os.path.abspath(path))[:-6]+"dbstacks" self.params['rundir'] = os.path.join(uppath, self.params['runname'])
def start(self): self.rootname = self.params['stackname'].split(".")[0] self.params['outputstack'] = os.path.join(self.params['rundir'], self.params['stackname']) if os.path.isfile(self.params['outputstack']): apFile.removeStack(self.params['outputstack']) if self.params['eotest'] is True: self.params['evenstack'] = os.path.splitext(self.params['outputstack'])[0]+'.even.hed' if os.path.isfile(self.params['evenstack']): apFile.removeStack(self.params['evenstack']) self.params['oddstack'] = os.path.splitext(self.params['outputstack'])[0]+'.odd.hed' if os.path.isfile(self.params['oddstack']): apFile.removeStack(self.params['oddstack']) classes = self.getClassData(self.params['reconid'], self.params['iter']) stackid = apStack.getStackIdFromRecon(self.params['reconid']) stackdata = apStack.getOnlyStackData(stackid) stackpath = os.path.join(stackdata['path']['path'], stackdata['name']) classkeys = classes.keys() classkeys.sort() classnum=0 keeplist = self.procKeepList() finallist = [] apDisplay.printMsg("Processing "+str(len(classes))+" classes") #loop through classes for key in classkeys: classnum+=1 if classnum%10 == 1: sys.stderr.write("\b\b\b\b\b\b\b\b\b\b\b\b\b\b") sys.stderr.write(str(classnum)+" of "+(str(len(classkeys)))) # loop through particles in class classfile = self.rootname+"-class.lst" classf = open(classfile, 'w') classf.write('#LST\n') nptcls=0 for ptcl in classes[key]['particles']: # translate DB into EMAN partnum = ptcl['particle']['particleNumber'] - 1 if partnum in keeplist: if ptcl['mirror']: mirror=1 else: mirror=0 rot = ptcl['euler3']*math.pi/180.0 classf.write( "%d\t%s\t%f,\t%f,%f,%f,%d\n" % (partnum, stackpath, ptcl['quality_factor'], rot, ptcl['shiftx'], ptcl['shifty'], mirror)) nptcls+=1 finallist.append(partnum) classf.close() if nptcls<1: continue self.makeClassAverages(classfile, self.params['outputstack'], classes[key], self.params['mask']) if self.params['eotest'] is True: self.makeEvenOddClasses(classfile, self.params['outputstack'], classes[key], self.params['mask']) apFile.removeFile(classfile) sys.stderr.write("\n") finalfilename = self.rootname+"-keep.lst" finalf = open(finalfilename, 'w') finallist.sort() for partnum in finallist: finalf.write('%d\n' % (partnum,) ) finalf.close() stackstr = str(stackdata.dbid) reconstr = str(self.params['reconid']) ### recon 3d volumes threedname = os.path.join(self.params['rundir'], self.rootname+"."+str(self.params['iter'])+"a.mrc") emancmd = ( "make3d "+self.params['outputstack']+" out=" +threedname+" hard=50 sym="+self.params['symmname']+" pad=240 mask="+str(self.params['mask'])+"; echo ''" ) #print emancmd apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True, logfile=self.rootname+"-eman.log") threednameb = os.path.join(self.params['rundir'], self.rootname+"."+str(self.params['iter'])+"b.mrc") emancmd = ( "proc3d "+threedname+" "+threednameb +" apix=1.63 norm=0,1 lp=8 origin=0,0,0 mask="+str(self.params['mask'])+"; echo '' " ) apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True, logfile=self.rootname+"-eman.log") if self.params['eotest'] is True: # even evenname = os.path.join(self.params['rundir'], self.rootname+"-even."+str(self.params['iter'])+"a.mrc") if os.path.isfile(self.params['evenstack']): evenemancmd = ( "make3d "+self.params['evenstack']+" out=" +evenname+" hard=50 sym="+self.params['symmname']+" pad=240 mask="+str(self.params['mask'])+"; echo ''" ) #print evenemancmd apEMAN.executeEmanCmd(evenemancmd, verbose=False, showcmd=True, logfile=self.rootname+"-eveneman.log") else: apDisplay.printWarning("file "+self.params['evenstack']+" does not exist") # odd oddname = os.path.join(self.params['rundir'], self.rootname+"-odd."+str(self.params['iter'])+"a.mrc") if os.path.isfile(self.params['oddstack']): oddemancmd = ( "make3d "+self.params['oddstack']+" out=" +oddname+" hard=50 sym="+self.params['symmname']+" pad=240 mask="+str(self.params['mask'])+"; echo ''" ) #print oddemancmd apEMAN.executeEmanCmd(oddemancmd, verbose=False, showcmd=True, logfile=self.rootname+"-oddeman.log") else: apDisplay.printWarning("file "+self.params['oddstack']+" does not exist") #eotest fscout = os.path.join(self.params['rundir'], self.rootname+"-fsc.eotest") if os.path.isfile(oddname) and os.path.isfile(evenname): eotestcmd = "proc3d "+oddname+" "+evenname+" fsc="+fscout apEMAN.executeEmanCmd(eotestcmd, verbose=True, showcmd=True) else: apDisplay.printWarning("could not perform eotest") if os.path.isfile(fscout): res = apRecon.getResolutionFromFSCFile(fscout, 160.0, 1.63) apDisplay.printColor( ("resolution: %.5f" % (res)), "cyan") resfile = self.rootname+"-res.txt" f = open(resfile, 'a') f.write("[ %s ]\nresolution: %.5f\n" % (time.asctime(), res)) f.close()
def start(self): self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### test insert to make sure data is not overwritten self.params['runtime'] = 0 #self.checkDuplicateRefBasedRun() ### set box size self.boxsize = int( math.floor(self.stack['boxsize'] / self.params['bin'] / 2.0)) * 2 ### convert stack to spider spiderstack = self.createSpiderFile() ### create template stack templatestack = self.createTemplateStack() ### run the alignment aligntime = time.time() usestack = spiderstack oldpartlist = None for i in range(self.params['numiter']): iternum = i + 1 apDisplay.printColor("\n\nITERATION " + str(iternum), "green") alignedstack, partlist = alignment.refBasedAlignParticles( usestack, templatestack, spiderstack, self.params['xysearch'], self.params['xystep'], self.params['numpart'], self.params['numtemplate'], self.params['firstring'], self.params['lastring'], iternum=iternum, oldpartlist=oldpartlist) oldpartlist = partlist usestack = alignedstack templatestack = self.updateTemplateStack(alignedstack, partlist, iternum) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ### remove large, worthless stack spiderstack = os.path.join(self.params['rundir'], "start.spi") apDisplay.printMsg("Removing un-aligned stack: " + spiderstack) apFile.removeFile(spiderstack, warn=True) ### convert aligned stack to imagic finalspistack = "aligned.spi" shutil.move(alignedstack, finalspistack) imagicstack = "aligned.hed" apFile.removeStack(imagicstack) emancmd = "proc2d " + finalspistack + " " + imagicstack apEMAN.executeEmanCmd(emancmd, verbose=True) ### average stack apStack.averageStack(imagicstack) ### calculate resolution for each reference apix = self.stack['apix'] * self.params['bin'] self.calcResolution(partlist, imagicstack, apix) if self.params['commit'] is True: apDisplay.printMsg("committing results to DB") self.params['runtime'] = aligntime self.insertRefBasedRun(partlist, imagicstack, insert=True) else: apDisplay.printWarning("not committing results to DB") ### remove temporary files apFile.removeFilePattern("alignments/alignedstack*.spi") apFile.removeFile(finalspistack)
def start(self): if self.params['rundir'] is None or not os.path.isdir( self.params['rundir']): apDisplay.printError("upload directory does not exist: " + str(self.params['rundir'])) ### create temp directory for extracting data self.params['tmpdir'] = os.path.join(self.params['rundir'], "temp") apParam.createDirectory(self.params['tmpdir'], warning=True) ### make sure that the stack & model IDs exist in database emanJobFile = self.findEmanJobFile() self.params['stack'] = apStack.getOnlyStackData(self.params['stackid']) self.stackmapping = apRecon.partnum2defid(self.params['stackid']) self.params['model'] = appiondata.ApInitialModelData.direct_query( self.params['modelid']) self.params['boxsize'] = apStack.getStackBoxsize( self.params['stackid']) ### parse out the refinement parameters from the log file self.parseLogFile() ### parse out the message passing subclassification parameters from the job/log file if self.params['package'] == 'EMAN/MsgP': self.parseMsgPassingParams() ### convert class average files from old to new format self.convertClassAvgFiles() ### get a list of the files in the directory self.listFiles() ### create a refinementRun entry in the database self.insertRefinementRun() if self.params['euleronly'] is False: ### insert the Iteration info for iteration in self.iterationdatas: ### if only uploading one iteration, skip to that one if self.params['oneiter'] and int( iteration['num']) != self.params['oneiter']: continue ### if beginning at later iteration, skip to that one if self.params['startiter'] and int( iteration['num']) < self.params['startiter']: continue ### if beginning at later iteration, skip to that one if self.params['enditer'] and int( iteration['num']) > self.params['enditer']: continue apDisplay.printColor( "\nUploading iteration " + str(iteration['num']) + " of " + str(len(self.iterationdatas)) + "\n", "green") for i in range(75): sys.stderr.write("#") sys.stderr.write("\n") self.insertIteration(iteration) ### calculate euler jumps if self.params['commit'] is True: reconrunid = self.params['refineRun'].dbid stackid = self.params['stack'].dbid if self.params['oneiter'] is None and len(self.iterationdatas) > 1: apDisplay.printMsg("calculating euler jumpers for recon=" + str(reconrunid)) eulerjump = apEulerJump.ApEulerJump() eulerjump.calculateEulerJumpsForEntireRecon( reconrunid, stackid) ### coran keep plot if self.params['package'] == 'EMAN/SpiCoran': apCoranPlot.makeCoranKeepPlot(reconrunid) apRecon.setGoodBadParticlesFromReconId(reconrunid)
def insertAlignmentRun(self, insert=False): ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '" + self.params['runname'] + "' and path already exist in database") ### setup Reference preparation parameters, if given MRAq = appiondata.ApMultiRefAlignRunData() if self.params['refs'] is True: MRAq['lowpass_refs'] = self.params['lowpass_refs'] MRAq['thresh_refs'] = self.params['thresh_refs'] MRAq['maskrad_refs'] = self.params['maskrad_refs'] ### setup Multi Reference Alignment Run MRAq['mirror'] = self.params['mirror'] MRAq['center'] = self.params['center'] MRAq['alignment_type'] = self.params['alignment_type'] if self.params['alignment_type'].lower() == "all": MRAq['first_alignment'] = self.params['first_alignment'] if self.params['alignment_type'].lower() == "brute_force": MRAq['num_orientations'] = self.params['num_orientations'] MRAq['max_shift_orig'] = self.params['max_shift_orig'] if self.params['center'] is True: MRAq['max_shift_this'] = self.params['max_shift_this'] MRAq['samp_param'] = self.params['samp_param'] MRAq['min_radius'] = self.params['minrad'] MRAq['max_radius'] = self.params['minrad'] MRAq['numiter'] = self.params['numiter'] ### finish alignment run alignrunq['imagicMRA'] = MRAq alignrunq['hidden'] = False alignrunq['description'] = self.params['description'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['bin'] = self.params['bin'] ### setup alignment stack alignstackq = appiondata.ApAlignStackData() alignstackq['imagicfile'] = "alignstack.hed" alignstackq['avgmrcfile'] = "average.mrc" alignstackq['refstackfile'] = "references.hed" alignstackq['iteration'] = self.params['numiter'] alignstackq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) alignstackq['alignrun'] = alignrunq ### check to make sure files exist imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find stack file: " + imagicfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: " + refstackfile) alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackId']) alignstackq['boxsize'] = self.params['boxsize'] alignstackq['pixelsize'] = self.params['apix'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.params['numpart'] ### insert if self.params['commit'] is True: alignstackq.insert() self.alignstackdata = alignstackq return
def start(self): ### get stack data notstackdata = apStack.getOnlyStackData(self.params['notstackid']) tiltstackdata = apStack.getOnlyStackData(self.params['tiltstackid']) ### get good particle numbers includeParticle, tiltParticlesData = self.getGoodAlignParticles() self.numpart = len(includeParticle) ### make doc file of Euler angles eulerfile = self.makeEulerDoc(tiltParticlesData) ### write kept particles to file self.params['keepfile'] = os.path.join( self.params['rundir'], "keepfile" + self.timestamp + ".lst") apDisplay.printMsg("writing to keepfile " + self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum) + "\n") kf.close() ### make new stack of tilted particle from that run tiltstackfile = os.path.join(tiltstackdata['path']['path'], tiltstackdata['name']) rctstackfile = os.path.join(self.params['rundir'], "rctstack" + self.timestamp + ".hed") apFile.removeStack(rctstackfile, warn=False) apStack.makeNewStack(tiltstackfile, rctstackfile, self.params['keepfile']) spiderstack = self.convertStackToSpider(rctstackfile) #self.mirrorParticles(tiltParticlesData, spiderstack) ### iterations over volume creation ### back project particles into filter volume volfile = os.path.join(self.params['rundir'], "volume%s-%03d.spi" % (self.timestamp, 0)) backproject.backprojectCG(spiderstack, eulerfile, volfile, numpart=self.numpart, pixrad=self.params['radius']) alignstack = spiderstack ### center/convert the volume file mrcvolfile = self.processVolume(volfile, 0) for i in range(self.params['numiters']): looptime = time.time() iternum = i + 1 apDisplay.printMsg("running backprojection iteration " + str(iternum)) ### xy-shift particles to volume projections alignstack = backproject.rctParticleShift( volfile, alignstack, eulerfile, iternum, numpart=self.numpart, pixrad=self.params['radius'], timestamp=self.timestamp) apFile.removeFile(volfile) ### back project particles into better volume volfile = os.path.join( self.params['rundir'], "volume%s-%03d.spi" % (self.timestamp, iternum)) backproject.backproject3F(alignstack, eulerfile, volfile, numpart=self.numpart) ### center/convert the volume file mrcvolfile = self.processVolume(volfile, iternum) apDisplay.printColor( "finished volume refinement loop in " + apDisplay.timeString(time.time() - looptime), "cyan") ### optimize Euler angles #NOT IMPLEMENTED YET ### perform eotest if self.params['eotest'] is True: self.runEoTest(alignstack, eulerfile) self.runRmeasure() ### insert volumes into DB self.insertRctRun(mrcvolfile)
def start(self): ### get stack parameteres self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackId']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackId']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackId']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### copy stack into working directory if os.path.isfile(self.stack['file']): apDisplay.printColor("copying stack into running directoroy", "cyan") if self.stack['file'][-4:] == ".img" or self.stack['file'][ -4:] == ".hed": strippedfile = self.stack['file'][:-4] else: strippedfile = self.stack['file'] while os.path.isfile( os.path.join(self.params['rundir'], "start.img")): apFile.removeStack( os.path.join(self.params['rundir'], "start.img")) emancmd = "proc2d "+strippedfile+".hed "+os.path.join(self.params['rundir'], "start.hed ")+\ "first=0 last="+str(self.params['numpart']-1) apParam.runCmd(emancmd, "EMAN") else: apDisplay.printError("stack not found in database") ### get template stack parameters self.templatestack = {} self.templatestack[ 'data'] = appiondata.ApTemplateStackData.direct_query( self.params['templateStackId']) self.templatestack['apix'] = self.templatestack['data']['apix'] self.templatestack['boxsize'] = self.templatestack['data']['boxsize'] self.templatestack['file'] = os.path.join( self.templatestack['data']['path']['path'], self.templatestack['data']['templatename']) self.templatestack['numimages'] = self.templatestack['data'][ 'numimages'] ### copy templates into working directory if os.path.isfile(self.templatestack['file']): apDisplay.printColor("copying templates into running directoroy", "cyan") ts = os.path.join(self.params['rundir'], "references.img") while os.path.isfile(ts): apFile.removeStack(ts) if self.templatestack['file'][-4:] == ".img" or self.templatestack[ 'file'][-4:] == ".hed": strippedfile = self.templatestack['file'][:-4] else: strippedfile = self.templatestack['file'] emancmd = "proc2d " + strippedfile + ".img " + ts apParam.runCmd(emancmd, "EMAN") else: apDisplay.printError("template stack not found in database") ### set new pixelsize if self.params['bin'] is not None and self.params['bin'] != 0: self.params['apix'] = float(self.stack['apix']) * int( self.params['bin']) else: self.params['apix'] = self.stack['apix'] ### scale, low-pass, and high-pass filter stack ... do this with imagic, because it determines the appropriate boxsizes scalingbatchfile = self.createImagicBatchFileScaling() preptime = time.time() proc = subprocess.Popen("chmod 775 " + str(scalingbatchfile), shell=True) proc.wait() os.chdir(self.params['rundir']) apParam.runCmd(scalingbatchfile, "IMAGIC") apIMAGIC.checkLogFileForErrors( os.path.join(self.params['rundir'], "prepareStack.log")) apDisplay.printColor( "finished IMAGIC in " + apDisplay.timeString(time.time() - preptime), "cyan") ### set new boxsize, done only after scaling is complete if self.params['bin'] is not None: self.params['boxsize'] = apFile.getBoxSize( os.path.join(self.params['rundir'], "start.hed"))[0] else: self.params['boxsize'] = self.stack['boxsize'] ### make sure template stack boxsize matches that of the input stack if self.params['apix'] != self.templatestack['apix'] or self.params[ 'boxsize'] != self.templatestack['boxsize']: self.scaleTemplates() starttime = time.time() print self.params print "... stack pixel size: " + str(self.params['apix']) print "... stack box size: " + str(self.params['boxsize']) apDisplay.printColor( "Running IMAGIC .batch file: See multiReferenceAlignment.log file for details", "cyan") ### create IMAGIC batch file batchfile = self.createImagicBatchFileMRA() ### execute IMAGIC batch file aligntime0 = time.time() proc = subprocess.Popen("chmod 775 " + str(batchfile), shell=True) proc.wait() os.chdir(self.params['rundir']) apParam.runCmd(batchfile, "IMAGIC") apIMAGIC.checkLogFileForErrors( os.path.join(self.params['rundir'], "multiReferenceAlignment.log")) apDisplay.printColor( "finished IMAGIC in " + apDisplay.timeString(time.time() - aligntime0), "cyan") ### get particle parameters (shift, rotate, refnum, mirror, ccc) partparams = self.getParticleParams() ### average stack alignstack = os.path.join(self.params['rundir'], "alignstack.hed") apStack.averageStack(alignstack) ### normalize particles (otherwise found problems in viewing with stackviewer) emancmd = "proc2d " + alignstack + " " + alignstack + ".norm.hed norm" while os.path.isfile(alignstack + ".norm.img"): apFile.removeStack(alignstack + ".norm.img") apParam.runCmd(emancmd, "EMAN") os.rename(alignstack + ".norm.hed", alignstack) os.rename(alignstack + ".norm.img", alignstack[:-4] + ".img") ### normalize references emancmd = "proc2d " + ts + " " + ts + ".norm.hed norm" while os.path.isfile(ts + ".norm.img"): apFile.removeStack(ts + ".norm.img") apParam.runCmd(emancmd, "EMAN") os.rename(ts + ".norm.hed", ts) os.rename(ts + ".norm.img", ts[:-4] + ".img") ### remove copied stack while os.path.isfile(os.path.join(self.params['rundir'], "start.img")): apFile.removeStack(os.path.join(self.params['rundir'], "start.img")) ### insert run into database self.insertAlignmentRun(insert=True) self.insertParticlesIntoDatabase(partparams, insert=True)
def start(self): #new stack path stackdata = apStack.getOnlyStackData(self.params['stackid']) apStackFormat.linkFormattedStack(stackdata, self.params['format'], 'test')
def createStackData(self): apDisplay.printColor("Starting upload of stack", "blue") pathq = appiondata.ApPathData() pathq['path'] = self.params['rundir'] manq = appiondata.ApManualParamsData() manq['diam'] = self.params['diameter'] manq['trace'] = False selectq = appiondata.ApSelectionRunData() selectq['name'] = 'fakestack_' + self.params['runname'] selectq['hidden'] = True selectq['path'] = pathq selectq['session'] = self.sessiondata selectq['manparams'] = manq stackq = appiondata.ApStackData() stackq['name'] = "start.hed" stackq['path'] = pathq stackq['description'] = self.params['description'] stackq['hidden'] = False stackq['pixelsize'] = self.params['apix'] * 1e-10 stackq['boxsize'] = self.boxsize stackq['centered'] = False stackparamq = appiondata.ApStackParamsData() stackparamq['boxSize'] = self.boxsize stackparamq['bin'] = 1 stackparamq['phaseFlipped'] = self.params['ctfcorrect'] if self.params['ctfcorrect'] is True: stackparamq['fileType'] = "manual" stackparamq['fileType'] = "imagic" stackparamq['normalized'] = self.params['normalize'] stackparamq['lowpass'] = 0 stackparamq['highpass'] = 0 stackrunq = appiondata.ApStackRunData() stackrunq['stackRunName'] = self.params['runname'] stackrunq['stackParams'] = stackparamq stackrunq['selectionrun'] = selectq stackrunq['session'] = self.sessiondata runsinstackq = appiondata.ApRunsInStackData() runsinstackq['stack'] = stackq runsinstackq['stackRun'] = stackrunq if self.params['commit'] is True: runsinstackq.insert() if 'syncstackid' in self.params.keys( ) and self.params['syncstackid']: stackdata = runsinstackq['stack'] stack2data = apStack.getOnlyStackData( self.params['syncstackid']) syncq = appiondata.ApSyncStackData( stack1=stackdata, stack2=stack2data, synctype=self.params['synctype']) syncq.insert() ### for each particle sys.stderr.write("Starting particle upload") for i in range(self.numpart): if i % 100 == 0: sys.stderr.write(".") partq = appiondata.ApParticleData() partq['image'] = None #We have no image, see if this works??? partq['selectionrun'] = selectq partq['xcoord'] = int(i % 1000) partq['ycoord'] = int(i / 1000) partq['diameter'] = self.params['diameter'] stackpartq = appiondata.ApStackParticleData() stackpartq['particleNumber'] = i + 1 stackpartq['stack'] = stackq stackpartq['stackRun'] = stackrunq stackpartq['particle'] = partq stackpartq['mean'] = 0.0 stackpartq['stdev'] = 1.0 if self.params['commit'] is True: stackpartq.insert() sys.stderr.write("\n") return
def insertAlignStackRunIntoDatabase(self, alignimagicfile): apDisplay.printMsg("Inserting CL2D Run into DB") ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.runparams['runname'] alignrunq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) # if uniquerun: # apDisplay.printError("Run name '"+self.runparams['runname']+"' and path already exist in database") ### setup cl2d run cl2dq = appiondata.ApCL2DRunData() cl2dq['runname'] = self.runparams['runname'] cl2dq['run_seconds'] = self.runparams['runtime'] cl2dq['fast'] = self.runparams['fast'] self.cl2dqdata = cl2dq ### finish alignment run alignrunq['cl2drun'] = cl2dq alignrunq['hidden'] = False alignrunq['runname'] = self.runparams['runname'] alignrunq['description'] = self.runparams['description'] alignrunq['lp_filt'] = self.runparams['lowpass'] alignrunq['hp_filt'] = self.runparams['highpass'] alignrunq['bin'] = self.runparams['bin'] ### setup alignment stack alignstackq = appiondata.ApAlignStackData() if self.runparams['align'] is True: ### option to create aligned stack alignstackq['imagicfile'] = alignimagicfile alignstackq['avgmrcfile'] = "average.mrc" alignstackq['refstackfile'] = "part" + self.params[ 'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1) alignstackq['iteration'] = self.runparams['maxiter'] alignstackq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) alignstackq['alignrun'] = alignrunq ### check to make sure files exist if self.runparams['align'] is True: ### option to create aligned stack alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(alignimagicfilepath): apDisplay.printError("could not find stack file: " + alignimagicfilepath) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: " + avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: " + refstackfile) alignstackq['stack'] = apStack.getOnlyStackData( self.runparams['stackid']) alignstackq['boxsize'] = self.boxsize alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId( self.runparams['stackid']) * self.runparams['bin'] alignstackq['description'] = self.runparams['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.runparams['numpart'] ### insert if self.params['commit'] is True: alignstackq.insert() self.alignstackdata = alignstackq return
def start(self): #old stack size stacksize = apStack.getNumberStackParticlesFromId( self.params['stackid']) # if exclude or include lists are not defined... if self.params['exclude'] is None and self.params['include'] is None: # if first and last are specified, create a file if self.params['first'] is not None and self.params[ 'last'] is not None: stp = str(self.params['first']) enp = str(self.params['last']) fname = 'sub' + str( self.params['stackid']) + '_' + stp + '-' + enp + '.lst' self.params['keepfile'] = os.path.join(self.params['rundir'], fname) apDisplay.printMsg("Creating keep list: " + self.params['keepfile']) f = open(self.params['keepfile'], 'w') for i in range(self.params['first'], self.params['last'] + 1): f.write('%d\n' % (int(i) - 1)) f.close() # generate the random list by giving number and create the file elif self.params['random'] is not None: #numOfRandomParticles = str(self.params['random']) #fname = 'random'+str(self.params['stackid'])+'_'+numOfRandomParticles+'.lst' fname = "random%d_%d.lst" % (self.params['stackid'], self.params['random']) self.params['keepfile'] = os.path.join(self.params['rundir'], fname) apDisplay.printMsg("Creating keep list: " + self.params['keepfile']) # create a file f = open(self.params['keepfile'], 'w') # generate a random sequence by giving size randomList = random.sample(xrange(self.params['last']), self.params['random']) randomList.sort() for partnum in randomList: f.write('%d\n' % partnum) f.close() # if splitting, create files containing the split values elif self.params['split'] > 1: for i in range(self.params['split']): fname = 'sub' + str( self.params['stackid']) + '.' + str(i + 1) + '.lst' self.params['keepfile'] = os.path.join( self.params['rundir'], fname) apDisplay.printMsg("Creating keep list: " + self.params['keepfile']) f = open(self.params['keepfile'], 'w') for p in range(stacksize): if (p % self.params['split']) - i == 0: f.write('%i\n' % p) f.close() # if exclude-from option is specified, convert particles to exclude elif self.params['excludefile'] is True: oldkf = open(self.params['keepfile']) partlist = [] for line in oldkf: particle = line.strip() try: particle = int(particle) except: continue partlist.append(particle) oldkf.close() # create new list excluding the particles apDisplay.printMsg("Converting keep file to exclude file") newkeepfile = "tmpnewkeepfile.txt" newkf = open(newkeepfile, 'w') for p in range(stacksize): if p not in partlist: newkf.write("%i\n" % p) newkf.close() self.params['keepfile'] = os.path.abspath(newkeepfile) # otherwise, just copy the file elif not os.path.isfile(os.path.basename(self.params['keepfile'])): shutil.copy(self.params['keepfile'], os.path.basename(self.params['keepfile'])) # if either exclude or include lists is defined elif self.params['exclude'] or self.params['include']: ### list of particles to be excluded excludelist = [] if self.params['exclude'] is not None: excludestrlist = self.params['exclude'].split(",") for excld in excludestrlist: excludelist.append(int(excld.strip())) apDisplay.printMsg("Exclude list: " + str(excludelist)) ### list of particles to be included includelist = [] if self.params['include'] is not None: includestrlist = self.params['include'].split(",") for incld in includestrlist: includelist.append(int(incld.strip())) apDisplay.printMsg("Include list: " + str(includelist)) #new stack path stackdata = apStack.getOnlyStackData(self.params['stackid']) newname = stackdata['name'] oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) #if include or exclude list is given... if self.params['include'] is not None or self.params[ 'exclude'] is not None: includeParticle = [] excludeParticle = 0 for partnum in range(stacksize): if includelist and partnum in includelist: includeParticle.append(partnum) elif excludelist and not partnum in excludelist: includeParticle.append(partnum) else: excludeParticle += 1 includeParticle.sort() ### write kept particles to file self.params['keepfile'] = os.path.join( self.params['rundir'], "keepfile-" + self.timestamp + ".list") apDisplay.printMsg("writing to keepfile " + self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum) + "\n") kf.close() #get number of particles numparticles = len(includeParticle) if excludelist: self.params['description'] += ( " ... %d particle substack of stackid %d" % (numparticles, self.params['stackid'])) elif includelist: self.params['description'] += ( " ... %d particle substack of stackid %d" % (numparticles, self.params['stackid'])) ogdescr = self.params['description'] for i in range(self.params['split']): ### always do this, if not splitting split=1 sb = os.path.splitext(stackdata['name']) if self.params['first'] is not None and self.params[ 'last'] is not None: newname = sb[0] + '.' + str(self.params['first']) + '-' + str( self.params['last']) + sb[-1] elif self.params['random'] is not None: newname = "%s-random%d%s" % (sb[0], self.params['random'], sb[-1]) elif self.params['split'] > 1: fname = 'sub' + str( self.params['stackid']) + '.' + str(i + 1) + '.lst' self.params['keepfile'] = os.path.join(self.params['rundir'], fname) newname = sb[0] + '.' + str(i + 1) + 'of' + str( self.params['split']) + sb[-1] newstack = os.path.join(self.params['rundir'], newname) apStack.checkForPreviousStack(newstack) #get number of particles f = open(self.params['keepfile'], "r") numparticles = len(f.readlines()) f.close() self.params['description'] = ogdescr self.params['description'] += ( (" ... %d particle substack of stackid %d" % (numparticles, self.params['stackid']))) #if splitting, add to description if self.params['split'] > 1: self.params['description'] += (" (%i of %i)" % (i + 1, self.params['split'])) #create the new sub stack if not self.params['correctbeamtilt']: apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=True) else: apBeamTilt.makeCorrectionStack(self.params['stackid'], oldstack, newstack) if not os.path.isfile(newstack): apDisplay.printError("No stack was created") apStack.commitSubStack(self.params, newname, sorted=False) apStack.averageStack(stack=newstack) newstackid = apStack.getStackIdFromPath(newstack) if self.params['meanplot'] is True: apDisplay.printMsg( "creating Stack Mean Plot montage for stackid") apStackMeanPlot.makeStackMeanPlot(newstackid)
def start(self): self.runtime = 0 self.partlist = [] self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.clipsize = int( math.floor(self.stack['boxsize'] / self.params['bin'] / 2.0) * self.params['bin'] * 2.0) #self.checkRunNamePath() ### convert stack to spider spiderstack = self.createSpiderFile() #create template stack templatestack = self.createTemplateStack() #create orientation reference orientref = self.createOrientationReference() ################################################################### aligntime = time.time() ### create batch file batchfilepath = self.setupBatchFile(spiderstack, templatestack, orientref) ### run the spider alignment apDisplay.printColor( "Running iterative ref-classification and free-alignment with spider", "cyan") self.runSpiderBatch(batchfilepath, spiderstack) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ################################################################### ### remove unaligned spider stack apDisplay.printMsg("Removing un-aligned stack: " + spiderstack) apFile.removeFile(spiderstack, warn=False) ### check to be sure files exist avgfile = os.path.join(self.params['rundir'], "alignstack.spi") #class averages if not os.path.isfile(avgfile): apDisplay.printError( "Final stack of aligned particles does not exist.") ### convert stacks to imagic self.convertStack2Imagic("alignstack.spi") self.convertStack2Imagic("avg.spi") ### make alignment average in mrc format emancmd = "proc2d avg.spi average.mrc average" apEMAN.executeEmanCmd(emancmd) inserttime = time.time() if self.params['commit'] is True: apDisplay.printWarning("committing results to DB") self.runtime = aligntime self.insertEdIterRun(insert=True) else: apDisplay.printWarning("not committing results to DB") inserttime = time.time() - inserttime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) apDisplay.printMsg("Database Insertion time: " + apDisplay.timeString(inserttime))
def start(self): ### check for existing run selectrunq = appiondata.ApSelectionRunData() selectrunq['name'] = self.params['runname'] selectrunq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) selectrundata = selectrunq.query(readimages=False) if selectrundata: apDisplay.printError("Runname already exists") if self.params['ddstack']: self.other_ddstack_used = [] self.dd = apDDprocess.DDStackProcessing() self.dd.setDDStackRun(self.params['ddstack']) self.newddstackrun = self.dd.getDDStackRun(show_msg=True) ### stack data stackdata = apStack.getOnlyStackData(self.params['stackid']) ### stack particles stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True) stackparts.reverse() ### selection run for first particle oldselectrun = stackparts[0]['particle']['selectionrun'] ### set selection run manualparamsq = appiondata.ApManualParamsData() manualparamsq['diam'] = self.getDiamFromSelectionRun(oldselectrun) manualparamsq['oldselectionrun'] = oldselectrun manualparamsq['trace'] = False selectrunq = appiondata.ApSelectionRunData() selectrunq['name'] = self.params['runname'] selectrunq['hidden'] = False selectrunq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) selectrunq['session'] = apStack.getSessionDataFromStackId( self.params['stackid']) selectrunq['manparams'] = manualparamsq ### insert particles apDisplay.printMsg("Inserting particles into database") count = 0 t0 = time.time() startmem = mem.active() numpart = len(stackparts) for stackpart in stackparts: count += 1 if count > 10 and count % 100 == 0: perpart = (time.time() - t0) / float(count + 1) apDisplay.printColor( "part %d of %d :: %.1fM mem :: %s/part :: %s remain" % (count, numpart, (mem.active() - startmem) / 1024., apDisplay.timeString(perpart), apDisplay.timeString(perpart * (numpart - count))), "blue") oldpartdata = stackpart['particle'] newpartq = appiondata.ApParticleData(initializer=oldpartdata) newpartq['selectionrun'] = selectrunq if self.params['ddstack']: newimagedata = self.getNewImageFromDDStack( oldpartdata['image']) if newimagedata is False: # no pick transferred continue newpartq['image'] = newimagedata if self.params['commit'] is True: newpartq.insert() apDisplay.printMsg("Completed in %s" % (apDisplay.timeString(time.time() - t0)))
def start(self): self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) # symmetry info if self.params['sym'] == 'Icos': self.params['symtype'] = 'I' self.params['symfold'] = None else: self.params['symtype'] = self.params['sym'][0] self.params['symfold'] = int(self.params['sym'][1:]) # eman "d" symmetry is spider "ci" if self.params['symtype'].upper() == "D": self.params['symtype'] = "CI" # create symmetry doc file sydoc = "sym.spi" refine.symmetryDoc(self.params['symtype'], self.params['symfold'], sydoc) # convert incr to array of increments ang_inc = self.params['incr'].split(',') self.params['numiter'] = len(ang_inc) self.params['increments'] = [] for i in range(0, self.params['numiter']): self.params['increments'].append(int(ang_inc[i])) # convert stack to spider stack spiderstack = os.path.join(self.params['rundir'], 'start.spi') operations.stackToSpiderStack(self.stack['file'], spiderstack, apix=self.stack['apix'], boxsize=self.stack['boxsize'], numpart=self.params['numpart']) # create filtered stack spiderstackfilt = spiderstack if (self.params['lowpass'] + self.params['highpass']) > 0: spiderstackfilt = os.path.join(self.params['rundir'], 'start_filt.spi') operations.stackToSpiderStack(self.stack['file'], spiderstackfilt, apix=self.stack['apix'], boxsize=self.stack['boxsize'], lp=self.params['lowpass'], hp=self.params['highpass'], numpart=self.params['numpart']) # rescale initial model if necessary outvol = os.path.join(self.params['rundir'], "vol000.spi") apModel.rescaleModel(self.params['modelid'], outvol, self.stack['boxsize'], self.stack['apix'], spider=True) self.params['itervol'] = outvol for iter in range(1, self.params['numiter'] + 1): # create projections for projection matching apDisplay.printMsg("creating reference projections of volume: %s" % self.params['itervol']) projs, numprojs, ang, sel = refine.createProjections( incr=self.params['increments'][iter - 1], boxsz=self.stack['boxsize'], symfold=self.params['symfold'], invol=self.params['itervol'], rad=self.params['rad'], ) # run reference-based alignment apDisplay.printMsg("running reference-based alignment (AP MQ)") apmqfile = "apmq%03d.spi" % iter outang = "angular%03d.spi" % iter shf = "shifts%03d.spi" % iter shiftedStack = "parts_shifted.spi" apFile.removeFile(shf) refine.spiderAPMQ( projs=projs, numprojs=numprojs, tsearch=self.params['xysearch'], tstep=self.params['xystep'], firstRing=self.params['firstring'], lastRing=self.params['lastring'], stackfile=spiderstackfilt, nump=self.params['numpart'], ang=ang, apmqfile=apmqfile, outang=outang, nproc=self.params['proc'], ) # use cross-correlation to find the sub-pixel alignment # of the particles, # results will be saved in "peakfile.spi" apFile.removeFile(shiftedStack) # don't use MPI here - for some reason slower? mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False) apmqlist = refine.readDocFile(apmqfile) avgccrot = 0 apDisplay.printMsg("creating shifted stack") for p in range(0, self.params['numpart']): ref = int(float(apmqlist[p][2])) ccrot = float(apmqlist[p][3]) inplane = float(apmqlist[p][4]) avgccrot += ccrot # invert the sign - ref projs will be rotated inplane *= -1 # get corresponding projection if (ref <= 0): # mirror projection if necessary ref *= -1 refimg = spyder.fileFilter(projs) + "@" + str(ref) refine.mirrorImg(refimg, "_3", inMySpi=mySpi) img = "_3" else: img = spyder.fileFilter(projs) + "@" + str(ref) refine.rotAndShiftImg(img, "_2", inplane, inMySpi=mySpi) refine.maskImg("_2", "_3", self.params['rad'], "D", "E", center=int((self.stack['boxsize'] / 2) + 1), inMySpi=mySpi) # pad ref image & stack image to twice the size refine.padImg("_3", "_2", 2 * self.stack['boxsize'], "N", 1, 1, 0, inMySpi=mySpi) stackimg = spyder.fileFilter(spiderstack) + "@" + str(p + 1) refine.padImg(stackimg, "_1", 2 * self.stack['boxsize'], "B", 1, 1, inMySpi=mySpi) # calculate cross-correlation refine.getCC("_1", "_2", "_1", inMySpi=mySpi) # crop the correllation image to allowable shift amount shift = int(self.params['allowedShift'] * self.stack['boxsize']) dim = 2 * shift + 1 topleftx = self.stack['boxsize'] - shift + 1 refine.windowImg("_1", "_2", dim, topleftx, topleftx, inMySpi=mySpi) # find the sub-pixel location of cc peak mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17", "_2", "0") # create new stack of shifted particles shpos = spyder.fileFilter(shiftedStack) + "@" + str(p + 1) mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN") mySpi.toSpiderQuiet("GP x17", "_2", str(shift + 1) + "," + str(shift + 1)) refine.copyImg(stackimg, shpos, inMySpi=mySpi) mySpi.toSpiderQuiet("ELSE") #mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16") mySpi.toSpiderQuiet("SH F", stackimg, shpos, "-x15,-x16") mySpi.toSpiderQuiet("ENDIF") # save shifts to file mySpi.toSpiderQuiet("SD " + str(p + 1) + ",x15,x16,x17", spyder.fileFilter(shf)) mySpi.toSpiderQuiet("SD E", spyder.fileFilter(shf)) mySpi.close() # create class average images refine.createClassAverages( shiftedStack, projs, apmqfile, numprojs, self.stack['boxsize'], shifted=True, ) # rename class averages & variacnes for iteration cmd = "/bin/mv classes.hed classes.%d.hed;" % iter cmd += "/bin/mv classes.img classes.%d.img;" % iter cmd += "/bin/mv variances.hed variances.%d.hed;" % iter cmd += "/bin/mv variances.img variances.%d.img;" % iter proc = subprocess.Popen(cmd, shell=True) proc.wait() # calculate the stddev for the apmq cc's for throwing out particles avgccrot /= self.params['numpart'] stdccrot = 0 for p in range(0, self.params['numpart']): stdccrot += abs(float(apmqlist[p][3]) - avgccrot) stdccrot /= self.params['numpart'] cccutoff = avgccrot + (stdccrot * self.params['keepsig']) apDisplay.printMsg("average cc: %f" % avgccrot) apDisplay.printMsg("setting cutoff to: %f" % cccutoff) # create new selection file that only has particles with good cc's selectfile = "select%03d.spi" % iter apFile.removeFile(selectfile) mySpi = spyder.SpiderSession(nproc=self.params['proc'], dataext=".spi", logo=False, log=False) i = 1 for p in range(0, self.params['numpart']): ccrot = float(apmqlist[p][3]) if ccrot >= cccutoff: mySpi.toSpiderQuiet("x11=%d" % (p + 1)) mySpi.toSpiderQuiet("SD %d,x11" % i, spyder.fileFilter(selectfile)) i += 1 mySpi.close() # calculate the new 3d structure using centered projections # and the corrected angles from the angular doc file apDisplay.printMsg("creating 3d volume") out_rawvol = "vol_raw%03d.spi" % iter if self.params['voliter'] is not None: refine.iterativeBackProjection( shiftedStack, selectfile, rad=self.params['rad'], ang=outang, out=out_rawvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['bpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc'], ) else: refine.backProjection(shiftedStack, selectfile, ang=outang, out=out_rawvol, sym=sydoc, nproc=self.params['proc']) # create even & odd select files apDisplay.printMsg("creating even/odd volumes") oddfile = "selectodd%03d.spi" % iter evenfile = "selecteven%03d.spi" % iter refine.docSplit(selectfile, oddfile, evenfile) # get the even & odd volumesa oddvol = "vol1%03d.spi" % iter evenvol = "vol2%03d.spi" % iter if self.params['voliter'] is not None: refine.iterativeBackProjection( shiftedStack, oddfile, rad=self.params['rad'], ang=outang, out=oddvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['eobpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc']) refine.iterativeBackProjection( shiftedStack, evenfile, rad=self.params['rad'], ang=outang, out=evenvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['eobpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc']) else: refine.backProjection( shiftedStack, oddfile, ang=outang, out=oddvol, sym=sydoc, nproc=self.params['proc'], ) refine.backProjection( shiftedStack, evenfile, ang=outang, out=evenvol, sym=sydoc, nproc=self.params['proc'], ) # calculate the FSC apDisplay.printMsg("calculating FSC") fscfile = "fsc%03d.spi" % iter emanfsc = "fsc.eotest.%d" % iter refine.calcFSC(oddvol, evenvol, fscfile) # convert to eman-style fscfile refine.spiderFSCtoEMAN(fscfile, emanfsc) # calculate the resolution at 0.5 FSC & write to file res = apRecon.calcRes(emanfsc, self.stack['boxsize'], self.stack['apix']) restxt = "resolution.txt" if iter == 1 and os.path.isfile(restxt): os.remove(restxt) resfile = open(restxt, "a") resfile.write("iter %d:\t%.3f\n" % (iter, res)) resfile.close() # filter & normalize the volume to be used as a reference in the next round outvol = "vol%03d.spi" % iter emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % ( out_rawvol, outvol, self.stack['apix'], res, self.params['rad']) if self.params['imask'] is not None: emancmd += " imask=%d" % self.params['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) # create mrc files of volumes emancmd = "proc3d %s %s apix=%.3f mask=%d norm" % ( out_rawvol, "threed.%da.mrc" % iter, self.stack['apix'], self.params['rad']) if self.params['imask'] is not None: emancmd += " imask=%d" % self.params['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % ( out_rawvol, "threed.%da.lp.mrc" % iter, self.stack['apix'], res, self.params['rad']) if self.params['imask'] is not None: emancmd += " imask=%d" % self.param['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) # set this model as start for next iteration, remove previous os.remove(self.params['itervol']) os.remove(out_rawvol) self.params['itervol'] = outvol # clean up directory apDisplay.printMsg("cleaning up directory") if os.path.isfile(oddvol): os.remove(oddvol) if os.path.isfile(evenvol): os.remove(evenvol) if os.path.isfile(ang): os.remove(ang) if os.path.isfile(sel): os.remove(sel) if os.path.isfile(projs): os.remove(projs) if os.path.isfile(oddfile): os.remove(oddfile) if os.path.isfile(evenfile): os.remove(evenfile) if os.path.isfile(emanfsc) and os.path.isfile(fscfile): os.remove(fscfile) if os.path.isfile(shiftedStack): os.remove(shiftedStack) os.remove(self.params['itervol'])
def uploadResults(self): if self.params['commit'] is False: return # Get the new file order fh = open("sort_junk.sel", 'r') lines = fh.readlines() i = 0 fileorder = {} for line in lines: args = line.split() if (len(args) > 1): match = re.match('[A-Za-z]+([0-9]+)\.[A-Za-z]+', (args[0].split('/'))[-1]) if (match): filenumber = int(match.groups()[0]) fileorder[i] = filenumber i += 1 fh.close() # Produce a new stack oldstack = apStack.getOnlyStackData(self.params['stackid'], msg=False) newstack = appiondata.ApStackData() newstack['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) newstack['name'] = "sorted.hed" if newstack.query(results=1): apDisplay.printError( "A stack with these parameters already exists") # Fill in data and submit newstack['oldstack'] = oldstack newstack['hidden'] = False newstack['substackname'] = self.params['runname'] newstack['description'] = self.params['description'] newstack['pixelsize'] = oldstack['pixelsize'] newstack['boxsize'] = oldstack['boxsize'] newstack['junksorted'] = True newstack.insert() # Insert stack images apDisplay.printMsg("Inserting stack particles") count = 0 total = len(fileorder.keys()) if total == 0: apDisplay.printError( "No particles can be inserted in the sorted stack") for i in fileorder.keys(): count += 1 if count % 100 == 0: sys.stderr.write( "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b") sys.stderr.write( str(count) + " of " + (str(total)) + " complete") # Get particle from the old stack oldparticle = apStack.getStackParticle(self.params['stackid'], fileorder[i] + 1) # Insert particle newparticle = appiondata.ApStackParticleData() newparticle['particleNumber'] = i + 1 newparticle['stack'] = newstack newparticle['stackRun'] = oldparticle['stackRun'] newparticle['particle'] = oldparticle['particle'] newparticle['mean'] = oldparticle['mean'] newparticle['stdev'] = oldparticle['stdev'] newparticle.insert() apDisplay.printMsg( "\n" + str(total) + " particles have been inserted into the sorted stack") # Insert runs in stack apDisplay.printMsg("Inserting Runs in Stack") runsinstack = apStack.getRunsInStack(self.params['stackid']) for run in runsinstack: newrunsq = appiondata.ApRunsInStackData() newrunsq['stack'] = newstack newrunsq['stackRun'] = run['stackRun'] newrunsq.insert() apDisplay.printMsg("finished") return
def checkConflicts(self): ### first get all stack data self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### modify boxsize and pixelsize according to binning self.boxsize = self.stack['boxsize'] self.clipsize = int( math.floor(self.boxsize / float(self.params['bin'] * 2))) * 2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" % (self.params['clipsize'], self.clipsize)) self.clipsize = self.params['clipsize'] self.apix = self.stack['apix'] if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] if clipsize % 2 == 1: clipsize += 1 ### making sure that clipped boxsize is even self.boxsize = clipsize self.apix = self.apix * self.params['bin'] ### basic error checking if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['ncls'] is None: apDisplay.printError("a number of classes was not provided") maxparticles = 1000000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles " + str(self.params['numpart']) + " than available " + str(apFile.numImagesInStack(stackfile))) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) if self.params['numpart'] > 5000: apDisplay.printWarning( "initial model calculation may not work with less than 5000 particles" ) self.mpirun = self.checkMPI() # if self.mpirun is None: # apDisplay.printError("There is no MPI installed") if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() ### SIMPLE defaults and error checking if self.params['ring2'] is None: self.params['ring2'] = (self.boxsize / 2) - 2 if self.params['ncls'] > 2000: apDisplay.printError( "number of classes should be less than 2000 for subsequent ORIGAMI run to work" ) if self.params['ncls'] > self.params['numpart']: self.params['ncls'] = self.params['numpart'] / self.params['minp'] if self.params['mask'] is None: self.params['mask'] = (self.boxsize / 2) - 2 if self.params['mw'] is None: apDisplay.printError( "please specify the molecular weight (in kDa)")
def start(self): ### new stack path stackdata = apStack.getOnlyStackData(self.params['stackid']) oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) newstack = os.path.join(self.params['rundir'], stackdata['name']) apStack.checkForPreviousStack(newstack) includelist = [] excludelist = [] ### list of classes to be excluded if self.params['dropclasslist'] is not None: excludestrlist = self.params['dropclasslist'].split(",") for excludeitem in excludestrlist: excludelist.append(int(excludeitem.strip())) apDisplay.printMsg("Exclude list: "+str(excludelist)) ### list of classes to be included if self.params['keepclasslist'] is not None: includestrlist = self.params['keepclasslist'].split(",") for includeitem in includestrlist: includelist.append(int(includeitem.strip())) ### or read from keepfile elif self.params['keepfile'] is not None: keeplistfile = open(self.params['keepfile']) for line in keeplistfile: if self.params['excludefrom'] is True: excludelist.append(int(line.strip())) else: includelist.append(int(line.strip())) keeplistfile.close() apDisplay.printMsg("Include list: "+str(includelist)) ### get particles from align or cluster stack apDisplay.printMsg("Querying database for particles") q0 = time.time() if self.params['alignid'] is not None: # DIRECT SQL STUFF sqlcmd = "SELECT " + \ "apd.partnum, " + \ "apd.xshift, apd.yshift, " + \ "apd.rotation, apd.mirror, " + \ "apd.spread, apd.correlation, " + \ "apd.score, apd.bad, " + \ "spd.particleNumber, " + \ "ard.refnum "+ \ "FROM ApAlignParticleData apd " + \ "LEFT JOIN ApStackParticleData spd ON " + \ "(apd.`REF|ApStackParticleData|stackpart` = spd.DEF_id) " + \ "LEFT JOIN ApAlignReferenceData ard ON" + \ "(apd.`REF|ApAlignReferenceData|ref` = ard.DEF_id) " + \ "WHERE `REF|ApAlignStackData|alignstack` = %i"%(self.params['alignid']) # These are AlignParticles particles = sinedon.directq.complexMysqlQuery('appiondata',sqlcmd) elif self.params['clusterid'] is not None: clusterpartq = appiondata.ApClusteringParticleData() clusterpartq['clusterstack'] = self.clusterstackdata # These are ClusteringParticles particles = clusterpartq.query() apDisplay.printMsg("Completed in %s\n"%(apDisplay.timeString(time.time()-q0))) ### write included particles to text file includeParticle = [] excludeParticle = 0 badscore = 0 badshift = 0 badspread = 0 f = open("test.log", "w") count = 0 t0 = time.time() apDisplay.printMsg("Parsing particle information") # find out if there is alignparticle info: is_cluster_p = False # alignparticle is a key of any particle in particles if the latter is # a CluateringParticle if 'alignparticle' in particles[0]: is_cluster_p = True for part in particles: count += 1 if is_cluster_p: # alignpart is an item of ClusteringParticle alignpart = part['alignparticle'] try: classnum = int(part['refnum'])-1 except: apDisplay.printWarning("particle %d was not put into any class" % (part['partnum'])) emanstackpartnum = alignpart['stackpart']['particleNumber']-1 else: # particle has info from AlignedParticle as results of direct query alignpart = part try: classnum = int(alignpart['refnum'])-1 except: apDisplay.printWarning("particle %d was not put into any class" % (part['partnum'])) classnum = None emanstackpartnum = int(alignpart['particleNumber'])-1 ### check shift if self.params['maxshift'] is not None: shift = math.hypot(alignpart['xshift'], alignpart['yshift']) if shift > self.params['maxshift']: excludeParticle += 1 if classnum is not None: f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum)) else: f.write("%d\t%d\texclude\n"%(count, emanstackpartnum)) badshift += 1 continue if self.params['minscore'] is not None: ### check score if ( alignpart['score'] is not None and alignpart['score'] < self.params['minscore'] ): excludeParticle += 1 if classnum is not None: f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum)) else: f.write("%d\t%d\texclude\n"%(count, emanstackpartnum)) badscore += 1 continue ### check spread if ( alignpart['spread'] is not None and alignpart['spread'] < self.params['minscore'] ): excludeParticle += 1 if classnum is not None: f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum)) else: f.write("%d\t%d\texclude\n"%(count, emanstackpartnum)) badspread += 1 continue if classnum is not None: if includelist and (classnum in includelist): includeParticle.append(emanstackpartnum) f.write("%d\t%d\t%d\tinclude\n"%(count, emanstackpartnum, classnum)) elif excludelist and not (classnum in excludelist): includeParticle.append(emanstackpartnum) f.write("%d\t%d\t%d\tinclude\n"%(count, emanstackpartnum, classnum)) else: excludeParticle += 1 f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum)) else: excludeParticle += 1 f.write("%d\t%d\texclude\n"%(count, emanstackpartnum)) f.close() includeParticle.sort() if badshift > 0: apDisplay.printMsg("%d paricles had a large shift"%(badshift)) if badscore > 0: apDisplay.printMsg("%d paricles had a low score"%(badscore)) if badspread > 0: apDisplay.printMsg("%d paricles had a low spread"%(badspread)) apDisplay.printMsg("Completed in %s\n"%(apDisplay.timeString(time.time()-t0))) apDisplay.printMsg("Keeping "+str(len(includeParticle))+" and excluding "+str(excludeParticle)+" particles") ### write kept particles to file self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile-"+self.timestamp+".list") apDisplay.printMsg("writing to keepfile "+self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum)+"\n") kf.close() ### get number of particles numparticles = len(includeParticle) if excludelist: self.params['description'] += ( " ... %d particle substack with %s classes excluded" % (numparticles, self.params['dropclasslist'])) elif includelist: self.params['description'] += ( " ... %d particle substack with %s classes included" % (numparticles, self.params['keepclasslist'])) outavg = os.path.join(self.params['rundir'],"average.mrc") ### create the new sub stack # first check if virtual stack if not os.path.isfile(oldstack): vstackdata=apStack.getVirtualStackParticlesFromId(self.params['stackid']) vparts = vstackdata['particles'] oldstack = vstackdata['filename'] # get subset of virtualstack vpartlist = [int(vparts[p]['particleNumber'])-1 for p in includeParticle] if self.params['writefile'] is True: apStack.makeNewStack(oldstack, newstack, vpartlist, bad=self.params['savebad']) apStack.averageStack(stack=oldstack,outfile=outavg,partlist=vpartlist) else: if self.params['writefile'] is True: apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=self.params['savebad']) apStack.averageStack(stack=oldstack,outfile=outavg,partlist=includeParticle) if self.params['writefile'] is True and not os.path.isfile(newstack): apDisplay.printError("No stack was created") if self.params['commit'] is True: apStack.commitSubStack(self.params,included=includeParticle) newstackid = apStack.getStackIdFromPath(newstack) apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=4)
def start(self): self.insertMaxLikeJob() self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.estimateIterTime() self.dumpParameters() ### process stack to local temp file proccmd = "proc2d " + self.stack['file'] + " temp.hed apix=" + str( self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], clipsize, clipsize) proccmd += " last=" + str(self.params['numpart'] - 1) apEMAN.executeEmanCmd(proccmd, verbose=True) ### process stack to final file self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp + ".hed") proccmd = "proc2d temp.hed " + self.params[ 'localstack'] + " apix=" + str( self.stack['apix'] * self.params['bin']) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp=" + str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp=" + str(self.params['lowpass']) apEMAN.executeEmanCmd(proccmd, verbose=True) apFile.removeStack("temp.hed") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles( self.params['localstack']) ### convert stack into single spider files templateselfile = self.initializeTemplates() ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params['rundir'], self.partlistdocfile) + " -iter " + str(self.params['maxiter']) + " -ref " + templateselfile + " -o " + os.path.join(self.params['rundir'], "part" + self.timestamp) + " -psi_step " + str(self.params['psistep'])) ### fast mode if self.params['fast'] is True: xmippopts += " -fast " if self.params['fastmode'] == "narrow": xmippopts += " -C 1e-10 " elif self.params['fastmode'] == "wide": xmippopts += " -C 1e-18 " ### convergence criteria if self.params['converge'] == "fast": xmippopts += " -eps 5e-3 " elif self.params['converge'] == "slow": xmippopts += " -eps 5e-8 " else: xmippopts += " -eps 5e-5 " ### mirrors if self.params['mirror'] is True: xmippopts += " -mirror " ### normalization if self.params['norm'] is True: xmippopts += " -norm " ### find number of processors if self.params['nproc'] is None: nproc = nproc = apParam.getNumProcessors() else: nproc = self.params['nproc'] mpirun = self.checkMPI() if nproc > 2 and mpirun is not None: ### use multi-processor apDisplay.printColor("Using " + str(nproc - 1) + " processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True) mpiruncmd = mpirun + " -np " + str( nproc - 1) + " " + xmippexe + " " + xmippopts self.writeXmippLog(mpiruncmd) apEMAN.executeEmanCmd(mpiruncmd, verbose=True, showcmd=True) else: ### use single processor xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True) xmippcmd = xmippexe + " " + xmippopts self.writeXmippLog(xmippcmd) apEMAN.executeEmanCmd(xmippcmd, verbose=True, showcmd=True) aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ### minor post-processing self.createReferenceStack() self.readyUploadFlag() self.dumpParameters()
def start(self): ### get stack data stackdata = apStack.getOnlyStackData(self.params['partstackid']) ############################### # # # Andres's refinement steps # # # ############################### print "\n" apDisplay.printMsg("##################################") apDisplay.printMsg("Starting Andres' refinement steps") apDisplay.printMsg("##################################") print "\n" if self.params['initvol'] is not None: spidervol = self.convertVolToSpider(mrcvolfile=self.params['initvol'], apix=self.params['initvolapix']) if self.params['modelid'] is not None: spidervol = self.convertVolToSpider(modelid=self.params['modelid']) partstack = stackdata['path']['path']+"/"+stackdata['name'] spiderstack = self.convertStackToSpider(partstack) partsnum = apStack.getNumberStackParticlesFromId(self.params['partstackid']) for j in range(self.params['refineiters']): iternum = j+1 appiondata.ApPathData.direct_query(1) apDisplay.printMsg("Starting projection-matching refinement/XMIPP iteration "+str(iternum)) boxsize = self.getBoxSize() ### projection-matching refinement/XMIPP apshout, apshstack, apsheuler, projstack, numprojs = self.projMatchRefine(spidervol, spiderstack, boxsize, partsnum, self.params['radius'], iternum) apDisplay.printMsg("Calculating weighted cross-correlation coefficients") ### calculation of weighted cross-correlation coefficients apshout_weighted = self.cccAPSH(apshout, iternum) apDisplay.printMsg("Creating select files based on weighted cross-correlation coefficients") ### create select files based on calculated weighted-cross-correlation corrSelect = self.makecccAPSHselectFile(apshout_weighted, iternum, factor=0.1) ### create volume file names apshVolfile = os.path.join(self.params['rundir'], "apshVolume-BPCG-%03d.spi"%(iternum)) apshVolfile2 = os.path.join(self.params['rundir'], "apshVolume-BPRP-%03d.spi"%(iternum)) ### run BPRP on selected particles backprojectPWL.backprojectCG(apshstack, apsheuler, apshVolfile, partsnum, self.params['radius']) self.APSHbackProject(apshstack, apsheuler, apshVolfile2, partsnum) ### center volume filename = os.path.splitext(apshVolfile)[0] apshVolFileCentered = filename+"_centered.spi" backprojectPWL.centerVolume(apshVolfile, apshVolFileCentered) ### generate class averages backprojectPWL.createClassAverages(apshstack,projstack,apshout,numprojs,boxsize,outclass="classes",rotated=True,shifted=True,dataext=".spi") print "check~~!!!" sys.exit(1) ### calculate FSC ### generate odd and even select files for FSC calculation corrSelectOdd, corrSelectEven = self.splitOddEven(cnum, corrSelect, iternum) fscout = self.runEoTest(corrSelectOdd, corrSelectEven, cnum, apshstack, apsheuler, iternum) self.runRmeasure(apshVolFileCentered) ### filter volume backproject.butterworthFscLP(apshVolFileCentered, fscout) ### reset file names for next round volfile = apshVolFileCentered eulerfile = apsheuler mrcvolfile = self.processVolume(volfile, cnum, iternum) print "\n" apDisplay.printMsg("###########################") apDisplay.printMsg("Done with iteration "+str(j+1)+"") apDisplay.printMsg("###########################") print "\n" #if len(self.classlist) > 1: #get a list of all unique combinations of volumes # pairlist = self.computeClassVolPair() ### insert volumes into DB self.insertOtrRun(mrcvolfile)
def checkConflicts(self): ### unpickle results or parse logfile, set default parameters if missing os.chdir(os.path.abspath(self.params['rundir'])) if os.path.isdir(os.path.join(self.params['rundir'], "recon")): apDisplay.printWarning( 'recon dir already exist, no need to unpack') else: self.unpackResults() ''' These are required error checks, everything else should be possible to obtain from the timestamped pickle file ''' stackdata = apStack.getOnlyStackData(self.params['stackid']) self.runparams = {} self.runparams['original_apix'] = ( stackdata['pixelsize'] / 1e-10 ) # convert pixelsize to angstroms per pixel. self.runparams['original_boxsize'] = stackdata['boxsize'] # some functions in readRunParameters needs the above params self.runparams.update(self.readRunParameters()) ### parameters recovered from runparameter file(s) if not self.runparams.has_key('stackid'): if self.params['stackid'] is not None: self.runparams['stackid'] = self.params['stackid'] else: apDisplay.printError( "stack id must be specified for proper database insertion") if not self.runparams.has_key('modelid'): if self.params['modelid'] is not None: self.runparams['modelid'] = self.params['modelid'] else: apDisplay.printError( "model id must be specified for proper database insertion") if not self.runparams.has_key('NumberOfReferences'): if self.params['NumberOfReferences'] is not None: self.runparams['NumberOfReferences'] = self.params[ 'NumberOfReferences'] else: apDisplay.printError( "number of references produced during the refinement needs to be specified" ) if not self.runparams.has_key('numiter'): if self.params['numiter'] is not None: self.runparams['numiter'] = self.params['numiter'] else: apDisplay.printError( "number of iterations run during the refinement needs to be specified" ) if not self.runparams.has_key('boxsize'): if self.params['boxsize'] is not None: self.runparams['boxsize'] = self.params['boxsize'] else: apDisplay.printError( "boxsize of the map / particles submitted for refinement needs to be specified" ) if not self.runparams.has_key('apix'): if self.params['apix'] is not None: self.runparams['apix'] = self.params['apix'] else: apDisplay.printError( "pixelsize of the map / particles submitted for refinement needs to be specified" ) if not self.runparams.has_key('symmetry'): if self.params['symid'] is not None: self.runparams['symmetry'] = apSymmetry.getSymmetryDataFromID( self.params['symid']) else: apDisplay.printError( "symmetry ID must be specified, you can input --symid=25 for an asymmetric reconstruction" ) # access multiModelRefinementRun this way in case it is not present if 'multiModelRefinementRun' in vars(self): if not self.runparams.has_key( 'NumberOfReferences' ) and self.multiModelRefinementRun is True: if self.params['NumberOfReferences'] is not None: self.runparams['NumberOfReferences'] = self.params[ 'NumberOfReferences'] else: apDisplay.printError( "number of output models in refinement needs to be specified for multi-model run" ) else: if self.params['NumberOfReferences'] is not None: self.runparams['NumberOfReferences'] = self.params[ 'NumberOfReferences'] if self.runparams['NumberOfReferences'] > 1: self.multiModelRefinementRun = True else: self.multiModelRefinementRun = False else: apDisplay.printError( "number of output models (references) in refinement needs to be specified for multi-model run" ) if not self.runparams.has_key('rundir'): self.runparams['rundir'] = self.params['rundir'] if not self.runparams.has_key('reconstruction_working_dir'): self.runparams['reconstruction_working_dir'] = str( self.package) + "_results" if not self.runparams.has_key('mask'): self.runparams['mask'] = None if not self.runparams.has_key('imask'): self.runparams['imask'] = None if not self.runparams.has_key('alignmentInnerRadius'): self.runparams['alignmentInnerRadius'] = None if not self.runparams.has_key('alignmentOuterRadius'): self.runparams['alignmentOuterRadius'] = None if not self.runparams.has_key('angularSamplingRate'): self.runparams['angularSamplingRate'] = None ### parameters specified for upload if self.params['jobid'] is not None: self.params['jobinfo'] = appiondata.ApAppionJobData.direct_query( self.params['jobid']) else: jobid = self.tryToGetJobID() if jobid is not False: self.params[ 'jobinfo'] = appiondata.ApAppionJobData.direct_query( self.params['jobid']) else: self.params['jobinfo'] = None if self.params[ 'timestamp'] is None and self.package == "external_package": apDisplay.printError("a timestamp (or some identifier) must be specified with the files. For example, the 3D mrc file " \ "for iteration 1 should be named 'recon_YourIdentifier_it001_vol001.mrc, in which case the timestamp should be specified " \ "as --timestamp=YourIdentifier") elif self.params['timestamp'] is None and self.params[ 'jobid'] is None and self.package != "external_package": self.params['timestamp'] = apParam.makeTimestamp() elif self.params['timestamp'] is None and self.params[ 'jobid'] is not None: timestamp = self.getTimestamp() if timestamp is None: self.params['timestamp'] = apParam.makeTimestamp() # apDisplay.printError("please specify the timestamp associated with the refinement parameters, e.g. --timestamp=08nov25c07") ### basic refinement variables self.initializeRefinementUploadVariables() return
def start(self): self.params['output_fileformat'] = 'mrc' newstackname = 'framealigned.hed' stackdata = apStack.getStackParticlesFromId(self.params['stackid']) stackrundata = apStack.getOnlyStackData(self.params['stackid']) apix = stackrundata['pixelsize'] * 1e10 kev = stackdata[0]['particle']['image']['scope']['high tension'] / 1000 origstackpath = os.path.join(stackrundata['path']['path'], stackrundata['name']) boxsize = stackdata[0]['stackRun']['stackParams']['boxSize'] binning = stackdata[0]['stackRun']['stackParams']['bin'] #determine camera type cameratype = stackdata[0]['particle']['image']['camera']['ccdcamera'][ 'name'] if self.params['override_camera'] is not None: cameratype = self.params['override_camera'] #create sorted boxfiles imagedict = {} masterlist = [] for particle in stackdata: parentimage = particle['particle']['image']['filename'] if parentimage in imagedict.keys(): imagedict[parentimage].append(particle['particle']) else: imagedict[parentimage] = [] imagedict[parentimage].append(particle['particle']) index = len(imagedict[parentimage]) - 1 masterlist.append({ 'particle': particle, 'key': parentimage, 'index': index }) #print masterlist for key in imagedict: particlelst = imagedict[key] parentimage = key framespath = particlelst[0]['image']['session']['frame path'] print cameratype if 'Gatan' in cameratype: #prepare frames print framespath #prepare frame directory framespathname = os.path.join(self.params['rundir'], parentimage + '.frames') if os.path.exists(framespathname): pass else: os.mkdir(framespathname) print framespathname mrcframestackname = parentimage + '.frames.mrc' print mrcframestackname nframes = particlelst[0]['image']['camera']['nframes'] print "Extracting frames for", mrcframestackname for n in range(nframes): a = mrc.read(os.path.join(framespath, mrcframestackname), n) numpil.write(a, imfile=os.path.join(framespathname, 'RawImage_%d.tif' % (n)), format='tiff') elif 'DE' in cameratype: framespathname = os.path.join(framespath, parentimage + '.frames') print os.getcwd() print framespathname #generate DE script call if os.path.exists(framespathname): print "found frames for", parentimage nframes = particlelst[0]['image']['camera']['nframes'] boxname = parentimage + '.box' boxpath = os.path.join(framespathname, boxname) shiftdata = {'scale': 1, 'shiftx': 0, 'shifty': 0} #flatfield references brightrefpath = particlelst[0]['image']['bright']['session'][ 'image path'] brightrefname = particlelst[0]['image']['bright']['filename'] brightnframes = particlelst[0]['image']['bright']['camera'][ 'nframes'] darkrefpath = particlelst[0]['image']['dark']['session'][ 'image path'] darkrefname = particlelst[0]['image']['dark']['filename'] darknframes = particlelst[0]['image']['dark']['camera'][ 'nframes'] brightref = os.path.join(brightrefpath, brightrefname + '.mrc') darkref = os.path.join(darkrefpath, darkrefname + '.mrc') print brightref print darkref apBoxer.processParticleData(particle['particle']['image'], boxsize, particlelst, shiftdata, boxpath) print framespathname #set appion specific options self.params['gainreference_filename'] = brightref self.params['gainreference_framecount'] = brightnframes self.params['darkreference_filename'] = darkref self.params['darkreference_framecount'] = darknframes self.params['input_framecount'] = nframes self.params['boxes_fromfiles'] = 1 #self.params['run_verbosity']=3 self.params['output_invert'] = 0 #self.params['radiationdamage_apix=']=apix self.params['radiationdamage_voltage'] = kev #self.params['boxes_boxsize']=boxsize outpath = os.path.join(self.params['rundir'], key) if os.path.exists(outpath): shutil.rmtree(outpath) os.mkdir(outpath) command = ['deProcessFrames.py'] keys = self.params.keys() keys.sort() for key in keys: param = self.params[key] #print key, param, type(param) if param == None or param == '': pass else: option = '--%s=%s' % (key, param) command.append(option) command.append(outpath) command.append(framespathname) print command if self.params['dryrun'] is False: subprocess.call(command) #recreate particle stack for n, particledict in enumerate(masterlist): parentimage = particledict['key'] correctedpath = os.path.join(self.params['rundir'], parentimage) print correctedpath if os.path.exists(correctedpath): correctedparticle = glob.glob( os.path.join(correctedpath, ('%s.*.region_%03d.*' % (parentimage, particledict['index'])))) print os.path.join(correctedpath, ('%s.*.region_%03d.*' % (parentimage, particledict['index']))) print correctedparticle #sys.exit() command = ['proc2d', correctedparticle[0], newstackname] if self.params['output_rotation'] != 0: command.append('rot=%d' % self.params['output_rotation']) if self.params['show_DE_command'] is True: print command subprocess.call(command) else: print "did not find frames for ", parentimage command = [ 'proc2d', origstackpath, newstackname, ('first=%d' % n), ('last=%d' % n) ] print command if self.params['dryrun'] is False: subprocess.call(command) #upload stack #make keep file self.params['keepfile'] = 'keepfile.txt' f = open(self.params['keepfile'], 'w') for n in range(len(masterlist)): f.write('%d\n' % (n)) f.close() apStack.commitSubStack(self.params, newname=newstackname) apStack.averageStack(stack=newstackname) print "Done!!!!"
def checkConflicts(self): apDisplay.printMsg( "/bin/rm -f *.hed *.img *.doc *.vol *.mrc *.spi *.log *~ *.pickle *.hist *.proj *.xmp *.sel *.basis *.original; rm -fr volume*" ) ### check for missing and duplicate entries #if self.params['alignid'] is None and self.params['clusterid'] is None: # apDisplay.printError("Please provide either --cluster-id or --align-id") if self.params['alignid'] is not None and self.params[ 'clusterid'] is not None: apDisplay.printError( "Please provide only one of either --cluster-id or --align-id") if self.params['modelstr'] is None and self.params['nvol'] is None: apDisplay.printError( "Please provide model numbers or number of volumes") elif self.params['modelstr'] is not None: modellist = self.params['modelstr'].split(",") self.params['modelids'] = [] for modelid in modellist: self.params['modelids'].append(int(modelid)) ### get the stack ID from the other IDs if self.params['alignid'] is not None: self.alignstackdata = appiondata.ApAlignStackData.direct_query( self.params['alignid']) self.params['stackid'] = self.alignstackdata['stack'].dbid elif self.params['clusterid'] is not None: self.clusterstackdata = appiondata.ApClusteringStackData.direct_query( self.params['clusterid']) self.alignstackdata = self.clusterstackdata['clusterrun'][ 'alignstack'] self.params['stackid'] = self.alignstackdata['stack'].dbid if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") #if self.params['description'] is None: # apDisplay.printError("run description was not defined") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if not self.params['fastmode'] in self.fastmodes: apDisplay.printError("fast mode must be on of: " + str(self.fastmodes)) maxparticles = 150000 minparticles = 50 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) if self.params['numpart'] < minparticles: apDisplay.printError("not enough particles requested, min: " + str(minparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles " + str(self.params['numpart']) + " than available " + str(apFile.numImagesInStack(stackfile))) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) ### find number of processors if self.params['nproc'] is None: self.nproc = apParam.getNumProcessors() else: self.nproc = self.params['nproc'] self.mpirun = self.checkMPI()
def commitToDatabase(self): """ insert the results into the database """ ### expected result for an alignment run: ### 1. aligned particle stack in IMAGIC ### 2. rotation, shift, and quality parameters for each particle ### 3. which particles belongs to which class ### 4. stack file with the class averages alignedstack = os.path.join(self.params['rundir'], "ptcl.hed") refstack = os.path.join(self.params['rundir'], "iter.final.hed") averagemrc = os.path.join(self.params['rundir'], "average.mrc") apStack.averageStack(alignedstack, averagemrc) particlemapping = self.determineClassOwnership() ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError( "Run name '" + runparams['runname'] + "' and path already exisclassmappingt in database") ### setup eman refine2d run emanrefinetwodq = appiondata.ApEMANRefine2dRunData() emanrefinetwodq['runname'] = self.params['runname'] emanrefinetwodq['run_seconds'] = time.time() - self.t0 emanrefinetwodq['num_iters'] = self.params['numiter'] emanrefinetwodq['num_classes'] = self.params['numclasses'] ### finish alignment run alignrunq['refine2drun'] = emanrefinetwodq alignrunq['hidden'] = False alignrunq['runname'] = self.params['runname'] alignrunq['description'] = self.params['description'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['bin'] = self.params['bin'] ### setup alignment stackalignimagicfile alignstackq = appiondata.ApAlignStackData() alignstackq['imagicfile'] = os.path.basename(alignedstack) alignstackq['avgmrcfile'] = os.path.basename(averagemrc) alignstackq['refstackfile'] = os.path.basename(refstack) alignstackq['iteration'] = self.params['numiter'] alignstackq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) alignstackq['alignrun'] = alignrunq ### check to make sure files exist alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(alignimagicfilepath): apDisplay.printError("could not find stack file: " + alignimagicfilepath) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: " + avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printErrrefqor("could not find reference stack file: " + refstackfile) ### continue setting values alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackid']) alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0] alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) * self.params['bin'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = apFile.numImagesInStack( alignimagicfilepath) ### inserting particles and references apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for emanpartnum in range(self.params['numpart']): partnum = emanpartnum + 1 if partnum % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refnum = particlemapping[emanpartnum] refq['refnum'] = refnum refq['iteration'] = self.params['numiter'] refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = alignrunq ### TODO: create mrc file #refq['mrcfile'] = refbase+".mrc" #reffile = os.path.join(self.params['rundir'], refq['mrcfile']) #if not os.path.isfile(reffile): # emancmd = "proc2d "+refstack+".xmp "+refstack+".mrc" # apEMAN.executeEmanCmd(emancmd, verbose=False) #if not os.path.isfile(reffile): # apDisplay.printError("could not find reference file: "+reffile) ### TODO: get resolution #refq['ssnr_resolution'] = TODO ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partnum alignpartq['alignstack'] = alignstackq stackpartdata = apStack.getStackParticle(self.params['stackid'], partnum) alignpartq['stackpart'] = stackpartdata ### TODO: get the alignment parameters #alignpartq['xshift'] = partdict['xshift'] #alignpartq['yshift'] = partdict['yshift'] #alignpartq['rotation'] = partdict['inplane'] #alignpartq['mirror'] = partdict['mirror'] alignpartq['ref'] = refq ### TODO: get the score #alignpartq['score'] = partdict['score'] ### insert if self.params['commit'] is True: alignpartq.insert() return
def setRunDir(self): self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) path = self.stackdata['path']['path'] uppath = os.path.dirname(os.path.abspath(path)) self.params['rundir'] = os.path.join(uppath, self.params['runname'])
def start(self): commit = self.params['commit'] description = self.params['description'] processdir = self.params['fulltomodir'] runname = self.params['runname'] offsetz = self.params['offsetz'] subbin = self.params['bin'] invert = self.params['invert'] fulltomodata = apTomo.getFullTomoData(self.params['fulltomoId']) #subvolume making if (self.params['selexonId'] is not None or self.params['stackId']) and fulltomodata is not None: sessiondata = fulltomodata['session'] seriesname = fulltomodata['name'].rstrip('_full') fullbin = fulltomodata['bin'] if not fullbin: apDisplay.printWarning( "no binning in full tomogram, something is wrong, use alignment bin for now" ) fullbin = fulltomodata['aligner']['alignrun']['bin'] fulltomopath = os.path.join( fulltomodata['reconrun']['path']['path'], seriesname + "_full.rec") fulltomoheader = mrc.readHeaderFromFile(fulltomopath) fulltomoshape = fulltomoheader['shape'] if self.params['sizez'] > fulltomoshape[1] * fullbin: self.params['sizez'] = fulltomoshape[1] * fullbin subrunname = self.params['subrunname'] volumeindex = apTomo.getLastVolumeIndex(fulltomodata) + 1 dimension = { 'x': int(self.params['sizex']), 'y': int(self.params['sizey']), 'z': int(self.params['sizez']) } zprojimagedata = fulltomodata['zprojection'] apDisplay.printMsg("getting pixelsize") pixelsize = apTomo.getTomoPixelSize( zprojimagedata) * fullbin * subbin gtransform = [1, 0, 0, 1, 0, 0] if self.params['selexonId']: particles = apParticle.getParticles(zprojimagedata, self.params['selexonId']) if self.params['stackId']: particles, stackparticles = apStack.getImageParticles( zprojimagedata, self.params['stackId']) stackdata = apStack.getOnlyStackData(self.params['stackId']) for p, particle in enumerate(particles): print particle['xcoord'], particle['ycoord'], fullbin center = apTomo.transformParticleCenter( particle, fullbin, gtransform) size = (dimension['x'] / fullbin, dimension['y'] / fullbin, dimension['z'] / fullbin) volumename = 'volume%d' % (volumeindex, ) volumedir = os.path.join(processdir, subrunname + '/', volumename + '/') apParam.createDirectory(volumedir) apImod.trimVolume(processdir, subrunname, seriesname, volumename, center, offsetz, size, True) long_volumename = seriesname + '_' + volumename subvolumepath = os.path.join(processdir, runname + "/", volumename + "/", long_volumename + ".rec") if subbin > 1 or invert: apTomo.modifyVolume(subvolumepath, subbin, invert) if commit: subtomorundata = apTomo.insertSubTomoRun( sessiondata, self.params['selexonId'], self.params['stackId'], subrunname, invert, subbin) subtomodata = apTomo.insertSubTomogram( fulltomodata, subtomorundata, particle, offsetz, dimension, volumedir, long_volumename, volumeindex, pixelsize, description) apTomo.makeMovie(subvolumepath) apTomo.makeProjection(subvolumepath) volumeindex += 1
def start(self): self.runtime = 0 self.partlist = [] self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) self.checkNoRefRun() ### convert stack to spider spiderstack = self.createSpiderFile() ### create initialization template if self.params['initmethod'] == 'allaverage': templatefile = self.averageTemplate() elif self.params['initmethod'] == 'selectrand': templatefile = self.selectRandomParticles() elif self.params['initmethod'] == 'randpart': templatefile = self.pickRandomParticle() elif self.params['initmethod'] == 'template': templatefile = self.getTemplate() else: apDisplay.printError("unknown initialization method defined: " + str(self.params['initmethod']) + " not in " + str(self.initmethods)) apDisplay.printColor("Running spider this can take awhile", "cyan") ### run the alignment aligntime = time.time() pixrad = int( round(self.params['partrad'] / self.stack['apix'] / self.params['bin'])) alignedstack, self.partlist = alignment.refFreeAlignParticles( spiderstack, templatefile, self.params['numpart'], pixrad, self.params['firstring'], self.params['lastring'], rundir=".") aligntime = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) ### remove large, worthless stack spiderstack = os.path.join(self.params['rundir'], "start.spi") apDisplay.printMsg("Removing un-aligned stack: " + spiderstack) apFile.removeFile(spiderstack, warn=False) ### convert stack to imagic imagicstack = self.convertSpiderStack(alignedstack) apFile.removeFile(alignedstack) inserttime = time.time() if self.params['commit'] is True: self.runtime = aligntime self.insertNoRefRun(imagicstack, insert=True) else: apDisplay.printWarning("not committing results to DB") inserttime = time.time() - inserttime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(aligntime)) apDisplay.printMsg("Database Insertion time: " + apDisplay.timeString(inserttime))
def insertRctRun(self, volfile): ### setup resolutions fscresq = appiondata.ApResolutionData() fscresq['type'] = "fsc" fscresq['half'] = self.fscresolution fscresq['fscfile'] = "fscdata" + self.timestamp + ".fsc" rmeasureq = appiondata.ApResolutionData() rmeasureq['type'] = "rmeasure" rmeasureq['half'] = self.rmeasureresolution rmeasureq['fscfile'] = None ### insert rct run data rctrunq = appiondata.ApRctRunData() rctrunq['runname'] = self.params['runname'] classliststr = operations.intListToString(self.classlist) rctrunq['classnums'] = classliststr rctrunq['numiter'] = self.params['numiters'] rctrunq['maskrad'] = self.params['radius'] rctrunq['lowpassvol'] = self.params['lowpassvol'] rctrunq['highpasspart'] = self.params['highpasspart'] rctrunq['lowpasspart'] = self.params['lowpasspart'] rctrunq['median'] = self.params['median'] rctrunq['description'] = self.params['description'] rctrunq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) rctrunq['alignstack'] = self.alignstackdata rctrunq['tiltstack'] = apStack.getOnlyStackData( self.params['tiltstackid']) rctrunq['numpart'] = self.numpart rctrunq['fsc_resolution'] = fscresq rctrunq['rmeasure_resolution'] = rmeasureq if self.params['commit'] is True: rctrunq.insert() ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['rctrun'] = rctrunq densq['path'] = appiondata.ApPathData( path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25) densq['pixelsize'] = apStack.getStackPixelSizeFromStackId( self.params['tiltstackid']) * self.params['tiltbin'] densq['boxsize'] = self.getBoxSize() densq['lowpass'] = self.params['lowpassvol'] densq['highpass'] = self.params['highpasspart'] densq['mask'] = self.params['radius'] #densq['iterid'] = self.params['numiters'] densq['description'] = self.params['description'] densq['resolution'] = self.fscresolution densq['rmeasure'] = self.rmeasureresolution densq['session'] = apStack.getSessionDataFromStackId( self.params['tiltstackid']) densq['md5sum'] = apFile.md5sumfile(volfile) if self.params['commit'] is True: densq.insert() return
def uploadResults(self): if self.params['commit'] is False: return # Produce new stacks oldstack = apStack.getOnlyStackData(self.params['notstackid'], msg=False) notstack = appiondata.ApStackData() notstack['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) notstack['name'] = self.notstackdata['name'] if notstack.query(results=1): apDisplay.printError( "A stack with these parameters already exists") tiltstack = appiondata.ApStackData() tiltstack['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) tiltstack['name'] = self.tiltstackdata['name'] if tiltstack.query(results=1): apDisplay.printError( "A stack with these parameters already exists") # Fill in data and submit notstack['oldstack'] = oldstack notstack['hidden'] = False notstack['substackname'] = self.params['runname'] notstack['description'] = self.params[ 'description'] + " ... tilt stack sorted" notstack['pixelsize'] = oldstack['pixelsize'] notstack.insert() tiltstack['oldstack'] = oldstack tiltstack['hidden'] = False tiltstack['substackname'] = self.params['runname'] tiltstack['description'] = self.params[ 'description'] + " ... tilt stack sorted" tiltstack['pixelsize'] = oldstack['pixelsize'] tiltstack.insert() # Insert stack images apDisplay.printMsg("Inserting stack particles") count = 0 for partdict in parttree: count += 1 if count % 100 == 0: sys.stderr.write( "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b") sys.stderr.write( str(count) + " of " + (str(total)) + " complete") # Get particles from the old stacks oldnotparticle = apStack.getStackParticle( self.params['notstackid'], partdict['not']) oldtiltparticle = apStack.getStackParticle( self.params['tiltstackid'], partdict['tilt']) # Insert particle notparticle = appiondata.ApStackParticleData() notparticle.update(oldnotparticle) notparticle['particleNumber'] = count notparticle['stack'] = notstack notparticle.insert() tiltparticle = appiondata.ApStackParticleData() tiltparticle.update(oldtiltparticle) tiltparticle['particleNumber'] = count tiltparticle['stack'] = tiltstack tiltparticle.insert() apDisplay.printMsg( "\n%d particles have been inserted into the tilt synchronized stacks" % (count)) # Insert runs in stack apDisplay.printMsg("Inserting Runs in Stack") runsinstack = apStack.getRunsInStack(self.params['notstackid']) for run in runsinstack: newrunsq = appiondata.ApRunsInStackData() newrunsq['stack'] = notstack newrunsq['stackRun'] = run['stackRun'] newrunsq.insert() runsinstack = apStack.getRunsInStack(self.params['tiltstackid']) for run in runsinstack: newrunsq = appiondata.ApRunsInStackData() newrunsq['stack'] = tiltstack newrunsq['stackRun'] = run['stackRun'] newrunsq.insert() apDisplay.printMsg("finished") return
def insertRunIntoDatabase(self, alignimagicfile, runparams): apDisplay.printMsg("Inserting MaxLike Run into DB") ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = runparams['runname'] alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database") ### setup max like run maxlikeq = appiondata.ApMaxLikeRunData() maxlikeq['runname'] = runparams['runname'] maxlikeq['run_seconds'] = runparams['runtime'] #maxlikeq['mask_diam'] = 2.0*runparams['maskrad'] maxlikeq['fast'] = runparams['fast'] maxlikeq['fastmode'] = runparams['fastmode'] maxlikeq['mirror'] = runparams['mirror'] maxlikeq['student'] = bool(runparams['student']) maxlikeq['init_method'] = "xmipp default" maxlikeq['job'] = self.getMaxLikeJob(runparams) ### finish alignment run alignrunq['maxlikerun'] = maxlikeq alignrunq['hidden'] = False alignrunq['runname'] = runparams['runname'] alignrunq['description'] = runparams['description'] alignrunq['lp_filt'] = runparams['lowpass'] alignrunq['hp_filt'] = runparams['highpass'] alignrunq['bin'] = runparams['bin'] ### setup alignment stack alignstackq = appiondata.ApAlignStackData() alignstackq['imagicfile'] = alignimagicfile alignstackq['avgmrcfile'] = "average.mrc" alignstackq['refstackfile'] = "part"+self.params['timestamp']+"_average.hed" alignstackq['iteration'] = self.lastiter alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) alignstackq['alignrun'] = alignrunq ### check to make sure files exist alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(alignimagicfilepath): apDisplay.printError("could not find stack file: "+alignimagicfilepath) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: "+avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: "+refstackfile) alignstackq['stack'] = apStack.getOnlyStackData(runparams['stackid']) alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0] alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(runparams['stackid'])*runparams['bin'] alignstackq['description'] = runparams['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = runparams['numpart'] ### insert if self.params['commit'] is True: alignstackq.insert() self.alignstackdata = alignstackq return
def makeNewStacks(self, parttree): ### untilted stack self.notstackdata = apStack.getOnlyStackData(self.params['notstackid']) notstackfile = os.path.join(self.notstackdata['path']['path'], self.notstackdata['name']) ### tilted stack if not self.tiltstackdata: self.tiltstackdata = apStack.getOnlyStackData( self.params['tiltstackid']) tiltstackfile = os.path.join(self.tiltstackdata['path']['path'], self.tiltstackdata['name']) ### make doc file of Euler angles #eulerfile = self.makeEulerDoc(parttree) eulerfile = os.path.join(self.params['rundir'], "eulersdoc" + self.timestamp + ".spi") if os.path.isfile(eulerfile): apFile.removeFile(eulerfile) count = 0 notstacklist = [] tiltstacklist = [] sizelimit = 2048 notbox = apImagicFile.getBoxsize(notstackfile) tiltbox = apImagicFile.getBoxsize(tiltstackfile) tiltstacks = [] notstacks = [] t0 = time.time() for partdict in parttree: ### print friendly message if count % 100 == 0: backs = "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b" sys.stderr.write(backs + backs + backs + backs) if count > sizelimit: esttime = (len(parttree) / float(count) - 1.0) * (time.time() - t0) sys.stderr.write( str(count) + " particles of " + str(len(parttree)) + ", " + apDisplay.timeString(esttime) + " remaining") else: sys.stderr.write( str(count) + " particles of " + str(len(parttree))) ### save stacks to file to save memory if count % sizelimit == 0: if count > 1: apDisplay.printMsg("Writing stacks to file") t1 = time.time() tiltname = os.path.join(self.params['rundir'], "tiltstack%d.hed" % (count)) apFile.removeStack(tiltname) apImagicFile.writeImagic(tiltstacklist, tiltname, msg=False) tiltstacks.append(tiltname) apDisplay.printMsg("finished tilted stack in " + apDisplay.timeString(time.time() - t1)) t1 = time.time() notname = os.path.join(self.params['rundir'], "notstack%d.hed" % (count)) apFile.removeStack(notname) apImagicFile.writeImagic(notstacklist, notname, msg=False) notstacks.append(notname) apDisplay.printMsg("finished untilted stack in " + apDisplay.timeString(time.time() - t1)) ### reset stacks apDisplay.printMsg("Reset stacks in memory") notstacklist = [] tiltstacklist = [] ### increment count count += 1 ### write to Euler doc self.appendEulerDoc(eulerfile, partdict['tilt'], count) ### untilted stack notpartarray = apImagicFile.readSingleParticleFromStack( notstackfile, partdict['not'], notbox, False) notstacklist.append(notpartarray) ### tilted stack tiltpartarray = apImagicFile.readSingleParticleFromStack( tiltstackfile, partdict['tilt'], tiltbox, False) tiltstacklist.append(tiltpartarray) ### write remaining particles to stack if len(notstacklist) > 0: apDisplay.printMsg("Writing stacks to file") t1 = time.time() tiltname = os.path.join(self.params['rundir'], "tiltstack%d.hed" % (count)) apFile.removeStack(tiltname) apImagicFile.writeImagic(tiltstacklist, tiltname, msg=False) tiltstacks.append(tiltname) apDisplay.printMsg("finished tilted stack in " + apDisplay.timeString(time.time() - t1)) t1 = time.time() notname = os.path.join(self.params['rundir'], "notstack%d.hed" % (count)) apFile.removeStack(notname) apImagicFile.writeImagic(notstacklist, notname, msg=False) notstacks.append(notname) apDisplay.printMsg("finished untilted stack in " + apDisplay.timeString(time.time() - t1)) ### merge NOT stack notname = os.path.join(self.params['rundir'], "notstack.hed") apImagicFile.mergeStacks(notstacks, notname) for stackname in notstacks: apFile.removeStack(stackname, warn=False) ### merge TILT stack tiltname = os.path.join(self.params['rundir'], "tiltstack.hed") apImagicFile.mergeStacks(tiltstacks, tiltname) for stackname in tiltstacks: apFile.removeStack(stackname, warn=False) ### upload results if self.params['commit'] is True: self.uploadResults()
def start(self): # self.insertCL2DJob() self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) ### process stack to local file if self.params['timestamp'] is None: apDisplay.printMsg("creating timestamp") self.params['timestamp'] = self.timestamp self.params['localstack'] = os.path.join( self.params['rundir'], self.params['timestamp'] + ".hed") if os.path.isfile(self.params['localstack']): apFile.removeStack(self.params['localstack']) proccmd = "proc2d " + self.stack['file'] + " " + self.params[ 'localstack'] + " apix=" + str(self.stack['apix']) if self.params['bin'] > 1 or self.params['clipsize'] is not None: clipsize = int(self.clipsize) * self.params['bin'] if clipsize % 2 == 1: clipsize += 1 ### making sure that clipped boxsize is even proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], clipsize, clipsize) proccmd += " last=" + str(self.params['numpart'] - 1) if self.params['highpass'] is not None and self.params['highpass'] > 1: proccmd += " hp=" + str(self.params['highpass']) if self.params['lowpass'] is not None and self.params['lowpass'] > 1: proccmd += " lp=" + str(self.params['lowpass']) apParam.runCmd(proccmd, "EMAN", verbose=True) if self.params['numpart'] != apFile.numImagesInStack( self.params['localstack']): apDisplay.printError("Missing particles in stack") ### convert stack into single spider files self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles( self.params['localstack']) ### setup Xmipp command aligntime = time.time() xmippopts = ( " " + " -i " + os.path.join(self.params['rundir'], self.partlistdocfile) + " -codes " + str(self.params['numrefs']) + " -iter " + str(self.params['maxiter']) + " -o " + os.path.join( self.params['rundir'], "part" + self.params['timestamp'])) if self.params['fast']: xmippopts += " -fast " if self.params['correlation']: xmippopts += " -useCorrelation " if self.params['classical']: xmippopts += " -classicalMultiref " if self.params['align']: xmippopts += " -alignImages " ### use multi-processor command apDisplay.printColor( "Using " + str(self.params['nproc']) + " processors!", "green") xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True) mpiruncmd = self.mpirun + " -np " + str( self.params['nproc']) + " " + xmippexe + " " + xmippopts self.writeXmippLog(mpiruncmd) apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True, logfile="xmipp.std") self.params['runtime'] = time.time() - aligntime apDisplay.printMsg("Alignment time: " + apDisplay.timeString(self.params['runtime'])) ### minor post-processing self.createReferenceStack() self.parseOutput() self.clearIntermediateFiles() # self.readyUploadFlag() apParam.dumpParameters( self.params, "cl2d-" + self.params['timestamp'] + "-params.pickle") ### upload results ... this used to be two separate operations, I'm combining into one self.runparams = apParam.readRunParameters("cl2d-" + self.params['timestamp'] + "-params.pickle") self.apix = apStack.getStackPixelSizeFromStackId( self.runparams['stackid']) * self.runparams['bin'] self.Nlevels = len( glob.glob("part" + self.params['timestamp'] + "_level_??_.hed")) ### create average of aligned stacks & insert aligned stack info lastLevelStack = "part" + self.params[ 'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1) apStack.averageStack(lastLevelStack) self.boxsize = apFile.getBoxSize(lastLevelStack)[0] self.insertCL2DParamsIntoDatabase() if self.runparams['align'] is True: self.insertAlignStackRunIntoDatabase("alignedStack.hed") self.calcResolution(self.Nlevels - 1) self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1) ### loop over each class average stack & insert as clustering stacks self.insertClusterRunIntoDatabase() for level in range(self.Nlevels): ### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES if self.params['align'] is True: self.calcResolution(level) partdict = self.getClassificationAtLevel(level) for classnum in partdict: self.insertClusterStackIntoDatabase( "part" + self.params['timestamp'] + "_level_%02d_.hed" % level, classnum + 1, partdict[classnum], len(partdict))