def start(self): knownstackdata = apStack.getOnlyStackData(self.params['knownstackid']) fullstackdata = apStack.getOnlyStackData(self.params['fullstackid']) ### get good particle numbers includeParticle, tiltParticlesData = self.getGoodParticles() self.numpart = len(includeParticle) ### write kept particles to file self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile"+self.timestamp+".lst") apDisplay.printMsg("writing to keepfile "+self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum)+"\n") kf.close() ### make new stack of tilted particle from that run fullstackfile = os.path.join(fullstackdata['path']['path'], fullstackdata['name']) sb = os.path.splitext(fullstackdata['name']) newname = "tiltpairsub%d" % self.params['knownstackid']+sb[-1] newstackfile = os.path.join(self.params['rundir'], newname) apFile.removeStack(newstackfile, warn=False) apStack.makeNewStack(fullstackfile, newstackfile, self.params['keepfile']) if not os.path.isfile(newstackfile): apDisplay.printError("No stack was created") self.params['stackid'] = self.params['fullstackid'] apStack.commitSubStack(self.params, newname, sorted=False) apStack.averageStack(stack=newstackfile) newstackid = apStack.getStackIdFromPath(newstackfile) if self.params['meanplot'] is True: apDisplay.printMsg("creating Stack Mean Plot montage for stackid") apStackMeanPlot.makeStackMeanPlot(newstackid)
def postLoopFunctions(self): ### Delete CTF corrected images if self.params['keepall'] is False: pattern = os.path.join(self.params['rundir'], self.params['sessionname']+'*.dwn.mrc') apFile.removeFilePattern(pattern) ### remove Ace2 images pattern = os.path.join(self.params['rundir'], self.params['sessionname']+'*mrc.corrected.mrc') apFile.removeFilePattern(pattern) ### remove Spider images if self.params['fliptype'] == 'spiderimage': pattern = os.path.join(self.params['rundir'], self.params['sessionname']+'*_out.spi') apFile.removeFilePattern(pattern) pattern = os.path.join(self.params['rundir'], self.params['sessionname']+'*_tf.spi') apFile.removeFilePattern(pattern) if self.noimages is True: return stackpath = os.path.join(self.params['rundir'], self.params['single']) ### delete this after testing apStack.averageStack(stack = stackpath) ### Create Stack Mean Plot if self.params['commit'] is True and self.params['meanplot'] is True: stackid = apStack.getStackIdFromPath(stackpath) if stackid is not None: apStackMeanPlot.makeStackMeanPlot(stackid) apDisplay.printColor("Timing stats", "blue") self.printTimeStats("Batch Boxer", self.batchboxertimes) self.printTimeStats("Ctf Correction", self.ctftimes) self.printTimeStats("Stack Merging", self.mergestacktimes) self.printTimeStats("Mean/Std Read", self.meanreadtimes) self.printTimeStats("DB Insertion", self.insertdbtimes)
def start(self): stackparts = apStack.getStackParticlesFromId(self.params['stackid']) stackdata = apStack.getOnlyStackData(self.params['stackid']) newname = stackdata['name'] oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) newstack = os.path.join(self.params['rundir'], newname) # calculate slop and intercept from the four points given slope = (self.params['maxy'] - self.params['miny']) / ( self.params['maxx'] - self.params['minx']) intercept = self.params['miny'] - (slope * self.params['minx']) # print slope # print intercept numparticles = 0 self.params['keepfile'] = os.path.join( self.params['rundir'], "keepfile-" + self.timestamp + ".list") f = open(self.params['keepfile'], 'w') for stackpart in stackparts: #print str(stackpart['particleNumber'])+","+ str(stackpart['mean'])+","+str(stackpart['stdev']) if stackpart['mean'] > self.params['minx'] and stackpart[ 'mean'] < self.params['maxx']: #print str(stackpart['particleNumber'])+","+ str(stackpart['mean'])+","+str(stackpart['stdev']) calcY = slope * stackpart['mean'] + intercept if (calcY >= stackpart['stdev'] and self.params['keepabove'] is not True) or \ (calcY <= stackpart['stdev'] and self.params['keepabove'] is True): emanpartnum = stackpart['particleNumber'] - 1 f.write('%i\n' % emanpartnum) numparticles += 1 f.close() self.params['description'] += ( (" ... %d particle substack of stackid %d" % (numparticles, self.params['stackid']))) #create the new sub stack apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=True) if not os.path.isfile(newstack): apDisplay.printError("No stack was created") apStack.commitSubStack(self.params, newname, oldstackparts=stackparts) apStack.averageStack(stack=newstack) # stack mean plot newstackid = apStack.getStackIdFromPath(newstack) apDisplay.printMsg("creating Stack Mean Plot montage for stackid") apStackMeanPlot.makeStackMeanPlot(newstackid)
def start(self): stackparts = apStack.getStackParticlesFromId(self.params['stackid']) stackdata = apStack.getOnlyStackData(self.params['stackid']) newname = stackdata['name'] oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) newstack = os.path.join(self.params['rundir'], newname) # calculate slop and intercept from the four points given slope = (self.params['maxy'] - self.params['miny']) / (self.params['maxx'] - self.params['minx']) intercept = self.params['miny'] - (slope*self.params['minx']) # print slope # print intercept numparticles = 0 self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile-"+self.timestamp+".list") f=open(self.params['keepfile'],'w') for stackpart in stackparts: #print str(stackpart['particleNumber'])+","+ str(stackpart['mean'])+","+str(stackpart['stdev']) if stackpart['mean'] > self.params['minx'] and stackpart['mean'] < self.params['maxx']: #print str(stackpart['particleNumber'])+","+ str(stackpart['mean'])+","+str(stackpart['stdev']) calcY = slope*stackpart['mean']+intercept if (calcY >= stackpart['stdev'] and self.params['keepabove'] is not True) or \ (calcY <= stackpart['stdev'] and self.params['keepabove'] is True): emanpartnum = stackpart['particleNumber']-1 f.write('%i\n' % emanpartnum) numparticles+=1 f.close() self.params['description'] +=( (" ... %d particle substack of stackid %d" % (numparticles, self.params['stackid'])) ) #create the new sub stack apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=True) if not os.path.isfile(newstack): apDisplay.printError("No stack was created") apStack.commitSubStack(self.params, newname, oldstackparts=stackparts) apStack.averageStack(stack=newstack) # stack mean plot newstackid = apStack.getStackIdFromPath(newstack) apDisplay.printMsg("creating Stack Mean Plot montage for stackid") apStackMeanPlot.makeStackMeanPlot(newstackid)
stackdata = apStack.getOnlyStackData(params["stackid"]) stackpath = stackdata["path"]["path"] # generate stack if it doesn't exist. if not os.path.isdir(stackpath): os.makedirs(stackpath) fname = os.path.join(stackpath, stackdata["name"]) # check if stack file already exists if os.path.isfile(fname): apDisplay.printError("file: '%s' already exists" % fname) vstackdata = apStack.getVirtualStackParticlesFromId(params["stackid"]) plist = [int(p["particleNumber"]) - 1 for p in vstackdata["particles"]] a = proc2dLib.RunProc2d() a.setValue("infile", vstackdata["filename"]) a.setValue("outfile", fname) a.setValue("list", plist) a.setValue("apix", apStack.getStackPixelSizeFromStackId(params["stackid"])) apDisplay.printMsg("generating stack: '%s' with %i particles" % (fname, len(plist))) a.run() outavg = os.path.join(stackpath, "average.mrc") if not os.path.isfile(outavg): apStack.averageStack(stack=fname, outfile=outavg) montageimg = os.path.join(stackpath, "montage%i.png" % params["stackid"]) if not os.path.isfile(montageimg): apStackMeanPlot.makeStackMeanPlot(params["stackid"], gridpoints=4)
def start(self): ### new stack path stackdata = apStack.getOnlyStackData(self.params['stackid']) oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) newstack = os.path.join(self.params['rundir'], stackdata['name']) apStack.checkForPreviousStack(newstack) includelist = [] excludelist = [] ### list of classes to be excluded if self.params['dropclasslist'] is not None: excludestrlist = self.params['dropclasslist'].split(",") for excludeitem in excludestrlist: excludelist.append(int(excludeitem.strip())) apDisplay.printMsg("Exclude list: " + str(excludelist)) ### list of classes to be included if self.params['keepclasslist'] is not None: includestrlist = self.params['keepclasslist'].split(",") for includeitem in includestrlist: includelist.append(int(includeitem.strip())) ### or read from keepfile elif self.params['keepfile'] is not None: keeplistfile = open(self.params['keepfile']) for line in keeplistfile: if self.params['excludefrom'] is True: excludelist.append(int(line.strip())) else: includelist.append(int(line.strip())) keeplistfile.close() apDisplay.printMsg("Include list: " + str(includelist)) ### get particles from align or cluster stack apDisplay.printMsg("Querying database for particles") q0 = time.time() if self.params['alignid'] is not None: alignpartq = appiondata.ApAlignParticleData() alignpartq['alignstack'] = self.alignstackdata particles = alignpartq.query() elif self.params['clusterid'] is not None: clusterpartq = appiondata.ApClusteringParticleData() clusterpartq['clusterstack'] = self.clusterstackdata particles = clusterpartq.query() apDisplay.printMsg("Complete in " + apDisplay.timeString(time.time() - q0)) ### write included particles to text file includeParticle = [] excludeParticle = 0 badscore = 0 badshift = 0 badspread = 0 f = open("test.log", "w") count = 0 for part in particles: count += 1 #partnum = part['partnum']-1 if 'alignparticle' in part: alignpart = part['alignparticle'] classnum = int(part['refnum']) - 1 else: alignpart = part classnum = int(part['ref']['refnum']) - 1 emanstackpartnum = alignpart['stackpart']['particleNumber'] - 1 ### check shift if self.params['maxshift'] is not None: shift = math.hypot(alignpart['xshift'], alignpart['yshift']) if shift > self.params['maxshift']: excludeParticle += 1 f.write("%d\t%d\t%d\texclude\n" % (count, emanstackpartnum, classnum)) badshift += 1 continue if self.params['minscore'] is not None: ### check score if (alignpart['score'] is not None and alignpart['score'] < self.params['minscore']): excludeParticle += 1 f.write("%d\t%d\t%d\texclude\n" % (count, emanstackpartnum, classnum)) badscore += 1 continue ### check spread if (alignpart['spread'] is not None and alignpart['spread'] < self.params['minscore']): excludeParticle += 1 f.write("%d\t%d\t%d\texclude\n" % (count, emanstackpartnum, classnum)) badspread += 1 continue if includelist and classnum in includelist: includeParticle.append(emanstackpartnum) f.write("%d\t%d\t%d\tinclude\n" % (count, emanstackpartnum, classnum)) elif excludelist and not classnum in excludelist: includeParticle.append(emanstackpartnum) f.write("%d\t%d\t%d\tinclude\n" % (count, emanstackpartnum, classnum)) else: excludeParticle += 1 f.write("%d\t%d\t%d\texclude\n" % (count, emanstackpartnum, classnum)) f.close() includeParticle.sort() if badshift > 0: apDisplay.printMsg("%d paricles had a large shift" % (badshift)) if badscore > 0: apDisplay.printMsg("%d paricles had a low score" % (badscore)) if badspread > 0: apDisplay.printMsg("%d paricles had a low spread" % (badspread)) apDisplay.printMsg("Keeping " + str(len(includeParticle)) + " and excluding " + str(excludeParticle) + " particles") #print includeParticle ### write kept particles to file self.params['keepfile'] = os.path.join( self.params['rundir'], "keepfile-" + self.timestamp + ".list") apDisplay.printMsg("writing to keepfile " + self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum) + "\n") kf.close() ### get number of particles numparticles = len(includeParticle) if excludelist: self.params['description'] += ( " ... %d particle substack with %s classes excluded" % (numparticles, self.params['dropclasslist'])) elif includelist: self.params['description'] += ( " ... %d particle substack with %s classes included" % (numparticles, self.params['keepclasslist'])) ### create the new sub stack apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=self.params['savebad']) if not os.path.isfile(newstack): apDisplay.printError("No stack was created") apStack.averageStack(stack=newstack) if self.params['commit'] is True: apStack.commitSubStack(self.params) newstackid = apStack.getStackIdFromPath(newstack) apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=4)
def start(self): #old stack size stacksize = apStack.getNumberStackParticlesFromId(self.params['stackid']) # if exclude or include lists are not defined... if self.params['exclude'] is None and self.params['include'] is None: # if first and last are specified, create a file if self.params['first'] is not None and self.params['last'] is not None: stp = str(self.params['first']) enp = str(self.params['last']) fname = 'sub'+str(self.params['stackid'])+'_'+stp+'-'+enp+'.lst' self.params['keepfile'] = os.path.join(self.params['rundir'],fname) apDisplay.printMsg("Creating keep list: "+self.params['keepfile']) f=open(self.params['keepfile'],'w') for i in range(self.params['first'],self.params['last']+1): f.write('%d\n' % (int(i)-1)) f.close() # generate the random list by giving number and create the file elif self.params['random'] is not None: #numOfRandomParticles = str(self.params['random']) #fname = 'random'+str(self.params['stackid'])+'_'+numOfRandomParticles+'.lst' fname = "random%d_%d.lst"%(self.params['stackid'], self.params['random']) self.params['keepfile'] = os.path.join(self.params['rundir'],fname) apDisplay.printMsg("Creating keep list: "+self.params['keepfile']) # create a file f=open(self.params['keepfile'],'w') # generate a random sequence by giving size randomList = random.sample(xrange(self.params['last']), self.params['random']) randomList.sort() for partnum in randomList: f.write('%d\n' % partnum) f.close() # if splitting, create files containing the split values elif self.params['split'] > 1: for i in range(self.params['split']): fname = 'sub'+str(self.params['stackid'])+'.'+str(i+1)+'.lst' self.params['keepfile'] = os.path.join(self.params['rundir'],fname) apDisplay.printMsg("Creating keep list: "+self.params['keepfile']) f = open(self.params['keepfile'],'w') for p in range(stacksize): if (p % self.params['split'])-i==0: f.write('%i\n' % p) f.close() # if exclude-from option is specified, convert particles to exclude elif self.params['excludefile'] is True: oldkf = open(self.params['keepfile']) partlist = [] for line in oldkf: particle=line.strip() try: particle = int(particle) except: continue partlist.append(particle) oldkf.close() # create new list excluding the particles apDisplay.printMsg("Converting keep file to exclude file") newkeepfile = "tmpnewkeepfile.txt" newkf = open(newkeepfile,'w') for p in range(stacksize): if p not in partlist: newkf.write("%i\n"%p) newkf.close() self.params['keepfile'] = os.path.abspath(newkeepfile) # otherwise, just copy the file elif not os.path.isfile(os.path.basename(self.params['keepfile'])): shutil.copy(self.params['keepfile'], os.path.basename(self.params['keepfile'])) # if either exclude or include lists is defined elif self.params['exclude'] or self.params['include']: ### list of particles to be excluded excludelist = [] if self.params['exclude'] is not None: excludestrlist = self.params['exclude'].split(",") for excld in excludestrlist: excludelist.append(int(excld.strip())) apDisplay.printMsg("Exclude list: "+str(excludelist)) ### list of particles to be included includelist = [] if self.params['include'] is not None: includestrlist = self.params['include'].split(",") for incld in includestrlist: includelist.append(int(incld.strip())) apDisplay.printMsg("Include list: "+str(includelist)) #new stack path stackdata = apStack.getOnlyStackData(self.params['stackid']) newname = stackdata['name'] oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) #if include or exclude list is given... if self.params['include'] is not None or self.params['exclude'] is not None: includeParticle = [] excludeParticle = 0 for partnum in range(stacksize): if includelist and partnum in includelist: includeParticle.append(partnum) elif excludelist and not partnum in excludelist: includeParticle.append(partnum) else: excludeParticle += 1 includeParticle.sort() ### write kept particles to file self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile-"+self.timestamp+".list") apDisplay.printMsg("writing to keepfile "+self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum)+"\n") kf.close() #get number of particles numparticles = len(includeParticle) if excludelist: self.params['description'] += ( " ... %d particle substack of stackid %d" % (numparticles, self.params['stackid'])) elif includelist: self.params['description'] += ( " ... %d particle substack of stackid %d" % (numparticles, self.params['stackid'])) ogdescr = self.params['description'] for i in range(self.params['split']): ### always do this, if not splitting split=1 sb = os.path.splitext(stackdata['name']) if self.params['first'] is not None and self.params['last'] is not None: newname = sb[0]+'.'+str(self.params['first'])+'-'+str(self.params['last'])+sb[-1] elif self.params['random'] is not None: newname = "%s-random%d%s"%(sb[0], self.params['random'], sb[-1]) elif self.params['split'] > 1: fname = 'sub'+str(self.params['stackid'])+'.'+str(i+1)+'.lst' self.params['keepfile'] = os.path.join(self.params['rundir'],fname) newname = sb[0]+'.'+str(i+1)+'of'+str(self.params['split'])+sb[-1] newstack = os.path.join(self.params['rundir'], newname) apStack.checkForPreviousStack(newstack) #get number of particles f = open(self.params['keepfile'], "r") numparticles = len(f.readlines()) f.close() self.params['description'] = ogdescr self.params['description'] += ( (" ... %d particle substack of stackid %d" % (numparticles, self.params['stackid'])) ) #if splitting, add to description if self.params['split'] > 1: self.params['description'] += (" (%i of %i)" % (i+1, self.params['split'])) #create the new sub stack if not self.params['correctbeamtilt']: apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=True) else: apBeamTilt.makeCorrectionStack(self.params['stackid'], oldstack, newstack) if not os.path.isfile(newstack): apDisplay.printError("No stack was created") apStack.commitSubStack(self.params, newname, sorted=False) apStack.averageStack(stack=newstack) newstackid = apStack.getStackIdFromPath(newstack) if self.params['meanplot'] is True: apDisplay.printMsg("creating Stack Mean Plot montage for stackid") apStackMeanPlot.makeStackMeanPlot(newstackid)
stackpath = stackdata['path']['path'] # generate stack if it doesn't exist. if not os.path.isdir(stackpath): os.makedirs(stackpath) fname = os.path.join(stackpath, stackdata['name']) # check if stack file already exists if os.path.isfile(fname): apDisplay.printError("file: '%s' already exists" % fname) vstackdata = apStack.getVirtualStackParticlesFromId(params['stackid']) plist = [int(p['particleNumber']) - 1 for p in vstackdata['particles']] a = proc2dLib.RunProc2d() a.setValue('infile', vstackdata['filename']) a.setValue('outfile', fname) a.setValue('list', plist) a.setValue('apix', apStack.getStackPixelSizeFromStackId(params['stackid'])) apDisplay.printMsg("generating stack: '%s' with %i particles" % (fname, len(plist))) a.run() outavg = os.path.join(stackpath, "average.mrc") if not os.path.isfile(outavg): apStack.averageStack(stack=fname, outfile=outavg) montageimg = os.path.join(stackpath, "montage%i.png" % params['stackid']) if not os.path.isfile(montageimg): apStackMeanPlot.makeStackMeanPlot(params['stackid'], gridpoints=4)
def start(self): ### new stack path oldstack = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) newstack = os.path.join(self.params['rundir'], self.stackdata['name']) apStack.checkForPreviousStack(newstack) ### get particles from stack apDisplay.printMsg("Querying stack particles") t0 = time.time() stackpartq = appiondata.ApRefineParticleData() stackpartq['refineIter'] = self.iterdata particles = stackpartq.query() apDisplay.printMsg("Finished in " + apDisplay.timeString(time.time() - t0)) ### write included particles to text file includeParticle = [] excludeParticle = 0 f = open("test.log", "w") count = 0 apDisplay.printMsg("Processing stack particles") t0 = time.time() for part in particles: count += 1 if count % 500 == 0: sys.stderr.write(".") emanstackpartnum = part['particle']['particleNumber'] - 1 if part['postRefine_keep'] == 1: ### good particle includeParticle.append(emanstackpartnum) f.write("%d\t%d\tinclude\n" % (count, emanstackpartnum)) else: ### bad particle excludeParticle += 1 f.write("%d\t%d\texclude\n" % (count, emanstackpartnum)) sys.stderr.write("\n") apDisplay.printMsg("Finished in " + apDisplay.timeString(time.time() - t0)) f.close() includeParticle.sort() apDisplay.printMsg("Keeping " + str(len(includeParticle)) + " and excluding " + str(excludeParticle) + " particles") ### write kept particles to file self.params['keepfile'] = os.path.join( self.params['rundir'], "keepfile-" + self.timestamp + ".list") apDisplay.printMsg("writing to keepfile " + self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum) + "\n") kf.close() ### get number of particles numparticles = len(includeParticle) self.params['description'] += (" ... %d no jumpers substack" % (numparticles, )) ### create the new sub stack apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=True) if not os.path.isfile(newstack): apDisplay.printError("No stack was created") apStack.averageStack(stack=newstack) if self.params['commit'] is True: apStack.commitSubStack(self.params) newstackid = apStack.getStackIdFromPath(newstack) apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=6)
def start(self): ### new stack path oldstack = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) newstack = os.path.join(self.params['rundir'], self.stackdata['name']) apStack.checkForPreviousStack(newstack) ### get particles from stack apDisplay.printMsg("Querying stack particles") t0 = time.time() stackpartq = appiondata.ApStackParticleData() stackpartq['stack'] = self.stackdata particles = stackpartq.query() apDisplay.printMsg("Finished in "+apDisplay.timeString(time.time()-t0)) ### write included particles to text file includeParticle = [] excludeParticle = 0 f = open("test.log", "w") count = 0 apDisplay.printMsg("Processing stack particles") t0 = time.time() for part in particles: count += 1 if count%500 == 0: sys.stderr.write(".") emanstackpartnum = part['particleNumber']-1 ### get euler jump data jumpq = appiondata.ApEulerJumpData() jumpq['particle'] = part jumpq['refineRun'] = self.recondata jumpdatas = jumpq.query(results=1) if not jumpdatas or len(jumpdatas) < 1: ### no data continue jumpdata = jumpdatas[0] if jumpdata['median'] is None or jumpdata['median'] > self.params['maxjump']: ### bad particle excludeParticle += 1 f.write("%d\t%d\t%.1f\texclude\n"%(count, emanstackpartnum, jumpdata['median'])) else: ### good particle includeParticle.append(emanstackpartnum) f.write("%d\t%d\t%.1f\tinclude\n"%(count, emanstackpartnum, jumpdata['median'])) sys.stderr.write("\n") apDisplay.printMsg("Finished in "+apDisplay.timeString(time.time()-t0)) f.close() includeParticle.sort() apDisplay.printMsg("Keeping "+str(len(includeParticle)) +" and excluding "+str(excludeParticle)+" particles") #print includeParticle ### write kept particles to file self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile-"+self.timestamp+".list") apDisplay.printMsg("writing to keepfile "+self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum)+"\n") kf.close() ### get number of particles numparticles = len(includeParticle) self.params['description'] += ( " ... %d no jumpers substack" % (numparticles,)) ### create the new sub stack apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=True) if not os.path.isfile(newstack): apDisplay.printError("No stack was created") apStack.averageStack(stack=newstack) if self.params['commit'] is True: apStack.commitSubStack(self.params) newstackid = apStack.getStackIdFromPath(newstack) apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=6)
def start(self): self.ace2correct = self.getACE2Path() ### determine amount of memory needed for entire stack memorylimit = 0.3 bytelimit = memorylimit*(1024**3) writeiters = 1 partbytes = 4*self.params['box']*self.params['box'] if partbytes*self.params['projcount'] > bytelimit: writeiters = int(math.ceil(float(partbytes)*self.params['projcount'] / bytelimit)) partsperiter = int(float(self.params['projcount']) / writeiters) ### number of particles read each time ### some defaults, and workarounds for now self.params['projpergraph'] = 100 self.params['filesperdir'] = partsperiter if self.params['filesperdir'] > 2048: self.params['filesperdir'] = 2048 ### first create projections if self.params['preforient'] is True: filename = self.createProjections(pad=self.params['pad'], invert=self.params['invert']) else: filename = self.createProjectionsEmanProp(pad=self.params['pad'], invert=self.params['invert']) ### shift & rotate randomly if self.params['rotang']!=0 or self.params['shiftrad']!=0: shiftstackname = self.shift_images(filename) else: shiftstackname = filename ### read MRC stats to figure out noise level addition mean1, stdev1 = self.readFileStats(shiftstackname) ### determine noise multiplication factor to ensure that appropriate amount of noise gets added to particles inside circular mask multfactor = 1.0/((float(self.params['radius'])/self.params['box'])*(float(self.params['radius'])/self.params['box'])*math.pi) ### calculate noiselevel additions and add noise to an initial ratio of 1.8, simulating beam and structural damage ### NOTE: THERE ARE DIFFERENT DEFINITIONS OF SNR, see below noiselevel1 = math.sqrt((float(stdev1)*float(stdev1)) / float(self.params['snr1'])) # noiselevel1 = float(stdev1) / float(self.params['snr1']) noiselevel1 = noiselevel1 * multfactor noisystack = self.addNoise(shiftstackname, noiselevel1, SNR=self.params['snr1']) ### get list of defocus values self.getListOfDefoci(self.params['projcount']) ### apply envelope and ctf to each .mrc file, then correct based on how well ace2 works on raw micrographs ctfstack, ctfpartlist = self.applyEnvelopeAndCTF(noisystack) #recoverlists = self.recoverLists() ### read IMAGIC stats to figure out noise level addition mean2, stdev2 = self.readFileStats(ctfstack) ### cascading of noise processes according to Frank and Al-Ali (1975) & Baxter (2009) # snr2 = 1 / ((1+1/float(self.params['snrtot'])) / (1/float(self.params['snr1']) + 1) - 1) snr2 = (1+1/self.params['snr1'])/(1/self.params['snrtot']-1/self.params['snr1']) ### NOTE: THERE ARE DIFFERENT DEFINITIONS OF SNR, see below noiselevel2 = math.sqrt((float(stdev2)*float(stdev2)) / float(snr2)) # noiselevel2 = float(stdev2) / float(snr2) noiselevel2 = noiselevel2 * multfactor ### add a last layer of noise noisystack2 = self.addNoise(ctfstack, noiselevel2, SNR=self.params['snrtot']) ### low-pass / high-pass filter resulting stack, if specified if self.params['hpfilt'] is not None or self.params['lpfilt'] is not None or self.params['norm'] is True: filtstack = noisystack2[:-4] if self.params['norm'] is True: filtstack = filtstack+"_norm.hed" else: filtstack = filtstack+"_filt.hed" apFile.removeStack(filtstack) emancmd = "proc2d "+noisystack2+" "+filtstack+" apix="+str(self.params['apix'])+" " if self.params['hpfilt'] is not None: emancmd = emancmd+"hp="+str(self.params['hpfilt'])+" " if self.params['lpfilt'] is not None: emancmd = emancmd+"lp="+str(self.params['lpfilt'])+" " if self.params['norm'] is True: emancmd = emancmd+"norm="+str(self.params['norm'])+" " apParam.runCmd(emancmd, "EMAN") self.params['finalstack'] = os.path.basename(filtstack) finalstack = filtstack else: self.params['finalstack'] = os.path.basename(noisystack2) finalstack = noisystack2 ### post-processing: create average file for viewing on webpages apStack.averageStack(finalstack) ### upload if commit is checked self.uploadData(ctfpartlist) ### post-processing: Create Stack Mean Plot if self.params['commit'] is True: stackid = apStack.getStackIdFromPath(finalstack) if stackid is not None: apStackMeanPlot.makeStackMeanPlot(stackid, gridpoints=8)
def start(self): #old stack size stacksize = apStack.getNumberStackParticlesFromId( self.params['stackid']) # if exclude or include lists are not defined... if self.params['exclude'] is None and self.params['include'] is None: # if first and last are specified, create a file if self.params['first'] is not None and self.params[ 'last'] is not None: stp = str(self.params['first']) enp = str(self.params['last']) fname = 'sub' + str( self.params['stackid']) + '_' + stp + '-' + enp + '.lst' self.params['keepfile'] = os.path.join(self.params['rundir'], fname) apDisplay.printMsg("Creating keep list: " + self.params['keepfile']) f = open(self.params['keepfile'], 'w') for i in range(self.params['first'], self.params['last'] + 1): f.write('%d\n' % (int(i) - 1)) f.close() # generate the random list by giving number and create the file elif self.params['random'] is not None: #numOfRandomParticles = str(self.params['random']) #fname = 'random'+str(self.params['stackid'])+'_'+numOfRandomParticles+'.lst' fname = "random%d_%d.lst" % (self.params['stackid'], self.params['random']) self.params['keepfile'] = os.path.join(self.params['rundir'], fname) apDisplay.printMsg("Creating keep list: " + self.params['keepfile']) # create a file f = open(self.params['keepfile'], 'w') # generate a random sequence by giving size randomList = random.sample(xrange(self.params['last']), self.params['random']) randomList.sort() for partnum in randomList: f.write('%d\n' % partnum) f.close() # if splitting, create files containing the split values elif self.params['split'] > 1: for i in range(self.params['split']): fname = 'sub' + str( self.params['stackid']) + '.' + str(i + 1) + '.lst' self.params['keepfile'] = os.path.join( self.params['rundir'], fname) apDisplay.printMsg("Creating keep list: " + self.params['keepfile']) f = open(self.params['keepfile'], 'w') for p in range(stacksize): if (p % self.params['split']) - i == 0: f.write('%i\n' % p) f.close() # if exclude-from option is specified, convert particles to exclude elif self.params['excludefile'] is True: oldkf = open(self.params['keepfile']) partlist = [] for line in oldkf: particle = line.strip() try: particle = int(particle) except: continue partlist.append(particle) oldkf.close() # create new list excluding the particles apDisplay.printMsg("Converting keep file to exclude file") newkeepfile = "tmpnewkeepfile.txt" newkf = open(newkeepfile, 'w') for p in range(stacksize): if p not in partlist: newkf.write("%i\n" % p) newkf.close() self.params['keepfile'] = os.path.abspath(newkeepfile) # otherwise, just copy the file elif not os.path.isfile(os.path.basename(self.params['keepfile'])): shutil.copy(self.params['keepfile'], os.path.basename(self.params['keepfile'])) # if either exclude or include lists is defined elif self.params['exclude'] or self.params['include']: ### list of particles to be excluded excludelist = [] if self.params['exclude'] is not None: excludestrlist = self.params['exclude'].split(",") for excld in excludestrlist: excludelist.append(int(excld.strip())) apDisplay.printMsg("Exclude list: " + str(excludelist)) ### list of particles to be included includelist = [] if self.params['include'] is not None: includestrlist = self.params['include'].split(",") for incld in includestrlist: includelist.append(int(incld.strip())) apDisplay.printMsg("Include list: " + str(includelist)) #new stack path stackdata = apStack.getOnlyStackData(self.params['stackid']) newname = stackdata['name'] oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) #if include or exclude list is given... if self.params['include'] is not None or self.params[ 'exclude'] is not None: includeParticle = [] excludeParticle = 0 for partnum in range(stacksize): if includelist and partnum in includelist: includeParticle.append(partnum) elif excludelist and not partnum in excludelist: includeParticle.append(partnum) else: excludeParticle += 1 includeParticle.sort() ### write kept particles to file self.params['keepfile'] = os.path.join( self.params['rundir'], "keepfile-" + self.timestamp + ".list") apDisplay.printMsg("writing to keepfile " + self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum) + "\n") kf.close() #get number of particles numparticles = len(includeParticle) if excludelist: self.params['description'] += ( " ... %d particle substack of stackid %d" % (numparticles, self.params['stackid'])) elif includelist: self.params['description'] += ( " ... %d particle substack of stackid %d" % (numparticles, self.params['stackid'])) ogdescr = self.params['description'] for i in range(self.params['split']): ### always do this, if not splitting split=1 sb = os.path.splitext(stackdata['name']) if self.params['first'] is not None and self.params[ 'last'] is not None: newname = sb[0] + '.' + str(self.params['first']) + '-' + str( self.params['last']) + sb[-1] elif self.params['random'] is not None: newname = "%s-random%d%s" % (sb[0], self.params['random'], sb[-1]) elif self.params['split'] > 1: fname = 'sub' + str( self.params['stackid']) + '.' + str(i + 1) + '.lst' self.params['keepfile'] = os.path.join(self.params['rundir'], fname) newname = sb[0] + '.' + str(i + 1) + 'of' + str( self.params['split']) + sb[-1] newstack = os.path.join(self.params['rundir'], newname) apStack.checkForPreviousStack(newstack) #get number of particles f = open(self.params['keepfile'], "r") numparticles = len(f.readlines()) f.close() self.params['description'] = ogdescr self.params['description'] += ( (" ... %d particle substack of stackid %d" % (numparticles, self.params['stackid']))) #if splitting, add to description if self.params['split'] > 1: self.params['description'] += (" (%i of %i)" % (i + 1, self.params['split'])) #create the new sub stack if not self.params['correctbeamtilt']: apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=True) else: apBeamTilt.makeCorrectionStack(self.params['stackid'], oldstack, newstack) if not os.path.isfile(newstack): apDisplay.printError("No stack was created") apStack.commitSubStack(self.params, newname, sorted=False) apStack.averageStack(stack=newstack) newstackid = apStack.getStackIdFromPath(newstack) if self.params['meanplot'] is True: apDisplay.printMsg( "creating Stack Mean Plot montage for stackid") apStackMeanPlot.makeStackMeanPlot(newstackid)
def start(self): ### new stack path stackdata = apStack.getOnlyStackData(self.params["stackid"]) oldstack = os.path.join(stackdata["path"]["path"], stackdata["name"]) newstack = os.path.join(self.params["rundir"], stackdata["name"]) apStack.checkForPreviousStack(newstack) includelist = [] excludelist = [] ### list of classes to be excluded if self.params["dropclasslist"] is not None: excludestrlist = self.params["dropclasslist"].split(",") for excludeitem in excludestrlist: excludelist.append(int(excludeitem.strip())) apDisplay.printMsg("Exclude list: " + str(excludelist)) ### list of classes to be included if self.params["keepclasslist"] is not None: includestrlist = self.params["keepclasslist"].split(",") for includeitem in includestrlist: includelist.append(int(includeitem.strip())) ### or read from keepfile elif self.params["keepfile"] is not None: keeplistfile = open(self.params["keepfile"]) for line in keeplistfile: if self.params["excludefrom"] is True: excludelist.append(int(line.strip())) else: includelist.append(int(line.strip())) keeplistfile.close() apDisplay.printMsg("Include list: " + str(includelist)) ### get particles from align or cluster stack apDisplay.printMsg("Querying database for particles") q0 = time.time() if self.params["alignid"] is not None: # DIRECT SQL STUFF sqlcmd = ( "SELECT " + "apd.partnum, " + "apd.xshift, apd.yshift, " + "apd.rotation, apd.mirror, " + "apd.spread, apd.correlation, " + "apd.score, apd.bad, " + "spd.particleNumber, " + "ard.refnum " + "FROM ApAlignParticleData apd " + "LEFT JOIN ApStackParticleData spd ON " + "(apd.`REF|ApStackParticleData|stackpart` = spd.DEF_id) " + "LEFT JOIN ApAlignReferenceData ard ON" + "(apd.`REF|ApAlignReferenceData|ref` = ard.DEF_id) " + "WHERE `REF|ApAlignStackData|alignstack` = %i" % (self.params["alignid"]) ) # These are AlignParticles particles = sinedon.directq.complexMysqlQuery("appiondata", sqlcmd) elif self.params["clusterid"] is not None: clusterpartq = appiondata.ApClusteringParticleData() clusterpartq["clusterstack"] = self.clusterstackdata # These are ClusteringParticles particles = clusterpartq.query() apDisplay.printMsg("Completed in %s\n" % (apDisplay.timeString(time.time() - q0))) ### write included particles to text file includeParticle = [] excludeParticle = 0 badscore = 0 badshift = 0 badspread = 0 f = open("test.log", "w") count = 0 t0 = time.time() apDisplay.printMsg("Parsing particle information") # find out if there is alignparticle info: is_cluster_p = False # alignparticle is a key of any particle in particles if the latter is # a CluateringParticle if "alignparticle" in particles[0]: is_cluster_p = True for part in particles: count += 1 if is_cluster_p: # alignpart is an item of ClusteringParticle alignpart = part["alignparticle"] try: classnum = int(part["refnum"]) - 1 except: apDisplay.printWarning("particle %d was not put into any class" % (part["partnum"])) emanstackpartnum = alignpart["stackpart"]["particleNumber"] - 1 else: # particle has info from AlignedParticle as results of direct query alignpart = part try: classnum = int(alignpart["refnum"]) - 1 except: apDisplay.printWarning("particle %d was not put into any class" % (part["partnum"])) classnum = None emanstackpartnum = int(alignpart["particleNumber"]) - 1 ### check shift if self.params["maxshift"] is not None: shift = math.hypot(alignpart["xshift"], alignpart["yshift"]) if shift > self.params["maxshift"]: excludeParticle += 1 if classnum is not None: f.write("%d\t%d\t%d\texclude\n" % (count, emanstackpartnum, classnum)) else: f.write("%d\t%d\texclude\n" % (count, emanstackpartnum)) badshift += 1 continue if self.params["minscore"] is not None: ### check score if alignpart["score"] is not None and alignpart["score"] < self.params["minscore"]: excludeParticle += 1 if classnum is not None: f.write("%d\t%d\t%d\texclude\n" % (count, emanstackpartnum, classnum)) else: f.write("%d\t%d\texclude\n" % (count, emanstackpartnum)) badscore += 1 continue ### check spread if alignpart["spread"] is not None and alignpart["spread"] < self.params["minscore"]: excludeParticle += 1 if classnum is not None: f.write("%d\t%d\t%d\texclude\n" % (count, emanstackpartnum, classnum)) else: f.write("%d\t%d\texclude\n" % (count, emanstackpartnum)) badspread += 1 continue if classnum is not None: if includelist and (classnum in includelist): includeParticle.append(emanstackpartnum) f.write("%d\t%d\t%d\tinclude\n" % (count, emanstackpartnum, classnum)) elif excludelist and not (classnum in excludelist): includeParticle.append(emanstackpartnum) f.write("%d\t%d\t%d\tinclude\n" % (count, emanstackpartnum, classnum)) else: excludeParticle += 1 f.write("%d\t%d\t%d\texclude\n" % (count, emanstackpartnum, classnum)) else: excludeParticle += 1 f.write("%d\t%d\texclude\n" % (count, emanstackpartnum)) f.close() includeParticle.sort() if badshift > 0: apDisplay.printMsg("%d paricles had a large shift" % (badshift)) if badscore > 0: apDisplay.printMsg("%d paricles had a low score" % (badscore)) if badspread > 0: apDisplay.printMsg("%d paricles had a low spread" % (badspread)) apDisplay.printMsg("Completed in %s\n" % (apDisplay.timeString(time.time() - t0))) apDisplay.printMsg( "Keeping " + str(len(includeParticle)) + " and excluding " + str(excludeParticle) + " particles" ) ### write kept particles to file self.params["keepfile"] = os.path.join(self.params["rundir"], "keepfile-" + self.timestamp + ".list") apDisplay.printMsg("writing to keepfile " + self.params["keepfile"]) kf = open(self.params["keepfile"], "w") for partnum in includeParticle: kf.write(str(partnum) + "\n") kf.close() ### get number of particles numparticles = len(includeParticle) if excludelist: self.params["description"] += " ... %d particle substack with %s classes excluded" % ( numparticles, self.params["dropclasslist"], ) elif includelist: self.params["description"] += " ... %d particle substack with %s classes included" % ( numparticles, self.params["keepclasslist"], ) outavg = os.path.join(self.params["rundir"], "average.mrc") ### create the new sub stack # first check if virtual stack if not os.path.isfile(oldstack): vstackdata = apStack.getVirtualStackParticlesFromId(self.params["stackid"]) vparts = vstackdata["particles"] oldstack = vstackdata["filename"] # get subset of virtualstack vpartlist = [int(vparts[p]["particleNumber"]) - 1 for p in includeParticle] if self.params["writefile"] is True: apStack.makeNewStack(oldstack, newstack, vpartlist, bad=self.params["savebad"]) apStack.averageStack(stack=oldstack, outfile=outavg, partlist=vpartlist) else: if self.params["writefile"] is True: apStack.makeNewStack(oldstack, newstack, self.params["keepfile"], bad=self.params["savebad"]) apStack.averageStack(stack=oldstack, outfile=outavg, partlist=includeParticle) if self.params["writefile"] is True and not os.path.isfile(newstack): apDisplay.printError("No stack was created") if self.params["commit"] is True: apStack.commitSubStack(self.params, included=includeParticle) newstackid = apStack.getStackIdFromPath(newstack) apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=4)
def start(self): ### new stack path stackdata = apStack.getOnlyStackData(self.params['stackid']) oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) newstack = os.path.join(self.params['rundir'], stackdata['name']) apStack.checkForPreviousStack(newstack) includelist = [] excludelist = [] ### list of classes to be excluded if self.params['dropclasslist'] is not None: excludestrlist = self.params['dropclasslist'].split(",") for excludeitem in excludestrlist: excludelist.append(int(excludeitem.strip())) apDisplay.printMsg("Exclude list: "+str(excludelist)) ### list of classes to be included if self.params['keepclasslist'] is not None: includestrlist = self.params['keepclasslist'].split(",") for includeitem in includestrlist: includelist.append(int(includeitem.strip())) ### or read from keepfile elif self.params['keepfile'] is not None: keeplistfile = open(self.params['keepfile']) for line in keeplistfile: if self.params['excludefrom'] is True: excludelist.append(int(line.strip())) else: includelist.append(int(line.strip())) keeplistfile.close() apDisplay.printMsg("Include list: "+str(includelist)) ### get particles from align or cluster stack apDisplay.printMsg("Querying database for particles") q0 = time.time() if self.params['alignid'] is not None: # DIRECT SQL STUFF sqlcmd = "SELECT " + \ "apd.partnum, " + \ "apd.xshift, apd.yshift, " + \ "apd.rotation, apd.mirror, " + \ "apd.spread, apd.correlation, " + \ "apd.score, apd.bad, " + \ "spd.particleNumber, " + \ "ard.refnum "+ \ "FROM ApAlignParticleData apd " + \ "LEFT JOIN ApStackParticleData spd ON " + \ "(apd.`REF|ApStackParticleData|stackpart` = spd.DEF_id) " + \ "LEFT JOIN ApAlignReferenceData ard ON" + \ "(apd.`REF|ApAlignReferenceData|ref` = ard.DEF_id) " + \ "WHERE `REF|ApAlignStackData|alignstack` = %i"%(self.params['alignid']) # These are AlignParticles particles = sinedon.directq.complexMysqlQuery('appiondata',sqlcmd) elif self.params['clusterid'] is not None: clusterpartq = appiondata.ApClusteringParticleData() clusterpartq['clusterstack'] = self.clusterstackdata # These are ClusteringParticles particles = clusterpartq.query() apDisplay.printMsg("Completed in %s\n"%(apDisplay.timeString(time.time()-q0))) ### write included particles to text file includeParticle = [] excludeParticle = 0 badscore = 0 badshift = 0 badspread = 0 f = open("test.log", "w") count = 0 t0 = time.time() apDisplay.printMsg("Parsing particle information") # find out if there is alignparticle info: is_cluster_p = False # alignparticle is a key of any particle in particles if the latter is # a CluateringParticle if 'alignparticle' in particles[0]: is_cluster_p = True for part in particles: count += 1 if is_cluster_p: # alignpart is an item of ClusteringParticle alignpart = part['alignparticle'] try: classnum = int(part['refnum'])-1 except: apDisplay.printWarning("particle %d was not put into any class" % (part['partnum'])) emanstackpartnum = alignpart['stackpart']['particleNumber']-1 else: # particle has info from AlignedParticle as results of direct query alignpart = part try: classnum = int(alignpart['refnum'])-1 except: apDisplay.printWarning("particle %d was not put into any class" % (part['partnum'])) classnum = None emanstackpartnum = int(alignpart['particleNumber'])-1 ### check shift if self.params['maxshift'] is not None: shift = math.hypot(alignpart['xshift'], alignpart['yshift']) if shift > self.params['maxshift']: excludeParticle += 1 if classnum is not None: f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum)) else: f.write("%d\t%d\texclude\n"%(count, emanstackpartnum)) badshift += 1 continue if self.params['minscore'] is not None: ### check score if ( alignpart['score'] is not None and alignpart['score'] < self.params['minscore'] ): excludeParticle += 1 if classnum is not None: f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum)) else: f.write("%d\t%d\texclude\n"%(count, emanstackpartnum)) badscore += 1 continue ### check spread if ( alignpart['spread'] is not None and alignpart['spread'] < self.params['minscore'] ): excludeParticle += 1 if classnum is not None: f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum)) else: f.write("%d\t%d\texclude\n"%(count, emanstackpartnum)) badspread += 1 continue if classnum is not None: if includelist and (classnum in includelist): includeParticle.append(emanstackpartnum) f.write("%d\t%d\t%d\tinclude\n"%(count, emanstackpartnum, classnum)) elif excludelist and not (classnum in excludelist): includeParticle.append(emanstackpartnum) f.write("%d\t%d\t%d\tinclude\n"%(count, emanstackpartnum, classnum)) else: excludeParticle += 1 f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum)) else: excludeParticle += 1 f.write("%d\t%d\texclude\n"%(count, emanstackpartnum)) f.close() includeParticle.sort() if badshift > 0: apDisplay.printMsg("%d paricles had a large shift"%(badshift)) if badscore > 0: apDisplay.printMsg("%d paricles had a low score"%(badscore)) if badspread > 0: apDisplay.printMsg("%d paricles had a low spread"%(badspread)) apDisplay.printMsg("Completed in %s\n"%(apDisplay.timeString(time.time()-t0))) apDisplay.printMsg("Keeping "+str(len(includeParticle))+" and excluding "+str(excludeParticle)+" particles") ### write kept particles to file self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile-"+self.timestamp+".list") apDisplay.printMsg("writing to keepfile "+self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum)+"\n") kf.close() ### get number of particles numparticles = len(includeParticle) if excludelist: self.params['description'] += ( " ... %d particle substack with %s classes excluded" % (numparticles, self.params['dropclasslist'])) elif includelist: self.params['description'] += ( " ... %d particle substack with %s classes included" % (numparticles, self.params['keepclasslist'])) outavg = os.path.join(self.params['rundir'],"average.mrc") ### create the new sub stack # first check if virtual stack if not os.path.isfile(oldstack): vstackdata=apStack.getVirtualStackParticlesFromId(self.params['stackid']) vparts = vstackdata['particles'] oldstack = vstackdata['filename'] # get subset of virtualstack vpartlist = [int(vparts[p]['particleNumber'])-1 for p in includeParticle] if self.params['writefile'] is True: apStack.makeNewStack(oldstack, newstack, vpartlist, bad=self.params['savebad']) apStack.averageStack(stack=oldstack,outfile=outavg,partlist=vpartlist) else: if self.params['writefile'] is True: apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=self.params['savebad']) apStack.averageStack(stack=oldstack,outfile=outavg,partlist=includeParticle) if self.params['writefile'] is True and not os.path.isfile(newstack): apDisplay.printError("No stack was created") if self.params['commit'] is True: apStack.commitSubStack(self.params,included=includeParticle) newstackid = apStack.getStackIdFromPath(newstack) apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=4)
def start(self): ### new stack path stackdata = apStack.getOnlyStackData(self.params['stackid']) oldstack = os.path.join(stackdata['path']['path'], stackdata['name']) newstack = os.path.join(self.params['rundir'], stackdata['name']) apStack.checkForPreviousStack(newstack) includelist = [] excludelist = [] ### list of classes to be excluded if self.params['dropclasslist'] is not None: excludestrlist = self.params['dropclasslist'].split(",") for excludeitem in excludestrlist: excludelist.append(int(excludeitem.strip())) apDisplay.printMsg("Exclude list: "+str(excludelist)) ### list of classes to be included if self.params['keepclasslist'] is not None: includestrlist = self.params['keepclasslist'].split(",") for includeitem in includestrlist: includelist.append(int(includeitem.strip())) ### or read from keepfile elif self.params['keepfile'] is not None: keeplistfile = open(self.params['keepfile']) for line in keeplistfile: if self.params['excludefrom'] is True: excludelist.append(int(line.strip())) else: includelist.append(int(line.strip())) keeplistfile.close() apDisplay.printMsg("Include list: "+str(includelist)) ### get particles from align or cluster stack apDisplay.printMsg("Querying database for particles") q0 = time.time() if self.params['alignid'] is not None: alignpartq = appiondata.ApAlignParticleData() alignpartq['alignstack'] = self.alignstackdata particles = alignpartq.query() elif self.params['clusterid'] is not None: clusterpartq = appiondata.ApClusteringParticleData() clusterpartq['clusterstack'] = self.clusterstackdata particles = clusterpartq.query() apDisplay.printMsg("Complete in "+apDisplay.timeString(time.time()-q0)) ### write included particles to text file includeParticle = [] excludeParticle = 0 badscore = 0 badshift = 0 badspread = 0 f = open("test.log", "w") count = 0 for part in particles: count += 1 #partnum = part['partnum']-1 if 'alignparticle' in part: alignpart = part['alignparticle'] classnum = int(part['refnum'])-1 else: alignpart = part classnum = int(part['ref']['refnum'])-1 emanstackpartnum = alignpart['stackpart']['particleNumber']-1 ### check shift if self.params['maxshift'] is not None: shift = math.hypot(alignpart['xshift'], alignpart['yshift']) if shift > self.params['maxshift']: excludeParticle += 1 f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum)) badshift += 1 continue if self.params['minscore'] is not None: ### check score if ( alignpart['score'] is not None and alignpart['score'] < self.params['minscore'] ): excludeParticle += 1 f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum)) badscore += 1 continue ### check spread if ( alignpart['spread'] is not None and alignpart['spread'] < self.params['minscore'] ): excludeParticle += 1 f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum)) badspread += 1 continue if includelist and classnum in includelist: includeParticle.append(emanstackpartnum) f.write("%d\t%d\t%d\tinclude\n"%(count, emanstackpartnum, classnum)) elif excludelist and not classnum in excludelist: includeParticle.append(emanstackpartnum) f.write("%d\t%d\t%d\tinclude\n"%(count, emanstackpartnum, classnum)) else: excludeParticle += 1 f.write("%d\t%d\t%d\texclude\n"%(count, emanstackpartnum, classnum)) f.close() includeParticle.sort() if badshift > 0: apDisplay.printMsg("%d paricles had a large shift"%(badshift)) if badscore > 0: apDisplay.printMsg("%d paricles had a low score"%(badscore)) if badspread > 0: apDisplay.printMsg("%d paricles had a low spread"%(badspread)) apDisplay.printMsg("Keeping "+str(len(includeParticle))+" and excluding "+str(excludeParticle)+" particles") #print includeParticle ### write kept particles to file self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile-"+self.timestamp+".list") apDisplay.printMsg("writing to keepfile "+self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum)+"\n") kf.close() ### get number of particles numparticles = len(includeParticle) if excludelist: self.params['description'] += ( " ... %d particle substack with %s classes excluded" % (numparticles, self.params['dropclasslist'])) elif includelist: self.params['description'] += ( " ... %d particle substack with %s classes included" % (numparticles, self.params['keepclasslist'])) ### create the new sub stack apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=self.params['savebad']) if not os.path.isfile(newstack): apDisplay.printError("No stack was created") apStack.averageStack(stack=newstack) if self.params['commit'] is True: apStack.commitSubStack(self.params) newstackid = apStack.getStackIdFromPath(newstack) apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=4)
stackpath = stackdata['path']['path'] # generate stack if it doesn't exist. if not os.path.isdir(stackpath): os.makedirs(stackpath) fname = os.path.join(stackpath, stackdata['name']) # check if stack file already exists if os.path.isfile(fname): apDisplay.printError("file: '%s' already exists"%fname) vstackdata = apStack.getVirtualStackParticlesFromId(params['stackid']) plist = [int(p['particleNumber'])-1 for p in vstackdata['particles']] a = proc2dLib.RunProc2d() a.setValue('infile',vstackdata['filename']) a.setValue('outfile',fname) a.setValue('list',plist) a.setValue('apix',apStack.getStackPixelSizeFromStackId(params['stackid'])) apDisplay.printMsg("generating stack: '%s' with %i particles"%(fname,len(plist))) a.run() outavg = os.path.join(stackpath, "average.mrc") if not os.path.isfile(outavg): apStack.averageStack(stack=fname,outfile=outavg) montageimg=os.path.join(stackpath,"montage%i.png"%params['stackid']) if not os.path.isfile(montageimg): apStackMeanPlot.makeStackMeanPlot(params['stackid'],gridpoints=4)