def verifyNumberOfCompletedRefinements(self, multiModelRefinementRun=False): """ queries te database to determine how many iterations of individual refinements have been completed: returns a list of completed iterations in a dictionary of completed refinements """ refine_complete = {} if multiModelRefinementRun is False: iterdata = appiondata.ApRefineIterData() iterdata['refineRun'] = self.refinerunq itercompletedata = iterdata.query() iter_complete = [] for iter in itercompletedata: iter_complete.append(iter['iteration']) iter_complete.sort() refine_complete[ self.refinerunq['reference_number']] = iter_complete else: refinedata = appiondata.ApRefineRunData() refinedata['multiModelRefineRun'] = self.multimodelq refinecompletedata = refinedata.query() for i, refine in enumerate(refinecompletedata): iterdata = appiondata.ApRefineIterData() iterdata['refineRun'] = refine itercompletedata = iterdata.query() iter_complete = [] for iter in itercompletedata: iter_complete.append(iter['iteration']) iter_complete.sort() refine_complete[refine['reference_number']] = iter_complete return refine_complete
def getEulersForParticleSinedon(self, stackpartid, reconrunid): """ returns all classdata for a particular particle and refinement """ refrundata = appiondata.ApRefineRunData.direct_query(reconrunid) stackpartdata = appiondata.ApStackParticleData.direct_query( stackpartid) refmentq = appiondata.ApRefineIterData() refmentq['refineRun'] = refrundata particledata = stackpartdata partclassq = appiondata.ApRefineParticleData() partclassq['particle'] = particledata partclassq['refineIter'] = refmentq partclassdata = partclassq.query() eulertree = [] for data in partclassdata: try: euler = {} euler['stackpartid'] = int(data['particle'].dbid) euler['euler1'] = float(data['euler1']) euler['euler2'] = float(data['euler2']) euler['euler3'] = float(data['euler3']) euler['mirror'] = self.nullOrValue(data['mirror']) euler['reject'] = self.nullOrValue(data['refine_keep']) euler['iteration'] = int(data['refineIter']['iteration']) eulertree.append(euler) except: print euler import pprint pprint.pprint(data) apDisplay.printError("bad data entry") return eulertree
def setIterData(self, iterparams, volumeDensity): frealigniterq = appiondata.ApFrealignIterData() frealignkeys = ('wgh', 'xstd', 'pbc', 'itmax', 'ipmax', 'target', 'thresh', 'cs', 'rrec', 'highpass', 'lowpass', 'rbfact') for key in frealignkeys: frealigniterq[key] = iterparams[key] iternum = iterparams['iternum'] iterq = appiondata.ApRefineIterData() iterq['iteration'] = iternum iterq['exemplar'] = False ### mask values are in pixels iterq['mask'] = iterparams['mask']/self.apix iterq['imask'] = iterparams['imask']/self.apix iterq['volumeDensity'] = os.path.basename(volumeDensity) ### Frealign does not produce class averages :( iterq['refineClassAverages'] = None iterq['postRefineClassAverages'] = None iterq['classVariance'] = None iterq['symmetry'] = iterparams['symmdata'] iterq['refineRun'] = self.runq iterq['resolution'] = self.getResolutionData(iternum) iterq['rMeasure'] = self.getRMeasureData(volumeDensity) iterq['frealignParams'] = frealigniterq return iterq
def getParticleInfo(reconid, iteration): """ Get all particle data for given recon and iteration """ refinerundata = appiondata.ApRefineRunData.direct_query(reconid) if not refinerundata: apDisplay.printError("Could not find refinerundata for reconrun id=" + str(reconid)) refineq = appiondata.ApRefineIterData() refineq['refineRun'] = refinerundata refineq['iteration'] = iteration refinedata = refineq.query(results=1) if not refinedata: apDisplay.printError("Could not find refinedata for reconrun id=" + str(reconid) + " iter=" + str(iteration)) refinepartq = appiondata.ApRefineParticleData() refinepartq['refineIter'] = refinedata[0] t0 = time.time() apDisplay.printMsg("querying particles on " + time.asctime()) refineparticledata = refinepartq.query() apDisplay.printMsg("received " + str(len(refineparticledata)) + " particles in " + apDisplay.timeString(time.time() - t0)) return (refineparticledata, refinedata[0])
def getSymmetryFromReconRunId(reconrunid, msg=True): """ get the symmetry from the last iteration of a refinement """ refrundata = appiondata.ApRefineRunData.direct_query(reconrunid) refdataq = appiondata.ApRefineIterData() refdataq['refineRun'] = refrundata refdata = refdataq.query() uniqsym = refdata[0]['symmetry'] if uniqsym is None: apDisplay.printWarning("symmetry is not saved during reconstruction!") apDisplay.printWarning("Using the symmetry of the initial model") modeldata = refrundata['initialModel'] uniqsym = modeldata['symmetry'] else: for data in refdata: if uniqsym != data['symmetry']: apDisplay.printWarning( "symmetry is not consistent throughout reconstruction!") apDisplay.printWarning("Using symmetry of last iteration") uniqsym = data['symmetry'] if msg is True: apDisplay.printMsg( "selected symmetry group: " + apDisplay.colorString("'" + uniqsym['eman_name'] + "'", "cyan") + " for recon run: " + str(reconrunid)) return uniqsym
def checkConflicts(self): if self.params['reconid'] is None: apDisplay.printError( "Please provide a recon run id, e.g. --reconid=15") if self.params['iternum'] is None: refineiterq = appiondata.ApRefineIterData() refrun = appiondata.ApRefineRunData.direct_query( self.params['reconid']) refineiterq['refineRun'] = refrun refiterdata = refineiterq.query() lastiter = len(refiterdata) refineiterq = appiondata.ApRefineIterData() refineiterq['refineRun'] = refrun refineiterq['iteration'] = lastiter refiterdata = refineiterq.query(results=1) if not refiterdata: apDisplay.printError( "Could not get iteration, please provide a iter num, e.g. --iternum=15" ) self.params['iternum'] = lastiter
def setGoodBadParticlesFromReconId(reconid): """ Goes through existing recons and caches the number of good and bad particles """ import sinedon import MySQLdb dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) db.autocommit(True) cursor = db.cursor() refinerundata = appiondata.ApRefineRunData.direct_query(reconid) refineq = appiondata.ApRefineIterData() refineq['refineRun'] = refinerundata refineiterdatas = refineq.query() r0 = time.time() for refineiterdata in refineiterdatas: t0 = time.time() #print "Iteration %d"%(refineiterdata['iteration']) goodbadq = appiondata.ApRefineGoodBadParticleData() goodbadq['refine'] = refineiterdata goodbaddata = goodbadq.query() if goodbaddata: continue fields = { 'good_refine': getParticleCount(refineiterdata.dbid, cursor, 'refine_keep', True), 'bad_refine': getParticleCount(refineiterdata.dbid, cursor, 'refine_keep', False), 'good_postRefine': getParticleCount(refineiterdata.dbid, cursor, 'postRefine_keep', True), 'bad_postRefine': getParticleCount(refineiterdata.dbid, cursor, 'postRefine_keep', False), } #print fields goodbadq = appiondata.ApRefineGoodBadParticleData() goodbadq['refine'] = refineiterdata goodbadq['good_refine'] = fields['good_refine'] goodbadq['bad_refine'] = fields['bad_refine'] goodbadq['good_postRefine'] = fields['good_postRefine'] goodbadq['bad_postRefine'] = fields['bad_postRefine'] goodbadq.insert() if time.time() - t0 > 2.0: apDisplay.printMsg("Iter completed in %s" % (apDisplay.timeString(time.time() - t0))) if time.time() - r0 > 2.0: apDisplay.printMsg("Refine completed in %s" % (apDisplay.timeString(time.time() - r0))) return
def getNumIterationsFromRefineRunID(refinerunid): refrundata = appiondata.ApRefineRunData.direct_query(refinerunid) refq = appiondata.ApRefineIterData() refq['refineRun'] = refrundata refdatas = refq.query() if not refdatas: return 0 maxiter = 0 for refdata in refdatas: iternum = refdata['iteration'] if iternum > maxiter: maxiter = iternum return maxiter
def start(self): apDisplay.printMsg("\n\n") ### get particles refrun = appiondata.ApRefineRunData.direct_query( self.params['reconid']) refiterq = appiondata.ApRefineIterData() refiterq['refineRun'] = refrun refiterq['iteration'] = self.params['iternum'] refiterdatas = refiterq.query(results=1) refpartq = appiondata.ApRefineParticleData() refpartq['refineIter'] = refiterdatas[0] #this gets lots of data refpartdatas = refpartq.query() ### get session firstrefpart = refpartdatas[0] firstpart = firstrefpart['particle']['particle'] sessiondata = firstpart['selectionrun']['session'] ### create a selection run runq = appiondata.ApSelectionRunData() for key in firstpart['selectionrun'].keys(): runq[key] = firstpart['selectionrun'][key] runq['name'] = self.params['runname'] runq['session'] = sessiondata pathq = appiondata.ApPathData() pathq['path'] = self.params['rundir'] runq['path'] = pathq runq['description'] = ( "Corrected particles from refine id %d iter %d and selection %d" % (self.params['reconid'], self.params['iternum'], firstpart['selectionrun'].dbid)) count = 0 for refpartdata in refpartdatas: count += 1 if count % 10 == 0: sys.stderr.write(".") partdata = refpartdata['particle']['particle'] newpartq = appiondata.ApParticleData() for key in partdata.keys(): newpartq[key] = partdata[key] newpartq['xcoord'] = partdata['xcoord'] + refpartdata['shiftx'] newpartq['ycoord'] = partdata['ycoord'] + refpartdata['shifty'] newpartq['selectionrun'] = runq newpartq.insert()
def getParticleInfo(self, reconid, iteration): """ Get all particle data for given recon and iteration """ t0 = time.time() cachefile = os.path.join( self.params['rundir'], "refineparticledata-r" + str(reconid) + "-i" + str(iteration) + ".cache") if os.path.isfile(cachefile): apDisplay.printColor("loading refineparticledata from cache file", "cyan") f = open(cachefile, 'r') refineparticledata = cPickle.load(f) f.close() else: refinerundata = appiondata.ApRefineRunData.direct_query(reconid) if not refinerundata: apDisplay.printError( "Could not find refinerundata for reconrun id=" + str(reconid)) refineq = appiondata.ApRefineIterData() refineq['refineRun'] = refinerundata refineq['iteration'] = iteration refinedata = refineq.query(results=1) if not refinedata: apDisplay.printError( "Could not find refinedata for reconrun id=" + str(reconid) + " iter=" + str(iteration)) refinepartq = appiondata.ApRefineParticleData() refinepartq['refineIter'] = refinedata[0] apDisplay.printMsg("querying particles on " + time.asctime()) refineparticledata = refinepartq.query() apDisplay.printMsg("saving refineparticledata to cache file") f = open(cachefile, 'w') cPickle.dump(refineparticledata, f) f.close() apDisplay.printMsg("received " + str(len(refineparticledata)) + " particles in " + apDisplay.timeString(time.time() - t0)) return refineparticledata
fscfile, self.runparams['boxsize'], self.runparams['apix']) apDisplay.printColor("FSC 0.5 Resolution: " + str(fscRes), "cyan") resq = appiondata.ApResolutionData() resq['half'] = fscRes resq['fscfile'] = os.path.basename(fscfile) except Exception, e: print e apDisplay.printWarning( "FSC file does not exist or is unreadable") resq = None else: resq = None ### fill in ApRefineIterData object iterationParamsq = appiondata.ApRefineIterData() if package_table is not None and package_database_object is not None: iterationParamsq[str( package_table.split("|")[1])] = package_database_object iterationParamsq['refineRun'] = self.refinerunq iterationParamsq['iteration'] = iteration iterationParamsq['resolution'] = resq if not self.params['euleronly']: iterationParamsq['rMeasure'] = self.getRMeasureData( iteration, reference_number) else: iterationParamsq['rMeasure'] = None iterationParamsq['mask'] = apRecon.getComponentFromVector( self.runparams['mask'], iteration - 1) iterationParamsq['imask'] = apRecon.getComponentFromVector( self.runparams['imask'], iteration - 1)
def getRefinementsFromRun(refinerundata): refineitq = appiondata.ApRefineIterData() refineitq['refineRun'] = refinerundata return refineitq.query()
def insertIteration(self, iteration): refineparamsq = appiondata.ApEmanRefineIterData() refineparamsq['ang'] = iteration['ang'] refineparamsq['lpfilter'] = iteration['lpfilter'] refineparamsq['hpfilter'] = iteration['hpfilter'] refineparamsq['pad'] = iteration['pad'] refineparamsq['EMAN_maxshift'] = iteration['maxshift'] refineparamsq['EMAN_hard'] = iteration['hard'] refineparamsq['EMAN_classkeep'] = iteration['classkeep'] refineparamsq['EMAN_classiter'] = iteration['classiter'] refineparamsq['EMAN_filt3d'] = iteration['filt3d'] refineparamsq['EMAN_shrink'] = iteration['shrink'] refineparamsq['EMAN_euler2'] = iteration['euler2'] refineparamsq['EMAN_xfiles'] = iteration['xfiles'] refineparamsq['EMAN_median'] = iteration['median'] refineparamsq['EMAN_phasecls'] = iteration['phasecls'] refineparamsq['EMAN_fscls'] = iteration['fscls'] refineparamsq['EMAN_refine'] = iteration['refine'] refineparamsq['EMAN_goodbad'] = iteration['goodbad'] refineparamsq['EMAN_perturb'] = iteration['perturb'] refineparamsq['MsgP_cckeep'] = iteration['msgpasskeep'] refineparamsq['MsgP_minptls'] = iteration['msgpassminp'] #create Chimera snapshots fscfile = os.path.join(self.params['rundir'], "fsc.eotest." + iteration['num']) halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix']) if self.params['snapfilter']: halfres = self.params['snapfilter'] volumeDensity = 'threed.' + iteration['num'] + 'a.mrc' volDensPath = os.path.join(self.params['rundir'], volumeDensity) apChimera.filterAndChimera(volDensPath, halfres, self.params['apix'], self.params['boxsize'], 'snapshot', self.params['contour'], self.params['zoom'], sym=iteration['sym']['eman_name'], mass=self.params['mass']) ## uncommment this for chimera image only runs... if self.params['chimeraonly'] is True: return # insert resolution data if halfres != True: resData = self.getResolutionData(iteration) else: apDisplay.printWarning( "resolution reported as nan, not committing results to database" ) return if self.params['package'] == 'EMAN': refineclassavg = 'classes_eman.' + iteration['num'] + '.img' postrefineclassavg = None elif self.params['package'] == 'EMAN/SpiCoran': refineclassavg = 'classes_eman.' + iteration['num'] + '.img' postrefineclassavg = 'classes_coran.' + iteration['num'] + '.img' elif self.params['package'] == 'EMAN/MsgP': refineclassavg = 'classes_eman.' + iteration['num'] + '.img' postrefineclassavg = 'classes_msgp.' + iteration['num'] + '.img' else: apDisplay.printError("Refinement Package Not Valid") # insert refinement results refineq = appiondata.ApRefineIterData() refineq['refineRun'] = self.params['refineRun'] refineq['emanParams'] = refineparamsq refineq['iteration'] = iteration['num'] refineq['resolution'] = resData refineq['rMeasure'] = self.getRMeasureData(iteration) refineq['mask'] = iteration['mask'] refineq['imask'] = iteration['imask'] refineq['symmetry'] = iteration['sym'] refineq['exemplar'] = False classvar = 'classes.' + iteration['num'] + '.var.img' refineq['refineClassAverages'] = refineclassavg refineq['postRefineClassAverages'] = postrefineclassavg if classvar in self.params['classvars']: refineq['classVariance'] = classvar if volumeDensity in self.params['volumes']: refineq['volumeDensity'] = volumeDensity apDisplay.printMsg("inserting Refinement Data into database") if self.params['commit'] is True: refineq.insert() else: apDisplay.printWarning("not committing results to database") #insert FSC data fscfile = os.path.join(self.params['rundir'], "fsc.eotest." + iteration['num']) self.insertFSC(fscfile, refineq, self.params['commit']) halfres = apRecon.calcRes(fscfile, self.params['boxsize'], self.params['apix']) apDisplay.printColor("FSC 0.5 Resolution: " + str(halfres), "cyan") # get projections eulers for iteration: eulers = self.getEulersFromProj(iteration['num']) # get list of bad particles for this iteration badprtls = self.readParticleLog(self.params['rundir'], iteration['num']) # expand cls.*.tar into temp file clsf = os.path.join(self.params['rundir'], "cls." + iteration['num'] + ".tar") #print "reading",clsf clstar = tarfile.open(clsf) clslist = clstar.getmembers() clsnames = clstar.getnames() #print "extracting",clsf,"into temp directory" for clsfile in clslist: clstar.extract(clsfile, self.params['tmpdir']) clstar.close() # for each class, insert particle alignment info into database apDisplay.printColor( "Inserting Particle Classification Data for " + str(len(clsnames)) + " classes", "magenta") t0 = time.time() for cls in clsnames: self.insertRefineParticleData(cls, iteration, eulers, badprtls, refineq, len(clsnames)) apDisplay.printColor( "\nFinished in " + apDisplay.timeString(time.time() - t0), "magenta") # remove temp directory for file in os.listdir(self.params['tmpdir']): os.remove(os.path.join(self.params['tmpdir'], file)) os.rmdir(self.params['tmpdir']) #create euler freq map apDisplay.printMsg("creating euler frequency map") refrunid = int(self.params['refineRun'].dbid) iternum = int(iteration['num']) if self.params['package'] != 'EMAN': postrefine = True else: postrefine = False apEulerDraw.createEulerImages(refrunid, iternum, path=self.params['rundir'], postrefine=postrefine) return
def start(self): partdict = {} partlist = [] ### get Euler angles for each particle for iternum in self.iternums: ### get recon iter data reconiterq = appiondata.ApRefineIterData() reconiterq['refineRun'] = self.reconrundata reconiterq['iteration'] = iternum reconiterdata = reconiterq.query( results=1)[0] #this should be unique ### get particle data reconpartq = appiondata.ApRefineParticleData() reconpartq['refineIter'] = reconiterdata apDisplay.printMsg("Querying for particles at " + time.asctime()) reconpartdatas = reconpartq.query() ### group particle data for partdata in reconpartdatas: partnum = partdata['particle']['particleNumber'] if not partnum in partlist: partlist.append(partnum) partdict[(partnum, iternum)] = partdata ### run through particles and check Euler angles partlist.sort() eulerdict = {} eulercount = {} reject = 0 for partnum in partlist: e1d = {} e2d = {} for iternum in self.iternums: if not (partnum, iternum) in partdict: continue partdata = partdict[(partnum, iternum)] euler1 = "%.2f" % (partdata['euler1']) if not euler1 in e1d: e1d[euler1] = 1 else: e1d[euler1] += 1 euler2 = "%.2f" % (partdata['euler2']) if not euler2 in e2d: e2d[euler2] = 1 else: e2d[euler2] += 1 #print partnum, euler1, euler2 counts = [(val, key) for key, val in e1d.items()] e1count, euler1 = max(counts) counts = [(val, key) for key, val in e2d.items()] e2count, euler2 = max(counts) # reject indeterminant particles if e2count < 2 or e1count < 2: reject += 1 continue ### group particles by their Euler angles if not (euler1, euler2) in eulerdict: eulerdict[(euler1, euler2)] = [] eulercount[(euler1, euler2)] = 0 eulerdict[(euler1, euler2)].append(partnum) eulercount[(euler1, euler2)] += 1 print "Rejected %d particles" % (reject) values = eulercount.values() values.sort() print values ### run through Euler angles and count particles counts = [(val, key) for key, val in eulercount.items()] mincount, val = min(counts) self.params['mincount'] = max(self.params['mincount'], mincount) #print "Keeping %d of %d particles"%(mincount*len(eulercount.keys()), len(partlist)) print "Keeping %d of %d particles" % ( self.params['mincount'] * len(eulercount.keys()), len(partlist)) keeplist = [] for key in eulerdict.keys(): eulerpartlist = eulerdict[key] if len(partlist) < self.params['mincount']: keeplist.extend(eulerpartlist) else: keeplist.extend(eulerpartlist[:self.params['mincount']]) keeplist.sort() print "Keeping %d of %d particles" % (len(keeplist), len(partlist)) #need to set keepfile for commitSubStack self.params['keepfile'] = os.path.join(self.params['rundir'], "equalviews.lst") f = open(self.params['keepfile'], "w") for partnum in keeplist: f.write("%d\n" % (partnum - 1)) f.close() ### make a new stack using the keep particles oldstackdata = self.reconrundata['stack'] oldstack = os.path.join(oldstackdata['path']['path'], oldstackdata['name']) newstack = os.path.join(self.params['rundir'], "start.hed") apStack.makeNewStack(oldstack, newstack, listfile=self.params['keepfile'], remove=True, bad=True) if not os.path.isfile(newstack): apDisplay.printError("No stack was created") self.params[ 'stackid'] = oldstackdata.dbid #need to set stackid for commitSubStack apStack.commitSubStack(self.params, "start.hed") apStack.averageStack(stack=newstack)