def insertRefineParticleData(self, iterdata, parttree): apDisplay.printMsg("Inserting particle data") phase_threshold = float(iterdata['frealignParams']['thresh']) count = 0 for partdict in parttree: count += 1 if count % 1000 == 0: apDisplay.printMsg("Inserted %d particles"%(count)) partrefineq = appiondata.ApRefineParticleData() partrefineq['refineIter'] = iterdata partnum = partdict['partnum'] stackpartid = self.stackmapping[partnum] stackpartdata = appiondata.ApStackParticleData.direct_query(stackpartid) partrefineq['particle'] = stackpartdata partkeys = ('shiftx', 'shifty', 'euler1', 'euler2', 'euler3', 'phase_residual') for key in partkeys: partrefineq[key] = partdict[key] ### check if particle was rejected if partdict['phase_residual'] > iterdata['frealignParams']['thresh']: partrefineq['refine_keep'] = False else: partrefineq['refine_keep'] = True partrefineq['mirror'] = False partrefineq['postRefine_keep'] = False partrefineq['euler_convention'] = 'zyz' if self.params['commit'] is True: partrefineq.insert() return
def getEulersForParticleSinedon(self, stackpartid, reconrunid): """ returns all classdata for a particular particle and refinement """ refrundata = appiondata.ApRefineRunData.direct_query(reconrunid) stackpartdata = appiondata.ApStackParticleData.direct_query( stackpartid) refmentq = appiondata.ApRefineIterData() refmentq['refineRun'] = refrundata particledata = stackpartdata partclassq = appiondata.ApRefineParticleData() partclassq['particle'] = particledata partclassq['refineIter'] = refmentq partclassdata = partclassq.query() eulertree = [] for data in partclassdata: try: euler = {} euler['stackpartid'] = int(data['particle'].dbid) euler['euler1'] = float(data['euler1']) euler['euler2'] = float(data['euler2']) euler['euler3'] = float(data['euler3']) euler['mirror'] = self.nullOrValue(data['mirror']) euler['reject'] = self.nullOrValue(data['refine_keep']) euler['iteration'] = int(data['refineIter']['iteration']) eulertree.append(euler) except: print euler import pprint pprint.pprint(data) apDisplay.printError("bad data entry") return eulertree
def getParticleInfo(reconid, iteration): """ Get all particle data for given recon and iteration """ refinerundata = appiondata.ApRefineRunData.direct_query(reconid) if not refinerundata: apDisplay.printError("Could not find refinerundata for reconrun id=" + str(reconid)) refineq = appiondata.ApRefineIterData() refineq['refineRun'] = refinerundata refineq['iteration'] = iteration refinedata = refineq.query(results=1) if not refinedata: apDisplay.printError("Could not find refinedata for reconrun id=" + str(reconid) + " iter=" + str(iteration)) refinepartq = appiondata.ApRefineParticleData() refinepartq['refineIter'] = refinedata[0] t0 = time.time() apDisplay.printMsg("querying particles on " + time.asctime()) refineparticledata = refinepartq.query() apDisplay.printMsg("received " + str(len(refineparticledata)) + " particles in " + apDisplay.timeString(time.time() - t0)) return (refineparticledata, refinedata[0])
def getStackParticleEulersForIteration(self, pnum): """ find the eulers assigned to a stack particle during a refinement. This function will first find the particle id for the given stack particle, then find its position in the reference stack, and will get the eulers for that particle in the recon """ # get stack particle id stackp = apStack.getStackParticle(self.params['stackid'], pnum) particleid = stackp['particle'].dbid # find particle in reference stack refstackp = apStack.getStackParticleFromParticleId(particleid, self.oldreconstackid, noDie=True) if not refstackp: percentnoeuler = 100*self.noeulers/float(self.params['last']) apDisplay.printWarning('No eulers for particle %d (%.1f%%)' % (pnum, percentnoeuler)) self.noeulers += 1 if percentnoeuler > 10: apDisplay.printError('More than 10% of the particles have no euler, use a different reference reconstruction') pclass={ 'euler1': 0.0, 'euler2': 0.0, 'euler3': 0.0, 'mirror': False, 'shiftx': 0.0, 'shifty': 0.0, } else: pclassq = appiondata.ApRefineParticleData() pclassq['particle'] = refstackp pclassq['refineIter'] = appiondata.ApRefineIterData.direct_query(self.params['reconiterid']) pclasses = pclassq.query(results=1) pclass = pclasses[0] try: emaneuler={ 'alt': pclass['euler1'], 'az': pclass['euler2'], 'phi': pclass['euler3'], 'mirror': pclass['mirror'], 'shiftx': pclass['shiftx'], 'shifty': pclass['shifty'], } except: print pclass emaneuler={ 'alt': pclass['euler1'], 'az': pclass['euler2'], 'phi': pclass['euler3'], 'mirror': pclass['mirror'], 'shiftx': pclass['shiftx'], 'shifty': pclass['shifty'], } return emaneuler
def getStackParticleEulersForIteration(params, pnum): """ find the eulers assigned to a stack particle during a refinement. This function will first find the particle id for the given stack particle, then find its position in the reference stack, and will get the eulers for that particle in the recon """ # get stack particle id stackp = apStack.getStackParticle(params['stackid'], pnum) particleid = stackp['particle'].dbid # find particle in reference stack refstackid = apStack.getStackIdFromIterationId(params['reconiterid'], msg=False) refstackp = apStack.getStackParticleFromParticleId(particleid, refstackid, nodie=True) if not refstackp: apDisplay.printWarning( 'No classification for stack particle %d in reconstruction iteration id: %d' % (pnum, params['reconiterid'])) params['noClassification'] += 1 if params['noClassification'] > (float(params['last']) * 0.10): apDisplay.printError( 'More than 10% of the particles have no classification, use a different reference reconstruction' ) pclass = {} pclass['eulers'] = {} pclass['eulers']['euler1'] = 0.0 pclass['eulers']['euler2'] = 0.0 pclass['inplane_rotation'] = 0.0 pclass['mirror'] = False pclass['shiftx'] = 0.0 pclass['shifty'] = 0.0 else: pclassq = appiondata.ApRefineParticleData() pclassq['particle'] = refstackp pclassq['refineIter'] = appiondata.ApRefineIterData.direct_query( params['reconiterid']) pclass = pclassq.query() pclass = pclass[0] params['eman_orient'] = {} params['eman_orient']['alt'] = pclass['euler1'] params['eman_orient']['az'] = pclass['euler2'] params['eman_orient']['phi'] = pclass['euler3'] params['eman_orient']['mirror'] = pclass['mirror'] params['eman_orient']['shiftx'] = pclass['shiftx'] params['eman_orient']['shifty'] = pclass['shifty'] return params
def start(self): apDisplay.printMsg("\n\n") ### get particles refrun = appiondata.ApRefineRunData.direct_query( self.params['reconid']) refiterq = appiondata.ApRefineIterData() refiterq['refineRun'] = refrun refiterq['iteration'] = self.params['iternum'] refiterdatas = refiterq.query(results=1) refpartq = appiondata.ApRefineParticleData() refpartq['refineIter'] = refiterdatas[0] #this gets lots of data refpartdatas = refpartq.query() ### get session firstrefpart = refpartdatas[0] firstpart = firstrefpart['particle']['particle'] sessiondata = firstpart['selectionrun']['session'] ### create a selection run runq = appiondata.ApSelectionRunData() for key in firstpart['selectionrun'].keys(): runq[key] = firstpart['selectionrun'][key] runq['name'] = self.params['runname'] runq['session'] = sessiondata pathq = appiondata.ApPathData() pathq['path'] = self.params['rundir'] runq['path'] = pathq runq['description'] = ( "Corrected particles from refine id %d iter %d and selection %d" % (self.params['reconid'], self.params['iternum'], firstpart['selectionrun'].dbid)) count = 0 for refpartdata in refpartdatas: count += 1 if count % 10 == 0: sys.stderr.write(".") partdata = refpartdata['particle']['particle'] newpartq = appiondata.ApParticleData() for key in partdata.keys(): newpartq[key] = partdata[key] newpartq['xcoord'] = partdata['xcoord'] + refpartdata['shiftx'] newpartq['ycoord'] = partdata['ycoord'] + refpartdata['shifty'] newpartq['selectionrun'] = runq newpartq.insert()
def getParticleInfo(self, reconid, iteration): """ Get all particle data for given recon and iteration """ t0 = time.time() cachefile = os.path.join( self.params['rundir'], "refineparticledata-r" + str(reconid) + "-i" + str(iteration) + ".cache") if os.path.isfile(cachefile): apDisplay.printColor("loading refineparticledata from cache file", "cyan") f = open(cachefile, 'r') refineparticledata = cPickle.load(f) f.close() else: refinerundata = appiondata.ApRefineRunData.direct_query(reconid) if not refinerundata: apDisplay.printError( "Could not find refinerundata for reconrun id=" + str(reconid)) refineq = appiondata.ApRefineIterData() refineq['refineRun'] = refinerundata refineq['iteration'] = iteration refinedata = refineq.query(results=1) if not refinedata: apDisplay.printError( "Could not find refinedata for reconrun id=" + str(reconid) + " iter=" + str(iteration)) refinepartq = appiondata.ApRefineParticleData() refinepartq['refineIter'] = refinedata[0] apDisplay.printMsg("querying particles on " + time.asctime()) refineparticledata = refinepartq.query() apDisplay.printMsg("saving refineparticledata to cache file") f = open(cachefile, 'w') cPickle.dump(refineparticledata, f) f.close() apDisplay.printMsg("received " + str(len(refineparticledata)) + " particles in " + apDisplay.timeString(time.time() - t0)) return refineparticledata
def insertRefinementParticleData(self, particledata, iterationParamsq, referenceParamsq, euler_convention='zxz'): ''' inserts all particle data into database, based on parameters from text file ''' apDisplay.printMsg( "inserting particle data into database ... this may take some time" ) for i in range(len(particledata)): prtlq = appiondata.ApRefineParticleData() ### map particle to stack # prtlnum = particledata[i]['partnum']+1 ### offset by 1 prtlnum = particledata[i]['partnum'] defid = self.stackmapping[prtlnum] stackp = appiondata.ApStackParticleData.direct_query(defid) if not stackp: apDisplay.printError("particle " + str(prtlnum) + " not in stack id=" + str(self.runparams['stackid'])) ### convert icos convention to standard convention full_sym_name = iterationParamsq['symmetry']['symmetry'] if 'Icos' in full_sym_name: # Icos particle data from particle file is always in 235 full_sym_name = 'Icos (2 3 5) Viper/3DEM' phi, theta, omega = apEulerCalc.convert3DEMEulerToStandardSym( full_sym_name, particledata[i]['phi'], particledata[i]['theta'], particledata[i]['omega']) ### convert Euler angles from 3DEM to EMAN format (temporary fix) alt, az, phi = apXmipp.convertXmippEulersToEman(phi, theta, omega) if self.multiModelRefinementRun is True: prtlq['multiModelRefineRun'] = self.multimodelq prtlq['refineIter'] = iterationParamsq prtlq['reference_number'] = referenceParamsq prtlq['particle'] = stackp prtlq['euler1'] = float(alt) prtlq['euler2'] = float(az) prtlq['euler3'] = float(phi) prtlq['shiftx'] = particledata[i]['shiftx'] prtlq['shifty'] = particledata[i]['shifty'] prtlq['mirror'] = 0 # prtlq['mirror'] = particledata[i]['mirror'] prtlq['3Dref_num'] = particledata[i]['refnum'] try: prtlq['2Dclass_num'] = particledata[i]['clsnum'] except: pass try: prtlq['quality_factor'] = particledata[i]['quality'] except: pass try: prtlq['refine_keep'] = particledata[i]['refine_keep'] except: pass try: prtlq['postRefine_keep'] = particledata[i]['postRefine_keep'] except: pass prtlq['euler_convention'] = euler_convention if self.params['commit'] is True: prtlq.insert() return
def getStackParticleEulersForIteration(self, stackPart): """ find the eulers assigned to a stack particle during a refinement. This function will first find the particle id for the given stack particle, then find its position in the reference stack, and will get the eulers for that particle in the recon assumes recon with FREALIGN """ # get stack particle id stackPartId = stackPart.dbid partId = stackPart['particle'].dbid # find particle in reference stack refStackId = apStack.getStackIdFromIterationId( self.params['reconiterid'], msg=False) refStackPart = apStack.getStackParticleFromParticleId( partId, refStackId) if not refStackPart: apDisplay.printWarning( 'No classification for stack particle %d in reconstruction iteration id: %d' % (refStackId, self.params['reconiterid'])) self.noClassification += 1 if self.noClassification > (float(params['last']) * 0.10): apDisplay.printError( 'More than 10% of the particles have no classification, use a different reference reconstruction' ) eulerDict = { 'psi': 0.0, 'theta': 0.0, 'phi': 0.0, 'shiftx': 0.0, 'shifty': 0.0, } return eulerDict refIterData = appiondata.ApRefineIterData.direct_query( self.params['reconiterid']) if refStackPart.dbid != stackPartId: self.mismatch += 1 refinePartQuery = appiondata.ApRefineParticleData() refinePartQuery['particle'] = refStackPart refinePartQuery['refineIter'] = refIterData refinePartDatas = refinePartQuery.query() refinePartData = refinePartDatas[0] emanEulerDict = { 'alt': refinePartData['euler1'], 'az': refinePartData['euler2'], 'phi': refinePartData['euler3'], 'shiftx': refinePartData['shiftx'], 'shifty': refinePartData['shifty'], 'mirror': refinePartData['mirror'], } eulerDict = apFrealign.convertAppionEmanEulersToFrealign( emanEulerDict, self.symmetryName) eulerDict['shiftx'] = emanEulerDict['shiftx'] * self.params['bin'] eulerDict['shifty'] = emanEulerDict['shifty'] * self.params['bin'] if refinePartData['mirror'] is True: eulerDict['shiftx'] *= -1 return eulerDict
def start(self): ### new stack path oldstack = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) newstack = os.path.join(self.params['rundir'], self.stackdata['name']) apStack.checkForPreviousStack(newstack) ### get particles from stack apDisplay.printMsg("Querying stack particles") t0 = time.time() stackpartq = appiondata.ApRefineParticleData() stackpartq['refineIter'] = self.iterdata particles = stackpartq.query() apDisplay.printMsg("Finished in " + apDisplay.timeString(time.time() - t0)) ### write included particles to text file includeParticle = [] excludeParticle = 0 f = open("test.log", "w") count = 0 apDisplay.printMsg("Processing stack particles") t0 = time.time() for part in particles: count += 1 if count % 500 == 0: sys.stderr.write(".") emanstackpartnum = part['particle']['particleNumber'] - 1 if part['postRefine_keep'] == 1: ### good particle includeParticle.append(emanstackpartnum) f.write("%d\t%d\tinclude\n" % (count, emanstackpartnum)) else: ### bad particle excludeParticle += 1 f.write("%d\t%d\texclude\n" % (count, emanstackpartnum)) sys.stderr.write("\n") apDisplay.printMsg("Finished in " + apDisplay.timeString(time.time() - t0)) f.close() includeParticle.sort() apDisplay.printMsg("Keeping " + str(len(includeParticle)) + " and excluding " + str(excludeParticle) + " particles") ### write kept particles to file self.params['keepfile'] = os.path.join( self.params['rundir'], "keepfile-" + self.timestamp + ".list") apDisplay.printMsg("writing to keepfile " + self.params['keepfile']) kf = open(self.params['keepfile'], "w") for partnum in includeParticle: kf.write(str(partnum) + "\n") kf.close() ### get number of particles numparticles = len(includeParticle) self.params['description'] += (" ... %d no jumpers substack" % (numparticles, )) ### create the new sub stack apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=True) if not os.path.isfile(newstack): apDisplay.printError("No stack was created") apStack.averageStack(stack=newstack) if self.params['commit'] is True: apStack.commitSubStack(self.params) newstackid = apStack.getStackIdFromPath(newstack) apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=6)
def insertRefineParticleData(self, cls, iteration, eulers, badprtls, refineq, numcls, euler_convention='zxz'): # get the corresponding proj number & eulers from filename replace = re.compile('\D') projnum = int(replace.sub('', cls)) clsfilename = os.path.join(self.params['tmpdir'], cls) sys.stderr.write(".") #f=open(clsfilename) #apDisplay.printMsg("Class "+str(projnum+1)+" of "+str(numcls)+": inserting " # +str(len(f.readlines())-2)+" particles") #f.close() # for each cls file get alignments for particles f = open(clsfilename) coranfail = False for line in f: # skip line if not a particle if re.search("start", line): prtlaliq = appiondata.ApRefineParticleData() # gather alignment data from line ali = line.split() prtlnum = int(ali[0]) # check if bad particle if prtlnum in badprtls: prtlaliq['refine_keep'] = False else: prtlaliq['refine_keep'] = True prtlnum += 1 # offset for EMAN qualf = float(ali[2].strip(',')) other = ali[3].split(',') rot = float(other[0]) * 180. / math.pi shx = float(other[1]) shy = float(other[2]) if (other[3] == '1'): prtlaliq['mirror'] = True # SPIDER coran kept particle corank = None if self.params['package'] == 'EMAN/SpiCoran': if len(other) > 4: corank = bool(int(other[4])) else: if coranfail is False: apDisplay.printWarning( "Coran failed on this iteration") coranfail = True # message passing kept particle if self.params['package'] == 'EMAN/MsgP' and len(ali) > 4: msgk = bool(int(ali[4])) else: msgk = None # find particle in stack database defid = self.stackmapping[prtlnum] stackp = appiondata.ApStackParticleData.direct_query(defid) if not stackp: apDisplay.printError("particle " + str(prtlnum) + " not in stack id=" + str(self.params['stack'].dbid)) # insert classification info prtlaliq['refineIter'] = refineq prtlaliq['particle'] = stackp prtlaliq['shiftx'] = shx prtlaliq['shifty'] = shy prtlaliq['euler1'] = eulers[projnum][0] prtlaliq['euler2'] = eulers[projnum][1] prtlaliq['euler3'] = rot prtlaliq['quality_factor'] = qualf if self.params['package'] == 'EMAN/MsgP': prtlaliq['postRefine_keep'] = msgk else: prtlaliq['postRefine_keep'] = corank prtlaliq['euler_convention'] = euler_convention #apDisplay.printMsg("inserting Particle Classification Data into database") if self.params['commit'] is True: prtlaliq.insert() f.close() return
def start(self): partdict = {} partlist = [] ### get Euler angles for each particle for iternum in self.iternums: ### get recon iter data reconiterq = appiondata.ApRefineIterData() reconiterq['refineRun'] = self.reconrundata reconiterq['iteration'] = iternum reconiterdata = reconiterq.query( results=1)[0] #this should be unique ### get particle data reconpartq = appiondata.ApRefineParticleData() reconpartq['refineIter'] = reconiterdata apDisplay.printMsg("Querying for particles at " + time.asctime()) reconpartdatas = reconpartq.query() ### group particle data for partdata in reconpartdatas: partnum = partdata['particle']['particleNumber'] if not partnum in partlist: partlist.append(partnum) partdict[(partnum, iternum)] = partdata ### run through particles and check Euler angles partlist.sort() eulerdict = {} eulercount = {} reject = 0 for partnum in partlist: e1d = {} e2d = {} for iternum in self.iternums: if not (partnum, iternum) in partdict: continue partdata = partdict[(partnum, iternum)] euler1 = "%.2f" % (partdata['euler1']) if not euler1 in e1d: e1d[euler1] = 1 else: e1d[euler1] += 1 euler2 = "%.2f" % (partdata['euler2']) if not euler2 in e2d: e2d[euler2] = 1 else: e2d[euler2] += 1 #print partnum, euler1, euler2 counts = [(val, key) for key, val in e1d.items()] e1count, euler1 = max(counts) counts = [(val, key) for key, val in e2d.items()] e2count, euler2 = max(counts) # reject indeterminant particles if e2count < 2 or e1count < 2: reject += 1 continue ### group particles by their Euler angles if not (euler1, euler2) in eulerdict: eulerdict[(euler1, euler2)] = [] eulercount[(euler1, euler2)] = 0 eulerdict[(euler1, euler2)].append(partnum) eulercount[(euler1, euler2)] += 1 print "Rejected %d particles" % (reject) values = eulercount.values() values.sort() print values ### run through Euler angles and count particles counts = [(val, key) for key, val in eulercount.items()] mincount, val = min(counts) self.params['mincount'] = max(self.params['mincount'], mincount) #print "Keeping %d of %d particles"%(mincount*len(eulercount.keys()), len(partlist)) print "Keeping %d of %d particles" % ( self.params['mincount'] * len(eulercount.keys()), len(partlist)) keeplist = [] for key in eulerdict.keys(): eulerpartlist = eulerdict[key] if len(partlist) < self.params['mincount']: keeplist.extend(eulerpartlist) else: keeplist.extend(eulerpartlist[:self.params['mincount']]) keeplist.sort() print "Keeping %d of %d particles" % (len(keeplist), len(partlist)) #need to set keepfile for commitSubStack self.params['keepfile'] = os.path.join(self.params['rundir'], "equalviews.lst") f = open(self.params['keepfile'], "w") for partnum in keeplist: f.write("%d\n" % (partnum - 1)) f.close() ### make a new stack using the keep particles oldstackdata = self.reconrundata['stack'] oldstack = os.path.join(oldstackdata['path']['path'], oldstackdata['name']) newstack = os.path.join(self.params['rundir'], "start.hed") apStack.makeNewStack(oldstack, newstack, listfile=self.params['keepfile'], remove=True, bad=True) if not os.path.isfile(newstack): apDisplay.printError("No stack was created") self.params[ 'stackid'] = oldstackdata.dbid #need to set stackid for commitSubStack apStack.commitSubStack(self.params, "start.hed") apStack.averageStack(stack=newstack)