def insertAlignParticlesIntoDatabase(self, level): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") D = self.getClassificationAtLevel(level) for ref in D: ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = ref + 1 refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['iteration'] = self.params['maxiter'] if ref in self.resdict: refq['ssnr_resolution'] = self.resdict[ref] ### setup particle info ... NOTE: ALIGNMENT PARAMETERS ARE NOT SAVED IN XMIPP 2.4 for partnum in D[ref]: alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = int(partnum) + 1 alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle( self.runparams['stackid'], int(partnum) + 1) ### particle numbering starts with 0!!!!!!! alignpartq['stackpart'] = stackpartdata alignpartq['ref'] = refq ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() ### insert bad particles if len(self.badpartlist) > 0: for p in self.badpartlist: refq = appiondata.ApAlignReferenceData() refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['iteration'] = self.params['maxiter'] alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = int(p) + 1 alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle( self.runparams['stackid'], int(p) + 1) alignpartq['stackpart'] = stackpartdata alignpartq['bad'] = 1 if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(self.params['numpart']) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan") return
def insertAlignParticlesIntoDatabase(self, level): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") D=self.getClassificationAtLevel(level) for ref in D: ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = ref+1 refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['iteration'] = self.params['maxiter'] if ref in self.resdict: refq['ssnr_resolution'] = self.resdict[ref] ### setup particle info ... NOTE: ALIGNMENT PARAMETERS ARE NOT SAVED IN XMIPP 2.4 for partnum in D[ref]: # particle numbering in D[ref] starts with 0 alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = int(partnum)+1 alignpartq['alignstack'] = self.alignstackdata ### particle numbering in Appion db starts with 1 stackpartdata = apStack.getStackParticle(self.runparams['stackid'], int(partnum)+1) alignpartq['stackpart'] = stackpartdata alignpartq['ref'] = refq ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() ### insert bad particles if len(self.badpartlist) > 0: for p in self.badpartlist: # particle numbering in badpartlist starts with 0 refq = appiondata.ApAlignReferenceData() refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['iteration'] = self.params['maxiter'] alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = int(p)+1 alignpartq['alignstack'] = self.alignstackdata ### particle numbering in Appion db starts with 1 stackpartdata = apStack.getStackParticle(self.runparams['stackid'], int(p)+1) alignpartq['stackpart'] = stackpartdata alignpartq['bad'] = 1 if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(self.params['numpart'])+" particles into the database in " +apDisplay.timeString(time.time()-t0), "cyan") return
def getGoodParticles(self): ''' Get particle numbers in eman format in fullstack that has tilt pair in knownstack ''' includeParticle = [] tiltParticlesData = [] no_match_count = 0 apDisplay.printMsg("Finding particles from tilt pairs ") ### get stack data knownstackid = self.params['knownstackid'] fullstackid = self.params['fullstackid'] numpartl = apStack.getNumberStackParticlesFromId(self.params['knownstackid']) for partnum in range(1,numpartl+1): stpartdata = apStack.getStackParticle(knownstackid, partnum, True) # stpartdata and otherpartdata are related by tilt transform imgnum, transformdata, otherpartdata = apTiltPair.getTiltTransformFromParticle(stpartdata['particle']) apDisplay.printMsg('Mapping particle %d to %d' % (stpartdata['particle'].dbid,otherpartdata.dbid)) fullstackpartdata = apStack.getStackParticleFromData(fullstackid, otherpartdata, True) if fullstackpartdata: # eman particle number starts from 0, appion database number starts from 1 emantiltstackpartnum = fullstackpartdata['particleNumber']-1 includeParticle.append(emantiltstackpartnum) tiltParticlesData.append(fullstackpartdata) else: no_match_count += 1 apDisplay.printWarning('There are %d particles without a match in the requested full stack' % no_match_count) return includeParticle, tiltParticlesData
def makeEulerDoc(self, parttree): count = 0 eulerfile = os.path.join(self.params['rundir'], "eulersdoc" + self.timestamp + ".spi") eulerf = open(eulerfile, "w") apDisplay.printMsg("creating Euler doc file") starttime = time.time() for partdict in parttree: stackpartdata = apStack.getStackParticle( self.params['tiltstackid'], partdict['tilt']) count += 1 if count % 100 == 0: sys.stderr.write(".") eulerf.flush() gamma, theta, phi, tiltangle = apTiltPair.getParticleTiltRotationAngles( stackpartdata) if gamma is None: apDisplay.printWarning("Skipping " + str(stackpartdata)) continue line = operations.spiderOutLine(count, [phi, tiltangle, gamma]) eulerf.write(line) eulerf.close() apDisplay.printColor( "finished Euler doc file in " + apDisplay.timeString(time.time() - starttime), "cyan") return eulerfile
def appendEulerDoc(self, eulerfile, tiltpartnum, count): eulerf = open(eulerfile, "a") stackpartdata = apStack.getStackParticle(self.params['tiltstackid'], tiltpartnum) gamma, theta, phi, tiltangle = apTiltPair.getParticleTiltRotationAngles(stackpartdata) line = operations.spiderOutLine(count, [phi, tiltangle, -1.0*gamma]) eulerf.write(line) eulerf.close()
def insertParticlesIntoDatabase(self, stackid, partList): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in partList: count += 1 if count % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refq["refnum"] = partdict["refnum"] if refq["refnum"] < 1: apDisplay.printError("Classes must start with 1") refq["iteration"] = self.lastiter # refq['mrcfile'] = refbase+".mrc" ### need to create mrcs of each class??? refq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"])) refq["alignrun"] = self.alignstackdata["alignrun"] if partdict["refnum"] in self.resdict: refq["ssnr_resolution"] = self.resdict[partdict["refnum"]] refq["isac_generation"] = partdict["generation"] ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq["partnum"] = partdict["partnum"] if alignpartq["partnum"] < 1: apDisplay.printError("Particles must start with 1") alignpartq["alignstack"] = self.alignstackdata if self.params["alignstackid"] is None: stackpartdata = apStack.getStackParticle(stackid, partdict["origPartNum"]) else: stackpartdata = apStack.getStackParticleFromAlignParticle( self.params["alignstackid"], partdict["origPartNum"] ) alignpartq["stackpart"] = stackpartdata alignpartq["xshift"] = partdict["xshift"] alignpartq["yshift"] = partdict["yshift"] alignpartq["rotation"] = partdict["inplane"] alignpartq["mirror"] = partdict["mirror"] alignpartq["ref"] = refq alignpartq["score"] = partdict["peak"] ### insert if self.params["commit"] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(count) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan", ) return
def insertParticlesIntoDatabase(self, stackid, partlist): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in partlist: count += 1 if count % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = partdict['refnum'] refq['iteration'] = self.lastiter refsearch = "part" + self.params['timestamp'] + "_ref*" + str( partdict['refnum']) + "*" refbase = os.path.splitext(glob.glob(refsearch)[0])[0] refq['mrcfile'] = refbase + ".mrc" refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] if partdict['refnum'] in self.resdict: refq['ssnr_resolution'] = self.resdict[partdict['refnum']] reffile = os.path.join(self.params['rundir'], refq['mrcfile']) if not os.path.isfile(reffile): emancmd = "proc2d " + refbase + ".xmp " + refbase + ".mrc" apEMAN.executeEmanCmd(emancmd, verbose=False) if not os.path.isfile(reffile): apDisplay.printError("could not find reference file: " + reffile) ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partdict['partnum'] alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle(stackid, partdict['partnum']) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['rotation'] = partdict['inplane'] alignpartq['mirror'] = partdict['mirror'] alignpartq['ref'] = refq alignpartq['spread'] = partdict['spread'] ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(count) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan") return
def getStackParticleEulersForIteration(self, pnum): """ find the eulers assigned to a stack particle during a refinement. This function will first find the particle id for the given stack particle, then find its position in the reference stack, and will get the eulers for that particle in the recon """ # get stack particle id stackp = apStack.getStackParticle(self.params['stackid'], pnum) particleid = stackp['particle'].dbid # find particle in reference stack refstackp = apStack.getStackParticleFromParticleId(particleid, self.oldreconstackid, noDie=True) if not refstackp: percentnoeuler = 100*self.noeulers/float(self.params['last']) apDisplay.printWarning('No eulers for particle %d (%.1f%%)' % (pnum, percentnoeuler)) self.noeulers += 1 if percentnoeuler > 10: apDisplay.printError('More than 10% of the particles have no euler, use a different reference reconstruction') pclass={ 'euler1': 0.0, 'euler2': 0.0, 'euler3': 0.0, 'mirror': False, 'shiftx': 0.0, 'shifty': 0.0, } else: pclassq = appiondata.ApRefineParticleData() pclassq['particle'] = refstackp pclassq['refineIter'] = appiondata.ApRefineIterData.direct_query(self.params['reconiterid']) pclasses = pclassq.query(results=1) pclass = pclasses[0] try: emaneuler={ 'alt': pclass['euler1'], 'az': pclass['euler2'], 'phi': pclass['euler3'], 'mirror': pclass['mirror'], 'shiftx': pclass['shiftx'], 'shifty': pclass['shifty'], } except: print pclass emaneuler={ 'alt': pclass['euler1'], 'az': pclass['euler2'], 'phi': pclass['euler3'], 'mirror': pclass['mirror'], 'shiftx': pclass['shiftx'], 'shifty': pclass['shifty'], } return emaneuler
def appendEulerDoc(self, eulerfile, tiltpartnum, count): eulerf = open(eulerfile, "a") stackpartdata = apStack.getStackParticle(self.params['tiltstackid'], tiltpartnum) gamma, theta, phi, tiltangle = apTiltPair.getParticleTiltRotationAngles(stackpartdata) if gamma is None: apDisplay.printWarning("Skipping "+str(stackpartdata)) return line = operations.spiderOutLine(count, [phi, tiltangle, -1.0*gamma]) eulerf.write(line) eulerf.close()
def insertParticlesIntoDatabase(self, stackid, partList): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in partList: count += 1 if count % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = partdict['refnum'] if refq['refnum'] < 1: apDisplay.printError("Classes must start with 1") refq['iteration'] = self.lastiter #refq['mrcfile'] = refbase+".mrc" ### need to create mrcs of each class??? refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] if partdict['refnum'] in self.resdict: refq['ssnr_resolution'] = self.resdict[partdict['refnum']] refq['isac_generation'] = partdict['generation'] ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partdict['partnum'] if alignpartq['partnum'] < 1: apDisplay.printError("Particles must start with 1") alignpartq['alignstack'] = self.alignstackdata if self.params['alignstackid'] is None: stackpartdata = apStack.getStackParticle( stackid, partdict['origPartNum']) else: stackpartdata = apStack.getStackParticleFromAlignParticle( self.params['alignstackid'], partdict['origPartNum']) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['rotation'] = partdict['inplane'] alignpartq['mirror'] = partdict['mirror'] alignpartq['ref'] = refq alignpartq['score'] = partdict['peak'] ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(count) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan") return
def insertParticlesIntoDatabase(self, partlist, insert=False): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for particle in partlist: count += 1 ### get all particle parameters partnum = count rotation = particle[0] * -1 ### in spider negative rotation is clockwise shiftx = particle[1] shifty = particle[2] ccc = particle[3] refnum = int(particle[4]) mirror = particle[5] if count % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = refnum refq['iteration'] = self.params['numiter'] refq['imagicfile'] = "references.hed" refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['templatestack'] = appiondata.ApTemplateStackData.direct_query(self.params['templateStackId']) reffile = os.path.join(self.params['rundir'], refq['imagicfile']) if not os.path.isfile(reffile): apDisplay.printError("could not find reference file: "+reffile) ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partnum alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle(self.params['stackId'], partnum) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = shiftx alignpartq['yshift'] = shifty alignpartq['rotation'] = rotation alignpartq['mirror'] = mirror alignpartq['ref'] = refq alignpartq['correlation'] = ccc ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in " +apDisplay.timeString(time.time()-t0), "cyan") return
def getStackParticleEulersForIteration(params, pnum): """ find the eulers assigned to a stack particle during a refinement. This function will first find the particle id for the given stack particle, then find its position in the reference stack, and will get the eulers for that particle in the recon """ # get stack particle id stackp = apStack.getStackParticle(params['stackid'], pnum) particleid = stackp['particle'].dbid # find particle in reference stack refstackid = apStack.getStackIdFromIterationId(params['reconiterid'], msg=False) refstackp = apStack.getStackParticleFromParticleId(particleid, refstackid, nodie=True) if not refstackp: apDisplay.printWarning( 'No classification for stack particle %d in reconstruction iteration id: %d' % (pnum, params['reconiterid'])) params['noClassification'] += 1 if params['noClassification'] > (float(params['last']) * 0.10): apDisplay.printError( 'More than 10% of the particles have no classification, use a different reference reconstruction' ) pclass = {} pclass['eulers'] = {} pclass['eulers']['euler1'] = 0.0 pclass['eulers']['euler2'] = 0.0 pclass['inplane_rotation'] = 0.0 pclass['mirror'] = False pclass['shiftx'] = 0.0 pclass['shifty'] = 0.0 else: pclassq = appiondata.ApRefineParticleData() pclassq['particle'] = refstackp pclassq['refineIter'] = appiondata.ApRefineIterData.direct_query( params['reconiterid']) pclass = pclassq.query() pclass = pclass[0] params['eman_orient'] = {} params['eman_orient']['alt'] = pclass['euler1'] params['eman_orient']['az'] = pclass['euler2'] params['eman_orient']['phi'] = pclass['euler3'] params['eman_orient']['mirror'] = pclass['mirror'] params['eman_orient']['shiftx'] = pclass['shiftx'] params['eman_orient']['shifty'] = pclass['shifty'] return params
def insertParticlesIntoDatabase(self, partlist, insert=False): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for particle in partlist: count += 1 ### get all particle parameters partnum = count rotation = particle[0] * -1 ### in spider negative rotation is clockwise shiftx = particle[1] shifty = particle[2] ccc = particle[3] refnum = int(particle[4]) mirror = particle[5] if count % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = refnum refq['iteration'] = self.params['numiter'] refq['imagicfile'] = "references.hed" refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['templatestack'] = appiondata.ApTemplateStackData.direct_query(self.params['templateStackId']) reffile = os.path.join(self.params['rundir'], refq['imagicfile']) if not os.path.isfile(reffile): apDisplay.printError("could not find reference file: "+reffile) ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partnum alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle(self.params['stackId'], partnum) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = shiftx alignpartq['yshift'] = shifty alignpartq['rotation'] = rotation alignpartq['mirror'] = mirror alignpartq['ref'] = refq alignpartq['correlation'] = ccc ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in " +apDisplay.timeString(time.time()-t0), "cyan") return
def insertAlignParticlesIntoDatabase(self): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for ref in self.classD: ### setup reference refq = appiondata.ApAlignReferenceData() refq["refnum"] = ref refq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"])) refq["alignrun"] = self.alignstackdata["alignrun"] refq["iteration"] = 10 # if ref in self.resdict: # refq['ssnr_resolution'] = self.resdict[ref] ### setup particle info for partnum in self.classD[ref]: alignpartq = appiondata.ApAlignParticleData() alignpartq["partnum"] = int(partnum) alignpartq["alignstack"] = self.alignstackdata ### particle numbering starts with 1 in SIMPLE stackpartdata = apStack.getStackParticle(self.runparams["stackid"], int(partnum)) alignpartq["stackpart"] = stackpartdata alignpartq["ref"] = refq try: alignpartq["xshift"] = self.alignD[partnum - 1]["shx"] except: pass try: alignpartq["yshift"] = self.alignD[partnum - 1]["shy"] except: pass try: alignpartq["mirror"] = self.alignD[partnum - 1]["mirror"] except: pass alignpartq["rotation"] = self.alignD[partnum - 1]["rot"] ### insert if self.params["commit"] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(self.params["numpart"]) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan", ) return
def appendEulerDoc(self, eulerfile, tiltpartnum, count): eulerf = open(eulerfile, "a") stackpartdata = apStack.getStackParticle(self.params['tiltstackid'], tiltpartnum) gamma, theta, phi, tiltangle = apTiltPair.getParticleTiltRotationAngles( stackpartdata) if gamma is None: apDisplay.printWarning("Skipping " + str(stackpartdata)) return line = operations.spiderOutLine(count, [phi, tiltangle, -1.0 * gamma]) eulerf.write(line) eulerf.close()
def getStackParticleEulersForIteration(self, pnum): """ find the eulers assigned to a stack particle during a refinement. This function will first find the particle id for the given stack particle, then find its position in the reference stack, and will get the eulers for that particle in the recon """ # get stack particle id stackp = apStack.getStackParticle(self.params["stackid"], pnum) particleid = stackp["particle"].dbid # find particle in reference stack refstackp = apStack.getStackParticleFromParticleId(particleid, self.oldreconstackid, nodie=True) if not refstackp: percentnoeuler = 100 * self.noeulers / float(self.params["last"]) apDisplay.printWarning("No eulers for particle %d (%.1f%%)" % (pnum, percentnoeuler)) self.noeulers += 1 if percentnoeuler > 10: apDisplay.printError( "More than 10% of the particles have no euler, use a different reference reconstruction" ) pclass = {"euler1": 0.0, "euler2": 0.0, "euler3": 0.0, "mirror": False, "shiftx": 0.0, "shifty": 0.0} else: pclassq = appiondata.ApRefineParticleData() pclassq["particle"] = refstackp pclassq["refineIter"] = appiondata.ApRefineIterData.direct_query(self.params["reconiterid"]) pclasses = pclassq.query(results=1) pclass = pclasses[0] try: emaneuler = { "alt": pclass["euler1"], "az": pclass["euler2"], "phi": pclass["euler3"], "mirror": pclass["mirror"], "shiftx": pclass["shiftx"], "shifty": pclass["shifty"], } except: print pclass emaneuler = { "alt": pclass["euler1"], "az": pclass["euler2"], "phi": pclass["euler3"], "mirror": pclass["mirror"], "shiftx": pclass["shiftx"], "shifty": pclass["shifty"], } return emaneuler
def getStackParticleTiltPair(stackid, partnum, tiltstackid=None): """ takes a stack id and particle number (1+) spider-style returns the stack particle number for the tilt pair """ #print stackid, partnum if tiltstackid is None: tiltstackid = stackid t0 = time.time() stackpartdata1 = apStack.getStackParticle(stackid, partnum) partdata = stackpartdata1['particle'] ### figure out if its particle 1 or 2 tiltpartq1 = appiondata.ApTiltParticlePairData() tiltpartq1['particle1'] = partdata tiltpartdatas1 = tiltpartq1.query(results=1, readimages=False) tiltpartq2 = appiondata.ApTiltParticlePairData() tiltpartq2['particle2'] = partdata tiltpartdatas2 = tiltpartq2.query(results=1, readimages=False) if not tiltpartdatas1 and tiltpartdatas2: #print "image1" otherpart = tiltpartdatas2[0]['particle1'] elif tiltpartdatas1 and not tiltpartdatas2: #print "image2" otherpart = tiltpartdatas1[0]['particle2'] else: print partdata print tiltpartdatas1 print tiltpartdatas2 apDisplay.printError("failed to get tilt pair data") ### get tilt stack particle tiltstackdata = apStack.getOnlyStackData(tiltstackid, msg=False) stackpartq = appiondata.ApStackParticleData() stackpartq['stack'] = tiltstackdata stackpartq['particle'] = otherpart stackpartdatas2 = stackpartq.query(results=1, readimages=False) if not stackpartdatas2: #print otherpart.dbid #apDisplay.printError("particle "+str(partnum)+" has no tilt pair in stackid="+str(tiltstackid)) return None stackpartdata = stackpartdatas2[0] #print partnum,"-->",stackpartnum if time.time() - t0 > 1.0: apDisplay.printMsg("long getStackPartTiltPair " + apDisplay.timeString(time.time() - t0)) return stackpartdata
def insertAlignParticlesIntoDatabase(self): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for ref in self.classD: ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = ref refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['iteration'] = 10 # if ref in self.resdict: # refq['ssnr_resolution'] = self.resdict[ref] ### setup particle info for partnum in self.classD[ref]: alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = int(partnum) alignpartq['alignstack'] = self.alignstackdata ### particle numbering starts with 1 in SIMPLE stackpartdata = apStack.getStackParticle( self.runparams['stackid'], int(partnum)) alignpartq['stackpart'] = stackpartdata alignpartq['ref'] = refq try: alignpartq['xshift'] = self.alignD[partnum - 1]['shx'] except: pass try: alignpartq['yshift'] = self.alignD[partnum - 1]['shy'] except: pass try: alignpartq['mirror'] = self.alignD[partnum - 1]['mirror'] except: pass alignpartq['rotation'] = self.alignD[partnum - 1]['rot'] ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(self.params['numpart']) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan") return
def getStackParticleTiltPair(stackid, partnum, tiltstackid=None): """ takes a stack id and particle number (1+) spider-style returns the stack particle number for the tilt pair """ #print stackid, partnum if tiltstackid is None: tiltstackid = stackid t0 = time.time() stackpartdata1 = apStack.getStackParticle(stackid, partnum) partdata = stackpartdata1['particle'] ### figure out if its particle 1 or 2 tiltpartq1 = appiondata.ApTiltParticlePairData() tiltpartq1['particle1'] = partdata tiltpartdatas1 = tiltpartq1.query(results=1, readimages=False) tiltpartq2 = appiondata.ApTiltParticlePairData() tiltpartq2['particle2'] = partdata tiltpartdatas2 = tiltpartq2.query(results=1, readimages=False) if not tiltpartdatas1 and tiltpartdatas2: #print "image1" otherpart = tiltpartdatas2[0]['particle1'] elif tiltpartdatas1 and not tiltpartdatas2: #print "image2" otherpart = tiltpartdatas1[0]['particle2'] else: print partdata print tiltpartdatas1 print tiltpartdatas2 apDisplay.printError("failed to get tilt pair data") ### get tilt stack particle tiltstackdata = apStack.getOnlyStackData(tiltstackid, msg=False) stackpartq = appiondata.ApStackParticleData() stackpartq['stack'] = tiltstackdata stackpartq['particle'] = otherpart stackpartdatas2 = stackpartq.query(results=1, readimages=False) if not stackpartdatas2: #print otherpart.dbid #apDisplay.printError("particle "+str(partnum)+" has no tilt pair in stackid="+str(tiltstackid)) return None stackpartdata = stackpartdatas2[0] #print partnum,"-->",stackpartnum if time.time()-t0 > 1.0: apDisplay.printMsg("long getStackPartTiltPair "+apDisplay.timeString(time.time()-t0)) return stackpartdata
def insertParticlesIntoDatabase(self, stackid, partlist): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in partlist: count += 1 if count % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = partdict['refnum'] refq['iteration'] = self.lastiter refsearch = "part"+self.params['timestamp']+"_ref*"+str(partdict['refnum'])+"*" refbase = os.path.splitext(glob.glob(refsearch)[0])[0] refq['mrcfile'] = refbase+".mrc" refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] if partdict['refnum'] in self.resdict: refq['ssnr_resolution'] = self.resdict[partdict['refnum']] reffile = os.path.join(self.params['rundir'], refq['mrcfile']) if not os.path.isfile(reffile): emancmd = "proc2d "+refbase+".xmp "+refbase+".mrc" apEMAN.executeEmanCmd(emancmd, verbose=False) if not os.path.isfile(reffile): apDisplay.printError("could not find reference file: "+reffile) ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partdict['partnum'] alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle(stackid, partdict['partnum']) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['rotation'] = partdict['inplane'] alignpartq['mirror'] = partdict['mirror'] alignpartq['ref'] = refq alignpartq['spread'] = partdict['spread'] ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in " +apDisplay.timeString(time.time()-t0), "cyan") return
def getStackParticlesInOrder(params): partorderfile = os.path.join(params['rundir'],'stackpartorder.list') stackid = params['stackid'] if not os.path.isfile(partorderfile): return apStack.getStackParticlesFromId(stackid) partfile = open(partorderfile,'r') lines = partfile.readlines() partorder = map((lambda x:int(x[:-1])),lines) partsort = list(partorder) partsort.sort() if partsort == partorder: return apStack.getStackParticlesFromId(stackid) apDisplay.printMsg("Preped stack has different order from the original stack. Getting information by the order of the preped stack") stpartdatas = [] for partnum in partorder: stpartdatas.append(apStack.getStackParticle(stackid, partnum)) return stpartdatas
def getStackParticleEulersForIteration(params,pnum): """ find the eulers assigned to a stack particle during a refinement. This function will first find the particle id for the given stack particle, then find its position in the reference stack, and will get the eulers for that particle in the recon """ # get stack particle id stackp = apStack.getStackParticle(params['stackid'],pnum) particleid = stackp['particle'].dbid # find particle in reference stack refstackid = apStack.getStackIdFromIterationId(params['reconiterid'],msg=False) refstackp = apStack.getStackParticleFromParticleId(particleid,refstackid, nodie=True) if not refstackp: apDisplay.printWarning('No classification for stack particle %d in reconstruction iteration id: %d' % (pnum, params['reconiterid'])) params['noClassification']+=1 if params['noClassification'] > (float(params['last'])*0.10): apDisplay.printError('More than 10% of the particles have no classification, use a different reference reconstruction') pclass={} pclass['eulers']={} pclass['eulers']['euler1']=0.0 pclass['eulers']['euler2']=0.0 pclass['inplane_rotation']=0.0 pclass['mirror']=False pclass['shiftx']=0.0 pclass['shifty']=0.0 else: pclassq = appiondata.ApRefineParticleData() pclassq['particle'] = refstackp pclassq['refineIter'] = appiondata.ApRefineIterData.direct_query(params['reconiterid']) pclass = pclassq.query() pclass=pclass[0] params['eman_orient']={} params['eman_orient']['alt']=pclass['euler1'] params['eman_orient']['az']=pclass['euler2'] params['eman_orient']['phi']=pclass['euler3'] params['eman_orient']['mirror']=pclass['mirror'] params['eman_orient']['shiftx']=pclass['shiftx'] params['eman_orient']['shifty']=pclass['shifty'] return params
def makeEulerDoc(self, parttree): count = 0 eulerfile = os.path.join(self.params['rundir'], "eulersdoc"+self.timestamp+".spi") eulerf = open(eulerfile, "w") apDisplay.printMsg("creating Euler doc file") starttime = time.time() for partdict in parttree: stackpartdata = apStack.getStackParticle(self.params['tiltstackid'], partdict['tilt']) count += 1 if count%100 == 0: sys.stderr.write(".") eulerf.flush() gamma, theta, phi, tiltangle = apTiltPair.getParticleTiltRotationAngles(stackpartdata) line = operations.spiderOutLine(count, [phi, tiltangle, gamma]) eulerf.write(line) eulerf.close() apDisplay.printColor("finished Euler doc file in "+apDisplay.timeString(time.time()-starttime), "cyan") return eulerfile
def getStackParticlesInOrder(params): partorderfile = os.path.join(params['rundir'], 'stackpartorder.list') stackid = params['stackid'] if not os.path.isfile(partorderfile): return apStack.getStackParticlesFromId(stackid) partfile = open(partorderfile, 'r') lines = partfile.readlines() partorder = map((lambda x: int(x[:-1])), lines) partsort = list(partorder) partsort.sort() if partsort == partorder: return apStack.getStackParticlesFromId(stackid) apDisplay.printMsg( "Preped stack has different order from the original stack. Getting information by the order of the preped stack" ) stpartdatas = [] for partnum in partorder: stpartdatas.append(apStack.getStackParticle(stackid, partnum)) return stpartdatas
def insertNoRefRun(self, imagicstack, insert=False): ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '" + self.params['runname'] + "' and path already exist in database") # create a norefParam object norefq = appiondata.ApSpiderNoRefRunData() norefq['runname'] = self.params['runname'] norefq['particle_diam'] = 2.0 * self.params['partrad'] norefq['first_ring'] = self.params['firstring'] norefq['last_ring'] = self.params['lastring'] norefq['init_method'] = self.params['initmethod'] norefq['run_seconds'] = self.runtime ### finish alignment run alignrunq['norefrun'] = norefq alignrunq['hidden'] = False alignrunq['bin'] = self.params['bin'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['description'] = self.params['description'] # STOP HERE ### setup alignment stack alignstackq = appiondata.ApAlignStackData() alignstackq['alignrun'] = alignrunq alignstackq['imagicfile'] = os.path.basename(imagicstack) alignstackq['avgmrcfile'] = "average.mrc" alignstackq['alignrun'] = alignrunq alignstackq['iteration'] = 0 alignstackq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) ### check to make sure files exist imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find stack file: " + imagicfile) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average file: " + avgmrcfile) alignstackq['stack'] = self.stack['data'] alignstackq['boxsize'] = math.floor(self.stack['boxsize'] / self.params['bin']) alignstackq['pixelsize'] = self.stack['apix'] * self.params['bin'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.params['numpart'] if insert is True: alignstackq.insert() ### create reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = 1 refq['iteration'] = 0 refq['mrcfile'] = "template.mrc" #refpath = os.path.abspath(os.path.join(self.params['rundir'], "alignment")) refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = alignrunq ### insert particle data apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in self.partlist: ### see apSpider.alignment.alignStack() for more info """ partdict.keys() 'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,... 'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,... 'mirror': checkMirror(templatenum), 'score': float(data[3]), 'rot': float(data[4]), 'xshift': float(data[5]), 'yshift': float(data[6]), """ alignpartq = appiondata.ApAlignParticleData() alignpartq['ref'] = refq alignpartq['partnum'] = partdict['num'] alignpartq['alignstack'] = alignstackq stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num']) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['rotation'] = partdict['rot'] #alignpartq['score'] = partdict['score'] if insert is True: alignpartq.insert() return
def insertRefBasedRun(self, partlist, imagicstack, insert=False): apDisplay.printMsg("committing results to DB") ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database") ### setup ref based run refrunq = appiondata.ApRefBasedRunData() refrunq['runname'] = self.params['runname'] refrunq['xysearch'] = self.params['xysearch'] refrunq['xystep'] = self.params['xystep'] refrunq['first_ring'] = self.params['firstring'] refrunq['last_ring'] = self.params['lastring'] refrunq['num_iter'] = self.params['numiter'] refrunq['invert_templs'] = self.params['inverttemplates'] refrunq['num_templs'] = self.params['numtemplate'] #refrunq['csym', int), refrunq['run_seconds'] = self.params['runtime'] ### finish alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['refbasedrun'] = refrunq alignrunq['hidden'] = False alignrunq['bin'] = self.params['bin'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['runname'] = self.params['runname'] alignrunq['description'] = self.params['description'] ### setup alignment stack alignstackq = appiondata.ApAlignStackData() alignstackq['alignrun'] = alignrunq alignstackq['imagicfile'] = imagicstack alignstackq['avgmrcfile'] = "average.mrc" emancmd = "proc2d templatestack%02d.spi templatestack%02d.hed"%(self.params['numiter'],self.params['numiter']) apEMAN.executeEmanCmd(emancmd) alignstackq['refstackfile'] = ("templatestack%02d.hed"%(self.params['numiter'])) alignstackq['alignrun'] = alignrunq alignstackq['iteration'] = self.params['numiter'] alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) ### check to make sure files exist imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find stack file: "+imagicfile) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: "+avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: "+refstackfile) alignstackq['stack'] = self.stack['data'] alignstackq['boxsize'] = self.boxsize alignstackq['pixelsize'] = self.stack['apix']*self.params['bin'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.params['numpart'] if insert is True: alignstackq.insert() ### insert reference data reflist = [] for j in range(self.params['numiter']): iternum = j+1 for i in range(len(self.templatelist)): refnum = i+1 templateid = self.templatelist[i] refq = appiondata.ApAlignReferenceData() refq['refnum'] = refnum refq['iteration'] = iternum refq['template'] = apTemplate.getTemplateFromId(templateid) refq['mrcfile'] = ("templateavg%02d-%02d.mrc"%(iternum,refnum)) refpath = os.path.join(self.params['rundir'], "templates") refq['path'] = appiondata.ApPathData(path=os.path.abspath(refpath)) refq['alignrun'] = alignrunq if refnum in self.resdict: refq['ssnr_resolution'] = self.resdict[refnum] if insert is True: refq.insert() if iternum == self.params['numiter']: reflist.append(refq) #refq['varmrcfile', str), #refq['frc_resolution', float), ### insert particle data apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in partlist: ### see apSpider.alignment.alignStack() for more info """ partdict.keys() 'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,... 'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,... 'mirror': checkMirror(templatenum), 'score': float(data[3]), 'rot': float(data[4]), 'xshift': float(data[5]), 'yshift': float(data[6]), """ alignpartq = appiondata.ApAlignParticleData() alignpartq['ref'] = reflist[partdict['template']-1] alignpartq['partnum'] = partdict['num'] alignpartq['alignstack'] = alignstackq stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num']) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['rotation'] = partdict['rot'] alignpartq['score'] = partdict['score'] alignpartq['mirror'] = partdict['mirror'] if insert is True: alignpartq.insert() return
def insertRefBasedRun(self, partlist, imagicstack, insert=False): apDisplay.printMsg("committing results to DB") ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database") ### setup ref based run refrunq = appiondata.ApRefBasedRunData() refrunq['runname'] = self.params['runname'] refrunq['xysearch'] = self.params['xysearch'] refrunq['xystep'] = self.params['xystep'] refrunq['first_ring'] = self.params['firstring'] refrunq['last_ring'] = self.params['lastring'] refrunq['num_iter'] = self.params['numiter'] refrunq['invert_templs'] = self.params['inverttemplates'] refrunq['num_templs'] = self.params['numtemplate'] #refrunq['csym', int), refrunq['run_seconds'] = self.params['runtime'] ### finish alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['refbasedrun'] = refrunq alignrunq['hidden'] = False alignrunq['bin'] = self.params['bin'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['runname'] = self.params['runname'] alignrunq['description'] = self.params['description'] ### setup alignment stack alignstackq = appiondata.ApAlignStackData() alignstackq['alignrun'] = alignrunq alignstackq['imagicfile'] = imagicstack alignstackq['avgmrcfile'] = "average.mrc" emancmd = "proc2d templatestack%02d.spi templatestack%02d.hed"%(self.params['numiter'],self.params['numiter']) apEMAN.executeEmanCmd(emancmd) alignstackq['refstackfile'] = ("templatestack%02d.hed"%(self.params['numiter'])) alignstackq['alignrun'] = alignrunq alignstackq['iteration'] = self.params['numiter'] alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) ### check to make sure files exist imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find stack file: "+imagicfile) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: "+avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: "+refstackfile) alignstackq['stack'] = self.stack['data'] alignstackq['boxsize'] = self.boxsize alignstackq['pixelsize'] = self.stack['apix']*self.params['bin'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.params['numpart'] if insert is True: alignstackq.insert() ### insert reference data reflist = [] for j in range(self.params['numiter']): iternum = j+1 for i in range(len(self.templatelist)): refnum = i+1 templateid = self.templatelist[i] refq = appiondata.ApAlignReferenceData() refq['refnum'] = refnum refq['iteration'] = iternum refq['template'] = apTemplate.getTemplateFromId(templateid) refq['mrcfile'] = ("templateavg%02d-%02d.mrc"%(iternum,refnum)) refpath = os.path.join(self.params['rundir'], "templates") refq['path'] = appiondata.ApPathData(path=os.path.abspath(refpath)) refq['alignrun'] = alignrunq if refnum in self.resdict: refq['ssnr_resolution'] = self.resdict[refnum] if insert is True: refq.insert() if iternum == self.params['numiter']: reflist.append(refq) #refq['varmrcfile', str), #refq['frc_resolution', float), ### insert particle data apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in partlist: ### see apSpider.alignment.alignStack() for more info """ partdict.keys() 'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,... 'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,... 'mirror': checkMirror(templatenum), 'score': float(data[3]), 'rot': float(data[4]), 'xshift': float(data[5]), 'yshift': float(data[6]), """ alignpartq = appiondata.ApAlignParticleData() alignpartq['ref'] = reflist[partdict['template']-1] alignpartq['partnum'] = partdict['num'] alignpartq['alignstack'] = alignstackq stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num']) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['rotation'] = partdict['rot'] alignpartq['score'] = partdict['score'] alignpartq['mirror'] = partdict['mirror'] if insert is True: alignpartq.insert() return
def insertEdIterRun(self, insert=False): ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '" + runparams['runname'] + "' and path already exist in database") ### setup ed-iter run editrunq = appiondata.ApEdIterRunData() editrunq['runname'] = self.params['runname'] editrunq['radius'] = self.params['partrad'] editrunq['num_iter'] = self.params['numiter'] #editrunq['freealigns'] = self.params['freealigns'] ### no longer an option editrunq['invert_templs'] = self.params['inverttemplates'] editrunq['num_templs'] = self.params['numtemplate'] #editrunq['csym', int), editrunq['run_seconds'] = self.runtime ### finish alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['editerrun'] = editrunq alignrunq['hidden'] = False alignrunq['bin'] = self.params['bin'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['runname'] = self.params['runname'] alignrunq['description'] = self.params['description'] ### setup aligned stack alignstackq = appiondata.ApAlignStackData() alignstackq['alignrun'] = alignrunq alignstackq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) # stack of aligned particles alignstackq['imagicfile'] = "alignstack.hed" # final class averages alignstackq['refstackfile'] = "avg.hed" # averaged results alignstackq['avgmrcfile'] = "average.mrc" # check to be sure dbpaths to files are correct refstackfile = os.path.join( self.params['rundir'], alignstackq['refstackfile']) #final averages if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: " + refstackfile) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) #averaged results if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: " + avgmrcfile) alignstackq['stack'] = self.stack['data'] alignstackq['iteration'] = self.params['numiter'] alignstackq['boxsize'] = math.floor(self.stack['boxsize'] / self.params['bin']) alignstackq['pixelsize'] = self.stack['apix'] * self.params['bin'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.params['numpart'] if insert is True: alignstackq.insert( ) #alignstackq contains alignrunq which contains editrunq ### setup data from each iteration apDisplay.printColor( "Inserting average and variance images from each iteration, please wait", "cyan") reflist = [] for j in range(self.params['numiter']): iternum = j + 1 for i in range(len(self.templatelist)): refnum = i + 1 templateid = self.templatelist[i] refq = appiondata.ApAlignReferenceData() refq['refnum'] = refnum refq['iteration'] = iternum refq['template'] = apTemplate.getTemplateFromId(templateid) refpath = os.path.join(self.params['rundir'], "templates") refq['path'] = appiondata.ApPathData( path=os.path.abspath(refpath)) refq['alignrun'] = alignrunq #refq['frc_resolution'] = #(float) avgname = os.path.join(self.params['rundir'], "r%02d/avg%03d" % (iternum, refnum)) varname = os.path.join(self.params['rundir'], "r%02d/var%03d" % (iternum, refnum)) if os.path.isfile(avgname + ".spi"): emancmd = "proc2d " + avgname + ".spi " + avgname + ".mrc" apEMAN.executeEmanCmd(emancmd) refq['mrcfile'] = (avgname + ".mrc") emancmd = "proc2d " + varname + ".spi " + varname + ".mrc" apEMAN.executeEmanCmd(emancmd) refq['varmrcfile'] = (varname + ".mrc") if insert is True: refq.insert() if iternum == self.params['numiter']: reflist.append(refq) else: reflist.append(None) ### insert particle data apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") partlist = self.readApshDocFile("apshdoc.spi", "apshdoc.pickle") for partdict in partlist: alignpartq = appiondata.ApAlignParticleData() alignpartq['ref'] = reflist[partdict['template'] - 1] alignpartq['partnum'] = partdict['num'] alignpartq['alignstack'] = alignstackq stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num']) alignpartq['stackpart'] = stackpartdata alignpartq['rotation'] = partdict['rot'] alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['mirror'] = partdict['mirror'] alignpartq['score'] = partdict['score'] if insert is True: alignpartq.insert() #insert each particle return
def uploadResults(self): if self.params['commit'] is False: return # Get the new file order fh = open("sort_junk.sel", 'r') lines = fh.readlines() i = 0 fileorder = {} for line in lines: args = line.split() if (len(args) > 1): match = re.match('[A-Za-z]+([0-9]+)\.[A-Za-z]+', (args[0].split('/'))[-1]) if (match): filenumber = int(match.groups()[0]) fileorder[i] = filenumber i += 1 fh.close() # Produce a new stack oldstack = apStack.getOnlyStackData(self.params['stackid'], msg=False) newstack = appiondata.ApStackData() newstack['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) newstack['name'] = "sorted.hed" if newstack.query(results=1): apDisplay.printError( "A stack with these parameters already exists") # Fill in data and submit newstack['oldstack'] = oldstack newstack['hidden'] = False newstack['substackname'] = self.params['runname'] newstack['description'] = self.params['description'] newstack['pixelsize'] = oldstack['pixelsize'] newstack['boxsize'] = oldstack['boxsize'] newstack['junksorted'] = True newstack.insert() # Insert stack images apDisplay.printMsg("Inserting stack particles") count = 0 total = len(fileorder.keys()) if total == 0: apDisplay.printError( "No particles can be inserted in the sorted stack") for i in fileorder.keys(): count += 1 if count % 100 == 0: sys.stderr.write( "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b") sys.stderr.write( str(count) + " of " + (str(total)) + " complete") # Get particle from the old stack oldparticle = apStack.getStackParticle(self.params['stackid'], fileorder[i] + 1) # Insert particle newparticle = appiondata.ApStackParticleData() newparticle['particleNumber'] = i + 1 newparticle['stack'] = newstack newparticle['stackRun'] = oldparticle['stackRun'] newparticle['particle'] = oldparticle['particle'] newparticle['mean'] = oldparticle['mean'] newparticle['stdev'] = oldparticle['stdev'] newparticle.insert() apDisplay.printMsg( "\n" + str(total) + " particles have been inserted into the sorted stack") # Insert runs in stack apDisplay.printMsg("Inserting Runs in Stack") runsinstack = apStack.getRunsInStack(self.params['stackid']) for run in runsinstack: newrunsq = appiondata.ApRunsInStackData() newrunsq['stack'] = newstack newrunsq['stackRun'] = run['stackRun'] newrunsq.insert() apDisplay.printMsg("finished") return
def commitToDatabase(self): """ insert the results into the database """ ### expected result for an alignment run: ### 1. aligned particle stack in IMAGIC ### 2. rotation, shift, and quality parameters for each particle ### 3. which particles belongs to which class ### 4. stack file with the class averages alignedstack = os.path.join(self.params['rundir'], "ptcl.hed") refstack = os.path.join(self.params['rundir'], "iter.final.hed") averagemrc = os.path.join(self.params['rundir'], "average.mrc") apStack.averageStack(alignedstack, averagemrc) particlemapping = self.determineClassOwnership() ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError( "Run name '" + runparams['runname'] + "' and path already exisclassmappingt in database") ### setup eman refine2d run emanrefinetwodq = appiondata.ApEMANRefine2dRunData() emanrefinetwodq['runname'] = self.params['runname'] emanrefinetwodq['run_seconds'] = time.time() - self.t0 emanrefinetwodq['num_iters'] = self.params['numiter'] emanrefinetwodq['num_classes'] = self.params['numclasses'] ### finish alignment run alignrunq['refine2drun'] = emanrefinetwodq alignrunq['hidden'] = False alignrunq['runname'] = self.params['runname'] alignrunq['description'] = self.params['description'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['bin'] = self.params['bin'] ### setup alignment stackalignimagicfile alignstackq = appiondata.ApAlignStackData() alignstackq['imagicfile'] = os.path.basename(alignedstack) alignstackq['avgmrcfile'] = os.path.basename(averagemrc) alignstackq['refstackfile'] = os.path.basename(refstack) alignstackq['iteration'] = self.params['numiter'] alignstackq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) alignstackq['alignrun'] = alignrunq ### check to make sure files exist alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(alignimagicfilepath): apDisplay.printError("could not find stack file: " + alignimagicfilepath) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: " + avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printErrrefqor("could not find reference stack file: " + refstackfile) ### continue setting values alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackid']) alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0] alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) * self.params['bin'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = apFile.numImagesInStack( alignimagicfilepath) ### inserting particles and references apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for emanpartnum in range(self.params['numpart']): partnum = emanpartnum + 1 if partnum % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refnum = particlemapping[emanpartnum] refq['refnum'] = refnum refq['iteration'] = self.params['numiter'] refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = alignrunq ### TODO: create mrc file #refq['mrcfile'] = refbase+".mrc" #reffile = os.path.join(self.params['rundir'], refq['mrcfile']) #if not os.path.isfile(reffile): # emancmd = "proc2d "+refstack+".xmp "+refstack+".mrc" # apEMAN.executeEmanCmd(emancmd, verbose=False) #if not os.path.isfile(reffile): # apDisplay.printError("could not find reference file: "+reffile) ### TODO: get resolution #refq['ssnr_resolution'] = TODO ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partnum alignpartq['alignstack'] = alignstackq stackpartdata = apStack.getStackParticle(self.params['stackid'], partnum) alignpartq['stackpart'] = stackpartdata ### TODO: get the alignment parameters #alignpartq['xshift'] = partdict['xshift'] #alignpartq['yshift'] = partdict['yshift'] #alignpartq['rotation'] = partdict['inplane'] #alignpartq['mirror'] = partdict['mirror'] alignpartq['ref'] = refq ### TODO: get the score #alignpartq['score'] = partdict['score'] ### insert if self.params['commit'] is True: alignpartq.insert() return
def insertNoRefRun(self, imagicstack, insert=False): ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq["runname"] = self.params["runname"] alignrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '" + self.params["runname"] + "' and path already exist in database") # create a norefParam object norefq = appiondata.ApSpiderNoRefRunData() norefq["runname"] = self.params["runname"] norefq["particle_diam"] = 2.0 * self.params["partrad"] norefq["first_ring"] = self.params["firstring"] norefq["last_ring"] = self.params["lastring"] norefq["init_method"] = self.params["initmethod"] norefq["run_seconds"] = self.runtime ### finish alignment run alignrunq["norefrun"] = norefq alignrunq["hidden"] = False alignrunq["bin"] = self.params["bin"] alignrunq["hp_filt"] = self.params["highpass"] alignrunq["lp_filt"] = self.params["lowpass"] alignrunq["description"] = self.params["description"] # STOP HERE ### setup alignment stack alignstackq = appiondata.ApAlignStackData() alignstackq["alignrun"] = alignrunq alignstackq["imagicfile"] = os.path.basename(imagicstack) alignstackq["avgmrcfile"] = "average.mrc" alignstackq["alignrun"] = alignrunq alignstackq["iteration"] = 0 alignstackq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"])) ### check to make sure files exist imagicfile = os.path.join(self.params["rundir"], alignstackq["imagicfile"]) if not os.path.isfile(imagicfile): apDisplay.printError("could not find stack file: " + imagicfile) avgmrcfile = os.path.join(self.params["rundir"], alignstackq["avgmrcfile"]) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average file: " + avgmrcfile) alignstackq["stack"] = self.stack["data"] alignstackq["boxsize"] = math.floor(self.stack["boxsize"] / self.params["bin"]) alignstackq["pixelsize"] = self.stack["apix"] * self.params["bin"] alignstackq["description"] = self.params["description"] alignstackq["hidden"] = False alignstackq["num_particles"] = self.params["numpart"] if insert is True: alignstackq.insert() ### create reference refq = appiondata.ApAlignReferenceData() refq["refnum"] = 1 refq["iteration"] = 0 refq["mrcfile"] = "template.mrc" # refpath = os.path.abspath(os.path.join(self.params['rundir'], "alignment")) refq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"])) refq["alignrun"] = alignrunq ### insert particle data apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in self.partlist: ### see apSpider.alignment.alignStack() for more info """ partdict.keys() 'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,... 'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,... 'mirror': checkMirror(templatenum), 'score': float(data[3]), 'rot': float(data[4]), 'xshift': float(data[5]), 'yshift': float(data[6]), """ alignpartq = appiondata.ApAlignParticleData() alignpartq["ref"] = refq alignpartq["partnum"] = partdict["num"] alignpartq["alignstack"] = alignstackq stackpartdata = apStack.getStackParticle(self.params["stackid"], partdict["num"]) alignpartq["stackpart"] = stackpartdata alignpartq["xshift"] = partdict["xshift"] alignpartq["yshift"] = partdict["yshift"] alignpartq["rotation"] = partdict["rot"] # alignpartq['score'] = partdict['score'] if insert is True: alignpartq.insert() return
def uploadResults(self): if self.params['commit'] is False: return # Produce new stacks oldstack = apStack.getOnlyStackData(self.params['notstackid'], msg=False) notstack = appiondata.ApStackData() notstack['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) notstack['name'] = self.notstackdata['name'] if notstack.query(results=1): apDisplay.printError("A stack with these parameters already exists") tiltstack = appiondata.ApStackData() tiltstack['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) tiltstack['name'] = self.tiltstackdata['name'] if tiltstack.query(results=1): apDisplay.printError("A stack with these parameters already exists") # Fill in data and submit notstack['oldstack'] = oldstack notstack['hidden'] = False notstack['substackname'] = self.params['runname'] notstack['description'] = self.params['description']+" ... tilt stack sorted" notstack['pixelsize'] = oldstack['pixelsize'] notstack.insert() tiltstack['oldstack'] = oldstack tiltstack['hidden'] = False tiltstack['substackname'] = self.params['runname'] tiltstack['description'] = self.params['description']+" ... tilt stack sorted" tiltstack['pixelsize'] = oldstack['pixelsize'] tiltstack.insert() # Insert stack images apDisplay.printMsg("Inserting stack particles") count=0 for partdict in parttree: count += 1 if count % 100 == 0: sys.stderr.write("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b") sys.stderr.write(str(count)+" of "+(str(total))+" complete") # Get particles from the old stacks oldnotparticle = apStack.getStackParticle(self.params['notstackid'], partdict['not']) oldtiltparticle = apStack.getStackParticle(self.params['tiltstackid'], partdict['tilt']) # Insert particle notparticle = appiondata.ApStackParticleData() notparticle.update(oldnotparticle) notparticle['particleNumber'] = count notparticle['stack'] = notstack notparticle.insert() tiltparticle = appiondata.ApStackParticleData() tiltparticle.update(oldtiltparticle) tiltparticle['particleNumber'] = count tiltparticle['stack'] = tiltstack tiltparticle.insert() apDisplay.printMsg("\n%d particles have been inserted into the tilt synchronized stacks"%(count)) # Insert runs in stack apDisplay.printMsg("Inserting Runs in Stack") runsinstack = apStack.getRunsInStack(self.params['notstackid']) for run in runsinstack: newrunsq = appiondata.ApRunsInStackData() newrunsq['stack'] = notstack newrunsq['stackRun'] = run['stackRun'] newrunsq.insert() runsinstack = apStack.getRunsInStack(self.params['tiltstackid']) for run in runsinstack: newrunsq = appiondata.ApRunsInStackData() newrunsq['stack'] = tiltstack newrunsq['stackRun'] = run['stackRun'] newrunsq.insert() apDisplay.printMsg("finished") return
def commitToDatabase(self): """ insert the results into the database """ ### expected result for an alignment run: ### 1. aligned particle stack in IMAGIC ### 2. rotation, shift, and quality parameters for each particle ### 3. which particles belongs to which class ### 4. stack file with the class averages alignedstack = os.path.join(self.params['rundir'], "ptcl.hed") refstack = os.path.join(self.params['rundir'], "iter.final.hed") averagemrc = os.path.join(self.params['rundir'], "average.mrc") apStack.averageStack(alignedstack, averagemrc) particlemapping = self.determineClassOwnership() ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '"+runparams['runname'] +"' and path already exisclassmappingt in database") ### setup eman refine2d run emanrefinetwodq = appiondata.ApEMANRefine2dRunData() emanrefinetwodq['runname'] = self.params['runname'] emanrefinetwodq['run_seconds'] = time.time() - self.t0 emanrefinetwodq['num_iters'] = self.params['numiter'] emanrefinetwodq['num_classes'] = self.params['numclasses'] ### finish alignment run alignrunq['refine2drun'] = emanrefinetwodq alignrunq['hidden'] = False alignrunq['runname'] = self.params['runname'] alignrunq['description'] = self.params['description'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['bin'] = self.params['bin'] ### setup alignment stackalignimagicfile alignstackq = appiondata.ApAlignStackData() alignstackq['imagicfile'] = os.path.basename(alignedstack) alignstackq['avgmrcfile'] = os.path.basename(averagemrc) alignstackq['refstackfile'] = os.path.basename(refstack) alignstackq['iteration'] = self.params['numiter'] alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) alignstackq['alignrun'] = alignrunq ### check to make sure files exist alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(alignimagicfilepath): apDisplay.printError("could not find stack file: "+alignimagicfilepath) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: "+avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printErrrefqor("could not find reference stack file: "+refstackfile) ### continue setting values alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackid']) alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0] alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])*self.params['bin'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = apFile.numImagesInStack(alignimagicfilepath) ### inserting particles and references apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for emanpartnum in range(self.params['numpart']): partnum = emanpartnum+1 if partnum % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refnum = particlemapping[emanpartnum] refq['refnum'] = refnum refq['iteration'] = self.params['numiter'] refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = alignrunq ### TODO: create mrc file #refq['mrcfile'] = refbase+".mrc" #reffile = os.path.join(self.params['rundir'], refq['mrcfile']) #if not os.path.isfile(reffile): # emancmd = "proc2d "+refstack+".xmp "+refstack+".mrc" # apEMAN.executeEmanCmd(emancmd, verbose=False) #if not os.path.isfile(reffile): # apDisplay.printError("could not find reference file: "+reffile) ### TODO: get resolution #refq['ssnr_resolution'] = TODO ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partnum alignpartq['alignstack'] = alignstackq stackpartdata = apStack.getStackParticle(self.params['stackid'], partnum) alignpartq['stackpart'] = stackpartdata ### TODO: get the alignment parameters #alignpartq['xshift'] = partdict['xshift'] #alignpartq['yshift'] = partdict['yshift'] #alignpartq['rotation'] = partdict['inplane'] #alignpartq['mirror'] = partdict['mirror'] alignpartq['ref'] = refq ### TODO: get the score #alignpartq['score'] = partdict['score'] ### insert if self.params['commit'] is True: alignpartq.insert() return
def insertEdIterRun(self, insert=False): ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database") ### setup ed-iter run editrunq = appiondata.ApEdIterRunData() editrunq['runname'] = self.params['runname'] editrunq['radius'] = self.params['partrad'] editrunq['num_iter'] = self.params['numiter'] #editrunq['freealigns'] = self.params['freealigns'] ### no longer an option editrunq['invert_templs'] = self.params['inverttemplates'] editrunq['num_templs'] = self.params['numtemplate'] #editrunq['csym', int), editrunq['run_seconds'] = self.runtime ### finish alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['editerrun'] = editrunq alignrunq['hidden'] = False alignrunq['bin'] = self.params['bin'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['runname'] = self.params['runname'] alignrunq['description'] = self.params['description'] ### setup aligned stack alignstackq = appiondata.ApAlignStackData() alignstackq['alignrun'] = alignrunq alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) # stack of aligned particles alignstackq['imagicfile'] = "alignstack.hed" # final class averages alignstackq['refstackfile'] = "avg.hed" # averaged results alignstackq['avgmrcfile'] = "average.mrc" # check to be sure dbpaths to files are correct refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) #final averages if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: "+refstackfile) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) #averaged results if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: "+avgmrcfile) alignstackq['stack'] = self.stack['data'] alignstackq['iteration'] = self.params['numiter'] alignstackq['boxsize'] = math.floor(self.stack['boxsize']/self.params['bin']) alignstackq['pixelsize'] = self.stack['apix']*self.params['bin'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.params['numpart'] if insert is True: alignstackq.insert() #alignstackq contains alignrunq which contains editrunq ### setup data from each iteration apDisplay.printColor("Inserting average and variance images from each iteration, please wait", "cyan") reflist = [] for j in range(self.params['numiter']): iternum = j+1 for i in range(len(self.templatelist)): refnum = i+1 templateid = self.templatelist[i] refq = appiondata.ApAlignReferenceData() refq['refnum'] = refnum refq['iteration'] = iternum refq['template'] = apTemplate.getTemplateFromId(templateid) refpath = os.path.join(self.params['rundir'], "templates") refq['path'] = appiondata.ApPathData(path=os.path.abspath(refpath)) refq['alignrun'] = alignrunq #refq['frc_resolution'] = #(float) avgname = os.path.join(self.params['rundir'], "r%02d/avg%03d"%(iternum,refnum) ) varname = os.path.join(self.params['rundir'], "r%02d/var%03d"%(iternum,refnum) ) if os.path.isfile(avgname+".spi"): emancmd = "proc2d "+avgname+".spi "+avgname+".mrc" apEMAN.executeEmanCmd(emancmd) refq['mrcfile'] = (avgname+".mrc") emancmd = "proc2d "+varname+".spi "+varname+".mrc" apEMAN.executeEmanCmd(emancmd) refq['varmrcfile'] = (varname+".mrc") if insert is True: refq.insert() if iternum == self.params['numiter']: reflist.append(refq) else: reflist.append(None) ### insert particle data apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") partlist = self.readApshDocFile("apshdoc.spi","apshdoc.pickle") for partdict in partlist: alignpartq = appiondata.ApAlignParticleData() alignpartq['ref'] = reflist[partdict['template']-1] alignpartq['partnum'] = partdict['num'] alignpartq['alignstack'] = alignstackq stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num']) alignpartq['stackpart'] = stackpartdata alignpartq['rotation'] = partdict['rot'] alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['mirror'] = partdict['mirror'] alignpartq['score'] = partdict['score'] if insert is True: alignpartq.insert() #insert each particle return
def uploadResults(self): if self.params['commit'] is False: return # Get the new file order fh=open("sort_junk.sel",'r') lines=fh.readlines() i=0; fileorder={}; for line in lines: args=line.split() if (len(args)>1): match=re.match('[A-Za-z]+([0-9]+)\.[A-Za-z]+', (args[0].split('/'))[-1]) if (match): filenumber=int(match.groups()[0]) fileorder[i]=filenumber i+=1 fh.close() # Produce a new stack oldstack = apStack.getOnlyStackData(self.params['stackid'],msg=False) newstack = appiondata.ApStackData() newstack['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) newstack['name'] = "sorted.hed" if newstack.query(results=1): apDisplay.printError("A stack with these parameters already exists") # Fill in data and submit newstack['oldstack'] = oldstack newstack['hidden'] = False newstack['substackname'] = self.params['runname'] newstack['description'] = self.params['description'] newstack['pixelsize'] = oldstack['pixelsize'] newstack['boxsize'] = oldstack['boxsize'] newstack['junksorted'] = True newstack.insert() # Insert stack images apDisplay.printMsg("Inserting stack particles") count=0 total=len(fileorder.keys()) if total==0: apDisplay.printError("No particles can be inserted in the sorted stack") for i in fileorder.keys(): count += 1 if count % 100 == 0: sys.stderr.write("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b") sys.stderr.write(str(count)+" of "+(str(total))+" complete") # Get particle from the old stack oldparticle = apStack.getStackParticle(self.params['stackid'], fileorder[i]+1) # Insert particle newparticle = appiondata.ApStackParticleData() newparticle['particleNumber'] = i+1 newparticle['stack'] = newstack newparticle['stackRun'] = oldparticle['stackRun'] newparticle['particle'] = oldparticle['particle'] newparticle['mean'] = oldparticle['mean'] newparticle['stdev'] = oldparticle['stdev'] newparticle.insert() apDisplay.printMsg("\n"+str(total)+" particles have been inserted into the sorted stack") # Insert runs in stack apDisplay.printMsg("Inserting Runs in Stack") runsinstack = apStack.getRunsInStack(self.params['stackid']) for run in runsinstack: newrunsq = appiondata.ApRunsInStackData() newrunsq['stack'] = newstack newrunsq['stackRun'] = run['stackRun'] newrunsq.insert() apDisplay.printMsg("finished") return