def getOneParticle(imgdata): """ returns paticles (as a list of dicts) for a given image ex: particles[0]['xcoord'] is the xcoord of particle 0 """ partq = appiondata.ApParticleData() partq['image'] = imgdata partd = partq.query(results=1) return partd
def getDefocPairParticles(imgdata, selectionid, particlelabel=None): ### get defocal pair if imgdata['preset'] is not None: defimgdata = apDefocalPairs.getDefocusPair(imgdata) else: # For pairs from Leginon Manual Application defimgdata = apDefocalPairs.getManualDefocusPair(imgdata) if defimgdata is None: apDisplay.printWarning( "Could not find defocal pair for image %s (id %d)" % (apDisplay.short(imgdata['filename']), imgdata.dbid)) return ([], {'shiftx': 0, 'shifty': 0, 'scale': 1}) apDisplay.printMsg( "Found defocus pair %s (id %d) for image %s (id %d)" % (apDisplay.short(defimgdata['filename']), defimgdata.dbid, apDisplay.short(imgdata['filename']), imgdata.dbid)) ### get particles partq = appiondata.ApParticleData() partq['image'] = defimgdata partq['selectionrun'] = appiondata.ApSelectionRunData.direct_query( selectionid) if particlelabel is not None: partq['label'] = particlelabel partdatas = partq.query() apDisplay.printMsg("Found %d particles for defocal pair %s (id %d)" % ( len(partdatas), apDisplay.short(defimgdata['filename']), defimgdata.dbid, )) if len(partdatas) == 0: return ([], {'shiftx': 0, 'shifty': 0, 'scale': 1}) ### get shift information has_tried = False while True: shiftq = appiondata.ApImageTransformationData() shiftq['image1'] = defimgdata shiftdatas = shiftq.query() if shiftdatas: shiftdata = shiftdatas[0] apDisplay.printMsg( "Shifting particles by %.1f,%.1f (%d X)" % (shiftdata['shiftx'], shiftdata['shifty'], shiftdata['scale'])) break else: if has_tried == True: apDisplay.printError( "No shift inserted to database after one try") apDisplay.printMsg("Calculating shift....") shiftpeak = apDefocalPairs.getShift(defimgdata, imgdata) apDefocalPairs.insertShift(defimgdata, imgdata, shiftpeak) has_tried = True return (partdatas, shiftdata)
def start(self): ### check for existing run selectrunq = appiondata.ApSelectionRunData() selectrunq['name'] = self.params['runname'] selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) selectrundata = selectrunq.query(readimages=False) if selectrundata: apDisplay.printError("Runname already exists") ### stack data stackdata = apStack.getOnlyStackData(self.params['stackid']) ### stack particles stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True) stackparts.reverse() ### selection run for first particle oldselectrun = stackparts[0]['particle']['selectionrun'] ### set selection run manualparamsq = appiondata.ApManualParamsData() manualparamsq['diam'] = self.getDiamFromSelectionRun(oldselectrun) manualparamsq['oldselectionrun'] = oldselectrun manualparamsq['trace'] = False selectrunq = appiondata.ApSelectionRunData() selectrunq['name'] = self.params['runname'] selectrunq['hidden'] = False selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) selectrunq['session'] = apStack.getSessionDataFromStackId(self.params['stackid']) selectrunq['manparams'] = manualparamsq ### insert particles apDisplay.printMsg("Inserting particles into database") count = 0 t0 = time.time() startmem = mem.active() numpart = len(stackparts) for stackpart in stackparts: count += 1 if count > 10 and count%100 == 0: perpart = (time.time()-t0)/float(count+1) apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"% (count, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart), apDisplay.timeString(perpart*(numpart-count))), "blue") oldpartdata = stackpart['particle'] newpartq = appiondata.ApParticleData(initializer=oldpartdata) newpartq['selectionrun'] = selectrunq if self.params['commit'] is True: newpartq.insert() apDisplay.printMsg("Completed in %s"%(apDisplay.timeString(time.time()-t0)))
def getParticlesForImageFromRunName(imgdata, runname): """ returns particles for a given image and selection run name """ srunq = appiondata.ApSelectionRunData() srunq['name'] = runname srunq['session'] = imgdata['session'] ptclq = appiondata.ApParticleData() ptclq['image'] = imgdata ptclq['selectionrun'] = srunq particles = ptclq.query() return particles
def getParticles(imgdata, selectionRunId, particlelabel=None): """ returns paticles (as a list of dicts) for a given image ex: particles[0]['xcoord'] is the xcoord of particle 0 """ selexonrun = appiondata.ApSelectionRunData.direct_query(selectionRunId) prtlq = appiondata.ApParticleData() prtlq['image'] = imgdata prtlq['selectionrun'] = selexonrun if particlelabel is not None: prtlq['label'] = particlelabel particles = prtlq.query() return particles
def start(self): apDisplay.printMsg("\n\n") ### get particles refrun = appiondata.ApRefineRunData.direct_query( self.params['reconid']) refiterq = appiondata.ApRefineIterData() refiterq['refineRun'] = refrun refiterq['iteration'] = self.params['iternum'] refiterdatas = refiterq.query(results=1) refpartq = appiondata.ApRefineParticleData() refpartq['refineIter'] = refiterdatas[0] #this gets lots of data refpartdatas = refpartq.query() ### get session firstrefpart = refpartdatas[0] firstpart = firstrefpart['particle']['particle'] sessiondata = firstpart['selectionrun']['session'] ### create a selection run runq = appiondata.ApSelectionRunData() for key in firstpart['selectionrun'].keys(): runq[key] = firstpart['selectionrun'][key] runq['name'] = self.params['runname'] runq['session'] = sessiondata pathq = appiondata.ApPathData() pathq['path'] = self.params['rundir'] runq['path'] = pathq runq['description'] = ( "Corrected particles from refine id %d iter %d and selection %d" % (self.params['reconid'], self.params['iternum'], firstpart['selectionrun'].dbid)) count = 0 for refpartdata in refpartdatas: count += 1 if count % 10 == 0: sys.stderr.write(".") partdata = refpartdata['particle']['particle'] newpartq = appiondata.ApParticleData() for key in partdata.keys(): newpartq[key] = partdata[key] newpartq['xcoord'] = partdata['xcoord'] + refpartdata['shiftx'] newpartq['ycoord'] = partdata['ycoord'] + refpartdata['shifty'] newpartq['selectionrun'] = runq newpartq.insert()
def getImageParticles(imagedata,stackid,nodie=True): """ Provided a Stack Id & imagedata, to find particles """ particleq = appiondata.ApParticleData(image=imagedata) stackpdata = appiondata.ApStackParticleData() stackpdata['particle'] = particleq stackpdata['stack'] = appiondata.ApStackData.direct_query(stackid) stackps = stackpdata.query() particles = [] if not stackps: if nodie is True: return particles,None apDisplay.printError("partnum="+str(particleid)+" was not found in stackid="+str(stackid)) for stackp in stackps: particles.append(stackp['particle']) return particles,stackps
def getParticlesFromStack(self, stackdata, imgdata, is_defocpair=False): """ For image (or defocal pair), imgdata get particles in corresponding stack """ if is_defocpair is True: sibling, shiftpeak = apDefocalPairs.getShiftFromImage( imgdata, self.params['sessionname']) if shiftpeak is None: return [], {'shiftx': 0, 'shifty': 0, 'scale': 1} shiftdata = { 'shiftx': shiftpeak['shift'][0], 'shifty': shiftpeak['shift'][1], 'scale': shiftpeak['scalefactor'] } searchimgdata = sibling else: searchimgdata = imgdata shiftdata = {'shiftx': 0, 'shifty': 0, 'scale': 1} partq = appiondata.ApParticleData() partq['image'] = searchimgdata stackpartq = appiondata.ApStackParticleData() stackpartq['stack'] = stackdata stackpartq['particle'] = partq stackpartdatas = stackpartq.query() partdatas = [] partorder = [] for stackpartdata in stackpartdatas: if self.params['partlimit'] and self.params[ 'partlimit'] < stackpartdata['particleNumber']: continue partdata = stackpartdata['particle'] partdatas.append(partdata) partorder.append(stackpartdata['particleNumber']) partdatas.reverse() partorder.reverse() self.writeStackParticleOrderFile(partorder) return partdatas, shiftdata
def createStackData(self): apDisplay.printColor("Starting upload of stack", "blue") pathq = appiondata.ApPathData() pathq['path'] = self.params['rundir'] manq = appiondata.ApManualParamsData() manq['diam'] = self.params['diameter'] manq['trace'] = False selectq = appiondata.ApSelectionRunData() selectq['name'] = 'fakestack_' + self.params['runname'] selectq['hidden'] = True selectq['path'] = pathq selectq['session'] = self.sessiondata selectq['manparams'] = manq stackq = appiondata.ApStackData() stackq['name'] = "start.hed" stackq['path'] = pathq stackq['description'] = self.params['description'] stackq['hidden'] = False stackq['pixelsize'] = self.params['apix'] * 1e-10 stackq['boxsize'] = self.boxsize stackq['centered'] = False stackparamq = appiondata.ApStackParamsData() stackparamq['boxSize'] = self.boxsize stackparamq['bin'] = 1 stackparamq['phaseFlipped'] = self.params['ctfcorrect'] if self.params['ctfcorrect'] is True: stackparamq['fileType'] = "manual" stackparamq['fileType'] = "imagic" stackparamq['normalized'] = self.params['normalize'] stackparamq['lowpass'] = 0 stackparamq['highpass'] = 0 stackrunq = appiondata.ApStackRunData() stackrunq['stackRunName'] = self.params['runname'] stackrunq['stackParams'] = stackparamq stackrunq['selectionrun'] = selectq stackrunq['session'] = self.sessiondata runsinstackq = appiondata.ApRunsInStackData() runsinstackq['stack'] = stackq runsinstackq['stackRun'] = stackrunq if self.params['commit'] is True: runsinstackq.insert() if 'syncstackid' in self.params.keys( ) and self.params['syncstackid']: stackdata = runsinstackq['stack'] stack2data = apStack.getOnlyStackData( self.params['syncstackid']) syncq = appiondata.ApSyncStackData( stack1=stackdata, stack2=stack2data, synctype=self.params['synctype']) syncq.insert() ### for each particle sys.stderr.write("Starting particle upload") for i in range(self.numpart): if i % 100 == 0: sys.stderr.write(".") partq = appiondata.ApParticleData() partq['image'] = None #We have no image, see if this works??? partq['selectionrun'] = selectq partq['xcoord'] = int(i % 1000) partq['ycoord'] = int(i / 1000) partq['diameter'] = self.params['diameter'] stackpartq = appiondata.ApStackParticleData() stackpartq['particleNumber'] = i + 1 stackpartq['stack'] = stackq stackpartq['stackRun'] = stackrunq stackpartq['particle'] = partq stackpartq['mean'] = 0.0 stackpartq['stdev'] = 1.0 if self.params['commit'] is True: stackpartq.insert() sys.stderr.write("\n") return
from appionlib import appiondata import leginon.leginondata import sys if __name__ == "__main__": if len(sys.argv) < 3: print "usage: ./pcount.py <session name> <particle selection run name>\n\n" sys.exit(1) ses_name = sys.argv[1] run_name = sys.argv[2] # query objects qsession = leginon.leginondata.SessionData(name=ses_name) qrun = appiondata.ApSelectionRunData(session=qsession, name=run_name) qpart = appiondata.ApParticleData(selectionrun=qrun) # all particles particles = qpart.query() # Find assessment of image that each particle is from. # This is messy because we want to query images without reading MRCs. assessments = {} akeep = [] areject = [] anone = [] for particle in particles: imgref = particle.special_getitem('image', dereference=False) imgid = imgref.dbid if imgid not in assessments: img = leginon.leginondata.AcquisitionImageData.direct_query(
def uploadData(self, ctfpartlist): ### read mean /stdev for uploading self.getPartMeanTree(os.path.join(self.params['rundir'], self.params['finalstack']), ctfpartlist) sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) if self.params['projectid'] is not None: projectnum = self.params['projectid'] else: projectnum = apProject.getProjectIdFromSessionName(self.params['sessionname']) ### create synthetic stack object ... not saving global params like runname, session, project, description, etc. here; that's in ApStackData syntheticq = appiondata.ApSyntheticStackParamsData() ### get number of fakestack runs numentries = len(syntheticq) syntheticq['modelid'] = appiondata.ApInitialModelData.direct_query(self.params['modelid']) syntheticq['boxsize'] = self.params['box'] syntheticq['apix'] = self.params['apix'] syntheticq['projcount'] = self.params['projcount'] syntheticq['projstdev'] = self.params['projstdev'] syntheticq['shiftrad'] = self.params['shiftrad'] syntheticq['rotang'] = self.params['rotang'] syntheticq['flip'] = self.params['flip'] syntheticq['kilovolts'] = self.params['kv'] syntheticq['spher_aber'] = self.params['cs'] syntheticq['defocus_x'] = self.params['df1'] syntheticq['defocus_y'] = self.params['df2'] syntheticq['randomdef'] = self.params['randomdef'] if self.params['randomdef'] is True: syntheticq['randomdef_std'] = self.params['randomdef_std'] syntheticq['astigmatism'] = self.params['astigmatism'] syntheticq['snr1'] = self.params['snr1'] syntheticq['snrtot'] = self.params['snrtot'] syntheticq['envelope'] = os.path.basename(self.params['envelopefile']) syntheticq['ace2correct'] = self.params['ace2correct'] syntheticq['ace2correct_rand'] = self.params['ace2correct_rand'] if self.params['ace2correct_rand'] is True: syntheticq['ace2correct_std'] = self.params['ace2correct_std'] syntheticq['ace2estimate'] = self.params['ace2estimate'] syntheticq['lowpass'] = self.params['lpfilt'] syntheticq['highpass'] = self.params['hpfilt'] syntheticq['norm'] = self.params['norm'] ### fill stack parameters stparamq = appiondata.ApStackParamsData() stparamq['boxSize'] = self.params['box'] stparamq['bin'] = 1 stparamq['fileType'] = "imagic" stparamq['defocpair'] = 0 stparamq['lowpass'] = self.params['lpfilt'] stparamq['highpass'] = self.params['hpfilt'] stparamq['norejects'] = 1 if self.params['invert'] is True: stparamq['inverted'] = 1 else: stparamq['inverted'] = 0 if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True: stparamq['phaseFlipped'] = 1 stparamq['fliptype'] = "ace2part" else: stparamq['phaseFlipped'] = 0 stparamq['normalized'] = self.params['norm'] paramslist = stparamq.query() ### create a stack object stackq = appiondata.ApStackData() stackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) ### see if stack already exists in the database (just checking path & name) uniqstackdatas = stackq.query(results=1) ### create a stackRun object runq = appiondata.ApStackRunData() runq['stackRunName'] = self.params['runname'] runq['session'] = sessiondata ### see if stack run already exists in the database (just checking runname & session) uniqrundatas = runq.query(results=1) ### finish stack object stackq['name'] = self.params['finalstack'] stackq['description'] = self.params['description'] stackq['hidden'] = 0 stackq['pixelsize'] = self.params['apix'] * 1e-10 stackq['boxsize'] = self.params['box'] self.stackdata = stackq ### finish stackRun object runq['stackParams'] = stparamq runq['syntheticStackParams'] = syntheticq self.stackrundata = runq ### create runinstack object rinstackq = appiondata.ApRunsInStackData() rinstackq['stackRun'] = runq ### if not in the database, make sure run doesn't already exist if not uniqstackdatas and not uniqrundatas: if self.params['commit'] is True: apDisplay.printColor("Inserting stack parameters into database", "cyan") rinstackq['stack'] = stackq rinstackq.insert() else: apDisplay.printWarning("NOT INSERTING stack parameters into database") elif uniqrundatas and not uniqstackdatas: apDisplay.printError("Weird, run data without stack already in the database") else: rinstack = rinstackq.query(results=1) prevrinstackq = appiondata.ApRunsInStackData() prevrinstackq['stackRun'] = uniqrundatas[0] prevrinstackq['stack'] = uniqstackdatas[0] prevrinstack = prevrinstackq.query(results=1) ## if no runinstack found, find out which parameters are wrong: if not rinstack: for i in uniqrundatas[0]: print "r =======",i,"========" if uniqrundatas[0][i] != runq[i]: apDisplay.printError("the value for parameter '"+str(i)+"' is different from before") else: print i,uniqrundatas[0][i],runq[i] for i in uniqrundatas[0]['stackParams']: print "p =======",i,"========" if uniqrundatas[0]['stackParams'][i] != stparamq[i]: apDisplay.printError("the value for parameter '"+str(i)+"' is different from before") else: print i, uniqrundatas[0]['stackParams'][i], stparamq[i] for i in uniqstackdatas[0]: print "s =======",i,"========" if uniqstackdatas[0][i] != stackq[i]: apDisplay.printError("the value for parameter '"+str(i)+"' is different from before") else: print i,uniqstackdatas[0][i],stackq[i] for i in prevrinstack[0]: print "rin =======",i,"========" if prevrinstack[0][i] != rinstackq[i]: print i,prevrinstack[0][i],rinstackq[i] apDisplay.printError("the value for parameter '"+str(i)+"' is different from before") else: print i,prevrinstack[0][i],rinstackq[i] apDisplay.printError("All parameters for a particular stack must be identical! \n"+\ "please check your parameter settings.") apDisplay.printWarning("Stack already exists in database! Will try and appending new particles to stack") ### create a fake selection run # selectq = appiondata.ApSelectionRunData() # selectq['session'] = sessiondata # selectq['name'] = "fakerun" # self.selectq = selectq if self.params['commit'] is True: apDisplay.printColor("Inserting fake selection parameters into the database", "cyan") # selectq.insert() else: apDisplay.printWarning("NOT INSERTING fake selection parameters into the database") partNumber = 0 ### loop over the particles and insert if self.params['commit'] is True: apDisplay.printColor("inserting particle parameters into database", "cyan") else: apDisplay.printWarning("NOT INSERTING particle parameters into database") for i in range(len(ctfpartlist)): partNumber += 1 partfile = ctfpartlist[i] partmeandict = self.partmeantree[i] partq = appiondata.ApParticleData() # partq['selectionrun'] = selectq partq['xcoord'] = partNumber stpartq = appiondata.ApStackParticleData() ### check unique params stpartq['stack'] = self.stackdata stpartq['stackRun'] = self.stackrundata stpartq['particleNumber'] = partNumber stpartdata = stpartq.query(results=1) if stpartdata: apDisplay.printError("trying to insert a duplicate particle") stpartq['particle'] = partq stpartq['mean'] = partmeandict['mean'] stpartq['stdev'] = partmeandict['stdev'] if self.params['commit'] is True: stpartq.insert() return
def insertParticlePeaks(peaktree, imgdata, runname, msg=False): """ takes an image data object (imgdata) and inserts particles into DB from peaktree """ #INFO sessiondata = imgdata['session'] imgname = imgdata['filename'] #GET RUN DATA runq = appiondata.ApSelectionRunData() runq['name'] = runname runq['session'] = sessiondata selectionruns = runq.query(results=1) if not selectionruns: apDisplay.printError("could not find selection run in database") ### WRITE PARTICLES TO DATABASE count = 0 t0 = time.time() for peakdict in peaktree: particlesq = appiondata.ApParticleData() particlesq['selectionrun'] = selectionruns[0] particlesq['image'] = imgdata if 'template' in peakdict and peakdict['template'] is not None: particlesq[ 'template'] = appiondata.ApTemplateImageData.direct_query( peakdict['template']) for key in 'correlation', 'peakmoment', 'peakstddev', 'peakarea', 'label': if key in peakdict and peakdict[key] is not None: if isinstance(peakdict[key], float): ### limit decimals particlesq[key] = round(peakdict[key], 6) else: particlesq[key] = peakdict[key] ### must be integers particlesq['xcoord'] = int(round(peakdict['xcoord'])) particlesq['ycoord'] = int(round(peakdict['ycoord'])) if 'angle' in peakdict: particlesq['angle'] = peakdict['angle'] if 'helixnum' in peakdict: particlesq['helixnum'] = peakdict['helixnum'] if 'helicalstep' in peakdict: particlesq['helicalstep'] = peakdict['helicalstep'] if 'diameter' in peakdict and peakdict['diameter'] is not None: peakdict['diameter'] = round(peakdict['diameter'], 6) if 'peakarea' in peakdict and peakdict[ 'peakarea'] is not None and peakdict['peakarea'] > 0: peakhasarea = True else: apDisplay.printWarning("peak has no area") peakhasarea = False if 'correlation' in peakdict and peakdict[ 'correlation'] is not None and peakdict['correlation'] > 2: apDisplay.printWarning("peak has correlation greater than 2.0") ### INSERT VALUES if peakhasarea is True: presult = particlesq.query() if not presult: count += 1 particlesq.insert() if msg is True: apDisplay.printMsg("inserted " + str(count) + " of " + str(len(peaktree)) + " peaks into database" + " in " + apDisplay.timeString(time.time() - t0)) return
def insertParticlePeakPairs(peaktree1, peaktree2, peakerrors, imgdata1, imgdata2, transdata, runname): """ takes both image data (imgdata) and inserts particle pairs into DB from peaktrees """ #INFO sessiondata = imgdata1['session'] legimgid1 = int(imgdata1.dbid) legimgid2 = int(imgdata2.dbid) imgname1 = imgdata1['filename'] imgname2 = imgdata2['filename'] #CHECK ARRAY LENGTHS len1 = len(peaktree1) len2 = len(peaktree2) len3 = len(peakerrors) if len1 != len2 or len2 != len3: apDisplay.printError("insertParticlePeakPairs particle arrays must have the same length "+\ str(len1)+" "+str(len2)+" "+str(len3)) #GET RUN DATA runq = appiondata.ApSelectionRunData() runq['name'] = runname runq['session'] = sessiondata selectionruns = runq.query(results=1) if not selectionruns: apDisplay.printError("could not find selection run in database") #GET TRANSFORM DATA transq = appiondata.ApImageTiltTransformData() transq['image1'] = imgdata1 transq['image2'] = imgdata2 transq['tiltrun'] = selectionruns[0] transids = transq.query(results=1) if not transids: apDisplay.printError("could not find transform id in database") ### WRITE PARTICLES TO DATABASE count = 0 t0 = time.time() last50 = time.time() apDisplay.printMsg("looping over " + str(len(peaktree1)) + " particles") for i in range(len(peaktree1)): remaining_peaks = len(peaktree1) - count if count and remaining_peaks and remaining_peaks % 50 == 0: #sys.stderr.write("<"+str(len(peaktree1)-count)) print( "%d particles remain, %s time remains, %s per particle, %s last 50 particles" % ( len(peaktree1) - count, apDisplay.timeString( (time.time() - t0) / count * (remaining_peaks)), apDisplay.timeString((time.time() - t0) / count), apDisplay.timeString(time.time() - last50), )) last50 = time.time() peakdict1 = peaktree1[i] peakdict2 = peaktree2[i] error = peakerrors[i] partq1 = appiondata.ApParticleData() partq1['selectionrun'] = selectionruns[0] partq1['image'] = imgdata1 partq1['xcoord'] = peakdict1['xcoord'] partq1['ycoord'] = peakdict1['ycoord'] partq1['peakarea'] = 1 partq2 = appiondata.ApParticleData() partq2['selectionrun'] = selectionruns[0] partq2['image'] = imgdata2 partq2['xcoord'] = peakdict2['xcoord'] partq2['ycoord'] = peakdict2['ycoord'] partq2['peakarea'] = 1 # I do NOT have to check if particles already exist, because this is a NEW selectionrun partpairq = appiondata.ApTiltParticlePairData() partpairq['particle1'] = partq1 partpairq['particle2'] = partq2 #NEED TO LOOK UP TRANSFORM DATA partpairq['transform'] = transdata #NEED TO CALCULATE ERROR, ALWAYS POSITIVE partpairq['error'] = error #presult = partpairq.query() #if not presult: count += 1 partq1.insert(force=True) partq2.insert(force=True) partpairq.insert(force=True) apDisplay.printMsg("inserted " + str(count) + " of " + str(len(peaktree1)) + " peaks into database" + " in " + apDisplay.timeString(time.time() - t0)) return