def start(self): ### check for existing run selectrunq = appiondata.ApSelectionRunData() selectrunq['name'] = self.params['runname'] selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) selectrundata = selectrunq.query(readimages=False) if selectrundata: apDisplay.printError("Runname already exists") ### stack data stackdata = apStack.getOnlyStackData(self.params['stackid']) ### stack particles stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True) stackparts.reverse() ### selection run for first particle oldselectrun = stackparts[0]['particle']['selectionrun'] ### set selection run manualparamsq = appiondata.ApManualParamsData() manualparamsq['diam'] = self.getDiamFromSelectionRun(oldselectrun) manualparamsq['oldselectionrun'] = oldselectrun manualparamsq['trace'] = False selectrunq = appiondata.ApSelectionRunData() selectrunq['name'] = self.params['runname'] selectrunq['hidden'] = False selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) selectrunq['session'] = apStack.getSessionDataFromStackId(self.params['stackid']) selectrunq['manparams'] = manualparamsq ### insert particles apDisplay.printMsg("Inserting particles into database") count = 0 t0 = time.time() startmem = mem.active() numpart = len(stackparts) for stackpart in stackparts: count += 1 if count > 10 and count%100 == 0: perpart = (time.time()-t0)/float(count+1) apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"% (count, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart), apDisplay.timeString(perpart*(numpart-count))), "blue") oldpartdata = stackpart['particle'] newpartq = appiondata.ApParticleData(initializer=oldpartdata) newpartq['selectionrun'] = selectrunq if self.params['commit'] is True: newpartq.insert() apDisplay.printMsg("Completed in %s"%(apDisplay.timeString(time.time()-t0)))
def checkPreviousTemplateRun(self): ### check if we have a previous selection run selectrunq = appiondata.ApSelectionRunData() selectrunq['name'] = self.params['runname'] selectrunq[ 'session'] = sessiondata = apDatabase.getSessionDataFromSessionName( self.params['sessionname']) rundatas = selectrunq.query(results=1) if not rundatas: return True rundata = rundatas[0] ### check if we have a previous template run templaterunq = appiondata.ApTemplateRunData(selectionrun=rundata) templatedatas = templaterunq.query() if not templatedatas: return True ### make sure of using same number of templates if len(self.params['templateIds']) != len(templatedatas): apDisplay.printError("different number of templates from last run") ### make sure we have same rotation parameters for i, templateid in enumerate(self.params['templateIds']): templaterunq = appiondata.ApTemplateRunData() templaterunq['selectionrun'] = rundata templaterunq[ 'template'] = appiondata.ApTemplateImageData.direct_query( templateid) ### this is wrong only check last template run not this run queryresult = templaterunq.query(results=1) if queryresult: templatedata = queryresult[0] return True
def getNumSelectionRunsFromSession(sessionname): sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) selectq = appiondata.ApSelectionRunData() selectq['session'] = sessiondata selectdatas = selectq.query() if not selectdatas: return 0 return len(selectdatas)
def getRecentSelectionIdFromSession(sessionname): sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) selectq = appiondata.ApSelectionRunData() selectq['session'] = sessiondata selectdatas = selectq.query(results=1) if not selectdatas: return None return selectdatas[0].dbid
def getImagePixelSizeFromContourId(contourid): # This returns pixelsize in Angstrom q = appiondata.ApSelectionRunData() rundata = q.direct_query(contourid) q = appiondata.ApContourData(selectionrun=rundata) r = q.query(results=1) lastimagedata = r[0]['image'] apix = apDatabase.getPixelSize(lastimagedata) return apix
def insertCenterSelectionRunData(self, sessiondata, selectionrunname): manparamsq = appiondata.ApManualParamsData() manparamsq['bin'] = self.fullbin manparamsq['tomocenter'] = True runq = appiondata.ApSelectionRunData(session=sessiondata, name=selectionrunname, manparams=manparamsq) runq.insert() self.params['selexonId'] = runq.dbid
def getContourPickerDataFileName(sessionname,contourid): q = appiondata.ApSelectionRunData() rundata = q.direct_query(contourid) if rundata: contourrunpath = rundata['path']['path'] datafilepath = os.path.join(contourrunpath,'contourpickerData-'+sessionname+'.txt') if os.path.isfile(datafilepath): return datafilepath return False
def commitSizingRun(params, method=None): q = appiondata.ApSelectionRunData() tracerundata = q.direct_query(params['contourid']) pathq = appiondata.ApPathData() pathq['path'] = params['rundir'] q = q.direct_query(params['contourid']) q = appiondata.ApSizingRunData() q['name'] = params['runname'] q['tracerun'] = tracerundata q['path'] = pathq q['method'] = method q.insert() return q
def getParticlesForImageFromRunName(imgdata, runname): """ returns particles for a given image and selection run name """ srunq = appiondata.ApSelectionRunData() srunq['name'] = runname srunq['session'] = imgdata['session'] ptclq = appiondata.ApParticleData() ptclq['image'] = imgdata ptclq['selectionrun'] = srunq particles = ptclq.query() return particles
def checkPreviousTemplateRun(self): ### check if we have a previous selection run selectrunq = appiondata.ApSelectionRunData() selectrunq['name'] = self.params['runname'] selectrunq[ 'session'] = sessiondata = apDatabase.getSessionDataFromSessionName( self.params['sessionname']) rundatas = selectrunq.query(results=1) if not rundatas: return True rundata = rundatas[0] ### check if we have a previous template run templaterunq = appiondata.ApTemplateRunData(selectionrun=rundata) templatedatas = templaterunq.query() if not templatedatas: return True ### make sure of using same number of templates if len(self.params['templateIds']) != len(templatedatas): apDisplay.printError("different number of templates from last run") ### make sure we have same rotation parameters for i, templateid in enumerate(self.params['templateIds']): templaterunq = appiondata.ApTemplateRunData() templaterunq['selectionrun'] = rundata templaterunq[ 'template'] = appiondata.ApTemplateImageData.direct_query( templateid) ### this is wrong only check last template run not this run templatedata = templaterunq.query(results=1)[0] if (templatedata['range_start'] != self.params["startang" + str(i + 1)] or templatedata['range_end'] != self.params["endang" + str(i + 1)] or templatedata['range_incr'] != self.params["incrang" + str(i + 1)]): print i + 1, templateid print templatedata['range_start'], self.params["startang" + str(i + 1)] print templatedata['range_end'], self.params["endang" + str(i + 1)] print templatedata['range_incr'], self.params["incrang" + str(i + 1)] apDisplay.printWarning( "different template search ranges from last run") return True
def start(self): apDisplay.printMsg("\n\n") ### get particles refrun = appiondata.ApRefineRunData.direct_query( self.params['reconid']) refiterq = appiondata.ApRefineIterData() refiterq['refineRun'] = refrun refiterq['iteration'] = self.params['iternum'] refiterdatas = refiterq.query(results=1) refpartq = appiondata.ApRefineParticleData() refpartq['refineIter'] = refiterdatas[0] #this gets lots of data refpartdatas = refpartq.query() ### get session firstrefpart = refpartdatas[0] firstpart = firstrefpart['particle']['particle'] sessiondata = firstpart['selectionrun']['session'] ### create a selection run runq = appiondata.ApSelectionRunData() for key in firstpart['selectionrun'].keys(): runq[key] = firstpart['selectionrun'][key] runq['name'] = self.params['runname'] runq['session'] = sessiondata pathq = appiondata.ApPathData() pathq['path'] = self.params['rundir'] runq['path'] = pathq runq['description'] = ( "Corrected particles from refine id %d iter %d and selection %d" % (self.params['reconid'], self.params['iternum'], firstpart['selectionrun'].dbid)) count = 0 for refpartdata in refpartdatas: count += 1 if count % 10 == 0: sys.stderr.write(".") partdata = refpartdata['particle']['particle'] newpartq = appiondata.ApParticleData() for key in partdata.keys(): newpartq[key] = partdata[key] newpartq['xcoord'] = partdata['xcoord'] + refpartdata['shiftx'] newpartq['ycoord'] = partdata['ycoord'] + refpartdata['shifty'] newpartq['selectionrun'] = runq newpartq.insert()
def insertManualParams(self): runq = appiondata.ApSelectionRunData() runq['name'] = self.params['runname'] runq['session'] = self.sessiondata runq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) rundatas = runq.query(results=1) if rundatas and rundatas[0]['params']['diam'] != self.params['diam']: apDisplay.printError("upload diameter not the same as last run") manparams = appiondata.ApSelectionParamsData() manparams['diam'] = self.params['diam'] runq['params'] = manparams if self.params['commit'] is True: apDisplay.printColor( "Inserting manual selection run into database", "green") runq.insert() return runq
def insertSubTomoRun(sessiondata, selectionrunid, stackid, name, invert=False, subbin=1): if selectionrunid: qpick = appiondata.ApSelectionRunData() pickdata = qpick.direct_query(selectionrunid) else: pickdata = None if stackid: qstack = appiondata.ApStackData() stackdata = qstack.direct_query(stackid) else: stackdata = None qrun = appiondata.ApSubTomogramRunData(session=sessiondata, pick=pickdata, stack=stackdata, runname=name, invert=invert, subbin=subbin) return publish(qrun)
def commitRunToDatabase(self, sessiondata, insert=True): ### this dict maps the contents of self.params to the sinedon object dbmap = { 'diam': 'diam', 'bin': 'bin', 'lp': 'lp_filt', 'hp': 'hp_filt', 'pixlimit': 'pixel_value_limit', 'invert': 'invert', #'thresh': 'threshold', # major bug, do not overwrite dict variables!!! 'thresh': 'manual_thresh', 'maxthresh': 'max_threshold', 'maxpeaks': 'max_peaks', 'maxsize': 'maxsize', 'median': 'median', 'defocpair': 'defocal_pairs', 'overlapmult': 'overlapmult', 'helicalinsert': 'helicalinsert', 'helicalstep': 'helicalstep', } paramQuery = self.getParticleParamsData() #insert common parameters for pkey, dbkey in dbmap.items(): if (dbkey in paramQuery and pkey in self.params and self.params[pkey] is not None): paramQuery[dbkey] = self.params[pkey] runq = appiondata.ApSelectionRunData() runq['name'] = self.params['runname'] runq['session'] = sessiondata rundatas = runq.query(results=1) if rundatas: #get previous params if isinstance(paramQuery, appiondata.ApSelectionParamsData): paramData = rundatas[0]['params'] elif isinstance(paramQuery, appiondata.ApDogParamsData): paramData = rundatas[0]['dogparams'] elif isinstance(paramQuery, appiondata.ApManualParamsData): paramData = rundatas[0]['manparams'] elif isinstance(paramQuery, appiondata.ApTiltAlignParamsData): paramData = rundatas[0]['tiltparams'] else: apDisplay.printError( "selection run does not have valid parameter data\n") #make sure all params are the same as previous session if not paramData: apDisplay.printError("No parameters\n") else: for key in paramQuery: if paramData[key] != paramQuery[key] and paramData[ key] is not None: try: data_dbid = paramData[key].dbid query_dbid = paramQuery[key].dbid if data_dbid != query_dbid: apDisplay.printWarning( str(key) + ":" + str(paramQuery[key].dbid) + " not equal to " + str(paramData[key].dbid)) apDisplay.printError( "All parameters for a picker run name must be identical\n" ) except: apDisplay.printWarning( str(key) + ":" + str(paramQuery[key]) + " not equal to " + str(paramData[key])) apDisplay.printError( "All parameters for a picker run name must be identical\n" ) #if I made it here all parameters are the same, so it isn't necessary to commit return rundatas[0] #set params for run if isinstance(paramQuery, appiondata.ApSelectionParamsData): runq['params'] = paramQuery elif isinstance(paramQuery, appiondata.ApDogParamsData): runq['dogparams'] = paramQuery elif isinstance(paramQuery, appiondata.ApManualParamsData): runq['manparams'] = paramQuery elif isinstance(paramQuery, appiondata.ApTiltAlignParamsData): runq['tiltparams'] = paramQuery else: apDisplay.printError( "self.getParticleParamsData() did not return valid parameter data\n" ) #create path runq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) #add description if self.params['description']: runq['description'] = self.params['description'] if insert is True: runq.insert() return runq
def insertTiltTransform(imgdata1, imgdata2, tiltparams, params): #First we need to sort imgdata #'07aug30b_a_00013gr_00010sq_v01_00002sq_v01_00016en_00' #'07aug30b_a_00013gr_00010sq_v01_00002sq_01_00016en_01' #last two digits confer order, but then the transform changes... bin = params['bin'] ### first find the runid runq = appiondata.ApSelectionRunData() runq['name'] = params['runname'] runq['session'] = imgdata1['session'] rundatas = runq.query(results=1, readimages=False) if not rundatas: apDisplay.printError("could not find runid in database") ### the order is specified by 1,2; so don't change it let makestack figure it out for imgdata in (imgdata1, imgdata2): for index in ("1","2"): transq = appiondata.ApImageTiltTransformData() transq["image"+index] = imgdata transq['tiltrun'] = rundatas[0] transdata = transq.query(readimages=False) if transdata: apDisplay.printWarning("Transform values already in database for "+imgdata['filename']) return transdata[0] ### prepare the insertion transq = appiondata.ApImageTiltTransformData() transq['image1'] = imgdata1 transq['image2'] = imgdata2 transq['tiltrun'] = rundatas[0] dbdict = tiltPickerToDbNames(tiltparams) if dbdict is None: return None #Can I do for key in appiondata.ApImageTiltTransformData() ro transq??? for key in ('image1_x','image1_y','image1_rotation','image2_x','image2_y','image2_rotation','scale_factor','tilt_angle', 'overlap'): if key not in dbdict: apDisplay.printError("Key: "+key+" was not found in transformation data") for key,val in dbdict.items(): #print key if re.match("image[12]_[xy]", key): transq[key] = round(val*bin,2) else: transq[key] = val #print i,v ### this overlap is wrong because the images are binned by 'bin' and now we give it the full image """ imgShape1 = numpy.asarray(imgdata1['image'].shape, dtype=numpy.int8)/params['bin'] image1 = numpy.ones(imgShape1) imgShape2 = numpy.asarray(imgdata2['image'].shape, dtype=numpy.int8)/params['bin'] image2 = numpy.ones(imgShape2) bestOverlap, tiltOverlap = apTiltTransform.getOverlapPercent(image1, image2, tiltparams) print "image overlaps", bestOverlap, tiltOverlap transq['overlap'] = round(bestOverlap,5) """ apDisplay.printMsg("Inserting transform between "+apDisplay.short(imgdata1['filename'])+\ " and "+apDisplay.short(imgdata2['filename'])+" into database") transq.insert() apDisplay.printMsg("done") return transq
def getSelectionRunDataFromName(imgdata, runname): srunq = appiondata.ApSelectionRunData() srunq['name'] = runname srunq['session'] = imgdata['session'] selectionrundata = srunq.query() return selectionrundata[0]
from appionlib import appiondata import leginon.leginondata import sys if __name__ == "__main__": if len(sys.argv) < 3: print "usage: ./pcount.py <session name> <particle selection run name>\n\n" sys.exit(1) ses_name = sys.argv[1] run_name = sys.argv[2] # query objects qsession = leginon.leginondata.SessionData(name=ses_name) qrun = appiondata.ApSelectionRunData(session=qsession, name=run_name) qpart = appiondata.ApParticleData(selectionrun=qrun) # all particles particles = qpart.query() # Find assessment of image that each particle is from. # This is messy because we want to query images without reading MRCs. assessments = {} akeep = [] areject = [] anone = [] for particle in particles: imgref = particle.special_getitem('image', dereference=False) imgid = imgref.dbid if imgid not in assessments:
def insertParticlePeaks(peaktree, imgdata, runname, msg=False): """ takes an image data object (imgdata) and inserts particles into DB from peaktree """ #INFO sessiondata = imgdata['session'] imgname = imgdata['filename'] #GET RUN DATA runq = appiondata.ApSelectionRunData() runq['name'] = runname runq['session'] = sessiondata selectionruns = runq.query(results=1) if not selectionruns: apDisplay.printError("could not find selection run in database") ### WRITE PARTICLES TO DATABASE count = 0 t0 = time.time() for peakdict in peaktree: particlesq = appiondata.ApParticleData() particlesq['selectionrun'] = selectionruns[0] particlesq['image'] = imgdata if 'template' in peakdict and peakdict['template'] is not None: particlesq[ 'template'] = appiondata.ApTemplateImageData.direct_query( peakdict['template']) for key in 'correlation', 'peakmoment', 'peakstddev', 'peakarea', 'label': if key in peakdict and peakdict[key] is not None: if isinstance(peakdict[key], float): ### limit decimals particlesq[key] = round(peakdict[key], 6) else: particlesq[key] = peakdict[key] ### must be integers particlesq['xcoord'] = int(round(peakdict['xcoord'])) particlesq['ycoord'] = int(round(peakdict['ycoord'])) if 'angle' in peakdict: particlesq['angle'] = peakdict['angle'] if 'helixnum' in peakdict: particlesq['helixnum'] = peakdict['helixnum'] if 'helicalstep' in peakdict: particlesq['helicalstep'] = peakdict['helicalstep'] if 'diameter' in peakdict and peakdict['diameter'] is not None: peakdict['diameter'] = round(peakdict['diameter'], 6) if 'peakarea' in peakdict and peakdict[ 'peakarea'] is not None and peakdict['peakarea'] > 0: peakhasarea = True else: apDisplay.printWarning("peak has no area") peakhasarea = False if 'correlation' in peakdict and peakdict[ 'correlation'] is not None and peakdict['correlation'] > 2: apDisplay.printWarning("peak has correlation greater than 2.0") ### INSERT VALUES if peakhasarea is True: presult = particlesq.query() if not presult: count += 1 particlesq.insert() if msg is True: apDisplay.printMsg("inserted " + str(count) + " of " + str(len(peaktree)) + " peaks into database" + " in " + apDisplay.timeString(time.time() - t0)) return
def runRelionPreprocess(self, newstackroot): ''' 1. Use stackIntoPicks.py to extract the particle locations from the selected stack. 2. Run makestack2.py without ctf correction or normalization using the stackIntoPicks result as the Particles run. 3. Run relion_preprocess with rescale and norm. Outputs .mrcs file. Neil: most of these steps could be done more generally ''' apDisplay.printWarning("Making a new stack from original images") # Build the stackIntoPicks command apDisplay.printMsg( 'Extracting the particle locations from your selected stack.') newstackrunname = self.params['runname'] + "_particles" newstackrundir = self.params['rundir'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid sessionid = int(self.params['expid']) cmd = "stackIntoPicks.py " cmd += (" --stackid=%d --projectid=%d --runname=%s --rundir=%s " % (stackid, projectid, newstackrunname, newstackrundir)) cmd += (" --commit --expId=%d --jobtype=stackintopicks " % (sessionid)) # Run the command logfilepath = os.path.join(newstackrundir, 'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd, logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Build the makestack2 command ''' This is the command we want to make a stack from a stackIntoPicks run makestack2.py --single=start.hed --selectionid=130 --invert --boxsize=320 --bin=2 --description="made from stackrun1 using stackintopicks" --runname=stack66 --rundir=/ami/data00/appion/zz07jul25b/stacks/stack66 --commit --preset=en --projectid=303 --session=zz07jul25b --no-rejects --no-wait --continue --expid=8556 --jobtype=makestack2 --ppn=1 --nodes=1 --walltime=240 --jobid=2016 ''' apDisplay.printMsg( 'Using selected stack particle locations to make a Relion ready stack....' ) # Get the ID of the stackIntoPicks run we just created #apParticle.getSelectionIdFromName(runname, sessionname) runq = appiondata.ApSelectionRunData() runq['name'] = newstackrunname runq['session'] = leginondata.SessionData.direct_query( self.params['expid']) rundatas = runq.query(results=1) print rundatas if rundatas: selectionid = rundatas[0].dbid else: apDisplay.printError( "Error creating Relion ready stack. Could not find stackIntoPicks.py data in database.\n" ) # Gather all the makestack parameters totalpart = self.originalStackData.numpart numpart = totalpart if not self.params['last'] else min( self.params['last'], totalpart) stackpathname = os.path.basename(self.originalStackData.path) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir, 'start.hed') presetname = self.originalStackData.preset # binning is combination of the original binning of the stack and the preparation binnning bin = self.originalStackData.bin * self.params['bin'] unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin lowpasstext = setArgText( 'lowpass', (self.params['lowpass'], self.originalStackData.lowpass), False) highpasstext = setArgText( 'highpass', (self.params['highpass'], self.originalStackData.highpass), True) partlimittext = setArgText('partlimit', (numpart, ), False) xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'], ), True) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid reversetext = '--reverse' if self.originalStackData.reverse else '' defoctext = '--defocpair' if self.originalStackData.defocpair else '' inverttext = '--no-invert' if not self.invert else '' # Build the makestack2 command cmd = "makestack2.py " cmd += (" --single=%s --selectionid=%d %s --boxsize=%d --bin=%d " % (os.path.basename(newstackimagicfile), selectionid, inverttext, unbinnedboxsize, bin)) cmd += ( " --description='Relion refinestack based on %s(id=%d)' --projectid=%d " % (stackpathname, stackid, projectid)) cmd += ( " --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d " % (presetname, newstackrunname, newstackrundir, sessionname, sessionid)) cmd += " --no-wait --no-commit --no-continue --jobtype=makestack2 " # Run the command logfilepath = os.path.join(newstackrundir, 'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd, logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Clean up boxfiles = glob.glob("*.box") for boxfile in boxfiles: apFile.removeFile(boxfile) # Make sure our new stack params reflects the changes made # Use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] self.stack['file'] = newstackroot + '.hed' # Run Relion pre-process command ''' Setup the follow relion preprocess command to normalize the new stack: relion_preprocess \ --o particles --norm --bg_radius 60 --white_dust -1 --black_dust -1 \ --operate_on /ami/data00/appion/zz07jul25b/stacks/stack63_no_xmipp_norm/start.hed ''' apDisplay.printMsg( 'Running Relion preprocessing to normalize your Relion Ready stack....' ) bg_radius = math.floor((self.stack['boxsize'] / 2) - 1) relioncmd = "relion_preprocess " relioncmd += " --o particles --norm --bg_radius %d " % (bg_radius) relioncmd += " --white_dust -1 --black_dust -1 --operate_on %s " % ( newstackimagicfile) apParam.runCmd(relioncmd, package="Relion", verbose=True, showcmd=True) self.stack['file'] = 'particles.mrcs'
def createStackData(self): apDisplay.printColor("Starting upload of stack", "blue") pathq = appiondata.ApPathData() pathq['path'] = self.params['rundir'] manq = appiondata.ApManualParamsData() manq['diam'] = self.params['diameter'] manq['trace'] = False selectq = appiondata.ApSelectionRunData() selectq['name'] = 'fakestack_' + self.params['runname'] selectq['hidden'] = True selectq['path'] = pathq selectq['session'] = self.sessiondata selectq['manparams'] = manq stackq = appiondata.ApStackData() stackq['name'] = "start.hed" stackq['path'] = pathq stackq['description'] = self.params['description'] stackq['hidden'] = False stackq['pixelsize'] = self.params['apix'] * 1e-10 stackq['boxsize'] = self.boxsize stackq['centered'] = False stackparamq = appiondata.ApStackParamsData() stackparamq['boxSize'] = self.boxsize stackparamq['bin'] = 1 stackparamq['phaseFlipped'] = self.params['ctfcorrect'] if self.params['ctfcorrect'] is True: stackparamq['fileType'] = "manual" stackparamq['fileType'] = "imagic" stackparamq['normalized'] = self.params['normalize'] stackparamq['lowpass'] = 0 stackparamq['highpass'] = 0 stackrunq = appiondata.ApStackRunData() stackrunq['stackRunName'] = self.params['runname'] stackrunq['stackParams'] = stackparamq stackrunq['selectionrun'] = selectq stackrunq['session'] = self.sessiondata runsinstackq = appiondata.ApRunsInStackData() runsinstackq['stack'] = stackq runsinstackq['stackRun'] = stackrunq if self.params['commit'] is True: runsinstackq.insert() if 'syncstackid' in self.params.keys( ) and self.params['syncstackid']: stackdata = runsinstackq['stack'] stack2data = apStack.getOnlyStackData( self.params['syncstackid']) syncq = appiondata.ApSyncStackData( stack1=stackdata, stack2=stack2data, synctype=self.params['synctype']) syncq.insert() ### for each particle sys.stderr.write("Starting particle upload") for i in range(self.numpart): if i % 100 == 0: sys.stderr.write(".") partq = appiondata.ApParticleData() partq['image'] = None #We have no image, see if this works??? partq['selectionrun'] = selectq partq['xcoord'] = int(i % 1000) partq['ycoord'] = int(i / 1000) partq['diameter'] = self.params['diameter'] stackpartq = appiondata.ApStackParticleData() stackpartq['particleNumber'] = i + 1 stackpartq['stack'] = stackq stackpartq['stackRun'] = stackrunq stackpartq['particle'] = partq stackpartq['mean'] = 0.0 stackpartq['stdev'] = 1.0 if self.params['commit'] is True: stackpartq.insert() sys.stderr.write("\n") return
def insertParticlePeakPairs(peaktree1, peaktree2, peakerrors, imgdata1, imgdata2, transdata, runname): """ takes both image data (imgdata) and inserts particle pairs into DB from peaktrees """ #INFO sessiondata = imgdata1['session'] legimgid1 = int(imgdata1.dbid) legimgid2 = int(imgdata2.dbid) imgname1 = imgdata1['filename'] imgname2 = imgdata2['filename'] #CHECK ARRAY LENGTHS len1 = len(peaktree1) len2 = len(peaktree2) len3 = len(peakerrors) if len1 != len2 or len2 != len3: apDisplay.printError("insertParticlePeakPairs particle arrays must have the same length "+\ str(len1)+" "+str(len2)+" "+str(len3)) #GET RUN DATA runq = appiondata.ApSelectionRunData() runq['name'] = runname runq['session'] = sessiondata selectionruns = runq.query(results=1) if not selectionruns: apDisplay.printError("could not find selection run in database") #GET TRANSFORM DATA transq = appiondata.ApImageTiltTransformData() transq['image1'] = imgdata1 transq['image2'] = imgdata2 transq['tiltrun'] = selectionruns[0] transids = transq.query(results=1) if not transids: apDisplay.printError("could not find transform id in database") ### WRITE PARTICLES TO DATABASE count = 0 t0 = time.time() last50 = time.time() apDisplay.printMsg("looping over " + str(len(peaktree1)) + " particles") for i in range(len(peaktree1)): remaining_peaks = len(peaktree1) - count if count and remaining_peaks and remaining_peaks % 50 == 0: #sys.stderr.write("<"+str(len(peaktree1)-count)) print( "%d particles remain, %s time remains, %s per particle, %s last 50 particles" % ( len(peaktree1) - count, apDisplay.timeString( (time.time() - t0) / count * (remaining_peaks)), apDisplay.timeString((time.time() - t0) / count), apDisplay.timeString(time.time() - last50), )) last50 = time.time() peakdict1 = peaktree1[i] peakdict2 = peaktree2[i] error = peakerrors[i] partq1 = appiondata.ApParticleData() partq1['selectionrun'] = selectionruns[0] partq1['image'] = imgdata1 partq1['xcoord'] = peakdict1['xcoord'] partq1['ycoord'] = peakdict1['ycoord'] partq1['peakarea'] = 1 partq2 = appiondata.ApParticleData() partq2['selectionrun'] = selectionruns[0] partq2['image'] = imgdata2 partq2['xcoord'] = peakdict2['xcoord'] partq2['ycoord'] = peakdict2['ycoord'] partq2['peakarea'] = 1 # I do NOT have to check if particles already exist, because this is a NEW selectionrun partpairq = appiondata.ApTiltParticlePairData() partpairq['particle1'] = partq1 partpairq['particle2'] = partq2 #NEED TO LOOK UP TRANSFORM DATA partpairq['transform'] = transdata #NEED TO CALCULATE ERROR, ALWAYS POSITIVE partpairq['error'] = error #presult = partpairq.query() #if not presult: count += 1 partq1.insert(force=True) partq2.insert(force=True) partpairq.insert(force=True) apDisplay.printMsg("inserted " + str(count) + " of " + str(len(peaktree1)) + " peaks into database" + " in " + apDisplay.timeString(time.time() - t0)) return