def insertAlignParticlesIntoDatabase(self, level): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") D = self.getClassificationAtLevel(level) for ref in D: ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = ref + 1 refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['iteration'] = self.params['maxiter'] if ref in self.resdict: refq['ssnr_resolution'] = self.resdict[ref] ### setup particle info ... NOTE: ALIGNMENT PARAMETERS ARE NOT SAVED IN XMIPP 2.4 for partnum in D[ref]: alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = int(partnum) + 1 alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle( self.runparams['stackid'], int(partnum) + 1) ### particle numbering starts with 0!!!!!!! alignpartq['stackpart'] = stackpartdata alignpartq['ref'] = refq ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() ### insert bad particles if len(self.badpartlist) > 0: for p in self.badpartlist: refq = appiondata.ApAlignReferenceData() refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['iteration'] = self.params['maxiter'] alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = int(p) + 1 alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle( self.runparams['stackid'], int(p) + 1) alignpartq['stackpart'] = stackpartdata alignpartq['bad'] = 1 if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(self.params['numpart']) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan") return
def insertParticlesIntoDatabase(self, stackid, partlist): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in partlist: count += 1 if count % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = partdict['refnum'] refq['iteration'] = self.lastiter refsearch = "part" + self.params['timestamp'] + "_ref*" + str( partdict['refnum']) + "*" refbase = os.path.splitext(glob.glob(refsearch)[0])[0] refq['mrcfile'] = refbase + ".mrc" refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] if partdict['refnum'] in self.resdict: refq['ssnr_resolution'] = self.resdict[partdict['refnum']] reffile = os.path.join(self.params['rundir'], refq['mrcfile']) if not os.path.isfile(reffile): emancmd = "proc2d " + refbase + ".xmp " + refbase + ".mrc" apEMAN.executeEmanCmd(emancmd, verbose=False) if not os.path.isfile(reffile): apDisplay.printError("could not find reference file: " + reffile) ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partdict['partnum'] alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle(stackid, partdict['partnum']) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['rotation'] = partdict['inplane'] alignpartq['mirror'] = partdict['mirror'] alignpartq['ref'] = refq alignpartq['spread'] = partdict['spread'] ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(count) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan") return
def insertParticlesIntoDatabase(self, stackid, partList): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in partList: count += 1 if count % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = partdict['refnum'] if refq['refnum'] < 1: apDisplay.printError("Classes must start with 1") refq['iteration'] = self.lastiter #refq['mrcfile'] = refbase+".mrc" ### need to create mrcs of each class??? refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] if partdict['refnum'] in self.resdict: refq['ssnr_resolution'] = self.resdict[partdict['refnum']] refq['isac_generation'] = partdict['generation'] ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partdict['partnum'] if alignpartq['partnum'] < 1: apDisplay.printError("Particles must start with 1") alignpartq['alignstack'] = self.alignstackdata if self.params['alignstackid'] is None: stackpartdata = apStack.getStackParticle( stackid, partdict['origPartNum']) else: stackpartdata = apStack.getStackParticleFromAlignParticle( self.params['alignstackid'], partdict['origPartNum']) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['rotation'] = partdict['inplane'] alignpartq['mirror'] = partdict['mirror'] alignpartq['ref'] = refq alignpartq['score'] = partdict['peak'] ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(count) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan") return
def insertParticlesIntoDatabase(self, partlist, insert=False): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for particle in partlist: count += 1 ### get all particle parameters partnum = count rotation = particle[0] * -1 ### in spider negative rotation is clockwise shiftx = particle[1] shifty = particle[2] ccc = particle[3] refnum = int(particle[4]) mirror = particle[5] if count % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = refnum refq['iteration'] = self.params['numiter'] refq['imagicfile'] = "references.hed" refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['templatestack'] = appiondata.ApTemplateStackData.direct_query(self.params['templateStackId']) reffile = os.path.join(self.params['rundir'], refq['imagicfile']) if not os.path.isfile(reffile): apDisplay.printError("could not find reference file: "+reffile) ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partnum alignpartq['alignstack'] = self.alignstackdata stackpartdata = apStack.getStackParticle(self.params['stackId'], partnum) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = shiftx alignpartq['yshift'] = shifty alignpartq['rotation'] = rotation alignpartq['mirror'] = mirror alignpartq['ref'] = refq alignpartq['correlation'] = ccc ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in " +apDisplay.timeString(time.time()-t0), "cyan") return
def insertAlignParticlesIntoDatabase(self): count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for ref in self.classD: ### setup reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = ref refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refq['iteration'] = 10 # if ref in self.resdict: # refq['ssnr_resolution'] = self.resdict[ref] ### setup particle info for partnum in self.classD[ref]: alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = int(partnum) alignpartq['alignstack'] = self.alignstackdata ### particle numbering starts with 1 in SIMPLE stackpartdata = apStack.getStackParticle( self.runparams['stackid'], int(partnum)) alignpartq['stackpart'] = stackpartdata alignpartq['ref'] = refq try: alignpartq['xshift'] = self.alignD[partnum - 1]['shx'] except: pass try: alignpartq['yshift'] = self.alignD[partnum - 1]['shy'] except: pass try: alignpartq['mirror'] = self.alignD[partnum - 1]['mirror'] except: pass alignpartq['rotation'] = self.alignD[partnum - 1]['rot'] ### insert if self.params['commit'] is True: inserted += 1 alignpartq.insert() apDisplay.printColor( "\ninserted " + str(inserted) + " of " + str(self.params['numpart']) + " particles into the database in " + apDisplay.timeString(time.time() - t0), "cyan") return
def insertRefBasedRun(self, partlist, imagicstack, insert=False): apDisplay.printMsg("committing results to DB") ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database") ### setup ref based run refrunq = appiondata.ApRefBasedRunData() refrunq['runname'] = self.params['runname'] refrunq['xysearch'] = self.params['xysearch'] refrunq['xystep'] = self.params['xystep'] refrunq['first_ring'] = self.params['firstring'] refrunq['last_ring'] = self.params['lastring'] refrunq['num_iter'] = self.params['numiter'] refrunq['invert_templs'] = self.params['inverttemplates'] refrunq['num_templs'] = self.params['numtemplate'] #refrunq['csym', int), refrunq['run_seconds'] = self.params['runtime'] ### finish alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['refbasedrun'] = refrunq alignrunq['hidden'] = False alignrunq['bin'] = self.params['bin'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['runname'] = self.params['runname'] alignrunq['description'] = self.params['description'] ### setup alignment stack alignstackq = appiondata.ApAlignStackData() alignstackq['alignrun'] = alignrunq alignstackq['imagicfile'] = imagicstack alignstackq['avgmrcfile'] = "average.mrc" emancmd = "proc2d templatestack%02d.spi templatestack%02d.hed"%(self.params['numiter'],self.params['numiter']) apEMAN.executeEmanCmd(emancmd) alignstackq['refstackfile'] = ("templatestack%02d.hed"%(self.params['numiter'])) alignstackq['alignrun'] = alignrunq alignstackq['iteration'] = self.params['numiter'] alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) ### check to make sure files exist imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find stack file: "+imagicfile) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: "+avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: "+refstackfile) alignstackq['stack'] = self.stack['data'] alignstackq['boxsize'] = self.boxsize alignstackq['pixelsize'] = self.stack['apix']*self.params['bin'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.params['numpart'] if insert is True: alignstackq.insert() ### insert reference data reflist = [] for j in range(self.params['numiter']): iternum = j+1 for i in range(len(self.templatelist)): refnum = i+1 templateid = self.templatelist[i] refq = appiondata.ApAlignReferenceData() refq['refnum'] = refnum refq['iteration'] = iternum refq['template'] = apTemplate.getTemplateFromId(templateid) refq['mrcfile'] = ("templateavg%02d-%02d.mrc"%(iternum,refnum)) refpath = os.path.join(self.params['rundir'], "templates") refq['path'] = appiondata.ApPathData(path=os.path.abspath(refpath)) refq['alignrun'] = alignrunq if refnum in self.resdict: refq['ssnr_resolution'] = self.resdict[refnum] if insert is True: refq.insert() if iternum == self.params['numiter']: reflist.append(refq) #refq['varmrcfile', str), #refq['frc_resolution', float), ### insert particle data apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in partlist: ### see apSpider.alignment.alignStack() for more info """ partdict.keys() 'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,... 'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,... 'mirror': checkMirror(templatenum), 'score': float(data[3]), 'rot': float(data[4]), 'xshift': float(data[5]), 'yshift': float(data[6]), """ alignpartq = appiondata.ApAlignParticleData() alignpartq['ref'] = reflist[partdict['template']-1] alignpartq['partnum'] = partdict['num'] alignpartq['alignstack'] = alignstackq stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num']) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['rotation'] = partdict['rot'] alignpartq['score'] = partdict['score'] alignpartq['mirror'] = partdict['mirror'] if insert is True: alignpartq.insert() return
def insertEdIterRun(self, insert=False): ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '" + runparams['runname'] + "' and path already exist in database") ### setup ed-iter run editrunq = appiondata.ApEdIterRunData() editrunq['runname'] = self.params['runname'] editrunq['radius'] = self.params['partrad'] editrunq['num_iter'] = self.params['numiter'] #editrunq['freealigns'] = self.params['freealigns'] ### no longer an option editrunq['invert_templs'] = self.params['inverttemplates'] editrunq['num_templs'] = self.params['numtemplate'] #editrunq['csym', int), editrunq['run_seconds'] = self.runtime ### finish alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['editerrun'] = editrunq alignrunq['hidden'] = False alignrunq['bin'] = self.params['bin'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['runname'] = self.params['runname'] alignrunq['description'] = self.params['description'] ### setup aligned stack alignstackq = appiondata.ApAlignStackData() alignstackq['alignrun'] = alignrunq alignstackq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) # stack of aligned particles alignstackq['imagicfile'] = "alignstack.hed" # final class averages alignstackq['refstackfile'] = "avg.hed" # averaged results alignstackq['avgmrcfile'] = "average.mrc" # check to be sure dbpaths to files are correct refstackfile = os.path.join( self.params['rundir'], alignstackq['refstackfile']) #final averages if not os.path.isfile(refstackfile): apDisplay.printError("could not find reference stack file: " + refstackfile) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) #averaged results if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: " + avgmrcfile) alignstackq['stack'] = self.stack['data'] alignstackq['iteration'] = self.params['numiter'] alignstackq['boxsize'] = math.floor(self.stack['boxsize'] / self.params['bin']) alignstackq['pixelsize'] = self.stack['apix'] * self.params['bin'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.params['numpart'] if insert is True: alignstackq.insert( ) #alignstackq contains alignrunq which contains editrunq ### setup data from each iteration apDisplay.printColor( "Inserting average and variance images from each iteration, please wait", "cyan") reflist = [] for j in range(self.params['numiter']): iternum = j + 1 for i in range(len(self.templatelist)): refnum = i + 1 templateid = self.templatelist[i] refq = appiondata.ApAlignReferenceData() refq['refnum'] = refnum refq['iteration'] = iternum refq['template'] = apTemplate.getTemplateFromId(templateid) refpath = os.path.join(self.params['rundir'], "templates") refq['path'] = appiondata.ApPathData( path=os.path.abspath(refpath)) refq['alignrun'] = alignrunq #refq['frc_resolution'] = #(float) avgname = os.path.join(self.params['rundir'], "r%02d/avg%03d" % (iternum, refnum)) varname = os.path.join(self.params['rundir'], "r%02d/var%03d" % (iternum, refnum)) if os.path.isfile(avgname + ".spi"): emancmd = "proc2d " + avgname + ".spi " + avgname + ".mrc" apEMAN.executeEmanCmd(emancmd) refq['mrcfile'] = (avgname + ".mrc") emancmd = "proc2d " + varname + ".spi " + varname + ".mrc" apEMAN.executeEmanCmd(emancmd) refq['varmrcfile'] = (varname + ".mrc") if insert is True: refq.insert() if iternum == self.params['numiter']: reflist.append(refq) else: reflist.append(None) ### insert particle data apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") partlist = self.readApshDocFile("apshdoc.spi", "apshdoc.pickle") for partdict in partlist: alignpartq = appiondata.ApAlignParticleData() alignpartq['ref'] = reflist[partdict['template'] - 1] alignpartq['partnum'] = partdict['num'] alignpartq['alignstack'] = alignstackq stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num']) alignpartq['stackpart'] = stackpartdata alignpartq['rotation'] = partdict['rot'] alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['mirror'] = partdict['mirror'] alignpartq['score'] = partdict['score'] if insert is True: alignpartq.insert() #insert each particle return
def commitToDatabase(self): """ insert the results into the database """ ### expected result for an alignment run: ### 1. aligned particle stack in IMAGIC ### 2. rotation, shift, and quality parameters for each particle ### 3. which particles belongs to which class ### 4. stack file with the class averages alignedstack = os.path.join(self.params['rundir'], "ptcl.hed") refstack = os.path.join(self.params['rundir'], "iter.final.hed") averagemrc = os.path.join(self.params['rundir'], "average.mrc") apStack.averageStack(alignedstack, averagemrc) particlemapping = self.determineClassOwnership() ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError( "Run name '" + runparams['runname'] + "' and path already exisclassmappingt in database") ### setup eman refine2d run emanrefinetwodq = appiondata.ApEMANRefine2dRunData() emanrefinetwodq['runname'] = self.params['runname'] emanrefinetwodq['run_seconds'] = time.time() - self.t0 emanrefinetwodq['num_iters'] = self.params['numiter'] emanrefinetwodq['num_classes'] = self.params['numclasses'] ### finish alignment run alignrunq['refine2drun'] = emanrefinetwodq alignrunq['hidden'] = False alignrunq['runname'] = self.params['runname'] alignrunq['description'] = self.params['description'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['bin'] = self.params['bin'] ### setup alignment stackalignimagicfile alignstackq = appiondata.ApAlignStackData() alignstackq['imagicfile'] = os.path.basename(alignedstack) alignstackq['avgmrcfile'] = os.path.basename(averagemrc) alignstackq['refstackfile'] = os.path.basename(refstack) alignstackq['iteration'] = self.params['numiter'] alignstackq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) alignstackq['alignrun'] = alignrunq ### check to make sure files exist alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(alignimagicfilepath): apDisplay.printError("could not find stack file: " + alignimagicfilepath) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average mrc file: " + avgmrcfile) refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) if not os.path.isfile(refstackfile): apDisplay.printErrrefqor("could not find reference stack file: " + refstackfile) ### continue setting values alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackid']) alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0] alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) * self.params['bin'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = apFile.numImagesInStack( alignimagicfilepath) ### inserting particles and references apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for emanpartnum in range(self.params['numpart']): partnum = emanpartnum + 1 if partnum % 100 == 0: sys.stderr.write(".") ### setup reference refq = appiondata.ApAlignReferenceData() refnum = particlemapping[emanpartnum] refq['refnum'] = refnum refq['iteration'] = self.params['numiter'] refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = alignrunq ### TODO: create mrc file #refq['mrcfile'] = refbase+".mrc" #reffile = os.path.join(self.params['rundir'], refq['mrcfile']) #if not os.path.isfile(reffile): # emancmd = "proc2d "+refstack+".xmp "+refstack+".mrc" # apEMAN.executeEmanCmd(emancmd, verbose=False) #if not os.path.isfile(reffile): # apDisplay.printError("could not find reference file: "+reffile) ### TODO: get resolution #refq['ssnr_resolution'] = TODO ### setup particle alignpartq = appiondata.ApAlignParticleData() alignpartq['partnum'] = partnum alignpartq['alignstack'] = alignstackq stackpartdata = apStack.getStackParticle(self.params['stackid'], partnum) alignpartq['stackpart'] = stackpartdata ### TODO: get the alignment parameters #alignpartq['xshift'] = partdict['xshift'] #alignpartq['yshift'] = partdict['yshift'] #alignpartq['rotation'] = partdict['inplane'] #alignpartq['mirror'] = partdict['mirror'] alignpartq['ref'] = refq ### TODO: get the score #alignpartq['score'] = partdict['score'] ### insert if self.params['commit'] is True: alignpartq.insert() return
def insertParticlesIntoDatabase(self, partlist, partrefdict): # insert particle alignment information into database count = 0 inserted = 0 t0 = time.time() apDisplay.printColor("\nPreparing to insert particle alignment data, please wait", "cyan") # get path data pathq = appiondata.ApPathData() pathq['path'] = os.path.abspath(self.params['rundir']) pathdata = pathq.query(results=1) pathid = pathdata[0].dbid # align run id alignrunid = self.alignstackdata['alignrun'].dbid # get stack particle ids stackpdbdict={} sqlcmd = "SELECT particleNumber,DEF_id "+ \ "FROM ApStackParticleData "+ \ "WHERE `REF|ApStackData|stack`=%i"%(self.params['stackid']) results = sinedon.directq.complexMysqlQuery('appiondata',sqlcmd) for part in results: pnum = int(part['particleNumber']) stackpdbdict[pnum]=int(part['DEF_id']) apDisplay.printColor("found "+str(len(results))+" particles in " +apDisplay.timeString(time.time()-t0), "cyan") t0 = time.time() apDisplay.printColor("\nInserting class averages into database","cyan") # insert reference image data reflistvals = [] for i in range(1,max(partrefdict.values())+1): sqlvals="(%i,%i,'%s',%i,%i)"%( \ i,self.params['currentiter'], \ os.path.basename(self.params['currentcls'])+".img", \ alignrunid,pathid) reflistvals.append(sqlvals) sqlcmd = "INSERT INTO `ApAlignReferenceData` ("+ \ "`refnum`,`iteration`,`imagicfile`,"+ \ "`REF|ApAlignRunData|alignrun`,`REF|ApPathData|path`) " sqlcmd += "VALUES "+",".join(reflistvals) sinedon.directq.complexMysqlQuery('appiondata',sqlcmd) # get DEF_ids from inserted references refq = appiondata.ApAlignReferenceData() refq['iteration'] = self.params['currentiter'] refq['imagicfile'] = os.path.basename(self.params['currentcls'])+".img" refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = self.alignstackdata['alignrun'] refresults = refq.query() # save DEF_ids to dictionary refdbiddict={} for ref in refresults: refdbiddict[ref['refnum']]=ref.dbid apDisplay.printColor("inserted "+str(len(refdbiddict))+" class averages in " +apDisplay.timeString(time.time()-t0), "cyan") t0 = time.time() apDisplay.printColor("\nAssembling database insertion command","cyan") partlistvals = [] for partdict in partlist: count += 1 if count % (len(partlist)/100) == 0: pleft = int(float(count)/len(partlist)*100) perpart = (time.time()-t0)/count tleft = (len(partlist)-count)*perpart sys.stderr.write("%3i%% complete, %s left \r"%(pleft,apDisplay.timeString(tleft))) partnum = int(partdict['partnum']) refnum = partrefdict[partnum] refnum_dbid = refdbiddict[refnum] stackpart_dbid = stackpdbdict[partnum] sqlvals = "(%i,%i,%i,%s,%s,%s,%i,%s,%i)"%( \ partdict['partnum'], alignrunid, stackpart_dbid, \ partdict['xshift'], partdict['yshift'], \ partdict['inplane'], partdict['mirror'], \ partdict['cc'],refnum_dbid) partlistvals.append(sqlvals) sys.stderr.write("100% complete\t\n") apDisplay.printColor("Inserting particle information into database","cyan") # start big insert cmd sqlstart = "INSERT INTO `ApAlignParticleData` (" + \ "`partnum`,`REF|ApAlignStackData|alignstack`," + \ "`REF|ApStackParticleData|stackpart`," + \ "`xshift`,`yshift`,`rotation`,`mirror`," + \ "`correlation`,`REF|ApAlignReferenceData|ref`) " + \ "VALUES " # break up command into groups of 100K inserts # this is a workaround for the max_allowed_packet at 16MB n = 100000 sqlinserts = [partlistvals[i:i+n] \ for i in range(0, len(partlistvals), n)] for sqlinsert in sqlinserts: sqlcmd=sqlstart+",".join(sqlinsert) sinedon.directq.complexMysqlQuery('appiondata',sqlcmd) apDisplay.printColor("\nInserted "+ str(count)+" particles into the database in " + apDisplay.timeString(time.time()-t0), "cyan")
def insertNoRefRun(self, imagicstack, insert=False): ### setup alignment run alignrunq = appiondata.ApAlignRunData() alignrunq['runname'] = self.params['runname'] alignrunq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) uniquerun = alignrunq.query(results=1) if uniquerun: apDisplay.printError("Run name '" + self.params['runname'] + "' and path already exist in database") # create a norefParam object norefq = appiondata.ApSpiderNoRefRunData() norefq['runname'] = self.params['runname'] norefq['particle_diam'] = 2.0 * self.params['partrad'] norefq['first_ring'] = self.params['firstring'] norefq['last_ring'] = self.params['lastring'] norefq['init_method'] = self.params['initmethod'] norefq['run_seconds'] = self.runtime ### finish alignment run alignrunq['norefrun'] = norefq alignrunq['hidden'] = False alignrunq['bin'] = self.params['bin'] alignrunq['hp_filt'] = self.params['highpass'] alignrunq['lp_filt'] = self.params['lowpass'] alignrunq['description'] = self.params['description'] # STOP HERE ### setup alignment stack alignstackq = appiondata.ApAlignStackData() alignstackq['alignrun'] = alignrunq alignstackq['imagicfile'] = os.path.basename(imagicstack) alignstackq['avgmrcfile'] = "average.mrc" alignstackq['alignrun'] = alignrunq alignstackq['iteration'] = 0 alignstackq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) ### check to make sure files exist imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find stack file: " + imagicfile) avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) if not os.path.isfile(avgmrcfile): apDisplay.printError("could not find average file: " + avgmrcfile) alignstackq['stack'] = self.stack['data'] alignstackq['boxsize'] = math.floor(self.stack['boxsize'] / self.params['bin']) alignstackq['pixelsize'] = self.stack['apix'] * self.params['bin'] alignstackq['description'] = self.params['description'] alignstackq['hidden'] = False alignstackq['num_particles'] = self.params['numpart'] if insert is True: alignstackq.insert() ### create reference refq = appiondata.ApAlignReferenceData() refq['refnum'] = 1 refq['iteration'] = 0 refq['mrcfile'] = "template.mrc" #refpath = os.path.abspath(os.path.join(self.params['rundir'], "alignment")) refq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) refq['alignrun'] = alignrunq ### insert particle data apDisplay.printColor("Inserting particle alignment data, please wait", "cyan") for partdict in self.partlist: ### see apSpider.alignment.alignStack() for more info """ partdict.keys() 'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,... 'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,... 'mirror': checkMirror(templatenum), 'score': float(data[3]), 'rot': float(data[4]), 'xshift': float(data[5]), 'yshift': float(data[6]), """ alignpartq = appiondata.ApAlignParticleData() alignpartq['ref'] = refq alignpartq['partnum'] = partdict['num'] alignpartq['alignstack'] = alignstackq stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num']) alignpartq['stackpart'] = stackpartdata alignpartq['xshift'] = partdict['xshift'] alignpartq['yshift'] = partdict['yshift'] alignpartq['rotation'] = partdict['rot'] #alignpartq['score'] = partdict['score'] if insert is True: alignpartq.insert() return