def commitScaledStack(stackdata, params): #make new params query newstackparamsq = appiondata.ApStackParamsData() for key in newstackparamsq.keys(): if key != 'bin': newstackparamsq[key] = stackdata[0]['stackRun']['stackParams'][key] newstackparamsq['bin'] = params['bin'] #make new stack query newstackq = appiondata.ApStackData() newstackq['path'] = appiondata.ApPathData( path=os.path.abspath(params['newstackpath'])) newstackq['name'] = params['newstackname'] newstackq['description'] = params['description'] newstackdata = newstackq.query() if newstackdata: print "A stack with these parameters already exists" return #make new run query #first check that run name doesn't already exist newstackrunq = appiondata.ApStackRunData() newstackrunq['stackRunName'] = os.path.basename( os.getcwd()) #use cwd for run name newstackrundata = newstackrunq.query() if newstackrundata: print "A stack run with this name (the current directory name) already exists. Exiting" sys.exit() newstackrunq = appiondata.ApStackRunData() newstackrunq['stackRunName'] = os.path.basename( os.getcwd()) #use cwd for run name newstackrunq['stackParams'] = newstackparamsq newstackrunq['session'] = stackdata[0]['stackRun']['session'] if "selectionrun" in stackdata[0]['stackRun'].keys: newstackrunq['selectionrun'] = stackdata[0]['stackRun']['selectionrun'] newstackrunq['syntheticStackParams'] = stackdata[0]['stackRun'][ 'syntheticStackParams'] #make new runs in stack query and insert also inserts stack and stack run newrisq = appiondata.ApRunsInStackData() newrisq['stack'] = newstackq newrisq['stackRun'] = newstackrunq newrisq.insert() #loop in reverse order so that order of ptcls in db is like that of orig for particle in range(len(stackdata) - 1, -1, -1): stackparticleq = appiondata.ApStackParticleData() stackparticleq['particleNumber'] = stackdata[particle][ 'particleNumber'] stackparticleq['stack'] = newstackq stackparticleq['stackRun'] = newstackrunq stackparticleq['particle'] = stackdata[particle]['particle'] #print stackparticleq stackparticleq.insert() return
def getNumAlignRunsFromSession(sessionname): sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) stackrunq = appiondata.ApStackRunData() stackrunq['session'] = sessiondata runsinstackq = appiondata.ApRunsInStackData() runsinstackq['stackRun'] = stackrunq runsindatas = runsinstackq.query() if not runsindatas: return 0 alignidlist = [] for runsindata in runsindatas: alignstackq = appiondata.ApAlignStackData() alignstackq['stack'] = runsindata['stack'] alignstackdatas = alignstackq.query() if not alignstackdatas: continue for alignstackdata in alignstackdatas: alignrunid = alignstackdata['alignrun'].dbid if not alignrunid in alignidlist: alignidlist.append(alignrunid) return len(alignidlist)
def getStackIdFromSubStackName(substackname, sessionname, msg=True): """ For a given run name and session name find stack id """ sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) stackrunq = appiondata.ApStackRunData() stackrunq['session'] = sessiondata stackq = appiondata.ApStackData() stackq['substackname'] = substackname runsinstackq = appiondata.ApRunsInStackData() runsinstackq['stackRun'] = stackrunq runsinstackq['stack'] = stackq runsindatas = runsinstackq.query() if not runsindatas: return None if len(runsindatas) == 1: ### simpe case stackid = runsindatas[0]['stack'].dbid else: for runsindata in runsindatas: print runsindata apDisplay.printError("Found too many sub-stacks for specified criteria") apDisplay.printMsg("Found stack id %d with substackname %s from session %s"%(stackid, substackname, sessionname)) return stackid
def hasRecentEntry(self): if self.appion_dbupgrade.tableExists('ApCtfData'): results = appiondata.ApCtfData().query(results=1) if results: ctfdata = results[0] if ctfdata.timestamp > self.checktime: return True if self.appion_dbupgrade.tableExists('ApStackRunData'): results = appiondata.ApStackRunData().query(results=1) if results: stackrundata = results[0] if stackrundata.timestamp > self.checktime and stackrundata[ 'stackParams']['phaseFlipped']: return True return False
def scanAppionDB(self): if self.appion_dbtools.tableExists('ApCtfData'): results = appiondata.ApCtfData().query(results=1) if results: ctfdata = results[0] if ctfdata.timestamp > self.checktime: print "\033[35m%s has new ApCtfData in %d days %d hours\033[0m" % (self.appion_dbtools.getDatabaseName(),-self.deltadays,-self.deltahours) if self.appion_dbtools.tableExists('ApStackRunData'): results = appiondata.ApStackRunData().query(results=1) if results: stackrundata = results[0] if stackrundata['stackParams']['phaseFlipped']: stackpartr = appiondata.ApStackParticleData(stackRun=stackrundata).query(results=1) if stackpartr: stackpartdata = stackpartr[0] if stackpartdata.timestamp > self.checktime: print "\033[35m%s has new particle inserted to Stack with phase flip in %d days %d hours\033[0m" % (self.appion_dbtools.getDatabaseName(),-self.deltadays,-self.deltahours)
def getNumStacksFromSession(sessionname): sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) stackrunq = appiondata.ApStackRunData() stackrunq['session'] = sessiondata runsinstackq = appiondata.ApRunsInStackData() runsinstackq['stackRun'] = stackrunq runsindatas = runsinstackq.query() if not runsindatas: return 0 stacklist = [] for runsindata in runsindatas: stackid = runsindata['stack'].dbid if not stackid in stacklist: stacklist.append(stackid) return len(stacklist)
def createStackData(self): apDisplay.printColor("Starting upload of stack", "blue") pathq = appiondata.ApPathData() pathq['path'] = self.params['rundir'] manq = appiondata.ApManualParamsData() manq['diam'] = self.params['diameter'] manq['trace'] = False selectq = appiondata.ApSelectionRunData() selectq['name'] = 'fakestack_' + self.params['runname'] selectq['hidden'] = True selectq['path'] = pathq selectq['session'] = self.sessiondata selectq['manparams'] = manq stackq = appiondata.ApStackData() stackq['name'] = "start.hed" stackq['path'] = pathq stackq['description'] = self.params['description'] stackq['hidden'] = False stackq['pixelsize'] = self.params['apix'] * 1e-10 stackq['boxsize'] = self.boxsize stackq['centered'] = False stackparamq = appiondata.ApStackParamsData() stackparamq['boxSize'] = self.boxsize stackparamq['bin'] = 1 stackparamq['phaseFlipped'] = self.params['ctfcorrect'] if self.params['ctfcorrect'] is True: stackparamq['fileType'] = "manual" stackparamq['fileType'] = "imagic" stackparamq['normalized'] = self.params['normalize'] stackparamq['lowpass'] = 0 stackparamq['highpass'] = 0 stackrunq = appiondata.ApStackRunData() stackrunq['stackRunName'] = self.params['runname'] stackrunq['stackParams'] = stackparamq stackrunq['selectionrun'] = selectq stackrunq['session'] = self.sessiondata runsinstackq = appiondata.ApRunsInStackData() runsinstackq['stack'] = stackq runsinstackq['stackRun'] = stackrunq if self.params['commit'] is True: runsinstackq.insert() if 'syncstackid' in self.params.keys( ) and self.params['syncstackid']: stackdata = runsinstackq['stack'] stack2data = apStack.getOnlyStackData( self.params['syncstackid']) syncq = appiondata.ApSyncStackData( stack1=stackdata, stack2=stack2data, synctype=self.params['synctype']) syncq.insert() ### for each particle sys.stderr.write("Starting particle upload") for i in range(self.numpart): if i % 100 == 0: sys.stderr.write(".") partq = appiondata.ApParticleData() partq['image'] = None #We have no image, see if this works??? partq['selectionrun'] = selectq partq['xcoord'] = int(i % 1000) partq['ycoord'] = int(i / 1000) partq['diameter'] = self.params['diameter'] stackpartq = appiondata.ApStackParticleData() stackpartq['particleNumber'] = i + 1 stackpartq['stack'] = stackq stackpartq['stackRun'] = stackrunq stackpartq['particle'] = partq stackpartq['mean'] = 0.0 stackpartq['stdev'] = 1.0 if self.params['commit'] is True: stackpartq.insert() sys.stderr.write("\n") return
def insertHipParticleData(self): self.particleNumber = 0 alldir = os.listdir(self.params['rundir']) ems_list = [] for file in alldir: if re.search(".ems.out", file): ems_list.append(file) ems_list.sort() avgdir = os.listdir(os.path.join(self.params['rundir'],"avgsnif1/avgsnif2/avg3")) fhlxavg_list = [] for file in avgdir: if re.search("f.hlxavg_dek", file): fhlxavg_list.append(file) fhlxavg_list.sort() nhlxavg_list = [] for file in avgdir: if re.search("n.hlxavg_dek", file): nhlxavg_list.append(file) nhlxavg_list.sort() for i in range(len(ems_list)): filename = ems_list[i] ### Will the filenames always be part000000.ext with apXmipp.breakStackIntoSingleFiles?? This is a risky way to get the file prefix. Find a better way! self.filep = filename[0:10] self.particleNumber += 1 HipParticle = appiondata.ApHipParticleData() HipParticle['hipRun'] = self.params['HipRun'] for file in fhlxavg_list: if re.search(self.filep, file): f = open(os.path.join(self.params['rundir'], "avgsnif1/avgsnif2/avg3", file), 'r') lines = f.readlines() keyline = lines[2] pzr_list = keyline.split() f.close() HipParticle['far_phi'] = pzr_list[2] HipParticle['far_z'] = pzr_list[3] HipParticle['far_rscale'] = pzr_list[4] HipParticle['far_ampscale'] = lines[-1] for file in nhlxavg_list: if re.search(self.filep, file): f = open(os.path.join(self.params['rundir'], "avgsnif1/avgsnif2/avg3", file), 'r') lines = f.readlines() keyline = lines[2] pzr_list = keyline.split() f.close() HipParticle['ner_phi'] = pzr_list[2] HipParticle['ner_z'] = pzr_list[3] HipParticle['ner_rscale'] = pzr_list[4] HipParticle['ner_ampscale'] = lines[-1] f = open(ems_list[i], 'r') lines = f.readlines() lastline = lines[-1] tsr_list = lastline.split() f.close() sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) HipParticle['session'] = sessiondata HipParticle['filename'] = self.filep HipParticle['stack'] = self.stackdata HipParticle['mrc_file'] = os.path.join(self.params['rundir'], self.filep + ".mrc") HipParticle['s_file'] = os.path.join(self.params['rundir'], self.filep + ".s") HipParticle['dft_file'] = os.path.join(self.params['rundir'], self.filep + ".dft") HipParticle['colb_file'] = os.path.join(self.params['rundir'], self.filep + ".colb") HipParticle['ner_file'] = os.path.join(self.params['rundir'], self.filep + ".ner") HipParticle['far_file'] = os.path.join(self.params['rundir'], self.filep + ".far") HipParticle['fft_file'] = os.path.join(self.params['rundir'], self.filep + ".fft") self.stackrundata = appiondata.ApStackRunData() HipParticle['stackRun'] = self.stackrundata HipParticle['particleNumber'] = self.particleNumber HipParticle['tilt'] = tsr_list[0] HipParticle['shift'] = tsr_list[1] HipParticle['resid'] = tsr_list[2] HipParticledata = HipParticle.query(results=1) if HipParticledata: apDisplay.printError("trying to insert a duplicate particle") if self.params['commit'] is True: HipParticle.insert() self.params['HipParticleID'] = HipParticle.dbid print "self.params['HipParticleID']",self.params['HipParticleID'] return
def uploadData(self, ctfpartlist): ### read mean /stdev for uploading self.getPartMeanTree(os.path.join(self.params['rundir'], self.params['finalstack']), ctfpartlist) sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) if self.params['projectid'] is not None: projectnum = self.params['projectid'] else: projectnum = apProject.getProjectIdFromSessionName(self.params['sessionname']) ### create synthetic stack object ... not saving global params like runname, session, project, description, etc. here; that's in ApStackData syntheticq = appiondata.ApSyntheticStackParamsData() ### get number of fakestack runs numentries = len(syntheticq) syntheticq['modelid'] = appiondata.ApInitialModelData.direct_query(self.params['modelid']) syntheticq['boxsize'] = self.params['box'] syntheticq['apix'] = self.params['apix'] syntheticq['projcount'] = self.params['projcount'] syntheticq['projstdev'] = self.params['projstdev'] syntheticq['shiftrad'] = self.params['shiftrad'] syntheticq['rotang'] = self.params['rotang'] syntheticq['flip'] = self.params['flip'] syntheticq['kilovolts'] = self.params['kv'] syntheticq['spher_aber'] = self.params['cs'] syntheticq['defocus_x'] = self.params['df1'] syntheticq['defocus_y'] = self.params['df2'] syntheticq['randomdef'] = self.params['randomdef'] if self.params['randomdef'] is True: syntheticq['randomdef_std'] = self.params['randomdef_std'] syntheticq['astigmatism'] = self.params['astigmatism'] syntheticq['snr1'] = self.params['snr1'] syntheticq['snrtot'] = self.params['snrtot'] syntheticq['envelope'] = os.path.basename(self.params['envelopefile']) syntheticq['ace2correct'] = self.params['ace2correct'] syntheticq['ace2correct_rand'] = self.params['ace2correct_rand'] if self.params['ace2correct_rand'] is True: syntheticq['ace2correct_std'] = self.params['ace2correct_std'] syntheticq['ace2estimate'] = self.params['ace2estimate'] syntheticq['lowpass'] = self.params['lpfilt'] syntheticq['highpass'] = self.params['hpfilt'] syntheticq['norm'] = self.params['norm'] ### fill stack parameters stparamq = appiondata.ApStackParamsData() stparamq['boxSize'] = self.params['box'] stparamq['bin'] = 1 stparamq['fileType'] = "imagic" stparamq['defocpair'] = 0 stparamq['lowpass'] = self.params['lpfilt'] stparamq['highpass'] = self.params['hpfilt'] stparamq['norejects'] = 1 if self.params['invert'] is True: stparamq['inverted'] = 1 else: stparamq['inverted'] = 0 if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True: stparamq['phaseFlipped'] = 1 stparamq['fliptype'] = "ace2part" else: stparamq['phaseFlipped'] = 0 stparamq['normalized'] = self.params['norm'] paramslist = stparamq.query() ### create a stack object stackq = appiondata.ApStackData() stackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) ### see if stack already exists in the database (just checking path & name) uniqstackdatas = stackq.query(results=1) ### create a stackRun object runq = appiondata.ApStackRunData() runq['stackRunName'] = self.params['runname'] runq['session'] = sessiondata ### see if stack run already exists in the database (just checking runname & session) uniqrundatas = runq.query(results=1) ### finish stack object stackq['name'] = self.params['finalstack'] stackq['description'] = self.params['description'] stackq['hidden'] = 0 stackq['pixelsize'] = self.params['apix'] * 1e-10 stackq['boxsize'] = self.params['box'] self.stackdata = stackq ### finish stackRun object runq['stackParams'] = stparamq runq['syntheticStackParams'] = syntheticq self.stackrundata = runq ### create runinstack object rinstackq = appiondata.ApRunsInStackData() rinstackq['stackRun'] = runq ### if not in the database, make sure run doesn't already exist if not uniqstackdatas and not uniqrundatas: if self.params['commit'] is True: apDisplay.printColor("Inserting stack parameters into database", "cyan") rinstackq['stack'] = stackq rinstackq.insert() else: apDisplay.printWarning("NOT INSERTING stack parameters into database") elif uniqrundatas and not uniqstackdatas: apDisplay.printError("Weird, run data without stack already in the database") else: rinstack = rinstackq.query(results=1) prevrinstackq = appiondata.ApRunsInStackData() prevrinstackq['stackRun'] = uniqrundatas[0] prevrinstackq['stack'] = uniqstackdatas[0] prevrinstack = prevrinstackq.query(results=1) ## if no runinstack found, find out which parameters are wrong: if not rinstack: for i in uniqrundatas[0]: print "r =======",i,"========" if uniqrundatas[0][i] != runq[i]: apDisplay.printError("the value for parameter '"+str(i)+"' is different from before") else: print i,uniqrundatas[0][i],runq[i] for i in uniqrundatas[0]['stackParams']: print "p =======",i,"========" if uniqrundatas[0]['stackParams'][i] != stparamq[i]: apDisplay.printError("the value for parameter '"+str(i)+"' is different from before") else: print i, uniqrundatas[0]['stackParams'][i], stparamq[i] for i in uniqstackdatas[0]: print "s =======",i,"========" if uniqstackdatas[0][i] != stackq[i]: apDisplay.printError("the value for parameter '"+str(i)+"' is different from before") else: print i,uniqstackdatas[0][i],stackq[i] for i in prevrinstack[0]: print "rin =======",i,"========" if prevrinstack[0][i] != rinstackq[i]: print i,prevrinstack[0][i],rinstackq[i] apDisplay.printError("the value for parameter '"+str(i)+"' is different from before") else: print i,prevrinstack[0][i],rinstackq[i] apDisplay.printError("All parameters for a particular stack must be identical! \n"+\ "please check your parameter settings.") apDisplay.printWarning("Stack already exists in database! Will try and appending new particles to stack") ### create a fake selection run # selectq = appiondata.ApSelectionRunData() # selectq['session'] = sessiondata # selectq['name'] = "fakerun" # self.selectq = selectq if self.params['commit'] is True: apDisplay.printColor("Inserting fake selection parameters into the database", "cyan") # selectq.insert() else: apDisplay.printWarning("NOT INSERTING fake selection parameters into the database") partNumber = 0 ### loop over the particles and insert if self.params['commit'] is True: apDisplay.printColor("inserting particle parameters into database", "cyan") else: apDisplay.printWarning("NOT INSERTING particle parameters into database") for i in range(len(ctfpartlist)): partNumber += 1 partfile = ctfpartlist[i] partmeandict = self.partmeantree[i] partq = appiondata.ApParticleData() # partq['selectionrun'] = selectq partq['xcoord'] = partNumber stpartq = appiondata.ApStackParticleData() ### check unique params stpartq['stack'] = self.stackdata stpartq['stackRun'] = self.stackrundata stpartq['particleNumber'] = partNumber stpartdata = stpartq.query(results=1) if stpartdata: apDisplay.printError("trying to insert a duplicate particle") stpartq['particle'] = partq stpartq['mean'] = partmeandict['mean'] stpartq['stdev'] = partmeandict['stdev'] if self.params['commit'] is True: stpartq.insert() return