def commitScaledStack(stackdata, params):
#make new params query
newstackparamsq = appiondata.ApStackParamsData()
for key in newstackparamsq.keys():
if key != 'bin':
newstackparamsq[key] = stackdata[0]['stackRun']['stackParams'][key]
newstackparamsq['bin'] = params['bin']
#make new stack query
newstackq = appiondata.ApStackData()
newstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(params['newstackpath']))
newstackq['name'] = params['newstackname']
newstackq['description'] = params['description']
newstackdata = newstackq.query()
if newstackdata:
print "A stack with these parameters already exists"
return
#make new run query
#first check that run name doesn't already exist
newstackrunq = appiondata.ApStackRunData()
newstackrunq['stackRunName'] = os.path.basename(
os.getcwd()) #use cwd for run name
newstackrundata = newstackrunq.query()
if newstackrundata:
print "A stack run with this name (the current directory name) already exists. Exiting"
sys.exit()
newstackrunq = appiondata.ApStackRunData()
newstackrunq['stackRunName'] = os.path.basename(
os.getcwd()) #use cwd for run name
newstackrunq['stackParams'] = newstackparamsq
newstackrunq['session'] = stackdata[0]['stackRun']['session']
if "selectionrun" in stackdata[0]['stackRun'].keys:
newstackrunq['selectionrun'] = stackdata[0]['stackRun']['selectionrun']
newstackrunq['syntheticStackParams'] = stackdata[0]['stackRun'][
'syntheticStackParams']
#make new runs in stack query and insert also inserts stack and stack run
newrisq = appiondata.ApRunsInStackData()
newrisq['stack'] = newstackq
newrisq['stackRun'] = newstackrunq
newrisq.insert()
#loop in reverse order so that order of ptcls in db is like that of orig
for particle in range(len(stackdata) - 1, -1, -1):
stackparticleq = appiondata.ApStackParticleData()
stackparticleq['particleNumber'] = stackdata[particle][
'particleNumber']
stackparticleq['stack'] = newstackq
stackparticleq['stackRun'] = newstackrunq
stackparticleq['particle'] = stackdata[particle]['particle']
#print stackparticleq
stackparticleq.insert()
return
def getNumAlignRunsFromSession(sessionname): sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) stackrunq = appiondata.ApStackRunData() stackrunq['session'] = sessiondata runsinstackq = appiondata.ApRunsInStackData() runsinstackq['stackRun'] = stackrunq runsindatas = runsinstackq.query() if not runsindatas: return 0 alignidlist = [] for runsindata in runsindatas: alignstackq = appiondata.ApAlignStackData() alignstackq['stack'] = runsindata['stack'] alignstackdatas = alignstackq.query() if not alignstackdatas: continue for alignstackdata in alignstackdatas: alignrunid = alignstackdata['alignrun'].dbid if not alignrunid in alignidlist: alignidlist.append(alignrunid) return len(alignidlist)
def getStackIdFromSubStackName(substackname, sessionname, msg=True):
"""
For a given run name and session name find stack id
"""
sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
stackrunq = appiondata.ApStackRunData()
stackrunq['session'] = sessiondata
stackq = appiondata.ApStackData()
stackq['substackname'] = substackname
runsinstackq = appiondata.ApRunsInStackData()
runsinstackq['stackRun'] = stackrunq
runsinstackq['stack'] = stackq
runsindatas = runsinstackq.query()
if not runsindatas:
return None
if len(runsindatas) == 1:
### simpe case
stackid = runsindatas[0]['stack'].dbid
else:
for runsindata in runsindatas:
print runsindata
apDisplay.printError("Found too many sub-stacks for specified criteria")
apDisplay.printMsg("Found stack id %d with substackname %s from session %s"%(stackid, substackname, sessionname))
return stackid
def hasRecentEntry(self):
if self.appion_dbupgrade.tableExists('ApCtfData'):
results = appiondata.ApCtfData().query(results=1)
if results:
ctfdata = results[0]
if ctfdata.timestamp > self.checktime:
return True
if self.appion_dbupgrade.tableExists('ApStackRunData'):
results = appiondata.ApStackRunData().query(results=1)
if results:
stackrundata = results[0]
if stackrundata.timestamp > self.checktime and stackrundata[
'stackParams']['phaseFlipped']:
return True
return False
def scanAppionDB(self):
if self.appion_dbtools.tableExists('ApCtfData'):
results = appiondata.ApCtfData().query(results=1)
if results:
ctfdata = results[0]
if ctfdata.timestamp > self.checktime:
print "\033[35m%s has new ApCtfData in %d days %d hours\033[0m" % (self.appion_dbtools.getDatabaseName(),-self.deltadays,-self.deltahours)
if self.appion_dbtools.tableExists('ApStackRunData'):
results = appiondata.ApStackRunData().query(results=1)
if results:
stackrundata = results[0]
if stackrundata['stackParams']['phaseFlipped']:
stackpartr = appiondata.ApStackParticleData(stackRun=stackrundata).query(results=1)
if stackpartr:
stackpartdata = stackpartr[0]
if stackpartdata.timestamp > self.checktime:
print "\033[35m%s has new particle inserted to Stack with phase flip in %d days %d hours\033[0m" % (self.appion_dbtools.getDatabaseName(),-self.deltadays,-self.deltahours)
def getNumStacksFromSession(sessionname):
sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
stackrunq = appiondata.ApStackRunData()
stackrunq['session'] = sessiondata
runsinstackq = appiondata.ApRunsInStackData()
runsinstackq['stackRun'] = stackrunq
runsindatas = runsinstackq.query()
if not runsindatas:
return 0
stacklist = []
for runsindata in runsindatas:
stackid = runsindata['stack'].dbid
if not stackid in stacklist:
stacklist.append(stackid)
return len(stacklist)
def createStackData(self):
apDisplay.printColor("Starting upload of stack", "blue")
pathq = appiondata.ApPathData()
pathq['path'] = self.params['rundir']
manq = appiondata.ApManualParamsData()
manq['diam'] = self.params['diameter']
manq['trace'] = False
selectq = appiondata.ApSelectionRunData()
selectq['name'] = 'fakestack_' + self.params['runname']
selectq['hidden'] = True
selectq['path'] = pathq
selectq['session'] = self.sessiondata
selectq['manparams'] = manq
stackq = appiondata.ApStackData()
stackq['name'] = "start.hed"
stackq['path'] = pathq
stackq['description'] = self.params['description']
stackq['hidden'] = False
stackq['pixelsize'] = self.params['apix'] * 1e-10
stackq['boxsize'] = self.boxsize
stackq['centered'] = False
stackparamq = appiondata.ApStackParamsData()
stackparamq['boxSize'] = self.boxsize
stackparamq['bin'] = 1
stackparamq['phaseFlipped'] = self.params['ctfcorrect']
if self.params['ctfcorrect'] is True:
stackparamq['fileType'] = "manual"
stackparamq['fileType'] = "imagic"
stackparamq['normalized'] = self.params['normalize']
stackparamq['lowpass'] = 0
stackparamq['highpass'] = 0
stackrunq = appiondata.ApStackRunData()
stackrunq['stackRunName'] = self.params['runname']
stackrunq['stackParams'] = stackparamq
stackrunq['selectionrun'] = selectq
stackrunq['session'] = self.sessiondata
runsinstackq = appiondata.ApRunsInStackData()
runsinstackq['stack'] = stackq
runsinstackq['stackRun'] = stackrunq
if self.params['commit'] is True:
runsinstackq.insert()
if 'syncstackid' in self.params.keys(
) and self.params['syncstackid']:
stackdata = runsinstackq['stack']
stack2data = apStack.getOnlyStackData(
self.params['syncstackid'])
syncq = appiondata.ApSyncStackData(
stack1=stackdata,
stack2=stack2data,
synctype=self.params['synctype'])
syncq.insert()
### for each particle
sys.stderr.write("Starting particle upload")
for i in range(self.numpart):
if i % 100 == 0:
sys.stderr.write(".")
partq = appiondata.ApParticleData()
partq['image'] = None #We have no image, see if this works???
partq['selectionrun'] = selectq
partq['xcoord'] = int(i % 1000)
partq['ycoord'] = int(i / 1000)
partq['diameter'] = self.params['diameter']
stackpartq = appiondata.ApStackParticleData()
stackpartq['particleNumber'] = i + 1
stackpartq['stack'] = stackq
stackpartq['stackRun'] = stackrunq
stackpartq['particle'] = partq
stackpartq['mean'] = 0.0
stackpartq['stdev'] = 1.0
if self.params['commit'] is True:
stackpartq.insert()
sys.stderr.write("\n")
return
def insertHipParticleData(self):
self.particleNumber = 0
alldir = os.listdir(self.params['rundir'])
ems_list = []
for file in alldir:
if re.search(".ems.out", file):
ems_list.append(file)
ems_list.sort()
avgdir = os.listdir(os.path.join(self.params['rundir'],"avgsnif1/avgsnif2/avg3"))
fhlxavg_list = []
for file in avgdir:
if re.search("f.hlxavg_dek", file):
fhlxavg_list.append(file)
fhlxavg_list.sort()
nhlxavg_list = []
for file in avgdir:
if re.search("n.hlxavg_dek", file):
nhlxavg_list.append(file)
nhlxavg_list.sort()
for i in range(len(ems_list)):
filename = ems_list[i]
### Will the filenames always be part000000.ext with apXmipp.breakStackIntoSingleFiles?? This is a risky way to get the file prefix. Find a better way!
self.filep = filename[0:10]
self.particleNumber += 1
HipParticle = appiondata.ApHipParticleData()
HipParticle['hipRun'] = self.params['HipRun']
for file in fhlxavg_list:
if re.search(self.filep, file):
f = open(os.path.join(self.params['rundir'], "avgsnif1/avgsnif2/avg3", file), 'r')
lines = f.readlines()
keyline = lines[2]
pzr_list = keyline.split()
f.close()
HipParticle['far_phi'] = pzr_list[2]
HipParticle['far_z'] = pzr_list[3]
HipParticle['far_rscale'] = pzr_list[4]
HipParticle['far_ampscale'] = lines[-1]
for file in nhlxavg_list:
if re.search(self.filep, file):
f = open(os.path.join(self.params['rundir'], "avgsnif1/avgsnif2/avg3", file), 'r')
lines = f.readlines()
keyline = lines[2]
pzr_list = keyline.split()
f.close()
HipParticle['ner_phi'] = pzr_list[2]
HipParticle['ner_z'] = pzr_list[3]
HipParticle['ner_rscale'] = pzr_list[4]
HipParticle['ner_ampscale'] = lines[-1]
f = open(ems_list[i], 'r')
lines = f.readlines()
lastline = lines[-1]
tsr_list = lastline.split()
f.close()
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
HipParticle['session'] = sessiondata
HipParticle['filename'] = self.filep
HipParticle['stack'] = self.stackdata
HipParticle['mrc_file'] = os.path.join(self.params['rundir'], self.filep + ".mrc")
HipParticle['s_file'] = os.path.join(self.params['rundir'], self.filep + ".s")
HipParticle['dft_file'] = os.path.join(self.params['rundir'], self.filep + ".dft")
HipParticle['colb_file'] = os.path.join(self.params['rundir'], self.filep + ".colb")
HipParticle['ner_file'] = os.path.join(self.params['rundir'], self.filep + ".ner")
HipParticle['far_file'] = os.path.join(self.params['rundir'], self.filep + ".far")
HipParticle['fft_file'] = os.path.join(self.params['rundir'], self.filep + ".fft")
self.stackrundata = appiondata.ApStackRunData()
HipParticle['stackRun'] = self.stackrundata
HipParticle['particleNumber'] = self.particleNumber
HipParticle['tilt'] = tsr_list[0]
HipParticle['shift'] = tsr_list[1]
HipParticle['resid'] = tsr_list[2]
HipParticledata = HipParticle.query(results=1)
if HipParticledata:
apDisplay.printError("trying to insert a duplicate particle")
if self.params['commit'] is True:
HipParticle.insert()
self.params['HipParticleID'] = HipParticle.dbid
print "self.params['HipParticleID']",self.params['HipParticleID']
return
def uploadData(self, ctfpartlist):
### read mean /stdev for uploading
self.getPartMeanTree(os.path.join(self.params['rundir'], self.params['finalstack']), ctfpartlist)
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
if self.params['projectid'] is not None:
projectnum = self.params['projectid']
else:
projectnum = apProject.getProjectIdFromSessionName(self.params['sessionname'])
### create synthetic stack object ... not saving global params like runname, session, project, description, etc. here; that's in ApStackData
syntheticq = appiondata.ApSyntheticStackParamsData()
### get number of fakestack runs
numentries = len(syntheticq)
syntheticq['modelid'] = appiondata.ApInitialModelData.direct_query(self.params['modelid'])
syntheticq['boxsize'] = self.params['box']
syntheticq['apix'] = self.params['apix']
syntheticq['projcount'] = self.params['projcount']
syntheticq['projstdev'] = self.params['projstdev']
syntheticq['shiftrad'] = self.params['shiftrad']
syntheticq['rotang'] = self.params['rotang']
syntheticq['flip'] = self.params['flip']
syntheticq['kilovolts'] = self.params['kv']
syntheticq['spher_aber'] = self.params['cs']
syntheticq['defocus_x'] = self.params['df1']
syntheticq['defocus_y'] = self.params['df2']
syntheticq['randomdef'] = self.params['randomdef']
if self.params['randomdef'] is True:
syntheticq['randomdef_std'] = self.params['randomdef_std']
syntheticq['astigmatism'] = self.params['astigmatism']
syntheticq['snr1'] = self.params['snr1']
syntheticq['snrtot'] = self.params['snrtot']
syntheticq['envelope'] = os.path.basename(self.params['envelopefile'])
syntheticq['ace2correct'] = self.params['ace2correct']
syntheticq['ace2correct_rand'] = self.params['ace2correct_rand']
if self.params['ace2correct_rand'] is True:
syntheticq['ace2correct_std'] = self.params['ace2correct_std']
syntheticq['ace2estimate'] = self.params['ace2estimate']
syntheticq['lowpass'] = self.params['lpfilt']
syntheticq['highpass'] = self.params['hpfilt']
syntheticq['norm'] = self.params['norm']
### fill stack parameters
stparamq = appiondata.ApStackParamsData()
stparamq['boxSize'] = self.params['box']
stparamq['bin'] = 1
stparamq['fileType'] = "imagic"
stparamq['defocpair'] = 0
stparamq['lowpass'] = self.params['lpfilt']
stparamq['highpass'] = self.params['hpfilt']
stparamq['norejects'] = 1
if self.params['invert'] is True:
stparamq['inverted'] = 1
else:
stparamq['inverted'] = 0
if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True:
stparamq['phaseFlipped'] = 1
stparamq['fliptype'] = "ace2part"
else:
stparamq['phaseFlipped'] = 0
stparamq['normalized'] = self.params['norm']
paramslist = stparamq.query()
### create a stack object
stackq = appiondata.ApStackData()
stackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
### see if stack already exists in the database (just checking path & name)
uniqstackdatas = stackq.query(results=1)
### create a stackRun object
runq = appiondata.ApStackRunData()
runq['stackRunName'] = self.params['runname']
runq['session'] = sessiondata
### see if stack run already exists in the database (just checking runname & session)
uniqrundatas = runq.query(results=1)
### finish stack object
stackq['name'] = self.params['finalstack']
stackq['description'] = self.params['description']
stackq['hidden'] = 0
stackq['pixelsize'] = self.params['apix'] * 1e-10
stackq['boxsize'] = self.params['box']
self.stackdata = stackq
### finish stackRun object
runq['stackParams'] = stparamq
runq['syntheticStackParams'] = syntheticq
self.stackrundata = runq
### create runinstack object
rinstackq = appiondata.ApRunsInStackData()
rinstackq['stackRun'] = runq
### if not in the database, make sure run doesn't already exist
if not uniqstackdatas and not uniqrundatas:
if self.params['commit'] is True:
apDisplay.printColor("Inserting stack parameters into database", "cyan")
rinstackq['stack'] = stackq
rinstackq.insert()
else:
apDisplay.printWarning("NOT INSERTING stack parameters into database")
elif uniqrundatas and not uniqstackdatas:
apDisplay.printError("Weird, run data without stack already in the database")
else:
rinstack = rinstackq.query(results=1)
prevrinstackq = appiondata.ApRunsInStackData()
prevrinstackq['stackRun'] = uniqrundatas[0]
prevrinstackq['stack'] = uniqstackdatas[0]
prevrinstack = prevrinstackq.query(results=1)
## if no runinstack found, find out which parameters are wrong:
if not rinstack:
for i in uniqrundatas[0]:
print "r =======",i,"========"
if uniqrundatas[0][i] != runq[i]:
apDisplay.printError("the value for parameter '"+str(i)+"' is different from before")
else:
print i,uniqrundatas[0][i],runq[i]
for i in uniqrundatas[0]['stackParams']:
print "p =======",i,"========"
if uniqrundatas[0]['stackParams'][i] != stparamq[i]:
apDisplay.printError("the value for parameter '"+str(i)+"' is different from before")
else:
print i, uniqrundatas[0]['stackParams'][i], stparamq[i]
for i in uniqstackdatas[0]:
print "s =======",i,"========"
if uniqstackdatas[0][i] != stackq[i]:
apDisplay.printError("the value for parameter '"+str(i)+"' is different from before")
else:
print i,uniqstackdatas[0][i],stackq[i]
for i in prevrinstack[0]:
print "rin =======",i,"========"
if prevrinstack[0][i] != rinstackq[i]:
print i,prevrinstack[0][i],rinstackq[i]
apDisplay.printError("the value for parameter '"+str(i)+"' is different from before")
else:
print i,prevrinstack[0][i],rinstackq[i]
apDisplay.printError("All parameters for a particular stack must be identical! \n"+\
"please check your parameter settings.")
apDisplay.printWarning("Stack already exists in database! Will try and appending new particles to stack")
### create a fake selection run
# selectq = appiondata.ApSelectionRunData()
# selectq['session'] = sessiondata
# selectq['name'] = "fakerun"
# self.selectq = selectq
if self.params['commit'] is True:
apDisplay.printColor("Inserting fake selection parameters into the database", "cyan")
# selectq.insert()
else:
apDisplay.printWarning("NOT INSERTING fake selection parameters into the database")
partNumber = 0
### loop over the particles and insert
if self.params['commit'] is True:
apDisplay.printColor("inserting particle parameters into database", "cyan")
else:
apDisplay.printWarning("NOT INSERTING particle parameters into database")
for i in range(len(ctfpartlist)):
partNumber += 1
partfile = ctfpartlist[i]
partmeandict = self.partmeantree[i]
partq = appiondata.ApParticleData()
# partq['selectionrun'] = selectq
partq['xcoord'] = partNumber
stpartq = appiondata.ApStackParticleData()
### check unique params
stpartq['stack'] = self.stackdata
stpartq['stackRun'] = self.stackrundata
stpartq['particleNumber'] = partNumber
stpartdata = stpartq.query(results=1)
if stpartdata:
apDisplay.printError("trying to insert a duplicate particle")
stpartq['particle'] = partq
stpartq['mean'] = partmeandict['mean']
stpartq['stdev'] = partmeandict['stdev']
if self.params['commit'] is True:
stpartq.insert()
return