def setRunDir(self):
"""
this function only runs if no rundir is defined at the command line
"""
if self.params['rundir'] is None:
if ('sessionname' in self.params and self.params['sessionname'] is not None ):
# command line users may use sessionname rather than expId
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.params['rundir'] = self.getDefaultBaseAppionDir(sessiondata,[self.processdirname,self.params['runname']])
else:
if ('expId' in self.params and self.params['expId']):
# expId should always be included from appionwrapper derived appionscript
sessiondata = apDatabase.getSessionDataFromSessionId(self.params['expId'])
self.params['rundir'] = self.getDefaultBaseAppionDir(sessiondata,[self.processdirname,self.params['runname']])
# The rest should not be needed with appionwrapper format
from appionlib import apStack
if ( self.params['rundir'] is None
and 'reconid' in self.params
and self.params['reconid'] is not None ):
self.params['stackid'] = apStack.getStackIdFromRecon(self.params['reconid'], msg=False)
if ( self.params['rundir'] is None
and 'stackid' in self.params
and self.params['stackid'] is not None ):
#auto set the run directory
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
path = os.path.abspath(stackdata['path']['path'])
path = os.path.dirname(path)
path = os.path.dirname(path)
self.params['rundir'] = os.path.join(path, self.processdirname, self.params['runname'])
self.params['outdir'] = self.params['rundir']
def checkConflicts(self):
if self.params["reconid"] is None:
apDisplay.printError("enter a reconstruction ID from the database")
if self.params["mask"] is None:
apDisplay.printError("enter a mask radius")
if self.params["iter"] is None:
apDisplay.printError("enter an iteration for the final Eulers")
self.params["stackid"] = apStack.getStackIdFromRecon(self.params["reconid"])
self.params["sessionname"] = apStack.getSessionDataFromStackId(self.params["stackid"])
def checkConflicts(self):
if self.params['reconid'] is None:
apDisplay.printError("enter a reconstruction ID from the database")
if self.params['mask'] is None:
apDisplay.printError("enter a mask radius")
if self.params['iter'] is None:
apDisplay.printError("enter an iteration for the final Eulers")
self.params['stackid'] = apStack.getStackIdFromRecon(
self.params['reconid'])
self.params['sessionname'] = apStack.getSessionDataFromStackId(
self.params['stackid'])['name']
def getStackPartID(self, stackpartnum, reconrunid, stackid=None):
if stackid is None:
stackid = apStack.getStackIdFromRecon(reconrunid, msg=False)
stackpartq = appiondata.ApStackParticleData()
stackpartq['stack'] = appiondata.ApStackData.direct_query(stackid)
stackpartq['particleNumber'] = stackpartnum
stackpartdata = stackpartq.query(results=1)
if not stackpartdata:
apDisplay.printError("Failed to get Stack Particle ID for Number "+str(partnum))
return stackpartdata[0].dbid
def getStackPartID(self, stackpartnum, reconrunid, stackid=None):
if stackid is None:
stackid = apStack.getStackIdFromRecon(reconrunid, msg=False)
stackpartq = appiondata.ApStackParticleData()
stackpartq['stack'] = appiondata.ApStackData.direct_query(stackid)
stackpartq['particleNumber'] = stackpartnum
stackpartdata = stackpartq.query(results=1)
if not stackpartdata:
apDisplay.printError(
"Failed to get Stack Particle ID for Number " + str(partnum))
return stackpartdata[0].dbid
def calculateEulerJumpsForEntireRecon(self, reconrunid, stackid=None, sym=None, multimodelrunid=None):
if sym is None:
sym = self.getSymmetry(reconrunid)
if re.match("^icos", sym.lower()):
apDisplay.printWarning("Processing Icos symmetry "+sym)
if not re.match("^[cd][0-9]+$", sym.lower()) and not re.match("^icos", sym.lower()):
apDisplay.printError("Cannot calculate euler jumps for symmetry: "+sym)
return
### get stack particles
if stackid is None:
stackid = apStack.getStackIdFromRecon(reconrunid, msg=False)
stackparts = apStack.getStackParticlesFromId(stackid)
stackparts.sort(self.sortStackParts)
numparts = len(stackparts)
### start loop
t0 = time.time()
medians = []
count = 0
miscount = 0
for stackpart in stackparts:
count += 1
if multimodelrunid is None:
jumpdata = self.getEulerJumpData(reconrunid, stackpartid=stackpart.dbid, stackid=stackid, sym=sym)
else:
jumpdata = self.getEulerJumpData(reconrunid, stackpartid=stackpart.dbid, stackid=stackid, sym=sym, multimodelrunid=multimodelrunid)
if jumpdata is None:
if miscount < 25:
continue
else:
break
medians.append(jumpdata['median'])
if count % 500 == 0:
timeremain = (time.time()-t0)/(count+1)*(numparts-count)
print ("particle=% 5d; median jump=% 3.2f, remain time= %s" % (stackpart['particleNumber'], jumpdata['median'],
apDisplay.timeString(timeremain)))
if len(medians) > 0:
### print stats
apDisplay.printMsg("complete "+str(len(stackparts))+" particles in "+apDisplay.timeString(time.time()-t0))
print "-- median euler jumper stats --"
medians = numpy.asarray(medians, dtype=numpy.float32)
print ("mean/std :: "+str(round(medians.mean(),2))+" +/- "
+str(round(medians.std(),2)))
print ("min/max :: "+str(round(medians.min(),2))+" <> "
+str(round(medians.max(),2)))
else:
apDisplay.printWarning("no Euler jumpers inserted into the database, make sure that the angles are read by the recon uploader")
return
def checkConflicts(self):
### check for missing and duplicate entries
if self.params['iterid'] is None:
apDisplay.printError("Please provide --iterid")
if self.params['description'] is None:
apDisplay.printError("Please provide --description")
if self.params['runname'] is None:
apDisplay.printError("Please provide --runname")
### get the stack ID from the other IDs
self.iterdata = appiondata.ApRefineIterData.direct_query(self.params['iterid'])
self.params['stackid'] = apStack.getStackIdFromRecon(self.iterdata['refineRun'].dbid)
self.stackdata = apStack.getOnlyStackData(self.params['stackid'])
### check and make sure we got the stack id
if self.params['stackid'] is None:
apDisplay.printError("Could not find stackid from iterid")
def checkConflicts(self):
### check for missing and duplicate entries
if self.params['iterid'] is None:
apDisplay.printError("Please provide --iterid")
if self.params['description'] is None:
apDisplay.printError("Please provide --description")
if self.params['runname'] is None:
apDisplay.printError("Please provide --runname")
### get the stack ID from the other IDs
self.iterdata = appiondata.ApRefineIterData.direct_query(self.params['iterid'])
self.params['stackid'] = apStack.getStackIdFromRecon(self.iterdata['refineRun'].dbid)
self.stackdata = apStack.getOnlyStackData(self.params['stackid'])
### check and make sure we got the stack id
if self.params['stackid'] is None:
apDisplay.printError("Could not find stackid from iterid")
def checkConflicts(self):
"""
make sure the necessary parameters are set correctly
"""
if self.cursor is None:
# connect
self.dbconf = sinedon.getConfig('appiondata')
self.db = MySQLdb.connect(**self.dbconf)
self.db.autocommit(True)
# create a cursor
self.cursor = self.db.cursor()
if not self.params['reconid']:
apDisplay.printError("Enter a Reconstruction Run ID, e.g. --reconid=243")
if not self.params['tiltrunid']:
self.params['tiltrunid'] = self.getTiltRunIDFromReconID(self.params['reconid'])
self.params['symmetry'] = apSymmetry.getSymmetryFromReconRunId(self.params['reconid'])
self.params['symmname'] = self.params['symmetry']['eman_name']
if not self.params['stackid']:
self.params['stackid'] = apStack.getStackIdFromRecon(self.params['reconid'])
def checkConflicts(self):
"""
make sure the necessary parameters are set correctly
"""
if self.cursor is None:
# connect
self.dbconf = sinedon.getConfig('appiondata')
self.db = MySQLdb.connect(**self.dbconf)
self.db.autocommit(True)
# create a cursor
self.cursor = self.db.cursor()
if not self.params['reconid']:
apDisplay.printError("Enter a Reconstruction Run ID, e.g. --reconid=243")
if not self.params['tiltrunid']:
self.params['tiltrunid'] = self.getTiltRunIDFromReconID(self.params['reconid'])
self.params['symmetry'] = apSymmetry.getSymmetryFromReconRunId(self.params['reconid'])
self.params['symmname'] = self.params['symmetry']['eman_name']
if not self.params['stackid']:
self.params['stackid'] = apStack.getStackIdFromRecon(self.params['reconid'])
def checkConflicts(self):
if self.params['reconid'] is None:
apDisplay.printError("enter a reconstruction ID from the database")
if self.params['mask'] is None:
apDisplay.printError("enter a mask radius")
if self.params['iter'] is None:
apDisplay.printError("enter an iteration for the final Eulers")
if self.params['keeplist'] is None:
apDisplay.printError("enter an keep list file")
self.params['keeplist'] = os.path.abspath(self.params['keeplist'])
if not os.path.isfile(self.params['keeplist']):
apDisplay.printError("could not find list file")
self.params['stackid'] = apStack.getStackIdFromRecon(self.params['reconid'])
if self.params['stackname'][-4:] != ".hed":
s = os.path.splitext(self.params['stackname'])[0]
s += ".hed"
self.params['stackname'] = s
apDisplay.printMsg("Stack name: "+self.params['stackname'])
self.params['symmetry'] = apSymmetry.getSymmetryFromReconRunId(self.params['reconid'])
self.params['symmname'] = self.params['symmetry']['eman_name']
def start(self):
self.rootname = self.params['stackname'].split(".")[0]
self.params['outputstack'] = os.path.join(self.params['rundir'],
self.params['stackname'])
if os.path.isfile(self.params['outputstack']):
apFile.removeStack(self.params['outputstack'])
if self.params['eotest'] is True:
self.params['evenstack'] = os.path.splitext(
self.params['outputstack'])[0] + '.even.hed'
if os.path.isfile(self.params['evenstack']):
apFile.removeStack(self.params['evenstack'])
self.params['oddstack'] = os.path.splitext(
self.params['outputstack'])[0] + '.odd.hed'
if os.path.isfile(self.params['oddstack']):
apFile.removeStack(self.params['oddstack'])
classes = self.getClassData(self.params['reconid'],
self.params['iter'])
stackid = apStack.getStackIdFromRecon(self.params['reconid'])
stackdata = apStack.getOnlyStackData(stackid)
stackpath = os.path.join(stackdata['path']['path'], stackdata['name'])
classkeys = classes.keys()
classkeys.sort()
classnum = 0
keeplist = self.procKeepList()
finallist = []
apDisplay.printMsg("Processing " + str(len(classes)) + " classes")
#loop through classes
for key in classkeys:
classnum += 1
if classnum % 10 == 1:
sys.stderr.write("\b\b\b\b\b\b\b\b\b\b\b\b\b\b")
sys.stderr.write(
str(classnum) + " of " + (str(len(classkeys))))
# loop through particles in class
classfile = self.rootname + "-class.lst"
classf = open(classfile, 'w')
classf.write('#LST\n')
nptcls = 0
for ptcl in classes[key]['particles']:
# translate DB into EMAN
partnum = ptcl['particle']['particleNumber'] - 1
if partnum in keeplist:
if ptcl['mirror']:
mirror = 1
else:
mirror = 0
rot = ptcl['euler3'] * math.pi / 180.0
classf.write("%d\t%s\t%f,\t%f,%f,%f,%d\n" %
(partnum, stackpath, ptcl['quality_factor'],
rot, ptcl['shiftx'], ptcl['shifty'], mirror))
nptcls += 1
finallist.append(partnum)
classf.close()
if nptcls < 1:
continue
self.makeClassAverages(classfile, self.params['outputstack'],
classes[key], self.params['mask'])
if self.params['eotest'] is True:
self.makeEvenOddClasses(classfile, self.params['outputstack'],
classes[key], self.params['mask'])
apFile.removeFile(classfile)
sys.stderr.write("\n")
finalfilename = self.rootname + "-keep.lst"
finalf = open(finalfilename, 'w')
finallist.sort()
for partnum in finallist:
finalf.write('%d\n' % (partnum, ))
finalf.close()
stackstr = str(stackdata.dbid)
reconstr = str(self.params['reconid'])
### recon 3d volumes
threedname = os.path.join(
self.params['rundir'],
self.rootname + "." + str(self.params['iter']) + "a.mrc")
emancmd = ("make3d " + self.params['outputstack'] + " out=" +
threedname + " hard=50 sym=" + self.params['symmname'] +
" pad=240 mask=" + str(self.params['mask']) + "; echo ''")
#print emancmd
apEMAN.executeEmanCmd(emancmd,
verbose=False,
showcmd=True,
logfile=self.rootname + "-eman.log")
threednameb = os.path.join(
self.params['rundir'],
self.rootname + "." + str(self.params['iter']) + "b.mrc")
emancmd = ("proc3d " + threedname + " " + threednameb +
" apix=1.63 norm=0,1 lp=8 origin=0,0,0 mask=" +
str(self.params['mask']) + "; echo '' ")
apEMAN.executeEmanCmd(emancmd,
verbose=False,
showcmd=True,
logfile=self.rootname + "-eman.log")
if self.params['eotest'] is True:
# even
evenname = os.path.join(
self.params['rundir'],
self.rootname + "-even." + str(self.params['iter']) + "a.mrc")
if os.path.isfile(self.params['evenstack']):
evenemancmd = ("make3d " + self.params['evenstack'] + " out=" +
evenname + " hard=50 sym=" +
self.params['symmname'] + " pad=240 mask=" +
str(self.params['mask']) + "; echo ''")
#print evenemancmd
apEMAN.executeEmanCmd(evenemancmd,
verbose=False,
showcmd=True,
logfile=self.rootname + "-eveneman.log")
else:
apDisplay.printWarning("file " + self.params['evenstack'] +
" does not exist")
# odd
oddname = os.path.join(
self.params['rundir'],
self.rootname + "-odd." + str(self.params['iter']) + "a.mrc")
if os.path.isfile(self.params['oddstack']):
oddemancmd = ("make3d " + self.params['oddstack'] + " out=" +
oddname + " hard=50 sym=" +
self.params['symmname'] + " pad=240 mask=" +
str(self.params['mask']) + "; echo ''")
#print oddemancmd
apEMAN.executeEmanCmd(oddemancmd,
verbose=False,
showcmd=True,
logfile=self.rootname + "-oddeman.log")
else:
apDisplay.printWarning("file " + self.params['oddstack'] +
" does not exist")
#eotest
fscout = os.path.join(self.params['rundir'],
self.rootname + "-fsc.eotest")
if os.path.isfile(oddname) and os.path.isfile(evenname):
eotestcmd = "proc3d " + oddname + " " + evenname + " fsc=" + fscout
apEMAN.executeEmanCmd(eotestcmd, verbose=True, showcmd=True)
else:
apDisplay.printWarning("could not perform eotest")
if os.path.isfile(fscout):
res = apRecon.getResolutionFromFSCFile(fscout, 160.0, 1.63)
apDisplay.printColor(("resolution: %.5f" % (res)), "cyan")
resfile = self.rootname + "-res.txt"
f = open(resfile, 'a')
f.write("[ %s ]\nresolution: %.5f\n" % (time.asctime(), res))
f.close()
def getSessionDataFromReconId(reconid):
stackid = apStack.getStackIdFromRecon(reconid)
partdata = apStack.getOneParticleFromStackId(stackid, msg=False)
sessiondata = partdata['particle']['selectionrun']['session']
return sessiondata
def calculateEulerJumpsForEntireRecon(self,
reconrunid,
stackid=None,
sym=None,
multimodelrunid=None):
if sym is None:
sym = self.getSymmetry(reconrunid)
if re.match("^icos", sym.lower()):
apDisplay.printWarning("Processing Icos symmetry " + sym)
if not re.match("^[cd][0-9]+$", sym.lower()) and not re.match(
"^icos", sym.lower()):
apDisplay.printError(
"Cannot calculate euler jumps for symmetry: " + sym)
return
### get stack particles
if stackid is None:
stackid = apStack.getStackIdFromRecon(reconrunid, msg=False)
stackparts = apStack.getStackParticlesFromId(stackid)
stackparts.sort(self.sortStackParts)
numparts = len(stackparts)
### start loop
t0 = time.time()
medians = []
count = 0
miscount = 0
for stackpart in stackparts:
count += 1
if multimodelrunid is None:
jumpdata = self.getEulerJumpData(reconrunid,
stackpartid=stackpart.dbid,
stackid=stackid,
sym=sym)
else:
jumpdata = self.getEulerJumpData(
reconrunid,
stackpartid=stackpart.dbid,
stackid=stackid,
sym=sym,
multimodelrunid=multimodelrunid)
if jumpdata is None:
if miscount < 25:
continue
else:
break
medians.append(jumpdata['median'])
if count % 500 == 0:
timeremain = (time.time() - t0) / (count + 1) * (numparts -
count)
print("particle=% 5d; median jump=% 3.2f, remain time= %s" %
(stackpart['particleNumber'], jumpdata['median'],
apDisplay.timeString(timeremain)))
if len(medians) > 0:
### print stats
apDisplay.printMsg("complete " + str(len(stackparts)) +
" particles in " +
apDisplay.timeString(time.time() - t0))
print "-- median euler jumper stats --"
medians = numpy.asarray(medians, dtype=numpy.float32)
print("mean/std :: " + str(round(medians.mean(), 2)) + " +/- " +
str(round(medians.std(), 2)))
print("min/max :: " + str(round(medians.min(), 2)) + " <> " +
str(round(medians.max(), 2)))
else:
apDisplay.printWarning(
"no Euler jumpers inserted into the database, make sure that the angles are read by the recon uploader"
)
return
def getSessionDataFromReconId(reconid):
stackid = apStack.getStackIdFromRecon(reconid)
partdata = apStack.getOneParticleFromStackId(stackid, msg=False)
sessiondata = partdata["particle"]["selectionrun"]["session"]
return sessiondata
def start(self):
self.rootname = self.params['stackname'].split(".")[0]
self.params['outputstack'] = os.path.join(self.params['rundir'], self.params['stackname'])
if os.path.isfile(self.params['outputstack']):
apFile.removeStack(self.params['outputstack'])
if self.params['eotest'] is True:
self.params['evenstack'] = os.path.splitext(self.params['outputstack'])[0]+'.even.hed'
if os.path.isfile(self.params['evenstack']):
apFile.removeStack(self.params['evenstack'])
self.params['oddstack'] = os.path.splitext(self.params['outputstack'])[0]+'.odd.hed'
if os.path.isfile(self.params['oddstack']):
apFile.removeStack(self.params['oddstack'])
classes = self.getClassData(self.params['reconid'], self.params['iter'])
stackid = apStack.getStackIdFromRecon(self.params['reconid'])
stackdata = apStack.getOnlyStackData(stackid)
stackpath = os.path.join(stackdata['path']['path'], stackdata['name'])
classkeys = classes.keys()
classkeys.sort()
classnum=0
keeplist = self.procKeepList()
finallist = []
apDisplay.printMsg("Processing "+str(len(classes))+" classes")
#loop through classes
for key in classkeys:
classnum+=1
if classnum%10 == 1:
sys.stderr.write("\b\b\b\b\b\b\b\b\b\b\b\b\b\b")
sys.stderr.write(str(classnum)+" of "+(str(len(classkeys))))
# loop through particles in class
classfile = self.rootname+"-class.lst"
classf = open(classfile, 'w')
classf.write('#LST\n')
nptcls=0
for ptcl in classes[key]['particles']:
# translate DB into EMAN
partnum = ptcl['particle']['particleNumber'] - 1
if partnum in keeplist:
if ptcl['mirror']:
mirror=1
else:
mirror=0
rot = ptcl['euler3']*math.pi/180.0
classf.write(
"%d\t%s\t%f,\t%f,%f,%f,%d\n" %
(partnum, stackpath, ptcl['quality_factor'],
rot, ptcl['shiftx'], ptcl['shifty'], mirror))
nptcls+=1
finallist.append(partnum)
classf.close()
if nptcls<1:
continue
self.makeClassAverages(classfile, self.params['outputstack'], classes[key], self.params['mask'])
if self.params['eotest'] is True:
self.makeEvenOddClasses(classfile, self.params['outputstack'], classes[key], self.params['mask'])
apFile.removeFile(classfile)
sys.stderr.write("\n")
finalfilename = self.rootname+"-keep.lst"
finalf = open(finalfilename, 'w')
finallist.sort()
for partnum in finallist:
finalf.write('%d\n' % (partnum,) )
finalf.close()
stackstr = str(stackdata.dbid)
reconstr = str(self.params['reconid'])
### recon 3d volumes
threedname = os.path.join(self.params['rundir'], self.rootname+"."+str(self.params['iter'])+"a.mrc")
emancmd = ( "make3d "+self.params['outputstack']+" out="
+threedname+" hard=50 sym="+self.params['symmname']+" pad=240 mask="+str(self.params['mask'])+"; echo ''" )
#print emancmd
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True, logfile=self.rootname+"-eman.log")
threednameb = os.path.join(self.params['rundir'], self.rootname+"."+str(self.params['iter'])+"b.mrc")
emancmd = ( "proc3d "+threedname+" "+threednameb
+" apix=1.63 norm=0,1 lp=8 origin=0,0,0 mask="+str(self.params['mask'])+"; echo '' " )
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True, logfile=self.rootname+"-eman.log")
if self.params['eotest'] is True:
# even
evenname = os.path.join(self.params['rundir'], self.rootname+"-even."+str(self.params['iter'])+"a.mrc")
if os.path.isfile(self.params['evenstack']):
evenemancmd = ( "make3d "+self.params['evenstack']+" out="
+evenname+" hard=50 sym="+self.params['symmname']+" pad=240 mask="+str(self.params['mask'])+"; echo ''" )
#print evenemancmd
apEMAN.executeEmanCmd(evenemancmd, verbose=False, showcmd=True, logfile=self.rootname+"-eveneman.log")
else:
apDisplay.printWarning("file "+self.params['evenstack']+" does not exist")
# odd
oddname = os.path.join(self.params['rundir'], self.rootname+"-odd."+str(self.params['iter'])+"a.mrc")
if os.path.isfile(self.params['oddstack']):
oddemancmd = ( "make3d "+self.params['oddstack']+" out="
+oddname+" hard=50 sym="+self.params['symmname']+" pad=240 mask="+str(self.params['mask'])+"; echo ''" )
#print oddemancmd
apEMAN.executeEmanCmd(oddemancmd, verbose=False, showcmd=True, logfile=self.rootname+"-oddeman.log")
else:
apDisplay.printWarning("file "+self.params['oddstack']+" does not exist")
#eotest
fscout = os.path.join(self.params['rundir'], self.rootname+"-fsc.eotest")
if os.path.isfile(oddname) and os.path.isfile(evenname):
eotestcmd = "proc3d "+oddname+" "+evenname+" fsc="+fscout
apEMAN.executeEmanCmd(eotestcmd, verbose=True, showcmd=True)
else:
apDisplay.printWarning("could not perform eotest")
if os.path.isfile(fscout):
res = apRecon.getResolutionFromFSCFile(fscout, 160.0, 1.63)
apDisplay.printColor( ("resolution: %.5f" % (res)), "cyan")
resfile = self.rootname+"-res.txt"
f = open(resfile, 'a')
f.write("[ %s ]\nresolution: %.5f\n" % (time.asctime(), res))
f.close()
