def verifyNumberOfCompletedRefinements(self,
multiModelRefinementRun=False):
"""
queries te database to determine how many iterations of individual refinements have been completed:
returns a list of completed iterations in a dictionary of completed refinements
"""
refine_complete = {}
if multiModelRefinementRun is False:
iterdata = appiondata.ApRefineIterData()
iterdata['refineRun'] = self.refinerunq
itercompletedata = iterdata.query()
iter_complete = []
for iter in itercompletedata:
iter_complete.append(iter['iteration'])
iter_complete.sort()
refine_complete[
self.refinerunq['reference_number']] = iter_complete
else:
refinedata = appiondata.ApRefineRunData()
refinedata['multiModelRefineRun'] = self.multimodelq
refinecompletedata = refinedata.query()
for i, refine in enumerate(refinecompletedata):
iterdata = appiondata.ApRefineIterData()
iterdata['refineRun'] = refine
itercompletedata = iterdata.query()
iter_complete = []
for iter in itercompletedata:
iter_complete.append(iter['iteration'])
iter_complete.sort()
refine_complete[refine['reference_number']] = iter_complete
return refine_complete
def getEulersForParticleSinedon(self, stackpartid, reconrunid):
"""
returns all classdata for a particular particle and refinement
"""
refrundata = appiondata.ApRefineRunData.direct_query(reconrunid)
stackpartdata = appiondata.ApStackParticleData.direct_query(
stackpartid)
refmentq = appiondata.ApRefineIterData()
refmentq['refineRun'] = refrundata
particledata = stackpartdata
partclassq = appiondata.ApRefineParticleData()
partclassq['particle'] = particledata
partclassq['refineIter'] = refmentq
partclassdata = partclassq.query()
eulertree = []
for data in partclassdata:
try:
euler = {}
euler['stackpartid'] = int(data['particle'].dbid)
euler['euler1'] = float(data['euler1'])
euler['euler2'] = float(data['euler2'])
euler['euler3'] = float(data['euler3'])
euler['mirror'] = self.nullOrValue(data['mirror'])
euler['reject'] = self.nullOrValue(data['refine_keep'])
euler['iteration'] = int(data['refineIter']['iteration'])
eulertree.append(euler)
except:
print euler
import pprint
pprint.pprint(data)
apDisplay.printError("bad data entry")
return eulertree
def setIterData(self, iterparams, volumeDensity):
frealigniterq = appiondata.ApFrealignIterData()
frealignkeys = ('wgh', 'xstd', 'pbc',
'itmax', 'ipmax', 'target', 'thresh', 'cs',
'rrec', 'highpass', 'lowpass', 'rbfact')
for key in frealignkeys:
frealigniterq[key] = iterparams[key]
iternum = iterparams['iternum']
iterq = appiondata.ApRefineIterData()
iterq['iteration'] = iternum
iterq['exemplar'] = False
### mask values are in pixels
iterq['mask'] = iterparams['mask']/self.apix
iterq['imask'] = iterparams['imask']/self.apix
iterq['volumeDensity'] = os.path.basename(volumeDensity)
### Frealign does not produce class averages :(
iterq['refineClassAverages'] = None
iterq['postRefineClassAverages'] = None
iterq['classVariance'] = None
iterq['symmetry'] = iterparams['symmdata']
iterq['refineRun'] = self.runq
iterq['resolution'] = self.getResolutionData(iternum)
iterq['rMeasure'] = self.getRMeasureData(volumeDensity)
iterq['frealignParams'] = frealigniterq
return iterq
def getParticleInfo(reconid, iteration):
"""
Get all particle data for given recon and iteration
"""
refinerundata = appiondata.ApRefineRunData.direct_query(reconid)
if not refinerundata:
apDisplay.printError("Could not find refinerundata for reconrun id=" +
str(reconid))
refineq = appiondata.ApRefineIterData()
refineq['refineRun'] = refinerundata
refineq['iteration'] = iteration
refinedata = refineq.query(results=1)
if not refinedata:
apDisplay.printError("Could not find refinedata for reconrun id=" +
str(reconid) + " iter=" + str(iteration))
refinepartq = appiondata.ApRefineParticleData()
refinepartq['refineIter'] = refinedata[0]
t0 = time.time()
apDisplay.printMsg("querying particles on " + time.asctime())
refineparticledata = refinepartq.query()
apDisplay.printMsg("received " + str(len(refineparticledata)) +
" particles in " +
apDisplay.timeString(time.time() - t0))
return (refineparticledata, refinedata[0])
def getSymmetryFromReconRunId(reconrunid, msg=True):
"""
get the symmetry from the last iteration of a refinement
"""
refrundata = appiondata.ApRefineRunData.direct_query(reconrunid)
refdataq = appiondata.ApRefineIterData()
refdataq['refineRun'] = refrundata
refdata = refdataq.query()
uniqsym = refdata[0]['symmetry']
if uniqsym is None:
apDisplay.printWarning("symmetry is not saved during reconstruction!")
apDisplay.printWarning("Using the symmetry of the initial model")
modeldata = refrundata['initialModel']
uniqsym = modeldata['symmetry']
else:
for data in refdata:
if uniqsym != data['symmetry']:
apDisplay.printWarning(
"symmetry is not consistent throughout reconstruction!")
apDisplay.printWarning("Using symmetry of last iteration")
uniqsym = data['symmetry']
if msg is True:
apDisplay.printMsg(
"selected symmetry group: " +
apDisplay.colorString("'" + uniqsym['eman_name'] + "'", "cyan") +
" for recon run: " + str(reconrunid))
return uniqsym
def checkConflicts(self):
if self.params['reconid'] is None:
apDisplay.printError(
"Please provide a recon run id, e.g. --reconid=15")
if self.params['iternum'] is None:
refineiterq = appiondata.ApRefineIterData()
refrun = appiondata.ApRefineRunData.direct_query(
self.params['reconid'])
refineiterq['refineRun'] = refrun
refiterdata = refineiterq.query()
lastiter = len(refiterdata)
refineiterq = appiondata.ApRefineIterData()
refineiterq['refineRun'] = refrun
refineiterq['iteration'] = lastiter
refiterdata = refineiterq.query(results=1)
if not refiterdata:
apDisplay.printError(
"Could not get iteration, please provide a iter num, e.g. --iternum=15"
)
self.params['iternum'] = lastiter
def setGoodBadParticlesFromReconId(reconid):
"""
Goes through existing recons and caches the number of good and bad particles
"""
import sinedon
import MySQLdb
dbconf = sinedon.getConfig('appiondata')
db = MySQLdb.connect(**dbconf)
db.autocommit(True)
cursor = db.cursor()
refinerundata = appiondata.ApRefineRunData.direct_query(reconid)
refineq = appiondata.ApRefineIterData()
refineq['refineRun'] = refinerundata
refineiterdatas = refineq.query()
r0 = time.time()
for refineiterdata in refineiterdatas:
t0 = time.time()
#print "Iteration %d"%(refineiterdata['iteration'])
goodbadq = appiondata.ApRefineGoodBadParticleData()
goodbadq['refine'] = refineiterdata
goodbaddata = goodbadq.query()
if goodbaddata:
continue
fields = {
'good_refine':
getParticleCount(refineiterdata.dbid, cursor, 'refine_keep', True),
'bad_refine':
getParticleCount(refineiterdata.dbid, cursor, 'refine_keep',
False),
'good_postRefine':
getParticleCount(refineiterdata.dbid, cursor, 'postRefine_keep',
True),
'bad_postRefine':
getParticleCount(refineiterdata.dbid, cursor, 'postRefine_keep',
False),
}
#print fields
goodbadq = appiondata.ApRefineGoodBadParticleData()
goodbadq['refine'] = refineiterdata
goodbadq['good_refine'] = fields['good_refine']
goodbadq['bad_refine'] = fields['bad_refine']
goodbadq['good_postRefine'] = fields['good_postRefine']
goodbadq['bad_postRefine'] = fields['bad_postRefine']
goodbadq.insert()
if time.time() - t0 > 2.0:
apDisplay.printMsg("Iter completed in %s" %
(apDisplay.timeString(time.time() - t0)))
if time.time() - r0 > 2.0:
apDisplay.printMsg("Refine completed in %s" %
(apDisplay.timeString(time.time() - r0)))
return
def getNumIterationsFromRefineRunID(refinerunid):
refrundata = appiondata.ApRefineRunData.direct_query(refinerunid)
refq = appiondata.ApRefineIterData()
refq['refineRun'] = refrundata
refdatas = refq.query()
if not refdatas:
return 0
maxiter = 0
for refdata in refdatas:
iternum = refdata['iteration']
if iternum > maxiter:
maxiter = iternum
return maxiter
def start(self):
apDisplay.printMsg("\n\n")
### get particles
refrun = appiondata.ApRefineRunData.direct_query(
self.params['reconid'])
refiterq = appiondata.ApRefineIterData()
refiterq['refineRun'] = refrun
refiterq['iteration'] = self.params['iternum']
refiterdatas = refiterq.query(results=1)
refpartq = appiondata.ApRefineParticleData()
refpartq['refineIter'] = refiterdatas[0]
#this gets lots of data
refpartdatas = refpartq.query()
### get session
firstrefpart = refpartdatas[0]
firstpart = firstrefpart['particle']['particle']
sessiondata = firstpart['selectionrun']['session']
### create a selection run
runq = appiondata.ApSelectionRunData()
for key in firstpart['selectionrun'].keys():
runq[key] = firstpart['selectionrun'][key]
runq['name'] = self.params['runname']
runq['session'] = sessiondata
pathq = appiondata.ApPathData()
pathq['path'] = self.params['rundir']
runq['path'] = pathq
runq['description'] = (
"Corrected particles from refine id %d iter %d and selection %d" %
(self.params['reconid'], self.params['iternum'],
firstpart['selectionrun'].dbid))
count = 0
for refpartdata in refpartdatas:
count += 1
if count % 10 == 0:
sys.stderr.write(".")
partdata = refpartdata['particle']['particle']
newpartq = appiondata.ApParticleData()
for key in partdata.keys():
newpartq[key] = partdata[key]
newpartq['xcoord'] = partdata['xcoord'] + refpartdata['shiftx']
newpartq['ycoord'] = partdata['ycoord'] + refpartdata['shifty']
newpartq['selectionrun'] = runq
newpartq.insert()
def getParticleInfo(self, reconid, iteration):
"""
Get all particle data for given recon and iteration
"""
t0 = time.time()
cachefile = os.path.join(
self.params['rundir'], "refineparticledata-r" + str(reconid) +
"-i" + str(iteration) + ".cache")
if os.path.isfile(cachefile):
apDisplay.printColor("loading refineparticledata from cache file",
"cyan")
f = open(cachefile, 'r')
refineparticledata = cPickle.load(f)
f.close()
else:
refinerundata = appiondata.ApRefineRunData.direct_query(reconid)
if not refinerundata:
apDisplay.printError(
"Could not find refinerundata for reconrun id=" +
str(reconid))
refineq = appiondata.ApRefineIterData()
refineq['refineRun'] = refinerundata
refineq['iteration'] = iteration
refinedata = refineq.query(results=1)
if not refinedata:
apDisplay.printError(
"Could not find refinedata for reconrun id=" +
str(reconid) + " iter=" + str(iteration))
refinepartq = appiondata.ApRefineParticleData()
refinepartq['refineIter'] = refinedata[0]
apDisplay.printMsg("querying particles on " + time.asctime())
refineparticledata = refinepartq.query()
apDisplay.printMsg("saving refineparticledata to cache file")
f = open(cachefile, 'w')
cPickle.dump(refineparticledata, f)
f.close()
apDisplay.printMsg("received " + str(len(refineparticledata)) +
" particles in " +
apDisplay.timeString(time.time() - t0))
return refineparticledata
fscfile, self.runparams['boxsize'], self.runparams['apix'])
apDisplay.printColor("FSC 0.5 Resolution: " + str(fscRes),
"cyan")
resq = appiondata.ApResolutionData()
resq['half'] = fscRes
resq['fscfile'] = os.path.basename(fscfile)
except Exception, e:
print e
apDisplay.printWarning(
"FSC file does not exist or is unreadable")
resq = None
else:
resq = None
### fill in ApRefineIterData object
iterationParamsq = appiondata.ApRefineIterData()
if package_table is not None and package_database_object is not None:
iterationParamsq[str(
package_table.split("|")[1])] = package_database_object
iterationParamsq['refineRun'] = self.refinerunq
iterationParamsq['iteration'] = iteration
iterationParamsq['resolution'] = resq
if not self.params['euleronly']:
iterationParamsq['rMeasure'] = self.getRMeasureData(
iteration, reference_number)
else:
iterationParamsq['rMeasure'] = None
iterationParamsq['mask'] = apRecon.getComponentFromVector(
self.runparams['mask'], iteration - 1)
iterationParamsq['imask'] = apRecon.getComponentFromVector(
self.runparams['imask'], iteration - 1)
def getRefinementsFromRun(refinerundata):
refineitq = appiondata.ApRefineIterData()
refineitq['refineRun'] = refinerundata
return refineitq.query()
def insertIteration(self, iteration):
refineparamsq = appiondata.ApEmanRefineIterData()
refineparamsq['ang'] = iteration['ang']
refineparamsq['lpfilter'] = iteration['lpfilter']
refineparamsq['hpfilter'] = iteration['hpfilter']
refineparamsq['pad'] = iteration['pad']
refineparamsq['EMAN_maxshift'] = iteration['maxshift']
refineparamsq['EMAN_hard'] = iteration['hard']
refineparamsq['EMAN_classkeep'] = iteration['classkeep']
refineparamsq['EMAN_classiter'] = iteration['classiter']
refineparamsq['EMAN_filt3d'] = iteration['filt3d']
refineparamsq['EMAN_shrink'] = iteration['shrink']
refineparamsq['EMAN_euler2'] = iteration['euler2']
refineparamsq['EMAN_xfiles'] = iteration['xfiles']
refineparamsq['EMAN_median'] = iteration['median']
refineparamsq['EMAN_phasecls'] = iteration['phasecls']
refineparamsq['EMAN_fscls'] = iteration['fscls']
refineparamsq['EMAN_refine'] = iteration['refine']
refineparamsq['EMAN_goodbad'] = iteration['goodbad']
refineparamsq['EMAN_perturb'] = iteration['perturb']
refineparamsq['MsgP_cckeep'] = iteration['msgpasskeep']
refineparamsq['MsgP_minptls'] = iteration['msgpassminp']
#create Chimera snapshots
fscfile = os.path.join(self.params['rundir'],
"fsc.eotest." + iteration['num'])
halfres = apRecon.calcRes(fscfile, self.params['boxsize'],
self.params['apix'])
if self.params['snapfilter']:
halfres = self.params['snapfilter']
volumeDensity = 'threed.' + iteration['num'] + 'a.mrc'
volDensPath = os.path.join(self.params['rundir'], volumeDensity)
apChimera.filterAndChimera(volDensPath,
halfres,
self.params['apix'],
self.params['boxsize'],
'snapshot',
self.params['contour'],
self.params['zoom'],
sym=iteration['sym']['eman_name'],
mass=self.params['mass'])
## uncommment this for chimera image only runs...
if self.params['chimeraonly'] is True:
return
# insert resolution data
if halfres != True:
resData = self.getResolutionData(iteration)
else:
apDisplay.printWarning(
"resolution reported as nan, not committing results to database"
)
return
if self.params['package'] == 'EMAN':
refineclassavg = 'classes_eman.' + iteration['num'] + '.img'
postrefineclassavg = None
elif self.params['package'] == 'EMAN/SpiCoran':
refineclassavg = 'classes_eman.' + iteration['num'] + '.img'
postrefineclassavg = 'classes_coran.' + iteration['num'] + '.img'
elif self.params['package'] == 'EMAN/MsgP':
refineclassavg = 'classes_eman.' + iteration['num'] + '.img'
postrefineclassavg = 'classes_msgp.' + iteration['num'] + '.img'
else:
apDisplay.printError("Refinement Package Not Valid")
# insert refinement results
refineq = appiondata.ApRefineIterData()
refineq['refineRun'] = self.params['refineRun']
refineq['emanParams'] = refineparamsq
refineq['iteration'] = iteration['num']
refineq['resolution'] = resData
refineq['rMeasure'] = self.getRMeasureData(iteration)
refineq['mask'] = iteration['mask']
refineq['imask'] = iteration['imask']
refineq['symmetry'] = iteration['sym']
refineq['exemplar'] = False
classvar = 'classes.' + iteration['num'] + '.var.img'
refineq['refineClassAverages'] = refineclassavg
refineq['postRefineClassAverages'] = postrefineclassavg
if classvar in self.params['classvars']:
refineq['classVariance'] = classvar
if volumeDensity in self.params['volumes']:
refineq['volumeDensity'] = volumeDensity
apDisplay.printMsg("inserting Refinement Data into database")
if self.params['commit'] is True:
refineq.insert()
else:
apDisplay.printWarning("not committing results to database")
#insert FSC data
fscfile = os.path.join(self.params['rundir'],
"fsc.eotest." + iteration['num'])
self.insertFSC(fscfile, refineq, self.params['commit'])
halfres = apRecon.calcRes(fscfile, self.params['boxsize'],
self.params['apix'])
apDisplay.printColor("FSC 0.5 Resolution: " + str(halfres), "cyan")
# get projections eulers for iteration:
eulers = self.getEulersFromProj(iteration['num'])
# get list of bad particles for this iteration
badprtls = self.readParticleLog(self.params['rundir'],
iteration['num'])
# expand cls.*.tar into temp file
clsf = os.path.join(self.params['rundir'],
"cls." + iteration['num'] + ".tar")
#print "reading",clsf
clstar = tarfile.open(clsf)
clslist = clstar.getmembers()
clsnames = clstar.getnames()
#print "extracting",clsf,"into temp directory"
for clsfile in clslist:
clstar.extract(clsfile, self.params['tmpdir'])
clstar.close()
# for each class, insert particle alignment info into database
apDisplay.printColor(
"Inserting Particle Classification Data for " +
str(len(clsnames)) + " classes", "magenta")
t0 = time.time()
for cls in clsnames:
self.insertRefineParticleData(cls, iteration, eulers, badprtls,
refineq, len(clsnames))
apDisplay.printColor(
"\nFinished in " + apDisplay.timeString(time.time() - t0),
"magenta")
# remove temp directory
for file in os.listdir(self.params['tmpdir']):
os.remove(os.path.join(self.params['tmpdir'], file))
os.rmdir(self.params['tmpdir'])
#create euler freq map
apDisplay.printMsg("creating euler frequency map")
refrunid = int(self.params['refineRun'].dbid)
iternum = int(iteration['num'])
if self.params['package'] != 'EMAN':
postrefine = True
else:
postrefine = False
apEulerDraw.createEulerImages(refrunid,
iternum,
path=self.params['rundir'],
postrefine=postrefine)
return
def start(self):
partdict = {}
partlist = []
### get Euler angles for each particle
for iternum in self.iternums:
### get recon iter data
reconiterq = appiondata.ApRefineIterData()
reconiterq['refineRun'] = self.reconrundata
reconiterq['iteration'] = iternum
reconiterdata = reconiterq.query(
results=1)[0] #this should be unique
### get particle data
reconpartq = appiondata.ApRefineParticleData()
reconpartq['refineIter'] = reconiterdata
apDisplay.printMsg("Querying for particles at " + time.asctime())
reconpartdatas = reconpartq.query()
### group particle data
for partdata in reconpartdatas:
partnum = partdata['particle']['particleNumber']
if not partnum in partlist:
partlist.append(partnum)
partdict[(partnum, iternum)] = partdata
### run through particles and check Euler angles
partlist.sort()
eulerdict = {}
eulercount = {}
reject = 0
for partnum in partlist:
e1d = {}
e2d = {}
for iternum in self.iternums:
if not (partnum, iternum) in partdict:
continue
partdata = partdict[(partnum, iternum)]
euler1 = "%.2f" % (partdata['euler1'])
if not euler1 in e1d:
e1d[euler1] = 1
else:
e1d[euler1] += 1
euler2 = "%.2f" % (partdata['euler2'])
if not euler2 in e2d:
e2d[euler2] = 1
else:
e2d[euler2] += 1
#print partnum, euler1, euler2
counts = [(val, key) for key, val in e1d.items()]
e1count, euler1 = max(counts)
counts = [(val, key) for key, val in e2d.items()]
e2count, euler2 = max(counts)
# reject indeterminant particles
if e2count < 2 or e1count < 2:
reject += 1
continue
### group particles by their Euler angles
if not (euler1, euler2) in eulerdict:
eulerdict[(euler1, euler2)] = []
eulercount[(euler1, euler2)] = 0
eulerdict[(euler1, euler2)].append(partnum)
eulercount[(euler1, euler2)] += 1
print "Rejected %d particles" % (reject)
values = eulercount.values()
values.sort()
print values
### run through Euler angles and count particles
counts = [(val, key) for key, val in eulercount.items()]
mincount, val = min(counts)
self.params['mincount'] = max(self.params['mincount'], mincount)
#print "Keeping %d of %d particles"%(mincount*len(eulercount.keys()), len(partlist))
print "Keeping %d of %d particles" % (
self.params['mincount'] * len(eulercount.keys()), len(partlist))
keeplist = []
for key in eulerdict.keys():
eulerpartlist = eulerdict[key]
if len(partlist) < self.params['mincount']:
keeplist.extend(eulerpartlist)
else:
keeplist.extend(eulerpartlist[:self.params['mincount']])
keeplist.sort()
print "Keeping %d of %d particles" % (len(keeplist), len(partlist))
#need to set keepfile for commitSubStack
self.params['keepfile'] = os.path.join(self.params['rundir'],
"equalviews.lst")
f = open(self.params['keepfile'], "w")
for partnum in keeplist:
f.write("%d\n" % (partnum - 1))
f.close()
### make a new stack using the keep particles
oldstackdata = self.reconrundata['stack']
oldstack = os.path.join(oldstackdata['path']['path'],
oldstackdata['name'])
newstack = os.path.join(self.params['rundir'], "start.hed")
apStack.makeNewStack(oldstack,
newstack,
listfile=self.params['keepfile'],
remove=True,
bad=True)
if not os.path.isfile(newstack):
apDisplay.printError("No stack was created")
self.params[
'stackid'] = oldstackdata.dbid #need to set stackid for commitSubStack
apStack.commitSubStack(self.params, "start.hed")
apStack.averageStack(stack=newstack)
