def insertRefineParticleData(self, iterdata, parttree):
apDisplay.printMsg("Inserting particle data")
phase_threshold = float(iterdata['frealignParams']['thresh'])
count = 0
for partdict in parttree:
count += 1
if count % 1000 == 0:
apDisplay.printMsg("Inserted %d particles"%(count))
partrefineq = appiondata.ApRefineParticleData()
partrefineq['refineIter'] = iterdata
partnum = partdict['partnum']
stackpartid = self.stackmapping[partnum]
stackpartdata = appiondata.ApStackParticleData.direct_query(stackpartid)
partrefineq['particle'] = stackpartdata
partkeys = ('shiftx', 'shifty', 'euler1', 'euler2', 'euler3', 'phase_residual')
for key in partkeys:
partrefineq[key] = partdict[key]
### check if particle was rejected
if partdict['phase_residual'] > iterdata['frealignParams']['thresh']:
partrefineq['refine_keep'] = False
else:
partrefineq['refine_keep'] = True
partrefineq['mirror'] = False
partrefineq['postRefine_keep'] = False
partrefineq['euler_convention'] = 'zyz'
if self.params['commit'] is True:
partrefineq.insert()
return
def getEulersForParticleSinedon(self, stackpartid, reconrunid):
"""
returns all classdata for a particular particle and refinement
"""
refrundata = appiondata.ApRefineRunData.direct_query(reconrunid)
stackpartdata = appiondata.ApStackParticleData.direct_query(
stackpartid)
refmentq = appiondata.ApRefineIterData()
refmentq['refineRun'] = refrundata
particledata = stackpartdata
partclassq = appiondata.ApRefineParticleData()
partclassq['particle'] = particledata
partclassq['refineIter'] = refmentq
partclassdata = partclassq.query()
eulertree = []
for data in partclassdata:
try:
euler = {}
euler['stackpartid'] = int(data['particle'].dbid)
euler['euler1'] = float(data['euler1'])
euler['euler2'] = float(data['euler2'])
euler['euler3'] = float(data['euler3'])
euler['mirror'] = self.nullOrValue(data['mirror'])
euler['reject'] = self.nullOrValue(data['refine_keep'])
euler['iteration'] = int(data['refineIter']['iteration'])
eulertree.append(euler)
except:
print euler
import pprint
pprint.pprint(data)
apDisplay.printError("bad data entry")
return eulertree
def getParticleInfo(reconid, iteration):
"""
Get all particle data for given recon and iteration
"""
refinerundata = appiondata.ApRefineRunData.direct_query(reconid)
if not refinerundata:
apDisplay.printError("Could not find refinerundata for reconrun id=" +
str(reconid))
refineq = appiondata.ApRefineIterData()
refineq['refineRun'] = refinerundata
refineq['iteration'] = iteration
refinedata = refineq.query(results=1)
if not refinedata:
apDisplay.printError("Could not find refinedata for reconrun id=" +
str(reconid) + " iter=" + str(iteration))
refinepartq = appiondata.ApRefineParticleData()
refinepartq['refineIter'] = refinedata[0]
t0 = time.time()
apDisplay.printMsg("querying particles on " + time.asctime())
refineparticledata = refinepartq.query()
apDisplay.printMsg("received " + str(len(refineparticledata)) +
" particles in " +
apDisplay.timeString(time.time() - t0))
return (refineparticledata, refinedata[0])
def getStackParticleEulersForIteration(self, pnum):
"""
find the eulers assigned to a stack particle
during a refinement. This function will first
find the particle id for the given stack particle,
then find its position in the reference stack, and
will get the eulers for that particle in the recon
"""
# get stack particle id
stackp = apStack.getStackParticle(self.params['stackid'], pnum)
particleid = stackp['particle'].dbid
# find particle in reference stack
refstackp = apStack.getStackParticleFromParticleId(particleid, self.oldreconstackid, noDie=True)
if not refstackp:
percentnoeuler = 100*self.noeulers/float(self.params['last'])
apDisplay.printWarning('No eulers for particle %d (%.1f%%)' % (pnum, percentnoeuler))
self.noeulers += 1
if percentnoeuler > 10:
apDisplay.printError('More than 10% of the particles have no euler, use a different reference reconstruction')
pclass={
'euler1': 0.0,
'euler2': 0.0,
'euler3': 0.0,
'mirror': False,
'shiftx': 0.0,
'shifty': 0.0,
}
else:
pclassq = appiondata.ApRefineParticleData()
pclassq['particle'] = refstackp
pclassq['refineIter'] = appiondata.ApRefineIterData.direct_query(self.params['reconiterid'])
pclasses = pclassq.query(results=1)
pclass = pclasses[0]
try:
emaneuler={
'alt': pclass['euler1'],
'az': pclass['euler2'],
'phi': pclass['euler3'],
'mirror': pclass['mirror'],
'shiftx': pclass['shiftx'],
'shifty': pclass['shifty'],
}
except:
print pclass
emaneuler={
'alt': pclass['euler1'],
'az': pclass['euler2'],
'phi': pclass['euler3'],
'mirror': pclass['mirror'],
'shiftx': pclass['shiftx'],
'shifty': pclass['shifty'],
}
return emaneuler
def getStackParticleEulersForIteration(params, pnum):
"""
find the eulers assigned to a stack particle
during a refinement. This function will first
find the particle id for the given stack particle,
then find its position in the reference stack, and
will get the eulers for that particle in the recon
"""
# get stack particle id
stackp = apStack.getStackParticle(params['stackid'], pnum)
particleid = stackp['particle'].dbid
# find particle in reference stack
refstackid = apStack.getStackIdFromIterationId(params['reconiterid'],
msg=False)
refstackp = apStack.getStackParticleFromParticleId(particleid,
refstackid,
nodie=True)
if not refstackp:
apDisplay.printWarning(
'No classification for stack particle %d in reconstruction iteration id: %d'
% (pnum, params['reconiterid']))
params['noClassification'] += 1
if params['noClassification'] > (float(params['last']) * 0.10):
apDisplay.printError(
'More than 10% of the particles have no classification, use a different reference reconstruction'
)
pclass = {}
pclass['eulers'] = {}
pclass['eulers']['euler1'] = 0.0
pclass['eulers']['euler2'] = 0.0
pclass['inplane_rotation'] = 0.0
pclass['mirror'] = False
pclass['shiftx'] = 0.0
pclass['shifty'] = 0.0
else:
pclassq = appiondata.ApRefineParticleData()
pclassq['particle'] = refstackp
pclassq['refineIter'] = appiondata.ApRefineIterData.direct_query(
params['reconiterid'])
pclass = pclassq.query()
pclass = pclass[0]
params['eman_orient'] = {}
params['eman_orient']['alt'] = pclass['euler1']
params['eman_orient']['az'] = pclass['euler2']
params['eman_orient']['phi'] = pclass['euler3']
params['eman_orient']['mirror'] = pclass['mirror']
params['eman_orient']['shiftx'] = pclass['shiftx']
params['eman_orient']['shifty'] = pclass['shifty']
return params
def start(self):
apDisplay.printMsg("\n\n")
### get particles
refrun = appiondata.ApRefineRunData.direct_query(
self.params['reconid'])
refiterq = appiondata.ApRefineIterData()
refiterq['refineRun'] = refrun
refiterq['iteration'] = self.params['iternum']
refiterdatas = refiterq.query(results=1)
refpartq = appiondata.ApRefineParticleData()
refpartq['refineIter'] = refiterdatas[0]
#this gets lots of data
refpartdatas = refpartq.query()
### get session
firstrefpart = refpartdatas[0]
firstpart = firstrefpart['particle']['particle']
sessiondata = firstpart['selectionrun']['session']
### create a selection run
runq = appiondata.ApSelectionRunData()
for key in firstpart['selectionrun'].keys():
runq[key] = firstpart['selectionrun'][key]
runq['name'] = self.params['runname']
runq['session'] = sessiondata
pathq = appiondata.ApPathData()
pathq['path'] = self.params['rundir']
runq['path'] = pathq
runq['description'] = (
"Corrected particles from refine id %d iter %d and selection %d" %
(self.params['reconid'], self.params['iternum'],
firstpart['selectionrun'].dbid))
count = 0
for refpartdata in refpartdatas:
count += 1
if count % 10 == 0:
sys.stderr.write(".")
partdata = refpartdata['particle']['particle']
newpartq = appiondata.ApParticleData()
for key in partdata.keys():
newpartq[key] = partdata[key]
newpartq['xcoord'] = partdata['xcoord'] + refpartdata['shiftx']
newpartq['ycoord'] = partdata['ycoord'] + refpartdata['shifty']
newpartq['selectionrun'] = runq
newpartq.insert()
def getParticleInfo(self, reconid, iteration):
"""
Get all particle data for given recon and iteration
"""
t0 = time.time()
cachefile = os.path.join(
self.params['rundir'], "refineparticledata-r" + str(reconid) +
"-i" + str(iteration) + ".cache")
if os.path.isfile(cachefile):
apDisplay.printColor("loading refineparticledata from cache file",
"cyan")
f = open(cachefile, 'r')
refineparticledata = cPickle.load(f)
f.close()
else:
refinerundata = appiondata.ApRefineRunData.direct_query(reconid)
if not refinerundata:
apDisplay.printError(
"Could not find refinerundata for reconrun id=" +
str(reconid))
refineq = appiondata.ApRefineIterData()
refineq['refineRun'] = refinerundata
refineq['iteration'] = iteration
refinedata = refineq.query(results=1)
if not refinedata:
apDisplay.printError(
"Could not find refinedata for reconrun id=" +
str(reconid) + " iter=" + str(iteration))
refinepartq = appiondata.ApRefineParticleData()
refinepartq['refineIter'] = refinedata[0]
apDisplay.printMsg("querying particles on " + time.asctime())
refineparticledata = refinepartq.query()
apDisplay.printMsg("saving refineparticledata to cache file")
f = open(cachefile, 'w')
cPickle.dump(refineparticledata, f)
f.close()
apDisplay.printMsg("received " + str(len(refineparticledata)) +
" particles in " +
apDisplay.timeString(time.time() - t0))
return refineparticledata
def insertRefinementParticleData(self,
particledata,
iterationParamsq,
referenceParamsq,
euler_convention='zxz'):
''' inserts all particle data into database, based on parameters from text file '''
apDisplay.printMsg(
"inserting particle data into database ... this may take some time"
)
for i in range(len(particledata)):
prtlq = appiondata.ApRefineParticleData()
### map particle to stack
# prtlnum = particledata[i]['partnum']+1 ### offset by 1
prtlnum = particledata[i]['partnum']
defid = self.stackmapping[prtlnum]
stackp = appiondata.ApStackParticleData.direct_query(defid)
if not stackp:
apDisplay.printError("particle " + str(prtlnum) +
" not in stack id=" +
str(self.runparams['stackid']))
### convert icos convention to standard convention
full_sym_name = iterationParamsq['symmetry']['symmetry']
if 'Icos' in full_sym_name:
# Icos particle data from particle file is always in 235
full_sym_name = 'Icos (2 3 5) Viper/3DEM'
phi, theta, omega = apEulerCalc.convert3DEMEulerToStandardSym(
full_sym_name, particledata[i]['phi'],
particledata[i]['theta'], particledata[i]['omega'])
### convert Euler angles from 3DEM to EMAN format (temporary fix)
alt, az, phi = apXmipp.convertXmippEulersToEman(phi, theta, omega)
if self.multiModelRefinementRun is True:
prtlq['multiModelRefineRun'] = self.multimodelq
prtlq['refineIter'] = iterationParamsq
prtlq['reference_number'] = referenceParamsq
prtlq['particle'] = stackp
prtlq['euler1'] = float(alt)
prtlq['euler2'] = float(az)
prtlq['euler3'] = float(phi)
prtlq['shiftx'] = particledata[i]['shiftx']
prtlq['shifty'] = particledata[i]['shifty']
prtlq['mirror'] = 0
# prtlq['mirror'] = particledata[i]['mirror']
prtlq['3Dref_num'] = particledata[i]['refnum']
try:
prtlq['2Dclass_num'] = particledata[i]['clsnum']
except:
pass
try:
prtlq['quality_factor'] = particledata[i]['quality']
except:
pass
try:
prtlq['refine_keep'] = particledata[i]['refine_keep']
except:
pass
try:
prtlq['postRefine_keep'] = particledata[i]['postRefine_keep']
except:
pass
prtlq['euler_convention'] = euler_convention
if self.params['commit'] is True:
prtlq.insert()
return
def getStackParticleEulersForIteration(self, stackPart):
"""
find the eulers assigned to a stack particle
during a refinement. This function will first
find the particle id for the given stack particle,
then find its position in the reference stack, and
will get the eulers for that particle in the recon
assumes recon with FREALIGN
"""
# get stack particle id
stackPartId = stackPart.dbid
partId = stackPart['particle'].dbid
# find particle in reference stack
refStackId = apStack.getStackIdFromIterationId(
self.params['reconiterid'], msg=False)
refStackPart = apStack.getStackParticleFromParticleId(
partId, refStackId)
if not refStackPart:
apDisplay.printWarning(
'No classification for stack particle %d in reconstruction iteration id: %d'
% (refStackId, self.params['reconiterid']))
self.noClassification += 1
if self.noClassification > (float(params['last']) * 0.10):
apDisplay.printError(
'More than 10% of the particles have no classification, use a different reference reconstruction'
)
eulerDict = {
'psi': 0.0,
'theta': 0.0,
'phi': 0.0,
'shiftx': 0.0,
'shifty': 0.0,
}
return eulerDict
refIterData = appiondata.ApRefineIterData.direct_query(
self.params['reconiterid'])
if refStackPart.dbid != stackPartId:
self.mismatch += 1
refinePartQuery = appiondata.ApRefineParticleData()
refinePartQuery['particle'] = refStackPart
refinePartQuery['refineIter'] = refIterData
refinePartDatas = refinePartQuery.query()
refinePartData = refinePartDatas[0]
emanEulerDict = {
'alt': refinePartData['euler1'],
'az': refinePartData['euler2'],
'phi': refinePartData['euler3'],
'shiftx': refinePartData['shiftx'],
'shifty': refinePartData['shifty'],
'mirror': refinePartData['mirror'],
}
eulerDict = apFrealign.convertAppionEmanEulersToFrealign(
emanEulerDict, self.symmetryName)
eulerDict['shiftx'] = emanEulerDict['shiftx'] * self.params['bin']
eulerDict['shifty'] = emanEulerDict['shifty'] * self.params['bin']
if refinePartData['mirror'] is True:
eulerDict['shiftx'] *= -1
return eulerDict
def start(self):
### new stack path
oldstack = os.path.join(self.stackdata['path']['path'],
self.stackdata['name'])
newstack = os.path.join(self.params['rundir'], self.stackdata['name'])
apStack.checkForPreviousStack(newstack)
### get particles from stack
apDisplay.printMsg("Querying stack particles")
t0 = time.time()
stackpartq = appiondata.ApRefineParticleData()
stackpartq['refineIter'] = self.iterdata
particles = stackpartq.query()
apDisplay.printMsg("Finished in " +
apDisplay.timeString(time.time() - t0))
### write included particles to text file
includeParticle = []
excludeParticle = 0
f = open("test.log", "w")
count = 0
apDisplay.printMsg("Processing stack particles")
t0 = time.time()
for part in particles:
count += 1
if count % 500 == 0:
sys.stderr.write(".")
emanstackpartnum = part['particle']['particleNumber'] - 1
if part['postRefine_keep'] == 1:
### good particle
includeParticle.append(emanstackpartnum)
f.write("%d\t%d\tinclude\n" % (count, emanstackpartnum))
else:
### bad particle
excludeParticle += 1
f.write("%d\t%d\texclude\n" % (count, emanstackpartnum))
sys.stderr.write("\n")
apDisplay.printMsg("Finished in " +
apDisplay.timeString(time.time() - t0))
f.close()
includeParticle.sort()
apDisplay.printMsg("Keeping " + str(len(includeParticle)) +
" and excluding " + str(excludeParticle) +
" particles")
### write kept particles to file
self.params['keepfile'] = os.path.join(
self.params['rundir'], "keepfile-" + self.timestamp + ".list")
apDisplay.printMsg("writing to keepfile " + self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
for partnum in includeParticle:
kf.write(str(partnum) + "\n")
kf.close()
### get number of particles
numparticles = len(includeParticle)
self.params['description'] += (" ... %d no jumpers substack" %
(numparticles, ))
### create the new sub stack
apStack.makeNewStack(oldstack,
newstack,
self.params['keepfile'],
bad=True)
if not os.path.isfile(newstack):
apDisplay.printError("No stack was created")
apStack.averageStack(stack=newstack)
if self.params['commit'] is True:
apStack.commitSubStack(self.params)
newstackid = apStack.getStackIdFromPath(newstack)
apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=6)
def insertRefineParticleData(self,
cls,
iteration,
eulers,
badprtls,
refineq,
numcls,
euler_convention='zxz'):
# get the corresponding proj number & eulers from filename
replace = re.compile('\D')
projnum = int(replace.sub('', cls))
clsfilename = os.path.join(self.params['tmpdir'], cls)
sys.stderr.write(".")
#f=open(clsfilename)
#apDisplay.printMsg("Class "+str(projnum+1)+" of "+str(numcls)+": inserting "
# +str(len(f.readlines())-2)+" particles")
#f.close()
# for each cls file get alignments for particles
f = open(clsfilename)
coranfail = False
for line in f:
# skip line if not a particle
if re.search("start", line):
prtlaliq = appiondata.ApRefineParticleData()
# gather alignment data from line
ali = line.split()
prtlnum = int(ali[0])
# check if bad particle
if prtlnum in badprtls:
prtlaliq['refine_keep'] = False
else:
prtlaliq['refine_keep'] = True
prtlnum += 1 # offset for EMAN
qualf = float(ali[2].strip(','))
other = ali[3].split(',')
rot = float(other[0]) * 180. / math.pi
shx = float(other[1])
shy = float(other[2])
if (other[3] == '1'):
prtlaliq['mirror'] = True
# SPIDER coran kept particle
corank = None
if self.params['package'] == 'EMAN/SpiCoran':
if len(other) > 4:
corank = bool(int(other[4]))
else:
if coranfail is False:
apDisplay.printWarning(
"Coran failed on this iteration")
coranfail = True
# message passing kept particle
if self.params['package'] == 'EMAN/MsgP' and len(ali) > 4:
msgk = bool(int(ali[4]))
else:
msgk = None
# find particle in stack database
defid = self.stackmapping[prtlnum]
stackp = appiondata.ApStackParticleData.direct_query(defid)
if not stackp:
apDisplay.printError("particle " + str(prtlnum) +
" not in stack id=" +
str(self.params['stack'].dbid))
# insert classification info
prtlaliq['refineIter'] = refineq
prtlaliq['particle'] = stackp
prtlaliq['shiftx'] = shx
prtlaliq['shifty'] = shy
prtlaliq['euler1'] = eulers[projnum][0]
prtlaliq['euler2'] = eulers[projnum][1]
prtlaliq['euler3'] = rot
prtlaliq['quality_factor'] = qualf
if self.params['package'] == 'EMAN/MsgP':
prtlaliq['postRefine_keep'] = msgk
else:
prtlaliq['postRefine_keep'] = corank
prtlaliq['euler_convention'] = euler_convention
#apDisplay.printMsg("inserting Particle Classification Data into database")
if self.params['commit'] is True:
prtlaliq.insert()
f.close()
return
def start(self):
partdict = {}
partlist = []
### get Euler angles for each particle
for iternum in self.iternums:
### get recon iter data
reconiterq = appiondata.ApRefineIterData()
reconiterq['refineRun'] = self.reconrundata
reconiterq['iteration'] = iternum
reconiterdata = reconiterq.query(
results=1)[0] #this should be unique
### get particle data
reconpartq = appiondata.ApRefineParticleData()
reconpartq['refineIter'] = reconiterdata
apDisplay.printMsg("Querying for particles at " + time.asctime())
reconpartdatas = reconpartq.query()
### group particle data
for partdata in reconpartdatas:
partnum = partdata['particle']['particleNumber']
if not partnum in partlist:
partlist.append(partnum)
partdict[(partnum, iternum)] = partdata
### run through particles and check Euler angles
partlist.sort()
eulerdict = {}
eulercount = {}
reject = 0
for partnum in partlist:
e1d = {}
e2d = {}
for iternum in self.iternums:
if not (partnum, iternum) in partdict:
continue
partdata = partdict[(partnum, iternum)]
euler1 = "%.2f" % (partdata['euler1'])
if not euler1 in e1d:
e1d[euler1] = 1
else:
e1d[euler1] += 1
euler2 = "%.2f" % (partdata['euler2'])
if not euler2 in e2d:
e2d[euler2] = 1
else:
e2d[euler2] += 1
#print partnum, euler1, euler2
counts = [(val, key) for key, val in e1d.items()]
e1count, euler1 = max(counts)
counts = [(val, key) for key, val in e2d.items()]
e2count, euler2 = max(counts)
# reject indeterminant particles
if e2count < 2 or e1count < 2:
reject += 1
continue
### group particles by their Euler angles
if not (euler1, euler2) in eulerdict:
eulerdict[(euler1, euler2)] = []
eulercount[(euler1, euler2)] = 0
eulerdict[(euler1, euler2)].append(partnum)
eulercount[(euler1, euler2)] += 1
print "Rejected %d particles" % (reject)
values = eulercount.values()
values.sort()
print values
### run through Euler angles and count particles
counts = [(val, key) for key, val in eulercount.items()]
mincount, val = min(counts)
self.params['mincount'] = max(self.params['mincount'], mincount)
#print "Keeping %d of %d particles"%(mincount*len(eulercount.keys()), len(partlist))
print "Keeping %d of %d particles" % (
self.params['mincount'] * len(eulercount.keys()), len(partlist))
keeplist = []
for key in eulerdict.keys():
eulerpartlist = eulerdict[key]
if len(partlist) < self.params['mincount']:
keeplist.extend(eulerpartlist)
else:
keeplist.extend(eulerpartlist[:self.params['mincount']])
keeplist.sort()
print "Keeping %d of %d particles" % (len(keeplist), len(partlist))
#need to set keepfile for commitSubStack
self.params['keepfile'] = os.path.join(self.params['rundir'],
"equalviews.lst")
f = open(self.params['keepfile'], "w")
for partnum in keeplist:
f.write("%d\n" % (partnum - 1))
f.close()
### make a new stack using the keep particles
oldstackdata = self.reconrundata['stack']
oldstack = os.path.join(oldstackdata['path']['path'],
oldstackdata['name'])
newstack = os.path.join(self.params['rundir'], "start.hed")
apStack.makeNewStack(oldstack,
newstack,
listfile=self.params['keepfile'],
remove=True,
bad=True)
if not os.path.isfile(newstack):
apDisplay.printError("No stack was created")
self.params[
'stackid'] = oldstackdata.dbid #need to set stackid for commitSubStack
apStack.commitSubStack(self.params, "start.hed")
apStack.averageStack(stack=newstack)
