def insertAlignParticlesIntoDatabase(self, level):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
D = self.getClassificationAtLevel(level)
for ref in D:
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = ref + 1
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['iteration'] = self.params['maxiter']
if ref in self.resdict:
refq['ssnr_resolution'] = self.resdict[ref]
### setup particle info ... NOTE: ALIGNMENT PARAMETERS ARE NOT SAVED IN XMIPP 2.4
for partnum in D[ref]:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(partnum) + 1
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(
self.runparams['stackid'],
int(partnum) +
1) ### particle numbering starts with 0!!!!!!!
alignpartq['stackpart'] = stackpartdata
alignpartq['ref'] = refq
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
### insert bad particles
if len(self.badpartlist) > 0:
for p in self.badpartlist:
refq = appiondata.ApAlignReferenceData()
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['iteration'] = self.params['maxiter']
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(p) + 1
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(
self.runparams['stackid'],
int(p) + 1)
alignpartq['stackpart'] = stackpartdata
alignpartq['bad'] = 1
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " +
str(self.params['numpart']) + " particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def getAlignParticleData(self, partnum, alignstackdata):
#logging.debug('Inside getAlignParticleData')
alignpartq = appiondata.ApAlignParticleData()
alignpartq['alignstack'] = alignstackdata
alignpartq['partnum'] = partnum
alignparts = alignpartq.query(results=1)
return alignparts[0]
def getStackParticleFromAlignParticle(alignrunid, alignpartnum, noDie=False):
alignrundata = appiondata.ApAlignRunData.direct_query(alignrunid)
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrundata
alignstackdatas = alignstackq.query()
if not alignstackdatas:
if noDie is True:
return
apDisplay.printError("alignpartnum=" + str(alignpartnum) +
" was not found in alignrunid=" + str(alignrunid))
alignstackdata = alignstackdatas[0]
alignpartq = appiondata.ApAlignParticleData()
alignpartq['alignstack'] = alignstackdata
alignpartq['partnum'] = alignpartnum
alignpartdata = alignpartq.query()
if not alignpartdata:
if noDie is True:
return
apDisplay.printError("alignpartnum=" + str(alignpartnum) +
" was not found in alignrunid=" + str(alignrunid))
if len(alignpartdata) > 1:
apDisplay.printError(
"There's a problem with this align stack. More than one particle with the same number."
)
return alignpartdata[0]['stackpart']
def getParticleInPlaneRotation(self, tiltstackpartdata):
partid = tiltstackpartdata.dbid
if partid in self.rotmirrorcache:
### use cached value
return self.rotmirrorcache[partid]
partnum = tiltstackpartdata['particleNumber']
notstackpartdata = apTiltPair.getStackParticleTiltPair(
self.params['tiltstackid'], partnum, self.params['notstackid'])
alignpartq = appiondata.ApAlignParticleData()
alignpartq['stackpart'] = notstackpartdata
alignpartq['alignstack'] = self.alignstackdata
alignpartdatas = alignpartq.query()
if not alignpartdatas or len(alignpartdatas) != 1:
apDisplay.printError(
"could not get inplane rotation for particle %d" %
(tiltstackpartdata['particleNumber']))
inplane = alignpartdatas[0]['rotation']
mirror = alignpartdatas[0]['mirror']
if alignpartdatas[0]['alignstack']['alignrun'][
'maxlikerun'] is not None:
# maxlike does mirror then rotation, and its rotation is negative relative to spider
# April 6, 2009: rotation is negative independent of mirror
inplane = -1.0 * inplane
self.rotmirrorcache[partid] = (inplane, mirror)
return inplane, mirror
def getAlignParticle(stackpdata,alignstackdata): oldstack = stackpdata['stack']['oldstack'] particledata = stackpdata['particle'] oldstackpdata = appiondata.ApStackParticleData(stack=oldstack,particle=particledata) q = appiondata.ApAlignParticleData(alignstack=alignstackdata,stackpart=oldstackpdata) results = q.query(readimages=False) if results: return results[0]
def insertParticlesIntoDatabase(self, stackid, partlist):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for partdict in partlist:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = partdict['refnum']
refq['iteration'] = self.lastiter
refsearch = "part" + self.params['timestamp'] + "_ref*" + str(
partdict['refnum']) + "*"
refbase = os.path.splitext(glob.glob(refsearch)[0])[0]
refq['mrcfile'] = refbase + ".mrc"
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
if partdict['refnum'] in self.resdict:
refq['ssnr_resolution'] = self.resdict[partdict['refnum']]
reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
if not os.path.isfile(reffile):
emancmd = "proc2d " + refbase + ".xmp " + refbase + ".mrc"
apEMAN.executeEmanCmd(emancmd, verbose=False)
if not os.path.isfile(reffile):
apDisplay.printError("could not find reference file: " +
reffile)
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partdict['partnum']
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(stackid,
partdict['partnum'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['inplane']
alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
alignpartq['spread'] = partdict['spread']
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " + str(count) +
" particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def insertParticlesIntoDatabase(self, stackid, partList):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for partdict in partList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = partdict['refnum']
if refq['refnum'] < 1:
apDisplay.printError("Classes must start with 1")
refq['iteration'] = self.lastiter
#refq['mrcfile'] = refbase+".mrc" ### need to create mrcs of each class???
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
if partdict['refnum'] in self.resdict:
refq['ssnr_resolution'] = self.resdict[partdict['refnum']]
refq['isac_generation'] = partdict['generation']
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partdict['partnum']
if alignpartq['partnum'] < 1:
apDisplay.printError("Particles must start with 1")
alignpartq['alignstack'] = self.alignstackdata
if self.params['alignstackid'] is None:
stackpartdata = apStack.getStackParticle(
stackid, partdict['origPartNum'])
else:
stackpartdata = apStack.getStackParticleFromAlignParticle(
self.params['alignstackid'], partdict['origPartNum'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['inplane']
alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
alignpartq['score'] = partdict['peak']
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " + str(count) +
" particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def getInplaneRotations(self):
# get all the particle rotations
apDisplay.printMsg("reading alignment data from database")
alignpartq = appiondata.ApAlignParticleData()
alignpartq['alignstack'] = self.alignstackdata
alignpartdatas = alignpartq.query()
rotationlist = [None] * len(alignpartdatas)
for part in alignpartdatas:
rotationlist[part['partnum'] - 1] = -part['rotation']
return rotationlist
def insertParticlesIntoDatabase(self, partlist, insert=False):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for particle in partlist:
count += 1
### get all particle parameters
partnum = count
rotation = particle[0] * -1 ### in spider negative rotation is clockwise
shiftx = particle[1]
shifty = particle[2]
ccc = particle[3]
refnum = int(particle[4])
mirror = particle[5]
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['imagicfile'] = "references.hed"
refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['templatestack'] = appiondata.ApTemplateStackData.direct_query(self.params['templateStackId'])
reffile = os.path.join(self.params['rundir'], refq['imagicfile'])
if not os.path.isfile(reffile):
apDisplay.printError("could not find reference file: "+reffile)
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(self.params['stackId'], partnum)
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = shiftx
alignpartq['yshift'] = shifty
alignpartq['rotation'] = rotation
alignpartq['mirror'] = mirror
alignpartq['ref'] = refq
alignpartq['correlation'] = ccc
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in "
+apDisplay.timeString(time.time()-t0), "cyan")
return
def getNumAlignedParticles(self):
t0 = time.time()
partq = appiondata.ApAlignParticleData()
self.alignstackdata = appiondata.ApAlignStackData.direct_query(
self.params['alignstackid'])
partq['alignstack'] = self.alignstackdata
partdata = partq.query()
numpart = len(partdata)
del partdata
apDisplay.printMsg("numpart=" + str(numpart) + " in " +
apDisplay.timeString(time.time() - t0))
return numpart
def insertAlignParticlesIntoDatabase(self):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for ref in self.classD:
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = ref
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['iteration'] = 10
# if ref in self.resdict:
# refq['ssnr_resolution'] = self.resdict[ref]
### setup particle info
for partnum in self.classD[ref]:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(partnum)
alignpartq['alignstack'] = self.alignstackdata
### particle numbering starts with 1 in SIMPLE
stackpartdata = apStack.getStackParticle(
self.runparams['stackid'], int(partnum))
alignpartq['stackpart'] = stackpartdata
alignpartq['ref'] = refq
try:
alignpartq['xshift'] = self.alignD[partnum - 1]['shx']
except:
pass
try:
alignpartq['yshift'] = self.alignD[partnum - 1]['shy']
except:
pass
try:
alignpartq['mirror'] = self.alignD[partnum - 1]['mirror']
except:
pass
alignpartq['rotation'] = self.alignD[partnum - 1]['rot']
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " +
str(self.params['numpart']) + " particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def getParticleInPlaneRotation(self, tiltstackpartdata):
partid = tiltstackpartdata.dbid
if partid in self.rotmirrorcache:
### use cached value
return self.rotmirrorcache[partid]
partnum = tiltstackpartdata['particleNumber']
notstackpartdata = apTiltPair.getStackParticleTiltPair(self.params['tiltstackid'],
partnum, self.params['notstackid'])
alignpartq = appiondata.ApAlignParticleData()
alignpartq['stackpart'] = notstackpartdata
alignpartq['alignstack'] = self.alignstackdata
alignpartdatas = alignpartq.query()
if not alignpartdatas or len(alignpartdatas) != 1:
apDisplay.printError("could not get inplane rotation for particle %d"%(tiltstackpartdata['particleNumber']))
inplane = alignpartdatas[0]['rotation']
mirror = alignpartdatas[0]['mirror']
self.rotmirrorcache[partid] = (inplane, mirror)
return inplane, mirror
def getGoodAlignParticles(self):
includeParticle = []
tiltParticlesData = []
nopairParticle = 0
excludeParticle = 0
badmirror = 0
badscore = 0
apDisplay.printMsg("Sorting particles from classes at " +
time.asctime())
count = 0
startmem = mem.active()
t0 = time.time()
if self.params['clusterid'] is not None:
### method 1: get particles from clustering data
clusterpartq = appiondata.ApClusteringParticleData()
clusterpartq[
'clusterstack'] = appiondata.ApClusteringStackData.direct_query(
self.params['clusterid'])
clusterpartdatas = clusterpartq.query()
apDisplay.printMsg("Sorting " + str(len(clusterpartdatas)) +
" clustered particles")
for clustpart in clusterpartdatas:
count += 1
if count % 50 == 0:
sys.stderr.write(".")
memdiff = (mem.active() - startmem) / count / 1024.0
if memdiff > 3:
apDisplay.printColor(
"Memory increase: %d MB/part" % (memdiff), "red")
#write to text file
clustnum = clustpart['refnum'] - 1
if self.params['minscore'] is not None:
if (clustpart['alignparticle']['score'] is not None
and clustpart['alignparticle']['score'] <
self.params['minscore']):
badscore += 1
continue
elif (clustpart['alignparticle']['spread'] is not None
and clustpart['alignparticle']['spread'] <
self.params['minscore']):
badscore += 1
continue
if clustnum in self.classlist:
notstackpartnum = clustpart['alignparticle']['stackpart'][
'particleNumber']
tiltstackpartdata = apTiltPair.getStackParticleTiltPair(
self.params['notstackid'], notstackpartnum,
self.params['tiltstackid'])
if tiltstackpartdata is None:
nopairParticle += 1
continue
tiltrot, theta, notrot, tiltangle = apTiltPair.getParticleTiltRotationAngles(
tiltstackpartdata)
if tiltrot is None:
apDisplay.printWarning("BAD particle " +
str(tiltstackpartdata))
nopairParticle += 1
continue
else:
inplane, mirror = self.getParticleInPlaneRotation(
tiltstackpartdata)
if (self.params['mirror'] == "all" or
(self.params['mirror'] == "no" and mirror is False)
or (self.params['mirror'] == "yes"
and mirror is True)):
emantiltstackpartnum = tiltstackpartdata[
'particleNumber'] - 1
includeParticle.append(emantiltstackpartnum)
tiltParticlesData.append(tiltstackpartdata)
if self.params['numpart'] is not None and len(
includeParticle) > self.params['numpart']:
break
else:
badmirror += 1
else:
excludeParticle += 1
else:
### method 2: get particles from alignment data
alignpartq = appiondata.ApAlignParticleData()
alignpartq['alignstack'] = self.alignstackdata
alignpartdatas = alignpartq.query()
apDisplay.printMsg("Sorting " + str(len(alignpartdatas)) +
" aligned particles")
for alignpart in alignpartdatas:
count += 1
if count % 50 == 0:
sys.stderr.write(".")
memdiff = (mem.active() - startmem) / count / 1024.0
if memdiff > 3:
apDisplay.printColor(
"Memory increase: %d MB/part" % (memdiff), "red")
#write to text file
alignnum = alignpart['ref']['refnum'] - 1
if (self.params['minscore'] is not None
and alignpart['score'] is not None
and alignpart['score'] < self.params['minscore']):
badscore += 1
continue
if alignnum in self.classlist:
notstackpartnum = alignpart['stackpart']['particleNumber']
tiltstackpartdata = apTiltPair.getStackParticleTiltPair(
self.params['notstackid'], notstackpartnum,
self.params['tiltstackid'])
if tiltstackpartdata is None:
nopairParticle += 1
else:
inplane, mirror = self.getParticleInPlaneRotation(
tiltstackpartdata)
if (self.params['mirror'] == "all" or
(self.params['mirror'] == "no" and mirror is False)
or (self.params['mirror'] == "yes"
and mirror is True)):
emantiltstackpartnum = tiltstackpartdata[
'particleNumber'] - 1
includeParticle.append(emantiltstackpartnum)
tiltParticlesData.append(tiltstackpartdata)
if self.params['numpart'] is not None and len(
includeParticle) > self.params['numpart']:
break
else:
badmirror += 1
else:
excludeParticle += 1
### end methods
includeParticle.sort()
### messages
if time.time() - t0 > 1.0:
apDisplay.printMsg("\nSorting time: " +
apDisplay.timeString(time.time() - t0))
apDisplay.printMsg("Keeping " + str(len(includeParticle)) +
" and excluding \n\t" + str(excludeParticle) +
" particles with " + str(nopairParticle) +
" unpaired particles")
if badmirror > 0:
apDisplay.printMsg("Particles with bad mirrors: %d" % (badmirror))
if badscore > 0:
apDisplay.printColor("Particles with bad scores: %d" % (badscore),
"cyan")
if len(includeParticle) < 1:
apDisplay.printError("No particles were kept")
memdiff = (mem.active() - startmem) / count / 1024.0
if memdiff > 0.1:
apDisplay.printColor("Memory increase: %.2f MB/part" % (memdiff),
"red")
return includeParticle, tiltParticlesData
def insertRefBasedRun(self, partlist, imagicstack, insert=False):
apDisplay.printMsg("committing results to DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database")
### setup ref based run
refrunq = appiondata.ApRefBasedRunData()
refrunq['runname'] = self.params['runname']
refrunq['xysearch'] = self.params['xysearch']
refrunq['xystep'] = self.params['xystep']
refrunq['first_ring'] = self.params['firstring']
refrunq['last_ring'] = self.params['lastring']
refrunq['num_iter'] = self.params['numiter']
refrunq['invert_templs'] = self.params['inverttemplates']
refrunq['num_templs'] = self.params['numtemplate']
#refrunq['csym', int),
refrunq['run_seconds'] = self.params['runtime']
### finish alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['refbasedrun'] = refrunq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['imagicfile'] = imagicstack
alignstackq['avgmrcfile'] = "average.mrc"
emancmd = "proc2d templatestack%02d.spi templatestack%02d.hed"%(self.params['numiter'],self.params['numiter'])
apEMAN.executeEmanCmd(emancmd)
alignstackq['refstackfile'] = ("templatestack%02d.hed"%(self.params['numiter']))
alignstackq['alignrun'] = alignrunq
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: "+imagicfile)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = self.stack['data']
alignstackq['boxsize'] = self.boxsize
alignstackq['pixelsize'] = self.stack['apix']*self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert()
### insert reference data
reflist = []
for j in range(self.params['numiter']):
iternum = j+1
for i in range(len(self.templatelist)):
refnum = i+1
templateid = self.templatelist[i]
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = iternum
refq['template'] = apTemplate.getTemplateFromId(templateid)
refq['mrcfile'] = ("templateavg%02d-%02d.mrc"%(iternum,refnum))
refpath = os.path.join(self.params['rundir'], "templates")
refq['path'] = appiondata.ApPathData(path=os.path.abspath(refpath))
refq['alignrun'] = alignrunq
if refnum in self.resdict:
refq['ssnr_resolution'] = self.resdict[refnum]
if insert is True:
refq.insert()
if iternum == self.params['numiter']:
reflist.append(refq)
#refq['varmrcfile', str),
#refq['frc_resolution', float),
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for partdict in partlist:
### see apSpider.alignment.alignStack() for more info
"""
partdict.keys()
'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,...
'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,...
'mirror': checkMirror(templatenum),
'score': float(data[3]),
'rot': float(data[4]),
'xshift': float(data[5]),
'yshift': float(data[6]),
"""
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = reflist[partdict['template']-1]
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['rot']
alignpartq['score'] = partdict['score']
alignpartq['mirror'] = partdict['mirror']
if insert is True:
alignpartq.insert()
return
def insertEdIterRun(self, insert=False):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + runparams['runname'] +
"' and path already exist in database")
### setup ed-iter run
editrunq = appiondata.ApEdIterRunData()
editrunq['runname'] = self.params['runname']
editrunq['radius'] = self.params['partrad']
editrunq['num_iter'] = self.params['numiter']
#editrunq['freealigns'] = self.params['freealigns'] ### no longer an option
editrunq['invert_templs'] = self.params['inverttemplates']
editrunq['num_templs'] = self.params['numtemplate']
#editrunq['csym', int),
editrunq['run_seconds'] = self.runtime
### finish alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['editerrun'] = editrunq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
### setup aligned stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
# stack of aligned particles
alignstackq['imagicfile'] = "alignstack.hed"
# final class averages
alignstackq['refstackfile'] = "avg.hed"
# averaged results
alignstackq['avgmrcfile'] = "average.mrc"
# check to be sure dbpaths to files are correct
refstackfile = os.path.join(
self.params['rundir'],
alignstackq['refstackfile']) #final averages
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: " +
refstackfile)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile']) #averaged results
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
alignstackq['stack'] = self.stack['data']
alignstackq['iteration'] = self.params['numiter']
alignstackq['boxsize'] = math.floor(self.stack['boxsize'] /
self.params['bin'])
alignstackq['pixelsize'] = self.stack['apix'] * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert(
) #alignstackq contains alignrunq which contains editrunq
### setup data from each iteration
apDisplay.printColor(
"Inserting average and variance images from each iteration, please wait",
"cyan")
reflist = []
for j in range(self.params['numiter']):
iternum = j + 1
for i in range(len(self.templatelist)):
refnum = i + 1
templateid = self.templatelist[i]
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = iternum
refq['template'] = apTemplate.getTemplateFromId(templateid)
refpath = os.path.join(self.params['rundir'], "templates")
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(refpath))
refq['alignrun'] = alignrunq
#refq['frc_resolution'] = #(float)
avgname = os.path.join(self.params['rundir'],
"r%02d/avg%03d" % (iternum, refnum))
varname = os.path.join(self.params['rundir'],
"r%02d/var%03d" % (iternum, refnum))
if os.path.isfile(avgname + ".spi"):
emancmd = "proc2d " + avgname + ".spi " + avgname + ".mrc"
apEMAN.executeEmanCmd(emancmd)
refq['mrcfile'] = (avgname + ".mrc")
emancmd = "proc2d " + varname + ".spi " + varname + ".mrc"
apEMAN.executeEmanCmd(emancmd)
refq['varmrcfile'] = (varname + ".mrc")
if insert is True:
refq.insert()
if iternum == self.params['numiter']:
reflist.append(refq)
else:
reflist.append(None)
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
partlist = self.readApshDocFile("apshdoc.spi", "apshdoc.pickle")
for partdict in partlist:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = reflist[partdict['template'] - 1]
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['rotation'] = partdict['rot']
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['mirror'] = partdict['mirror']
alignpartq['score'] = partdict['score']
if insert is True:
alignpartq.insert() #insert each particle
return
def commitToDatabase(self):
"""
insert the results into the database
"""
### expected result for an alignment run:
### 1. aligned particle stack in IMAGIC
### 2. rotation, shift, and quality parameters for each particle
### 3. which particles belongs to which class
### 4. stack file with the class averages
alignedstack = os.path.join(self.params['rundir'], "ptcl.hed")
refstack = os.path.join(self.params['rundir'], "iter.final.hed")
averagemrc = os.path.join(self.params['rundir'], "average.mrc")
apStack.averageStack(alignedstack, averagemrc)
particlemapping = self.determineClassOwnership()
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError(
"Run name '" + runparams['runname'] +
"' and path already exisclassmappingt in database")
### setup eman refine2d run
emanrefinetwodq = appiondata.ApEMANRefine2dRunData()
emanrefinetwodq['runname'] = self.params['runname']
emanrefinetwodq['run_seconds'] = time.time() - self.t0
emanrefinetwodq['num_iters'] = self.params['numiter']
emanrefinetwodq['num_classes'] = self.params['numclasses']
### finish alignment run
alignrunq['refine2drun'] = emanrefinetwodq
alignrunq['hidden'] = False
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['bin'] = self.params['bin']
### setup alignment stackalignimagicfile
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = os.path.basename(alignedstack)
alignstackq['avgmrcfile'] = os.path.basename(averagemrc)
alignstackq['refstackfile'] = os.path.basename(refstack)
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: " +
alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
refstackfile = os.path.join(self.params['rundir'],
alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printErrrefqor("could not find reference stack file: " +
refstackfile)
### continue setting values
alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid']) * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = apFile.numImagesInStack(
alignimagicfilepath)
### inserting particles and references
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for emanpartnum in range(self.params['numpart']):
partnum = emanpartnum + 1
if partnum % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refnum = particlemapping[emanpartnum]
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = alignrunq
### TODO: create mrc file
#refq['mrcfile'] = refbase+".mrc"
#reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
#if not os.path.isfile(reffile):
# emancmd = "proc2d "+refstack+".xmp "+refstack+".mrc"
# apEMAN.executeEmanCmd(emancmd, verbose=False)
#if not os.path.isfile(reffile):
# apDisplay.printError("could not find reference file: "+reffile)
### TODO: get resolution
#refq['ssnr_resolution'] = TODO
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partnum)
alignpartq['stackpart'] = stackpartdata
### TODO: get the alignment parameters
#alignpartq['xshift'] = partdict['xshift']
#alignpartq['yshift'] = partdict['yshift']
#alignpartq['rotation'] = partdict['inplane']
#alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
### TODO: get the score
#alignpartq['score'] = partdict['score']
### insert
if self.params['commit'] is True:
alignpartq.insert()
return
def insertNoRefRun(self, imagicstack, insert=False):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + self.params['runname'] +
"' and path already exist in database")
# create a norefParam object
norefq = appiondata.ApSpiderNoRefRunData()
norefq['runname'] = self.params['runname']
norefq['particle_diam'] = 2.0 * self.params['partrad']
norefq['first_ring'] = self.params['firstring']
norefq['last_ring'] = self.params['lastring']
norefq['init_method'] = self.params['initmethod']
norefq['run_seconds'] = self.runtime
### finish alignment run
alignrunq['norefrun'] = norefq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['description'] = self.params['description']
# STOP HERE
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['imagicfile'] = os.path.basename(imagicstack)
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['alignrun'] = alignrunq
alignstackq['iteration'] = 0
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: " + imagicfile)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average file: " + avgmrcfile)
alignstackq['stack'] = self.stack['data']
alignstackq['boxsize'] = math.floor(self.stack['boxsize'] /
self.params['bin'])
alignstackq['pixelsize'] = self.stack['apix'] * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert()
### create reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = 1
refq['iteration'] = 0
refq['mrcfile'] = "template.mrc"
#refpath = os.path.abspath(os.path.join(self.params['rundir'], "alignment"))
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = alignrunq
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for partdict in self.partlist:
### see apSpider.alignment.alignStack() for more info
"""
partdict.keys()
'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,...
'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,...
'mirror': checkMirror(templatenum),
'score': float(data[3]),
'rot': float(data[4]),
'xshift': float(data[5]),
'yshift': float(data[6]),
"""
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = refq
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['rot']
#alignpartq['score'] = partdict['score']
if insert is True:
alignpartq.insert()
return
def getAlignParticleData(self, partnum):
alignpartq = appiondata.ApAlignParticleData()
alignpartq['alignstack'] = self.alignstackdata
alignpartq['partnum'] = partnum
alignparts = alignpartq.query(results=1)
return alignparts[0]
def start(self):
### new stack path
stackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
newstack = os.path.join(self.params['rundir'], stackdata['name'])
apStack.checkForPreviousStack(newstack)
includelist = []
excludelist = []
### list of classes to be excluded
if self.params['dropclasslist'] is not None:
excludestrlist = self.params['dropclasslist'].split(",")
for excludeitem in excludestrlist:
excludelist.append(int(excludeitem.strip()))
apDisplay.printMsg("Exclude list: " + str(excludelist))
### list of classes to be included
if self.params['keepclasslist'] is not None:
includestrlist = self.params['keepclasslist'].split(",")
for includeitem in includestrlist:
includelist.append(int(includeitem.strip()))
### or read from keepfile
elif self.params['keepfile'] is not None:
keeplistfile = open(self.params['keepfile'])
for line in keeplistfile:
if self.params['excludefrom'] is True:
excludelist.append(int(line.strip()))
else:
includelist.append(int(line.strip()))
keeplistfile.close()
apDisplay.printMsg("Include list: " + str(includelist))
### get particles from align or cluster stack
apDisplay.printMsg("Querying database for particles")
q0 = time.time()
if self.params['alignid'] is not None:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['alignstack'] = self.alignstackdata
particles = alignpartq.query()
elif self.params['clusterid'] is not None:
clusterpartq = appiondata.ApClusteringParticleData()
clusterpartq['clusterstack'] = self.clusterstackdata
particles = clusterpartq.query()
apDisplay.printMsg("Complete in " +
apDisplay.timeString(time.time() - q0))
### write included particles to text file
includeParticle = []
excludeParticle = 0
badscore = 0
badshift = 0
badspread = 0
f = open("test.log", "w")
count = 0
for part in particles:
count += 1
#partnum = part['partnum']-1
if 'alignparticle' in part:
alignpart = part['alignparticle']
classnum = int(part['refnum']) - 1
else:
alignpart = part
classnum = int(part['ref']['refnum']) - 1
emanstackpartnum = alignpart['stackpart']['particleNumber'] - 1
### check shift
if self.params['maxshift'] is not None:
shift = math.hypot(alignpart['xshift'], alignpart['yshift'])
if shift > self.params['maxshift']:
excludeParticle += 1
f.write("%d\t%d\t%d\texclude\n" %
(count, emanstackpartnum, classnum))
badshift += 1
continue
if self.params['minscore'] is not None:
### check score
if (alignpart['score'] is not None
and alignpart['score'] < self.params['minscore']):
excludeParticle += 1
f.write("%d\t%d\t%d\texclude\n" %
(count, emanstackpartnum, classnum))
badscore += 1
continue
### check spread
if (alignpart['spread'] is not None
and alignpart['spread'] < self.params['minscore']):
excludeParticle += 1
f.write("%d\t%d\t%d\texclude\n" %
(count, emanstackpartnum, classnum))
badspread += 1
continue
if includelist and classnum in includelist:
includeParticle.append(emanstackpartnum)
f.write("%d\t%d\t%d\tinclude\n" %
(count, emanstackpartnum, classnum))
elif excludelist and not classnum in excludelist:
includeParticle.append(emanstackpartnum)
f.write("%d\t%d\t%d\tinclude\n" %
(count, emanstackpartnum, classnum))
else:
excludeParticle += 1
f.write("%d\t%d\t%d\texclude\n" %
(count, emanstackpartnum, classnum))
f.close()
includeParticle.sort()
if badshift > 0:
apDisplay.printMsg("%d paricles had a large shift" % (badshift))
if badscore > 0:
apDisplay.printMsg("%d paricles had a low score" % (badscore))
if badspread > 0:
apDisplay.printMsg("%d paricles had a low spread" % (badspread))
apDisplay.printMsg("Keeping " + str(len(includeParticle)) +
" and excluding " + str(excludeParticle) +
" particles")
#print includeParticle
### write kept particles to file
self.params['keepfile'] = os.path.join(
self.params['rundir'], "keepfile-" + self.timestamp + ".list")
apDisplay.printMsg("writing to keepfile " + self.params['keepfile'])
kf = open(self.params['keepfile'], "w")
for partnum in includeParticle:
kf.write(str(partnum) + "\n")
kf.close()
### get number of particles
numparticles = len(includeParticle)
if excludelist:
self.params['description'] += (
" ... %d particle substack with %s classes excluded" %
(numparticles, self.params['dropclasslist']))
elif includelist:
self.params['description'] += (
" ... %d particle substack with %s classes included" %
(numparticles, self.params['keepclasslist']))
### create the new sub stack
apStack.makeNewStack(oldstack,
newstack,
self.params['keepfile'],
bad=self.params['savebad'])
if not os.path.isfile(newstack):
apDisplay.printError("No stack was created")
apStack.averageStack(stack=newstack)
if self.params['commit'] is True:
apStack.commitSubStack(self.params)
newstackid = apStack.getStackIdFromPath(newstack)
apStackMeanPlot.makeStackMeanPlot(newstackid, gridpoints=4)