def insertAlignParticlesIntoDatabase(self, level):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
D = self.getClassificationAtLevel(level)
for ref in D:
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = ref + 1
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['iteration'] = self.params['maxiter']
if ref in self.resdict:
refq['ssnr_resolution'] = self.resdict[ref]
### setup particle info ... NOTE: ALIGNMENT PARAMETERS ARE NOT SAVED IN XMIPP 2.4
for partnum in D[ref]:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(partnum) + 1
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(
self.runparams['stackid'],
int(partnum) +
1) ### particle numbering starts with 0!!!!!!!
alignpartq['stackpart'] = stackpartdata
alignpartq['ref'] = refq
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
### insert bad particles
if len(self.badpartlist) > 0:
for p in self.badpartlist:
refq = appiondata.ApAlignReferenceData()
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['iteration'] = self.params['maxiter']
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(p) + 1
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(
self.runparams['stackid'],
int(p) + 1)
alignpartq['stackpart'] = stackpartdata
alignpartq['bad'] = 1
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " +
str(self.params['numpart']) + " particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def insertAlignParticlesIntoDatabase(self, level):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
D=self.getClassificationAtLevel(level)
for ref in D:
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = ref+1
refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['iteration'] = self.params['maxiter']
if ref in self.resdict:
refq['ssnr_resolution'] = self.resdict[ref]
### setup particle info ... NOTE: ALIGNMENT PARAMETERS ARE NOT SAVED IN XMIPP 2.4
for partnum in D[ref]: # particle numbering in D[ref] starts with 0
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(partnum)+1
alignpartq['alignstack'] = self.alignstackdata
### particle numbering in Appion db starts with 1
stackpartdata = apStack.getStackParticle(self.runparams['stackid'], int(partnum)+1)
alignpartq['stackpart'] = stackpartdata
alignpartq['ref'] = refq
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
### insert bad particles
if len(self.badpartlist) > 0:
for p in self.badpartlist: # particle numbering in badpartlist starts with 0
refq = appiondata.ApAlignReferenceData()
refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['iteration'] = self.params['maxiter']
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(p)+1
alignpartq['alignstack'] = self.alignstackdata
### particle numbering in Appion db starts with 1
stackpartdata = apStack.getStackParticle(self.runparams['stackid'], int(p)+1)
alignpartq['stackpart'] = stackpartdata
alignpartq['bad'] = 1
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(self.params['numpart'])+" particles into the database in "
+apDisplay.timeString(time.time()-t0), "cyan")
return
def getGoodParticles(self):
'''
Get particle numbers in eman format in fullstack that
has tilt pair in knownstack
'''
includeParticle = []
tiltParticlesData = []
no_match_count = 0
apDisplay.printMsg("Finding particles from tilt pairs ")
### get stack data
knownstackid = self.params['knownstackid']
fullstackid = self.params['fullstackid']
numpartl = apStack.getNumberStackParticlesFromId(self.params['knownstackid'])
for partnum in range(1,numpartl+1):
stpartdata = apStack.getStackParticle(knownstackid, partnum, True)
# stpartdata and otherpartdata are related by tilt transform
imgnum, transformdata, otherpartdata = apTiltPair.getTiltTransformFromParticle(stpartdata['particle'])
apDisplay.printMsg('Mapping particle %d to %d' % (stpartdata['particle'].dbid,otherpartdata.dbid))
fullstackpartdata = apStack.getStackParticleFromData(fullstackid, otherpartdata, True)
if fullstackpartdata:
# eman particle number starts from 0, appion database number starts from 1
emantiltstackpartnum = fullstackpartdata['particleNumber']-1
includeParticle.append(emantiltstackpartnum)
tiltParticlesData.append(fullstackpartdata)
else:
no_match_count += 1
apDisplay.printWarning('There are %d particles without a match in the requested full stack' % no_match_count)
return includeParticle, tiltParticlesData
def makeEulerDoc(self, parttree):
count = 0
eulerfile = os.path.join(self.params['rundir'],
"eulersdoc" + self.timestamp + ".spi")
eulerf = open(eulerfile, "w")
apDisplay.printMsg("creating Euler doc file")
starttime = time.time()
for partdict in parttree:
stackpartdata = apStack.getStackParticle(
self.params['tiltstackid'], partdict['tilt'])
count += 1
if count % 100 == 0:
sys.stderr.write(".")
eulerf.flush()
gamma, theta, phi, tiltangle = apTiltPair.getParticleTiltRotationAngles(
stackpartdata)
if gamma is None:
apDisplay.printWarning("Skipping " + str(stackpartdata))
continue
line = operations.spiderOutLine(count, [phi, tiltangle, gamma])
eulerf.write(line)
eulerf.close()
apDisplay.printColor(
"finished Euler doc file in " +
apDisplay.timeString(time.time() - starttime), "cyan")
return eulerfile
def appendEulerDoc(self, eulerfile, tiltpartnum, count):
eulerf = open(eulerfile, "a")
stackpartdata = apStack.getStackParticle(self.params['tiltstackid'], tiltpartnum)
gamma, theta, phi, tiltangle = apTiltPair.getParticleTiltRotationAngles(stackpartdata)
line = operations.spiderOutLine(count, [phi, tiltangle, -1.0*gamma])
eulerf.write(line)
eulerf.close()
def insertParticlesIntoDatabase(self, stackid, partList):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for partdict in partList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq["refnum"] = partdict["refnum"]
if refq["refnum"] < 1:
apDisplay.printError("Classes must start with 1")
refq["iteration"] = self.lastiter
# refq['mrcfile'] = refbase+".mrc" ### need to create mrcs of each class???
refq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
refq["alignrun"] = self.alignstackdata["alignrun"]
if partdict["refnum"] in self.resdict:
refq["ssnr_resolution"] = self.resdict[partdict["refnum"]]
refq["isac_generation"] = partdict["generation"]
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq["partnum"] = partdict["partnum"]
if alignpartq["partnum"] < 1:
apDisplay.printError("Particles must start with 1")
alignpartq["alignstack"] = self.alignstackdata
if self.params["alignstackid"] is None:
stackpartdata = apStack.getStackParticle(stackid, partdict["origPartNum"])
else:
stackpartdata = apStack.getStackParticleFromAlignParticle(
self.params["alignstackid"], partdict["origPartNum"]
)
alignpartq["stackpart"] = stackpartdata
alignpartq["xshift"] = partdict["xshift"]
alignpartq["yshift"] = partdict["yshift"]
alignpartq["rotation"] = partdict["inplane"]
alignpartq["mirror"] = partdict["mirror"]
alignpartq["ref"] = refq
alignpartq["score"] = partdict["peak"]
### insert
if self.params["commit"] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted "
+ str(inserted)
+ " of "
+ str(count)
+ " particles into the database in "
+ apDisplay.timeString(time.time() - t0),
"cyan",
)
return
def insertParticlesIntoDatabase(self, stackid, partlist):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for partdict in partlist:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = partdict['refnum']
refq['iteration'] = self.lastiter
refsearch = "part" + self.params['timestamp'] + "_ref*" + str(
partdict['refnum']) + "*"
refbase = os.path.splitext(glob.glob(refsearch)[0])[0]
refq['mrcfile'] = refbase + ".mrc"
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
if partdict['refnum'] in self.resdict:
refq['ssnr_resolution'] = self.resdict[partdict['refnum']]
reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
if not os.path.isfile(reffile):
emancmd = "proc2d " + refbase + ".xmp " + refbase + ".mrc"
apEMAN.executeEmanCmd(emancmd, verbose=False)
if not os.path.isfile(reffile):
apDisplay.printError("could not find reference file: " +
reffile)
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partdict['partnum']
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(stackid,
partdict['partnum'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['inplane']
alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
alignpartq['spread'] = partdict['spread']
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " + str(count) +
" particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def getStackParticleEulersForIteration(self, pnum):
"""
find the eulers assigned to a stack particle
during a refinement. This function will first
find the particle id for the given stack particle,
then find its position in the reference stack, and
will get the eulers for that particle in the recon
"""
# get stack particle id
stackp = apStack.getStackParticle(self.params['stackid'], pnum)
particleid = stackp['particle'].dbid
# find particle in reference stack
refstackp = apStack.getStackParticleFromParticleId(particleid, self.oldreconstackid, noDie=True)
if not refstackp:
percentnoeuler = 100*self.noeulers/float(self.params['last'])
apDisplay.printWarning('No eulers for particle %d (%.1f%%)' % (pnum, percentnoeuler))
self.noeulers += 1
if percentnoeuler > 10:
apDisplay.printError('More than 10% of the particles have no euler, use a different reference reconstruction')
pclass={
'euler1': 0.0,
'euler2': 0.0,
'euler3': 0.0,
'mirror': False,
'shiftx': 0.0,
'shifty': 0.0,
}
else:
pclassq = appiondata.ApRefineParticleData()
pclassq['particle'] = refstackp
pclassq['refineIter'] = appiondata.ApRefineIterData.direct_query(self.params['reconiterid'])
pclasses = pclassq.query(results=1)
pclass = pclasses[0]
try:
emaneuler={
'alt': pclass['euler1'],
'az': pclass['euler2'],
'phi': pclass['euler3'],
'mirror': pclass['mirror'],
'shiftx': pclass['shiftx'],
'shifty': pclass['shifty'],
}
except:
print pclass
emaneuler={
'alt': pclass['euler1'],
'az': pclass['euler2'],
'phi': pclass['euler3'],
'mirror': pclass['mirror'],
'shiftx': pclass['shiftx'],
'shifty': pclass['shifty'],
}
return emaneuler
def appendEulerDoc(self, eulerfile, tiltpartnum, count):
eulerf = open(eulerfile, "a")
stackpartdata = apStack.getStackParticle(self.params['tiltstackid'], tiltpartnum)
gamma, theta, phi, tiltangle = apTiltPair.getParticleTiltRotationAngles(stackpartdata)
if gamma is None:
apDisplay.printWarning("Skipping "+str(stackpartdata))
return
line = operations.spiderOutLine(count, [phi, tiltangle, -1.0*gamma])
eulerf.write(line)
eulerf.close()
def insertParticlesIntoDatabase(self, stackid, partList):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for partdict in partList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = partdict['refnum']
if refq['refnum'] < 1:
apDisplay.printError("Classes must start with 1")
refq['iteration'] = self.lastiter
#refq['mrcfile'] = refbase+".mrc" ### need to create mrcs of each class???
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
if partdict['refnum'] in self.resdict:
refq['ssnr_resolution'] = self.resdict[partdict['refnum']]
refq['isac_generation'] = partdict['generation']
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partdict['partnum']
if alignpartq['partnum'] < 1:
apDisplay.printError("Particles must start with 1")
alignpartq['alignstack'] = self.alignstackdata
if self.params['alignstackid'] is None:
stackpartdata = apStack.getStackParticle(
stackid, partdict['origPartNum'])
else:
stackpartdata = apStack.getStackParticleFromAlignParticle(
self.params['alignstackid'], partdict['origPartNum'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['inplane']
alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
alignpartq['score'] = partdict['peak']
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " + str(count) +
" particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def insertParticlesIntoDatabase(self, partlist, insert=False):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for particle in partlist:
count += 1
### get all particle parameters
partnum = count
rotation = particle[0] * -1 ### in spider negative rotation is clockwise
shiftx = particle[1]
shifty = particle[2]
ccc = particle[3]
refnum = int(particle[4])
mirror = particle[5]
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['imagicfile'] = "references.hed"
refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['templatestack'] = appiondata.ApTemplateStackData.direct_query(self.params['templateStackId'])
reffile = os.path.join(self.params['rundir'], refq['imagicfile'])
if not os.path.isfile(reffile):
apDisplay.printError("could not find reference file: "+reffile)
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(self.params['stackId'], partnum)
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = shiftx
alignpartq['yshift'] = shifty
alignpartq['rotation'] = rotation
alignpartq['mirror'] = mirror
alignpartq['ref'] = refq
alignpartq['correlation'] = ccc
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in "
+apDisplay.timeString(time.time()-t0), "cyan")
return
def getStackParticleEulersForIteration(params, pnum):
"""
find the eulers assigned to a stack particle
during a refinement. This function will first
find the particle id for the given stack particle,
then find its position in the reference stack, and
will get the eulers for that particle in the recon
"""
# get stack particle id
stackp = apStack.getStackParticle(params['stackid'], pnum)
particleid = stackp['particle'].dbid
# find particle in reference stack
refstackid = apStack.getStackIdFromIterationId(params['reconiterid'],
msg=False)
refstackp = apStack.getStackParticleFromParticleId(particleid,
refstackid,
nodie=True)
if not refstackp:
apDisplay.printWarning(
'No classification for stack particle %d in reconstruction iteration id: %d'
% (pnum, params['reconiterid']))
params['noClassification'] += 1
if params['noClassification'] > (float(params['last']) * 0.10):
apDisplay.printError(
'More than 10% of the particles have no classification, use a different reference reconstruction'
)
pclass = {}
pclass['eulers'] = {}
pclass['eulers']['euler1'] = 0.0
pclass['eulers']['euler2'] = 0.0
pclass['inplane_rotation'] = 0.0
pclass['mirror'] = False
pclass['shiftx'] = 0.0
pclass['shifty'] = 0.0
else:
pclassq = appiondata.ApRefineParticleData()
pclassq['particle'] = refstackp
pclassq['refineIter'] = appiondata.ApRefineIterData.direct_query(
params['reconiterid'])
pclass = pclassq.query()
pclass = pclass[0]
params['eman_orient'] = {}
params['eman_orient']['alt'] = pclass['euler1']
params['eman_orient']['az'] = pclass['euler2']
params['eman_orient']['phi'] = pclass['euler3']
params['eman_orient']['mirror'] = pclass['mirror']
params['eman_orient']['shiftx'] = pclass['shiftx']
params['eman_orient']['shifty'] = pclass['shifty']
return params
def insertParticlesIntoDatabase(self, partlist, insert=False):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for particle in partlist:
count += 1
### get all particle parameters
partnum = count
rotation = particle[0] * -1 ### in spider negative rotation is clockwise
shiftx = particle[1]
shifty = particle[2]
ccc = particle[3]
refnum = int(particle[4])
mirror = particle[5]
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['imagicfile'] = "references.hed"
refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['templatestack'] = appiondata.ApTemplateStackData.direct_query(self.params['templateStackId'])
reffile = os.path.join(self.params['rundir'], refq['imagicfile'])
if not os.path.isfile(reffile):
apDisplay.printError("could not find reference file: "+reffile)
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(self.params['stackId'], partnum)
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = shiftx
alignpartq['yshift'] = shifty
alignpartq['rotation'] = rotation
alignpartq['mirror'] = mirror
alignpartq['ref'] = refq
alignpartq['correlation'] = ccc
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in "
+apDisplay.timeString(time.time()-t0), "cyan")
return
def insertAlignParticlesIntoDatabase(self):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for ref in self.classD:
### setup reference
refq = appiondata.ApAlignReferenceData()
refq["refnum"] = ref
refq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
refq["alignrun"] = self.alignstackdata["alignrun"]
refq["iteration"] = 10
# if ref in self.resdict:
# refq['ssnr_resolution'] = self.resdict[ref]
### setup particle info
for partnum in self.classD[ref]:
alignpartq = appiondata.ApAlignParticleData()
alignpartq["partnum"] = int(partnum)
alignpartq["alignstack"] = self.alignstackdata
### particle numbering starts with 1 in SIMPLE
stackpartdata = apStack.getStackParticle(self.runparams["stackid"], int(partnum))
alignpartq["stackpart"] = stackpartdata
alignpartq["ref"] = refq
try:
alignpartq["xshift"] = self.alignD[partnum - 1]["shx"]
except:
pass
try:
alignpartq["yshift"] = self.alignD[partnum - 1]["shy"]
except:
pass
try:
alignpartq["mirror"] = self.alignD[partnum - 1]["mirror"]
except:
pass
alignpartq["rotation"] = self.alignD[partnum - 1]["rot"]
### insert
if self.params["commit"] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted "
+ str(inserted)
+ " of "
+ str(self.params["numpart"])
+ " particles into the database in "
+ apDisplay.timeString(time.time() - t0),
"cyan",
)
return
def appendEulerDoc(self, eulerfile, tiltpartnum, count):
eulerf = open(eulerfile, "a")
stackpartdata = apStack.getStackParticle(self.params['tiltstackid'],
tiltpartnum)
gamma, theta, phi, tiltangle = apTiltPair.getParticleTiltRotationAngles(
stackpartdata)
if gamma is None:
apDisplay.printWarning("Skipping " + str(stackpartdata))
return
line = operations.spiderOutLine(count, [phi, tiltangle, -1.0 * gamma])
eulerf.write(line)
eulerf.close()
def getStackParticleEulersForIteration(self, pnum):
"""
find the eulers assigned to a stack particle
during a refinement. This function will first
find the particle id for the given stack particle,
then find its position in the reference stack, and
will get the eulers for that particle in the recon
"""
# get stack particle id
stackp = apStack.getStackParticle(self.params["stackid"], pnum)
particleid = stackp["particle"].dbid
# find particle in reference stack
refstackp = apStack.getStackParticleFromParticleId(particleid, self.oldreconstackid, nodie=True)
if not refstackp:
percentnoeuler = 100 * self.noeulers / float(self.params["last"])
apDisplay.printWarning("No eulers for particle %d (%.1f%%)" % (pnum, percentnoeuler))
self.noeulers += 1
if percentnoeuler > 10:
apDisplay.printError(
"More than 10% of the particles have no euler, use a different reference reconstruction"
)
pclass = {"euler1": 0.0, "euler2": 0.0, "euler3": 0.0, "mirror": False, "shiftx": 0.0, "shifty": 0.0}
else:
pclassq = appiondata.ApRefineParticleData()
pclassq["particle"] = refstackp
pclassq["refineIter"] = appiondata.ApRefineIterData.direct_query(self.params["reconiterid"])
pclasses = pclassq.query(results=1)
pclass = pclasses[0]
try:
emaneuler = {
"alt": pclass["euler1"],
"az": pclass["euler2"],
"phi": pclass["euler3"],
"mirror": pclass["mirror"],
"shiftx": pclass["shiftx"],
"shifty": pclass["shifty"],
}
except:
print pclass
emaneuler = {
"alt": pclass["euler1"],
"az": pclass["euler2"],
"phi": pclass["euler3"],
"mirror": pclass["mirror"],
"shiftx": pclass["shiftx"],
"shifty": pclass["shifty"],
}
return emaneuler
def getStackParticleTiltPair(stackid, partnum, tiltstackid=None):
"""
takes a stack id and particle number (1+) spider-style
returns the stack particle number for the tilt pair
"""
#print stackid, partnum
if tiltstackid is None:
tiltstackid = stackid
t0 = time.time()
stackpartdata1 = apStack.getStackParticle(stackid, partnum)
partdata = stackpartdata1['particle']
### figure out if its particle 1 or 2
tiltpartq1 = appiondata.ApTiltParticlePairData()
tiltpartq1['particle1'] = partdata
tiltpartdatas1 = tiltpartq1.query(results=1, readimages=False)
tiltpartq2 = appiondata.ApTiltParticlePairData()
tiltpartq2['particle2'] = partdata
tiltpartdatas2 = tiltpartq2.query(results=1, readimages=False)
if not tiltpartdatas1 and tiltpartdatas2:
#print "image1"
otherpart = tiltpartdatas2[0]['particle1']
elif tiltpartdatas1 and not tiltpartdatas2:
#print "image2"
otherpart = tiltpartdatas1[0]['particle2']
else:
print partdata
print tiltpartdatas1
print tiltpartdatas2
apDisplay.printError("failed to get tilt pair data")
### get tilt stack particle
tiltstackdata = apStack.getOnlyStackData(tiltstackid, msg=False)
stackpartq = appiondata.ApStackParticleData()
stackpartq['stack'] = tiltstackdata
stackpartq['particle'] = otherpart
stackpartdatas2 = stackpartq.query(results=1, readimages=False)
if not stackpartdatas2:
#print otherpart.dbid
#apDisplay.printError("particle "+str(partnum)+" has no tilt pair in stackid="+str(tiltstackid))
return None
stackpartdata = stackpartdatas2[0]
#print partnum,"-->",stackpartnum
if time.time() - t0 > 1.0:
apDisplay.printMsg("long getStackPartTiltPair " +
apDisplay.timeString(time.time() - t0))
return stackpartdata
def insertAlignParticlesIntoDatabase(self):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for ref in self.classD:
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = ref
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['iteration'] = 10
# if ref in self.resdict:
# refq['ssnr_resolution'] = self.resdict[ref]
### setup particle info
for partnum in self.classD[ref]:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(partnum)
alignpartq['alignstack'] = self.alignstackdata
### particle numbering starts with 1 in SIMPLE
stackpartdata = apStack.getStackParticle(
self.runparams['stackid'], int(partnum))
alignpartq['stackpart'] = stackpartdata
alignpartq['ref'] = refq
try:
alignpartq['xshift'] = self.alignD[partnum - 1]['shx']
except:
pass
try:
alignpartq['yshift'] = self.alignD[partnum - 1]['shy']
except:
pass
try:
alignpartq['mirror'] = self.alignD[partnum - 1]['mirror']
except:
pass
alignpartq['rotation'] = self.alignD[partnum - 1]['rot']
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " +
str(self.params['numpart']) + " particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def getStackParticleTiltPair(stackid, partnum, tiltstackid=None):
"""
takes a stack id and particle number (1+) spider-style
returns the stack particle number for the tilt pair
"""
#print stackid, partnum
if tiltstackid is None:
tiltstackid = stackid
t0 = time.time()
stackpartdata1 = apStack.getStackParticle(stackid, partnum)
partdata = stackpartdata1['particle']
### figure out if its particle 1 or 2
tiltpartq1 = appiondata.ApTiltParticlePairData()
tiltpartq1['particle1'] = partdata
tiltpartdatas1 = tiltpartq1.query(results=1, readimages=False)
tiltpartq2 = appiondata.ApTiltParticlePairData()
tiltpartq2['particle2'] = partdata
tiltpartdatas2 = tiltpartq2.query(results=1, readimages=False)
if not tiltpartdatas1 and tiltpartdatas2:
#print "image1"
otherpart = tiltpartdatas2[0]['particle1']
elif tiltpartdatas1 and not tiltpartdatas2:
#print "image2"
otherpart = tiltpartdatas1[0]['particle2']
else:
print partdata
print tiltpartdatas1
print tiltpartdatas2
apDisplay.printError("failed to get tilt pair data")
### get tilt stack particle
tiltstackdata = apStack.getOnlyStackData(tiltstackid, msg=False)
stackpartq = appiondata.ApStackParticleData()
stackpartq['stack'] = tiltstackdata
stackpartq['particle'] = otherpart
stackpartdatas2 = stackpartq.query(results=1, readimages=False)
if not stackpartdatas2:
#print otherpart.dbid
#apDisplay.printError("particle "+str(partnum)+" has no tilt pair in stackid="+str(tiltstackid))
return None
stackpartdata = stackpartdatas2[0]
#print partnum,"-->",stackpartnum
if time.time()-t0 > 1.0:
apDisplay.printMsg("long getStackPartTiltPair "+apDisplay.timeString(time.time()-t0))
return stackpartdata
def insertParticlesIntoDatabase(self, stackid, partlist):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for partdict in partlist:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = partdict['refnum']
refq['iteration'] = self.lastiter
refsearch = "part"+self.params['timestamp']+"_ref*"+str(partdict['refnum'])+"*"
refbase = os.path.splitext(glob.glob(refsearch)[0])[0]
refq['mrcfile'] = refbase+".mrc"
refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
if partdict['refnum'] in self.resdict:
refq['ssnr_resolution'] = self.resdict[partdict['refnum']]
reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
if not os.path.isfile(reffile):
emancmd = "proc2d "+refbase+".xmp "+refbase+".mrc"
apEMAN.executeEmanCmd(emancmd, verbose=False)
if not os.path.isfile(reffile):
apDisplay.printError("could not find reference file: "+reffile)
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partdict['partnum']
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(stackid, partdict['partnum'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['inplane']
alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
alignpartq['spread'] = partdict['spread']
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in "
+apDisplay.timeString(time.time()-t0), "cyan")
return
def getStackParticlesInOrder(params):
partorderfile = os.path.join(params['rundir'],'stackpartorder.list')
stackid = params['stackid']
if not os.path.isfile(partorderfile):
return apStack.getStackParticlesFromId(stackid)
partfile = open(partorderfile,'r')
lines = partfile.readlines()
partorder = map((lambda x:int(x[:-1])),lines)
partsort = list(partorder)
partsort.sort()
if partsort == partorder:
return apStack.getStackParticlesFromId(stackid)
apDisplay.printMsg("Preped stack has different order from the original stack. Getting information by the order of the preped stack")
stpartdatas = []
for partnum in partorder:
stpartdatas.append(apStack.getStackParticle(stackid, partnum))
return stpartdatas
def getStackParticleEulersForIteration(params,pnum):
"""
find the eulers assigned to a stack particle
during a refinement. This function will first
find the particle id for the given stack particle,
then find its position in the reference stack, and
will get the eulers for that particle in the recon
"""
# get stack particle id
stackp = apStack.getStackParticle(params['stackid'],pnum)
particleid = stackp['particle'].dbid
# find particle in reference stack
refstackid = apStack.getStackIdFromIterationId(params['reconiterid'],msg=False)
refstackp = apStack.getStackParticleFromParticleId(particleid,refstackid, nodie=True)
if not refstackp:
apDisplay.printWarning('No classification for stack particle %d in reconstruction iteration id: %d' % (pnum, params['reconiterid']))
params['noClassification']+=1
if params['noClassification'] > (float(params['last'])*0.10):
apDisplay.printError('More than 10% of the particles have no classification, use a different reference reconstruction')
pclass={}
pclass['eulers']={}
pclass['eulers']['euler1']=0.0
pclass['eulers']['euler2']=0.0
pclass['inplane_rotation']=0.0
pclass['mirror']=False
pclass['shiftx']=0.0
pclass['shifty']=0.0
else:
pclassq = appiondata.ApRefineParticleData()
pclassq['particle'] = refstackp
pclassq['refineIter'] = appiondata.ApRefineIterData.direct_query(params['reconiterid'])
pclass = pclassq.query()
pclass=pclass[0]
params['eman_orient']={}
params['eman_orient']['alt']=pclass['euler1']
params['eman_orient']['az']=pclass['euler2']
params['eman_orient']['phi']=pclass['euler3']
params['eman_orient']['mirror']=pclass['mirror']
params['eman_orient']['shiftx']=pclass['shiftx']
params['eman_orient']['shifty']=pclass['shifty']
return params
def makeEulerDoc(self, parttree):
count = 0
eulerfile = os.path.join(self.params['rundir'], "eulersdoc"+self.timestamp+".spi")
eulerf = open(eulerfile, "w")
apDisplay.printMsg("creating Euler doc file")
starttime = time.time()
for partdict in parttree:
stackpartdata = apStack.getStackParticle(self.params['tiltstackid'], partdict['tilt'])
count += 1
if count%100 == 0:
sys.stderr.write(".")
eulerf.flush()
gamma, theta, phi, tiltangle = apTiltPair.getParticleTiltRotationAngles(stackpartdata)
line = operations.spiderOutLine(count, [phi, tiltangle, gamma])
eulerf.write(line)
eulerf.close()
apDisplay.printColor("finished Euler doc file in "+apDisplay.timeString(time.time()-starttime), "cyan")
return eulerfile
def getStackParticlesInOrder(params):
partorderfile = os.path.join(params['rundir'], 'stackpartorder.list')
stackid = params['stackid']
if not os.path.isfile(partorderfile):
return apStack.getStackParticlesFromId(stackid)
partfile = open(partorderfile, 'r')
lines = partfile.readlines()
partorder = map((lambda x: int(x[:-1])), lines)
partsort = list(partorder)
partsort.sort()
if partsort == partorder:
return apStack.getStackParticlesFromId(stackid)
apDisplay.printMsg(
"Preped stack has different order from the original stack. Getting information by the order of the preped stack"
)
stpartdatas = []
for partnum in partorder:
stpartdatas.append(apStack.getStackParticle(stackid, partnum))
return stpartdatas
def insertNoRefRun(self, imagicstack, insert=False):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + self.params['runname'] +
"' and path already exist in database")
# create a norefParam object
norefq = appiondata.ApSpiderNoRefRunData()
norefq['runname'] = self.params['runname']
norefq['particle_diam'] = 2.0 * self.params['partrad']
norefq['first_ring'] = self.params['firstring']
norefq['last_ring'] = self.params['lastring']
norefq['init_method'] = self.params['initmethod']
norefq['run_seconds'] = self.runtime
### finish alignment run
alignrunq['norefrun'] = norefq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['description'] = self.params['description']
# STOP HERE
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['imagicfile'] = os.path.basename(imagicstack)
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['alignrun'] = alignrunq
alignstackq['iteration'] = 0
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: " + imagicfile)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average file: " + avgmrcfile)
alignstackq['stack'] = self.stack['data']
alignstackq['boxsize'] = math.floor(self.stack['boxsize'] /
self.params['bin'])
alignstackq['pixelsize'] = self.stack['apix'] * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert()
### create reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = 1
refq['iteration'] = 0
refq['mrcfile'] = "template.mrc"
#refpath = os.path.abspath(os.path.join(self.params['rundir'], "alignment"))
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = alignrunq
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for partdict in self.partlist:
### see apSpider.alignment.alignStack() for more info
"""
partdict.keys()
'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,...
'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,...
'mirror': checkMirror(templatenum),
'score': float(data[3]),
'rot': float(data[4]),
'xshift': float(data[5]),
'yshift': float(data[6]),
"""
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = refq
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['rot']
#alignpartq['score'] = partdict['score']
if insert is True:
alignpartq.insert()
return
def insertRefBasedRun(self, partlist, imagicstack, insert=False):
apDisplay.printMsg("committing results to DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database")
### setup ref based run
refrunq = appiondata.ApRefBasedRunData()
refrunq['runname'] = self.params['runname']
refrunq['xysearch'] = self.params['xysearch']
refrunq['xystep'] = self.params['xystep']
refrunq['first_ring'] = self.params['firstring']
refrunq['last_ring'] = self.params['lastring']
refrunq['num_iter'] = self.params['numiter']
refrunq['invert_templs'] = self.params['inverttemplates']
refrunq['num_templs'] = self.params['numtemplate']
#refrunq['csym', int),
refrunq['run_seconds'] = self.params['runtime']
### finish alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['refbasedrun'] = refrunq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['imagicfile'] = imagicstack
alignstackq['avgmrcfile'] = "average.mrc"
emancmd = "proc2d templatestack%02d.spi templatestack%02d.hed"%(self.params['numiter'],self.params['numiter'])
apEMAN.executeEmanCmd(emancmd)
alignstackq['refstackfile'] = ("templatestack%02d.hed"%(self.params['numiter']))
alignstackq['alignrun'] = alignrunq
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: "+imagicfile)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = self.stack['data']
alignstackq['boxsize'] = self.boxsize
alignstackq['pixelsize'] = self.stack['apix']*self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert()
### insert reference data
reflist = []
for j in range(self.params['numiter']):
iternum = j+1
for i in range(len(self.templatelist)):
refnum = i+1
templateid = self.templatelist[i]
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = iternum
refq['template'] = apTemplate.getTemplateFromId(templateid)
refq['mrcfile'] = ("templateavg%02d-%02d.mrc"%(iternum,refnum))
refpath = os.path.join(self.params['rundir'], "templates")
refq['path'] = appiondata.ApPathData(path=os.path.abspath(refpath))
refq['alignrun'] = alignrunq
if refnum in self.resdict:
refq['ssnr_resolution'] = self.resdict[refnum]
if insert is True:
refq.insert()
if iternum == self.params['numiter']:
reflist.append(refq)
#refq['varmrcfile', str),
#refq['frc_resolution', float),
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for partdict in partlist:
### see apSpider.alignment.alignStack() for more info
"""
partdict.keys()
'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,...
'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,...
'mirror': checkMirror(templatenum),
'score': float(data[3]),
'rot': float(data[4]),
'xshift': float(data[5]),
'yshift': float(data[6]),
"""
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = reflist[partdict['template']-1]
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['rot']
alignpartq['score'] = partdict['score']
alignpartq['mirror'] = partdict['mirror']
if insert is True:
alignpartq.insert()
return
def insertRefBasedRun(self, partlist, imagicstack, insert=False):
apDisplay.printMsg("committing results to DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database")
### setup ref based run
refrunq = appiondata.ApRefBasedRunData()
refrunq['runname'] = self.params['runname']
refrunq['xysearch'] = self.params['xysearch']
refrunq['xystep'] = self.params['xystep']
refrunq['first_ring'] = self.params['firstring']
refrunq['last_ring'] = self.params['lastring']
refrunq['num_iter'] = self.params['numiter']
refrunq['invert_templs'] = self.params['inverttemplates']
refrunq['num_templs'] = self.params['numtemplate']
#refrunq['csym', int),
refrunq['run_seconds'] = self.params['runtime']
### finish alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['refbasedrun'] = refrunq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['imagicfile'] = imagicstack
alignstackq['avgmrcfile'] = "average.mrc"
emancmd = "proc2d templatestack%02d.spi templatestack%02d.hed"%(self.params['numiter'],self.params['numiter'])
apEMAN.executeEmanCmd(emancmd)
alignstackq['refstackfile'] = ("templatestack%02d.hed"%(self.params['numiter']))
alignstackq['alignrun'] = alignrunq
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: "+imagicfile)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = self.stack['data']
alignstackq['boxsize'] = self.boxsize
alignstackq['pixelsize'] = self.stack['apix']*self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert()
### insert reference data
reflist = []
for j in range(self.params['numiter']):
iternum = j+1
for i in range(len(self.templatelist)):
refnum = i+1
templateid = self.templatelist[i]
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = iternum
refq['template'] = apTemplate.getTemplateFromId(templateid)
refq['mrcfile'] = ("templateavg%02d-%02d.mrc"%(iternum,refnum))
refpath = os.path.join(self.params['rundir'], "templates")
refq['path'] = appiondata.ApPathData(path=os.path.abspath(refpath))
refq['alignrun'] = alignrunq
if refnum in self.resdict:
refq['ssnr_resolution'] = self.resdict[refnum]
if insert is True:
refq.insert()
if iternum == self.params['numiter']:
reflist.append(refq)
#refq['varmrcfile', str),
#refq['frc_resolution', float),
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for partdict in partlist:
### see apSpider.alignment.alignStack() for more info
"""
partdict.keys()
'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,...
'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,...
'mirror': checkMirror(templatenum),
'score': float(data[3]),
'rot': float(data[4]),
'xshift': float(data[5]),
'yshift': float(data[6]),
"""
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = reflist[partdict['template']-1]
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['rot']
alignpartq['score'] = partdict['score']
alignpartq['mirror'] = partdict['mirror']
if insert is True:
alignpartq.insert()
return
def insertEdIterRun(self, insert=False):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + runparams['runname'] +
"' and path already exist in database")
### setup ed-iter run
editrunq = appiondata.ApEdIterRunData()
editrunq['runname'] = self.params['runname']
editrunq['radius'] = self.params['partrad']
editrunq['num_iter'] = self.params['numiter']
#editrunq['freealigns'] = self.params['freealigns'] ### no longer an option
editrunq['invert_templs'] = self.params['inverttemplates']
editrunq['num_templs'] = self.params['numtemplate']
#editrunq['csym', int),
editrunq['run_seconds'] = self.runtime
### finish alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['editerrun'] = editrunq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
### setup aligned stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
# stack of aligned particles
alignstackq['imagicfile'] = "alignstack.hed"
# final class averages
alignstackq['refstackfile'] = "avg.hed"
# averaged results
alignstackq['avgmrcfile'] = "average.mrc"
# check to be sure dbpaths to files are correct
refstackfile = os.path.join(
self.params['rundir'],
alignstackq['refstackfile']) #final averages
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: " +
refstackfile)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile']) #averaged results
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
alignstackq['stack'] = self.stack['data']
alignstackq['iteration'] = self.params['numiter']
alignstackq['boxsize'] = math.floor(self.stack['boxsize'] /
self.params['bin'])
alignstackq['pixelsize'] = self.stack['apix'] * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert(
) #alignstackq contains alignrunq which contains editrunq
### setup data from each iteration
apDisplay.printColor(
"Inserting average and variance images from each iteration, please wait",
"cyan")
reflist = []
for j in range(self.params['numiter']):
iternum = j + 1
for i in range(len(self.templatelist)):
refnum = i + 1
templateid = self.templatelist[i]
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = iternum
refq['template'] = apTemplate.getTemplateFromId(templateid)
refpath = os.path.join(self.params['rundir'], "templates")
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(refpath))
refq['alignrun'] = alignrunq
#refq['frc_resolution'] = #(float)
avgname = os.path.join(self.params['rundir'],
"r%02d/avg%03d" % (iternum, refnum))
varname = os.path.join(self.params['rundir'],
"r%02d/var%03d" % (iternum, refnum))
if os.path.isfile(avgname + ".spi"):
emancmd = "proc2d " + avgname + ".spi " + avgname + ".mrc"
apEMAN.executeEmanCmd(emancmd)
refq['mrcfile'] = (avgname + ".mrc")
emancmd = "proc2d " + varname + ".spi " + varname + ".mrc"
apEMAN.executeEmanCmd(emancmd)
refq['varmrcfile'] = (varname + ".mrc")
if insert is True:
refq.insert()
if iternum == self.params['numiter']:
reflist.append(refq)
else:
reflist.append(None)
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
partlist = self.readApshDocFile("apshdoc.spi", "apshdoc.pickle")
for partdict in partlist:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = reflist[partdict['template'] - 1]
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['rotation'] = partdict['rot']
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['mirror'] = partdict['mirror']
alignpartq['score'] = partdict['score']
if insert is True:
alignpartq.insert() #insert each particle
return
def uploadResults(self):
if self.params['commit'] is False:
return
# Get the new file order
fh = open("sort_junk.sel", 'r')
lines = fh.readlines()
i = 0
fileorder = {}
for line in lines:
args = line.split()
if (len(args) > 1):
match = re.match('[A-Za-z]+([0-9]+)\.[A-Za-z]+',
(args[0].split('/'))[-1])
if (match):
filenumber = int(match.groups()[0])
fileorder[i] = filenumber
i += 1
fh.close()
# Produce a new stack
oldstack = apStack.getOnlyStackData(self.params['stackid'], msg=False)
newstack = appiondata.ApStackData()
newstack['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
newstack['name'] = "sorted.hed"
if newstack.query(results=1):
apDisplay.printError(
"A stack with these parameters already exists")
# Fill in data and submit
newstack['oldstack'] = oldstack
newstack['hidden'] = False
newstack['substackname'] = self.params['runname']
newstack['description'] = self.params['description']
newstack['pixelsize'] = oldstack['pixelsize']
newstack['boxsize'] = oldstack['boxsize']
newstack['junksorted'] = True
newstack.insert()
# Insert stack images
apDisplay.printMsg("Inserting stack particles")
count = 0
total = len(fileorder.keys())
if total == 0:
apDisplay.printError(
"No particles can be inserted in the sorted stack")
for i in fileorder.keys():
count += 1
if count % 100 == 0:
sys.stderr.write(
"\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b")
sys.stderr.write(
str(count) + " of " + (str(total)) + " complete")
# Get particle from the old stack
oldparticle = apStack.getStackParticle(self.params['stackid'],
fileorder[i] + 1)
# Insert particle
newparticle = appiondata.ApStackParticleData()
newparticle['particleNumber'] = i + 1
newparticle['stack'] = newstack
newparticle['stackRun'] = oldparticle['stackRun']
newparticle['particle'] = oldparticle['particle']
newparticle['mean'] = oldparticle['mean']
newparticle['stdev'] = oldparticle['stdev']
newparticle.insert()
apDisplay.printMsg(
"\n" + str(total) +
" particles have been inserted into the sorted stack")
# Insert runs in stack
apDisplay.printMsg("Inserting Runs in Stack")
runsinstack = apStack.getRunsInStack(self.params['stackid'])
for run in runsinstack:
newrunsq = appiondata.ApRunsInStackData()
newrunsq['stack'] = newstack
newrunsq['stackRun'] = run['stackRun']
newrunsq.insert()
apDisplay.printMsg("finished")
return
def commitToDatabase(self):
"""
insert the results into the database
"""
### expected result for an alignment run:
### 1. aligned particle stack in IMAGIC
### 2. rotation, shift, and quality parameters for each particle
### 3. which particles belongs to which class
### 4. stack file with the class averages
alignedstack = os.path.join(self.params['rundir'], "ptcl.hed")
refstack = os.path.join(self.params['rundir'], "iter.final.hed")
averagemrc = os.path.join(self.params['rundir'], "average.mrc")
apStack.averageStack(alignedstack, averagemrc)
particlemapping = self.determineClassOwnership()
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError(
"Run name '" + runparams['runname'] +
"' and path already exisclassmappingt in database")
### setup eman refine2d run
emanrefinetwodq = appiondata.ApEMANRefine2dRunData()
emanrefinetwodq['runname'] = self.params['runname']
emanrefinetwodq['run_seconds'] = time.time() - self.t0
emanrefinetwodq['num_iters'] = self.params['numiter']
emanrefinetwodq['num_classes'] = self.params['numclasses']
### finish alignment run
alignrunq['refine2drun'] = emanrefinetwodq
alignrunq['hidden'] = False
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['bin'] = self.params['bin']
### setup alignment stackalignimagicfile
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = os.path.basename(alignedstack)
alignstackq['avgmrcfile'] = os.path.basename(averagemrc)
alignstackq['refstackfile'] = os.path.basename(refstack)
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: " +
alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
refstackfile = os.path.join(self.params['rundir'],
alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printErrrefqor("could not find reference stack file: " +
refstackfile)
### continue setting values
alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid']) * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = apFile.numImagesInStack(
alignimagicfilepath)
### inserting particles and references
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for emanpartnum in range(self.params['numpart']):
partnum = emanpartnum + 1
if partnum % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refnum = particlemapping[emanpartnum]
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = alignrunq
### TODO: create mrc file
#refq['mrcfile'] = refbase+".mrc"
#reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
#if not os.path.isfile(reffile):
# emancmd = "proc2d "+refstack+".xmp "+refstack+".mrc"
# apEMAN.executeEmanCmd(emancmd, verbose=False)
#if not os.path.isfile(reffile):
# apDisplay.printError("could not find reference file: "+reffile)
### TODO: get resolution
#refq['ssnr_resolution'] = TODO
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partnum)
alignpartq['stackpart'] = stackpartdata
### TODO: get the alignment parameters
#alignpartq['xshift'] = partdict['xshift']
#alignpartq['yshift'] = partdict['yshift']
#alignpartq['rotation'] = partdict['inplane']
#alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
### TODO: get the score
#alignpartq['score'] = partdict['score']
### insert
if self.params['commit'] is True:
alignpartq.insert()
return
def insertNoRefRun(self, imagicstack, insert=False):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq["runname"] = self.params["runname"]
alignrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + self.params["runname"] + "' and path already exist in database")
# create a norefParam object
norefq = appiondata.ApSpiderNoRefRunData()
norefq["runname"] = self.params["runname"]
norefq["particle_diam"] = 2.0 * self.params["partrad"]
norefq["first_ring"] = self.params["firstring"]
norefq["last_ring"] = self.params["lastring"]
norefq["init_method"] = self.params["initmethod"]
norefq["run_seconds"] = self.runtime
### finish alignment run
alignrunq["norefrun"] = norefq
alignrunq["hidden"] = False
alignrunq["bin"] = self.params["bin"]
alignrunq["hp_filt"] = self.params["highpass"]
alignrunq["lp_filt"] = self.params["lowpass"]
alignrunq["description"] = self.params["description"]
# STOP HERE
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq["alignrun"] = alignrunq
alignstackq["imagicfile"] = os.path.basename(imagicstack)
alignstackq["avgmrcfile"] = "average.mrc"
alignstackq["alignrun"] = alignrunq
alignstackq["iteration"] = 0
alignstackq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
### check to make sure files exist
imagicfile = os.path.join(self.params["rundir"], alignstackq["imagicfile"])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: " + imagicfile)
avgmrcfile = os.path.join(self.params["rundir"], alignstackq["avgmrcfile"])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average file: " + avgmrcfile)
alignstackq["stack"] = self.stack["data"]
alignstackq["boxsize"] = math.floor(self.stack["boxsize"] / self.params["bin"])
alignstackq["pixelsize"] = self.stack["apix"] * self.params["bin"]
alignstackq["description"] = self.params["description"]
alignstackq["hidden"] = False
alignstackq["num_particles"] = self.params["numpart"]
if insert is True:
alignstackq.insert()
### create reference
refq = appiondata.ApAlignReferenceData()
refq["refnum"] = 1
refq["iteration"] = 0
refq["mrcfile"] = "template.mrc"
# refpath = os.path.abspath(os.path.join(self.params['rundir'], "alignment"))
refq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
refq["alignrun"] = alignrunq
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for partdict in self.partlist:
### see apSpider.alignment.alignStack() for more info
"""
partdict.keys()
'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,...
'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,...
'mirror': checkMirror(templatenum),
'score': float(data[3]),
'rot': float(data[4]),
'xshift': float(data[5]),
'yshift': float(data[6]),
"""
alignpartq = appiondata.ApAlignParticleData()
alignpartq["ref"] = refq
alignpartq["partnum"] = partdict["num"]
alignpartq["alignstack"] = alignstackq
stackpartdata = apStack.getStackParticle(self.params["stackid"], partdict["num"])
alignpartq["stackpart"] = stackpartdata
alignpartq["xshift"] = partdict["xshift"]
alignpartq["yshift"] = partdict["yshift"]
alignpartq["rotation"] = partdict["rot"]
# alignpartq['score'] = partdict['score']
if insert is True:
alignpartq.insert()
return
def uploadResults(self):
if self.params['commit'] is False:
return
# Produce new stacks
oldstack = apStack.getOnlyStackData(self.params['notstackid'], msg=False)
notstack = appiondata.ApStackData()
notstack['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
notstack['name'] = self.notstackdata['name']
if notstack.query(results=1):
apDisplay.printError("A stack with these parameters already exists")
tiltstack = appiondata.ApStackData()
tiltstack['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
tiltstack['name'] = self.tiltstackdata['name']
if tiltstack.query(results=1):
apDisplay.printError("A stack with these parameters already exists")
# Fill in data and submit
notstack['oldstack'] = oldstack
notstack['hidden'] = False
notstack['substackname'] = self.params['runname']
notstack['description'] = self.params['description']+" ... tilt stack sorted"
notstack['pixelsize'] = oldstack['pixelsize']
notstack.insert()
tiltstack['oldstack'] = oldstack
tiltstack['hidden'] = False
tiltstack['substackname'] = self.params['runname']
tiltstack['description'] = self.params['description']+" ... tilt stack sorted"
tiltstack['pixelsize'] = oldstack['pixelsize']
tiltstack.insert()
# Insert stack images
apDisplay.printMsg("Inserting stack particles")
count=0
for partdict in parttree:
count += 1
if count % 100 == 0:
sys.stderr.write("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b")
sys.stderr.write(str(count)+" of "+(str(total))+" complete")
# Get particles from the old stacks
oldnotparticle = apStack.getStackParticle(self.params['notstackid'], partdict['not'])
oldtiltparticle = apStack.getStackParticle(self.params['tiltstackid'], partdict['tilt'])
# Insert particle
notparticle = appiondata.ApStackParticleData()
notparticle.update(oldnotparticle)
notparticle['particleNumber'] = count
notparticle['stack'] = notstack
notparticle.insert()
tiltparticle = appiondata.ApStackParticleData()
tiltparticle.update(oldtiltparticle)
tiltparticle['particleNumber'] = count
tiltparticle['stack'] = tiltstack
tiltparticle.insert()
apDisplay.printMsg("\n%d particles have been inserted into the tilt synchronized stacks"%(count))
# Insert runs in stack
apDisplay.printMsg("Inserting Runs in Stack")
runsinstack = apStack.getRunsInStack(self.params['notstackid'])
for run in runsinstack:
newrunsq = appiondata.ApRunsInStackData()
newrunsq['stack'] = notstack
newrunsq['stackRun'] = run['stackRun']
newrunsq.insert()
runsinstack = apStack.getRunsInStack(self.params['tiltstackid'])
for run in runsinstack:
newrunsq = appiondata.ApRunsInStackData()
newrunsq['stack'] = tiltstack
newrunsq['stackRun'] = run['stackRun']
newrunsq.insert()
apDisplay.printMsg("finished")
return
def commitToDatabase(self):
"""
insert the results into the database
"""
### expected result for an alignment run:
### 1. aligned particle stack in IMAGIC
### 2. rotation, shift, and quality parameters for each particle
### 3. which particles belongs to which class
### 4. stack file with the class averages
alignedstack = os.path.join(self.params['rundir'], "ptcl.hed")
refstack = os.path.join(self.params['rundir'], "iter.final.hed")
averagemrc = os.path.join(self.params['rundir'], "average.mrc")
apStack.averageStack(alignedstack, averagemrc)
particlemapping = self.determineClassOwnership()
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']
+"' and path already exisclassmappingt in database")
### setup eman refine2d run
emanrefinetwodq = appiondata.ApEMANRefine2dRunData()
emanrefinetwodq['runname'] = self.params['runname']
emanrefinetwodq['run_seconds'] = time.time() - self.t0
emanrefinetwodq['num_iters'] = self.params['numiter']
emanrefinetwodq['num_classes'] = self.params['numclasses']
### finish alignment run
alignrunq['refine2drun'] = emanrefinetwodq
alignrunq['hidden'] = False
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['bin'] = self.params['bin']
### setup alignment stackalignimagicfile
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = os.path.basename(alignedstack)
alignstackq['avgmrcfile'] = os.path.basename(averagemrc)
alignstackq['refstackfile'] = os.path.basename(refstack)
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: "+alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printErrrefqor("could not find reference stack file: "+refstackfile)
### continue setting values
alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])*self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = apFile.numImagesInStack(alignimagicfilepath)
### inserting particles and references
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for emanpartnum in range(self.params['numpart']):
partnum = emanpartnum+1
if partnum % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refnum = particlemapping[emanpartnum]
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = alignrunq
### TODO: create mrc file
#refq['mrcfile'] = refbase+".mrc"
#reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
#if not os.path.isfile(reffile):
# emancmd = "proc2d "+refstack+".xmp "+refstack+".mrc"
# apEMAN.executeEmanCmd(emancmd, verbose=False)
#if not os.path.isfile(reffile):
# apDisplay.printError("could not find reference file: "+reffile)
### TODO: get resolution
#refq['ssnr_resolution'] = TODO
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'], partnum)
alignpartq['stackpart'] = stackpartdata
### TODO: get the alignment parameters
#alignpartq['xshift'] = partdict['xshift']
#alignpartq['yshift'] = partdict['yshift']
#alignpartq['rotation'] = partdict['inplane']
#alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
### TODO: get the score
#alignpartq['score'] = partdict['score']
### insert
if self.params['commit'] is True:
alignpartq.insert()
return
def insertEdIterRun(self, insert=False):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database")
### setup ed-iter run
editrunq = appiondata.ApEdIterRunData()
editrunq['runname'] = self.params['runname']
editrunq['radius'] = self.params['partrad']
editrunq['num_iter'] = self.params['numiter']
#editrunq['freealigns'] = self.params['freealigns'] ### no longer an option
editrunq['invert_templs'] = self.params['inverttemplates']
editrunq['num_templs'] = self.params['numtemplate']
#editrunq['csym', int),
editrunq['run_seconds'] = self.runtime
### finish alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['editerrun'] = editrunq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
### setup aligned stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
# stack of aligned particles
alignstackq['imagicfile'] = "alignstack.hed"
# final class averages
alignstackq['refstackfile'] = "avg.hed"
# averaged results
alignstackq['avgmrcfile'] = "average.mrc"
# check to be sure dbpaths to files are correct
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile']) #final averages
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile']) #averaged results
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
alignstackq['stack'] = self.stack['data']
alignstackq['iteration'] = self.params['numiter']
alignstackq['boxsize'] = math.floor(self.stack['boxsize']/self.params['bin'])
alignstackq['pixelsize'] = self.stack['apix']*self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert() #alignstackq contains alignrunq which contains editrunq
### setup data from each iteration
apDisplay.printColor("Inserting average and variance images from each iteration, please wait", "cyan")
reflist = []
for j in range(self.params['numiter']):
iternum = j+1
for i in range(len(self.templatelist)):
refnum = i+1
templateid = self.templatelist[i]
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = iternum
refq['template'] = apTemplate.getTemplateFromId(templateid)
refpath = os.path.join(self.params['rundir'], "templates")
refq['path'] = appiondata.ApPathData(path=os.path.abspath(refpath))
refq['alignrun'] = alignrunq
#refq['frc_resolution'] = #(float)
avgname = os.path.join(self.params['rundir'], "r%02d/avg%03d"%(iternum,refnum) )
varname = os.path.join(self.params['rundir'], "r%02d/var%03d"%(iternum,refnum) )
if os.path.isfile(avgname+".spi"):
emancmd = "proc2d "+avgname+".spi "+avgname+".mrc"
apEMAN.executeEmanCmd(emancmd)
refq['mrcfile'] = (avgname+".mrc")
emancmd = "proc2d "+varname+".spi "+varname+".mrc"
apEMAN.executeEmanCmd(emancmd)
refq['varmrcfile'] = (varname+".mrc")
if insert is True:
refq.insert()
if iternum == self.params['numiter']:
reflist.append(refq)
else:
reflist.append(None)
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
partlist = self.readApshDocFile("apshdoc.spi","apshdoc.pickle")
for partdict in partlist:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = reflist[partdict['template']-1]
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['rotation'] = partdict['rot']
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['mirror'] = partdict['mirror']
alignpartq['score'] = partdict['score']
if insert is True:
alignpartq.insert() #insert each particle
return
def uploadResults(self):
if self.params['commit'] is False:
return
# Get the new file order
fh=open("sort_junk.sel",'r')
lines=fh.readlines()
i=0;
fileorder={};
for line in lines:
args=line.split()
if (len(args)>1):
match=re.match('[A-Za-z]+([0-9]+)\.[A-Za-z]+',
(args[0].split('/'))[-1])
if (match):
filenumber=int(match.groups()[0])
fileorder[i]=filenumber
i+=1
fh.close()
# Produce a new stack
oldstack = apStack.getOnlyStackData(self.params['stackid'],msg=False)
newstack = appiondata.ApStackData()
newstack['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
newstack['name'] = "sorted.hed"
if newstack.query(results=1):
apDisplay.printError("A stack with these parameters already exists")
# Fill in data and submit
newstack['oldstack'] = oldstack
newstack['hidden'] = False
newstack['substackname'] = self.params['runname']
newstack['description'] = self.params['description']
newstack['pixelsize'] = oldstack['pixelsize']
newstack['boxsize'] = oldstack['boxsize']
newstack['junksorted'] = True
newstack.insert()
# Insert stack images
apDisplay.printMsg("Inserting stack particles")
count=0
total=len(fileorder.keys())
if total==0:
apDisplay.printError("No particles can be inserted in the sorted stack")
for i in fileorder.keys():
count += 1
if count % 100 == 0:
sys.stderr.write("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b")
sys.stderr.write(str(count)+" of "+(str(total))+" complete")
# Get particle from the old stack
oldparticle = apStack.getStackParticle(self.params['stackid'], fileorder[i]+1)
# Insert particle
newparticle = appiondata.ApStackParticleData()
newparticle['particleNumber'] = i+1
newparticle['stack'] = newstack
newparticle['stackRun'] = oldparticle['stackRun']
newparticle['particle'] = oldparticle['particle']
newparticle['mean'] = oldparticle['mean']
newparticle['stdev'] = oldparticle['stdev']
newparticle.insert()
apDisplay.printMsg("\n"+str(total)+" particles have been inserted into the sorted stack")
# Insert runs in stack
apDisplay.printMsg("Inserting Runs in Stack")
runsinstack = apStack.getRunsInStack(self.params['stackid'])
for run in runsinstack:
newrunsq = appiondata.ApRunsInStackData()
newrunsq['stack'] = newstack
newrunsq['stackRun'] = run['stackRun']
newrunsq.insert()
apDisplay.printMsg("finished")
return
