def calcFSC(vol1, vol2, fscfile, dataext=".spi", inMySpi=False):
"""
Calculate the differential 3-D phase residual and the
Fourier Shell Correlation between two volumes.
The Differential Phase Residual over a shell with thickness
given by shell width and the Fourier Shell Correlation
between shells of specified widths are computed and stored
in the document file.
"""
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
else:
mySpi = inMySpi
apFile.removeFile(fscfile)
v1 = spyder.fileFilter(vol1)
v2 = spyder.fileFilter(vol2)
fsc = spyder.fileFilter(fscfile)
mySpi.toSpiderQuiet("RF 3", v1, v2, "1", "0.5,1.5", "C", "90", "3", fsc)
if inMySpi is False:
mySpi.close()
def rotshiftStack(origstackfile, rotShiftFile, timestamp, iternum, classnum=None, dataext=".spi"):
origstackfile = spyder.fileFilter(origstackfile)
if classnum is not None:
apshstackfile = str(classnum)+"/"+"apshstack%s.spi"%(timestamp)
else:
apshstackfile = "apshstack%s.spi"%(timestamp)
apshstackfile = spyder.fileFilter(apshstackfile)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
mySpider.toSpider("RT SQ",
origstackfile+"@*****", #stack and particle number
apshstackfile+"@*****", #output stack and particle
"("+str(rotation)+")", #rotation
"("+str(Xshift)+","+str(Yshift)+")", #shift parameters
)
mySpider.close()
return
def spiderAPMQ(projs,
numprojs,
tsearch,
tstep,
lastRing,
stackfile,
nump,
ang,
firstRing=1,
startp=1,
apmqfile="apmq.spi",
nproc=1,
outang="angular.spi",
dataext=".spi"):
apFile.removeFile(apmqfile)
mySpi = spyder.SpiderSession(nproc=nproc,
dataext=dataext,
logo=False,
log=True)
mySpi.toSpiderQuiet(
"AP MQ",
spyder.fileFilter(projs) + "@*****",
"1-" + str(numprojs),
str(tsearch) + "," + str(tstep),
str(firstRing) + "," + str(lastRing),
spyder.fileFilter(stackfile) + "@******",
str(startp) + "-" + str(nump),
spyder.fileFilter(apmqfile),
)
apFile.removeFile(outang)
mySpi.toSpiderQuiet(
"VO MD",
spyder.fileFilter(ang),
spyder.fileFilter(apmqfile),
spyder.fileFilter(outang),
)
mySpi.close()
return outang
def createProjections(incr,
boxsz,
symfold,
invol,
rad,
sel="selvoea.spi",
ang="angvoea.spi",
projs="proj.spi",
nproc=1,
dataext=".spi"):
# create doc file containing euler angles
spiderVOEA(incr, ang, symfold)
projangles = readDocFile(ang)
numprojs = len(projangles)
mySpi = spyder.SpiderSession(nproc=nproc,
dataext=dataext,
logo=False,
log=True)
apFile.removeFile(sel)
mySpi.toSpiderQuiet(
"DOC CREATE",
spyder.fileFilter(sel),
"1",
"1-" + str(numprojs),
)
apFile.removeFile(projs)
mySpi.toSpiderQuiet(
"PJ 3Q",
spyder.fileFilter(invol),
str(rad),
spyder.fileFilter(sel),
spyder.fileFilter(ang),
spyder.fileFilter(projs) + "@*****",
)
mySpi.close()
return projs, numprojs, ang, sel
def commonLines(stackfile,
maskdiam=None,
minfreq=0,
maxfreq=0.25,
ang=5.0,
numiter=200,
outdocfile=None,
numpart=None,
dataext=".spi"):
"""
performs common lines on a input spider stack
"""
if numpart is None or numpart < 3:
apDisplay.printError("undefined number of particles")
if maskdiam is None:
apDisplay.printError("undefined mask diameter")
starttime = time.time()
if dataext in stackfile:
stackfile = stackfile[:-4]
randdocfile = generateRandomAngles(numpart)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
mySpider.toSpider(
"OP",
stackfile + "@*****", #stack file
"1-%d" % (numpart), #number of particles
str(maskdiam), #mark diameter
"%.3f,%.3f" %
(minfreq, maxfreq), #frequency range for line projections
str(ang), #angular increment or accuracy
str(numiter), #number of iterations
randdocfile, #random angles doc file
outdocfile, #output angles doc file
)
mySpider.close()
apDisplay.printColor(
"finished common lines in " +
apDisplay.timeString(time.time() - starttime), "cyan")
return
def createGaussianSphere(self, volfile, boxsize):
stdev = boxsize / 5.0
randdev = boxsize / 20.0
halfbox = boxsize / 2.0
#width of Gaussian
gaussstr = (
"%.3f,%.3f,%.3f" %
(stdev + randdev * random.random(), stdev +
randdev * random.random(), stdev + randdev * random.random()))
apDisplay.printMsg("Creating Gaussian volume with stdev=" + gaussstr)
mySpider = spyder.SpiderSession(logo=False)
mySpider.toSpiderQuiet(
"MO 3",
spyder.fileFilter(volfile),
"%d,%d,%d" % (boxsize, boxsize, boxsize),
"G", #G for Gaussian
"%.3f,%.3f,%.3f" %
(halfbox, halfbox, halfbox), #center of Gaussian
gaussstr,
)
mySpider.close()
return
def rotAndShiftVol(invol,
outvol,
rot=(0, 0, 0),
center=(0, 0, 0),
shift=(0.0, 0.0, 0.0),
dataext=".spi",
inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
else:
mySpi = inMySpi
vol1 = spyder.fileFilter(invol)
tmpvol = spyder.fileFilter('temp')
vol3 = spyder.fileFilter(outvol)
cleanlist = []
invol = vol1
if rot != (0, 0, 0):
outvol = tmpvol
rot = tuple(map((lambda x: float(x)), rot))
rotstr = '%.2f,%.2f,%.2f' % rot
center = tuple(map((lambda x: float(x)), center))
centerstr = '%.1f,%.1f,%.1f' % center
mySpi.toSpider("RT 3A", invol, outvol, rotstr, centerstr)
cleanlist.append(invol)
invol = tmpvol
if shift != (0, 0, 0):
outvol = vol3
shift = tuple(map((lambda x: float(x)), shift))
shiftstr = '%f.1,%f.1,%f.1' % shift
mySpi.toSpider("SH", invol, outvol, shiftstr)
cleanlist.append(invol)
invol = outvol
if outvol != vol3:
mySpi.toSpider("CP", invol, vol3)
if inMySpi is False:
mySpi.close()
for cleanfile in cleanlist:
os.remove(cleanfile + '.spi')
def boxMask(self, infile, outfile, spirots=None):
from appionlib.apSpider import operations
# boxmask the particles
apDisplay.printMsg("masking the particles with a rectangular box")
nump = apStack.getNumberStackParticlesFromId(self.params['stackid'])
box = self.stackdata['boxsize']
apix = self.stackdata['pixelsize'] * 1e10
if self.params['mask'] is None:
mask = box / 2 - 2
else:
mask = int(self.params['mask'] / apix)
imask = int(self.params['imask'] / apix)
length = int(self.params['length'] / apix)
falloff = self.params['falloff'] / apix
mask -= falloff / 2
length = (length / 2) - (falloff / 2)
# create blank image for mask
maskfile = "boxmask.spi"
operations.createBoxMask(maskfile, box, mask, length, falloff, imask)
mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
mySpi.toSpiderQuiet("CP", spyder.fileFilter(maskfile), "_4")
mySpi.toSpider("do x10=1,%i" % nump)
if self.params['vertical'] is not True:
mySpi.toSpider("UD IC x10,x30", spyder.fileFilter(spirots),
"x30 = -1*x30", "RT B", "_4", "_9", "(x30)", "(0)",
"MU",
spyder.fileFilter(infile) + "@{******x10}", "_9")
else:
mySpi.toSpider("MU",
spyder.fileFilter(infile) + "@{******x10}", "_4")
mySpi.toSpider(
spyder.fileFilter(outfile) + "@{******x10}", "*", "enddo")
if self.params['vertical'] is not True:
mySpi.toSpider("UD ICE", spyder.fileFilter(spirots))
mySpi.close()
def ClCla(alignedstack,
numpart=None,
numclasses=40,
factorlist=range(1, 5),
corandata="coran/corandata",
dataext=".spi"):
"""
this doesn't work
"""
if alignedstack[-4:] == dataext:
alignedstack = alignedstack[:-4]
rundir = "cluster"
classavg = rundir + "/" + ("classavgstack%03d" % numclasses)
classvar = rundir + "/" + ("classvarstack%03d" % numclasses)
apParam.createDirectory(rundir)
for i in range(numclasses):
apFile.removeFile(rundir + ("/classdoc%04d" % (i + 1)) + dataext)
apFile.removeFile(rundir + "/clusterdoc" + dataext)
factorstr, factorkey = operations.intListToString(factorlist)
### do hierarchical clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
mySpider.toSpider(
"CL CLA",
corandata, # path to coran data
rundir + "/clusterdoc", #clusterdoc file
factorstr, #factor numbers
"5,8",
"4",
"2", # minimum number of particles per class
"Y",
rundir + "/dendrogram.ps",
"Y",
rundir + "/dendrogramdoc",
)
mySpider.close()
def mirrorParticles(self, partdatas, spiderstack):
partnum = 0
mySpider = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
for stackpartdata in partdatas:
partnum += 1
inplane, mirror = self.getParticleInPlaneRotation(stackpartdata)
if mirror is True:
sys.stderr.write("m")
mySpider.toSpiderQuiet(
"MR",
spyder.fileFilter(spiderstack) + ("@%05d" % (partnum)),
"_9",
"Y",
)
mySpider.toSpiderQuiet(
"CP",
"_9",
spyder.fileFilter(spiderstack) + ("@%05d" % (partnum)),
)
else:
sys.stderr.write(".")
sys.stderr.write("\n")
mySpider.close()
def maskImg(inimg,
outimg,
outrad,
cutoff,
background="P",
inrad="0",
center="0",
dataext=".spi",
inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
else:
mySpi = inMySpi
img1 = spyder.fileFilter(inimg)
img2 = spyder.fileFilter(outimg)
mySpi.toSpiderQuiet("MA", img1, img2,
str(outrad) + "," + str(inrad), cutoff, background)
if background == "E":
mySpi.toSpiderQuiet("0", str(center) + "," + str(center))
if cutoff == "C" or cutoff == "G":
mySpi.toSpiderQuiet("3.5")
if inMySpi is False:
mySpi.close()
def correspondenceAnalysis(alignedstack, boxsize, maskpixrad, numpart, numfactors=8, dataext=".spi", nproc=1):
"""
inputs:
aligned stack
search params
outputs:
eigen images
eigen vectors
coran parameters
"""
### setup
if dataext in alignedstack:
alignedstack = alignedstack[:-4]
t0 = time.time()
rundir = "coran"
apParam.createDirectory(rundir)
### make template in memory
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False, nproc=nproc)
mySpider.toSpiderQuiet("MO", "_9", "%d,%d" % (boxsize, boxsize), "C", str(maskpixrad*2.0))
### performing correspondence analysis
apDisplay.printMsg("Performing correspondence analysis (long wait)")
mySpider.toSpider(
"CA S",
spyder.fileFilter(alignedstack)+"@******", "1-"+str(numpart),
"_9", str(numfactors), "C", "100",
rundir+"/corandata")
mySpider.close()
contriblist = analyzeEigenFactors(alignedstack, rundir, numpart, numfactors, dataext)
td1 = time.time()-t0
apDisplay.printMsg("completed correspondence analysis of "+str(numpart)
+" particles in "+apDisplay.timeString(td1))
return contriblist
def findRotation(self, avgimg):
# use radon transform in SPIDER to find rotation to orient the average image vertical
if os.path.isfile("angle.spi"):
os.remove("angle.spi")
box = self.alignstackdata['boxsize']
mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
# circular mask the average
mySpi.toSpider("MA",
spyder.fileFilter(avgimg) + "@1", "_1",
"%i,0" % ((box / 2) - 2), "C", "E", "0",
"%i,%i" % ((box / 2) + 1, (box / 2) + 1), "3")
# get power spectrum
mySpi.toSpider("PW", "_1", "_2")
# Radon transform
mySpi.toSpider("RM 2DN", "_2", "1", "_3", "%i" % box, "%i" % (box / 2),
"0,0", "N")
# mask in the X direction to only include equator
mySpi.toSpider("MA X", "_3", "_4", "6,0", "D", "E", "0",
"%i,%i" % ((box / 2), (box / 2)))
# find peak
mySpi.toSpider("PK x20,x21,x22", "_4", "1,0")
# save the angles to a file
mySpi.toSpider("SD 1, x21", "angle")
mySpi.toSpider("SD E", "angle")
mySpi.close()
f = open("angle.spi")
for line in f:
d = line.strip().split()
if d[0][0] == ";" or len(d) < 3:
continue
rot = float(d[2])
os.remove("angle.spi")
os.remove(avgimg)
return rot
def createBoxMask(maskfile,
boxsize,
maskx,
masky,
falloff,
imask=None,
dataext=".spi"):
"""
creates a rectangular mask with soft falloff
"""
apDisplay.printMsg("Creating %i x %i box mask" % (boxsize, boxsize))
mySpi = spyder.SpiderSession(dataext=dataext,
logo=False,
nproc=1,
log=False)
# create blank image for mask
mySpi.toSpiderQuiet("BL", "_1", "%i,%i" % (boxsize, boxsize), "N", "1")
# mask it in X
mySpi.toSpiderQuiet("MA X", "_1", "_2", "%i" % maskx, "C", "E", "0",
"%i,%i" % (boxsize / 2, boxsize / 2), "%.2f" % falloff)
# inner mask in X
if imask is not None:
mySpi.toSpiderQuiet("MA X", "_2", "_3", "%i,%i" % (boxsize / 2, imask),
"C", "E", "0",
"%i,%i" % (boxsize / 2, boxsize / 2),
"%.2f" % (falloff / 4))
mySpi.toSpiderQuiet("CP", "_3", "_2")
# mask in Y
mySpi.toSpiderQuiet("MA Y", "_2", spyder.fileFilter(maskfile),
"%i" % masky, "C", "E", "0",
"%i,%i" % (boxsize / 2, boxsize / 2), "%.2f" % falloff)
mySpi.close()
if not os.path.isfile(maskfile) or apFile.fileSize(maskfile) < 2:
apDisplay.printError("Failed to create mask file")
return
def butterworthLP(volfile, pixelsize, dataext=".spi"):
"""
inputs:
volume
outputs:
volume
"""
### setup
starttime = time.time()
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(volfile+dataext):
apDisplay.printError("volume file not found: "+volfile+dataext)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
mySpider.toSpider("FQ",
volfile, #filename for volume
"_1",
# volfile+"_filtered", #filename for output volume
"(7)",
"(%.5f,%.5f)" % (pixelsize/25,pixelsize/15), #pass-band and stop-band
)
mySpider.close()
apDisplay.printColor("finished filtering the volume "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def makeSpiderCoranBatch(params, filename, clsdir):
nfacts = 20
if params['nptcls'] < 21:
nfacts = params['nptcls'] - 1
f = open(os.path.join(clsdir, filename), 'w')
f.write('MD ; verbose off in spider log file\n')
f.write('VB OFF\n')
f.write('\n')
f.write('x99=%d ; number of particles in stack\n' % params['nptcls'])
f.write('x98=%d ; box size\n' % params['boxsize'])
f.write('x94=%d ; mask radius\n' % params['coranmask'])
haccut = params['haccut']
sp = spyder.SpiderSession()
if sp.version() >= 18.03:
haccut *= 100
sp.close()
f.write('x93=%f ; cutoff for hierarchical clustering\n' % haccut)
f.write('x92=20 ; additive constant for hierarchical clustering\n')
f.write('\n')
alignstack = 'aligned'
if params['hp'] is not None:
f.write('FR G ; aligned hp stack file\n')
f.write('[alignedhp]alignedhp\n')
alignstack = 'alignedhp'
f.write('FR G ; aligned stack file\n')
f.write('[aligned]aligned\n')
f.write('\n')
f.write(';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n')
f.write('\n')
f.write('FR G ; where to write class lists\n')
f.write('[clhc_cls]classes/clhc_cls\n')
f.write('\n')
f.write('FR G ; where to write alignment data\n')
f.write('[ali]alignment/\n')
f.write('\n')
f.write('VM\n')
f.write('mkdir alignment\n')
f.write('\n')
f.write(
';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n'
)
f.write(
';; create the sequential file and then use that file and do a hierarchical ;;\n'
)
f.write(
';; clustering. Run clhd and clhe to classify the particles into different ;;\n'
)
f.write(
';; groups. ;;\n'
)
f.write(
';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n'
)
f.write('\n')
f.write('VM\n')
f.write('echo Performing multivariate statistical analysis\n')
f.write('VM\n')
f.write('echo " making template file"\n')
f.write('\n')
f.write('MO ; make mask template\n')
f.write('_9 ; save template in memory\n')
f.write('x98,x98 ; box size\n')
f.write('c ; circle\n')
f.write('x94 ; radius of mask\n')
f.write('\n')
f.write('VM\n')
f.write('echo " doing correspondence analysis"\n')
f.write('\n')
f.write('CA S ; do correspondence analysis\n')
f.write('[%s]@***** ; aligned stack\n' % alignstack)
f.write('1-x99 ; particles to use\n')
f.write('_9 ; mask file\n')
f.write('%d ; number of factors to be used\n' % nfacts)
f.write('C ; Coran analysis\n')
f.write(
'x92 ; additive constant (since coran cannot have negative values)\n'
)
f.write('[ali]coran ; output file prefix\n')
f.write('\n')
f.write('\n')
f.write('DO LB14 x11=1,%d\n' % nfacts)
f.write('CA SRE\n')
f.write('[ali]coran\n')
f.write('x11\n')
f.write('[ali]sre@{***x11}\n')
f.write('LB14\n')
f.write('\n')
#f.write('VM\n')
#f.write('eigendoc.py alignment/coran_EIG.spi alignment/eigendoc.out 30\n')
#f.write('\n')
f.write('VM\n')
f.write('echo " clustering..."\n')
f.write('\n')
f.write('CL HC ; do hierarchical clustering\n')
f.write('[ali]coran_IMC ; coran image factor coordinate file\n')
f.write('1-3\n')
f.write(
'1.00 ; factor numbers to be included in clustering algorithm\n'
)
f.write('1.00 ; factor weights\n')
f.write('1.00 ; for each factor number\n')
f.write('5 ; use Wards method\n')
f.write('Y ; make a postscript of dendogram\n')
f.write('[ali]clhc.ps ; dendogram image file\n')
f.write('Y ; save dendogram doc file\n')
f.write('[ali]clhc_doc ; dendogram doc file\n')
f.write('\n')
f.write('\n')
f.write(';;;determine number of classes for given threshold\n')
f.write('CL HD\n')
f.write('x93\n')
f.write('[ali]clhc_doc\n')
f.write('clhc_classes\n')
f.write('\n')
f.write('UD N,x12\n')
f.write('clhc_classes\n')
f.write('\n')
f.write('VM\n')
f.write('mkdir classes\n')
f.write('\n')
f.write('VM\n')
f.write(
'echo "Creating {%F5.1%x12} classes using a threshold of {%F7.5%x93}"\n'
)
f.write(
'CL HE ; generate doc files containing particle numbers for classes\n'
)
f.write('x93 ; threshold (closer to 0=more classes)\n')
f.write('[ali]clhc_doc ; dendogram doc file\n')
f.write(
'[clhc_cls]**** ; selection doc file that will contain # of objects for classes\n'
)
f.write('\n')
f.write(';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n')
f.write(';; average aligned particles together ;;\n')
f.write(';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n')
f.write('\n')
f.write('VM\n')
f.write('echo Averaging particles into classes\n')
f.write('\n')
f.write('DO LB20 x81=1,x12\n')
f.write('AS R\n')
f.write('[aligned]@*****\n')
f.write('[clhc_cls]{****x81}\n')
f.write('A\n')
f.write('classes_avg@{****x81}\n')
f.write('classes_var@{****x81}\n')
f.write('LB20\n')
f.write('\n')
f.write('EN D\n')
def alignAPSH(volfile, origstackfile, eulerdocfile, classnum, boxsize, numpart, pixrad, timestamp, iternum, dataext=".spi"):
apshAngularFile = str(classnum)+"/"+"apshAngularFile-%03d.spi"%(iternum)
if os.path.isfile(apshAngularFile):
os.remove(apshAngularFile)
apDisplay.printColor("File exist! Removing file "+apshAngularFile, "cyan")
apshAngularFile = spyder.fileFilter(apshAngularFile)
apshListFile = str(classnum)+"/"+"apshListFile-%03d.spi"%(iternum)
if os.path.isfile(apshListFile):
os.remove(apshListFile)
apDisplay.printColor("File exist! Removing file "+apshListFile, "cyan")
apshListFile = spyder.fileFilter(apshListFile)
apshOutFile = str(classnum)+"/"+"apshOut-%03d.spi"%(iternum)
if os.path.isfile(apshOutFile):
os.remove(apshOutFile)
apDisplay.printColor("File exist! Removing file "+apshOutFile, "cyan")
apshOut = spyder.fileFilter(apshOutFile)
apshProjStack = str(classnum)+"/"+"apshProjStack-%03d.spi"%(iternum)
if os.path.isfile(apshProjStack):
os.remove(apshProjStack)
apDisplay.printColor("File exist! Removing file "+apshProjStack, "cyan")
apshProjStack = spyder.fileFilter(apshProjStack)
origstackfile = spyder.fileFilter(origstackfile)
volfile = spyder.fileFilter(volfile)
origeulerdocfile = spyder.fileFilter(eulerdocfile)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
mySpider.toSpider("VO EA,x53",
"10", #angular increment
"0,90.0", #Range of Theta
"0,359.9", #Range of Phi
apshAngularFile, #Angular file name
)
mySpider.toSpider("DOC CREATE",
apshListFile, #List file name
"1",
"1-x53"
)
#mySpider.toSpider("MS",
# "_9@", #reference projections
# str(boxsize),str(boxsize),"1", #boxsize
# "x53",
#)
mySpider.toSpider("PJ 3Q",
volfile, #input vol file
str(pixrad), #pixel radius
apshListFile, #number of particles
apshAngularFile, #Euler DOC file
apshProjStack+"@*****", #output projections
)
mySpider.toSpider("AP SH",
apshProjStack+"@*****", #reference projections
"1-x53", #reference numbers
"6,2", #translational search range, step size
"1,%d"%(boxsize/3), #first and last ring
apshAngularFile,
origstackfile+"@*****",
"1-%d"%(numpart),
origeulerdocfile,
"20", #"(0.0)", # no restrictions for alignment
"1", # mirror check?
apshOut, # output
)
mySpider.close()
return apshOutFile
def crossCorrelateAndShift(infile, reffile, alignfile, ccdocfile, numpart, dataext=".spi"):
### rewriten to do the whole thing in memory in SPIDER, it should be faster
starttime = time.time()
infile = spyder.fileFilter(infile)
reffile = spyder.fileFilter(reffile)
alignfile = spyder.fileFilter(alignfile)
partimg = "_4"
ccmap = "_5"
windccmap = "_6"
boxsize = apFile.getBoxSize(infile+dataext)
if not os.path.isfile(infile+dataext):
apDisplay.printError("input stack file not found: "+infile+dataext)
if not os.path.isfile(reffile+dataext):
apDisplay.printError("reference stack file not found: "+reffile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
### Allocate empty stack
mySpider.toSpiderQuiet(
"MS I", #command
"_2@", #name
"%d,%d,%d"%(boxsize), #boxsize
str(numpart+1), #num part to create in memory
str(numpart+1), #max particle number
)
partnum = 0
while partnum < numpart:
partnum+=1
mySpider.toSpiderQuiet("CP",
infile+("@%05d"%(partnum)), #picture
partimg,
)
### cross correlate images; reversed order to avoid -1*shift
mySpider.toSpiderQuiet("CC N",
reffile+("@%05d"%(partnum)), #reference
partimg, #picture
ccmap, #output file
)
### cannot shift more the 1/4 size of the image
mySpider.toSpiderQuiet("FI x52", partimg, "12" )
mySpider.toSpiderQuiet("x54=int(x52/2)") #window size
mySpider.toSpiderQuiet("x55=int(x52/4)") #window topleft
mySpider.toSpiderQuiet("WI",
ccmap, #input file
windccmap, #output file
"x54,x54", #window size
"x55,x55", #window origin
)
### find the cross-correlation peak
mySpider.toSpiderQuiet("x56=int(x52/4)+1") #center of window
mySpider.toSpiderQuiet("PK M x11,x12,x13,x14",
windccmap, #input ccmap file
"x56,x56", #origin coordinates
)
### save info to doc file
mySpider.toSpiderQuiet("SD %d,x13,x14"%(partnum),
ccdocfile, #input ccmap file
)
### shift the images images
mySpider.toSpiderQuiet("SH",
partimg, #old stack
("_2@%05d"%(partnum)), #new stack
"x13,x14", #shift value file
)
### finish up
#save stack to file
mySpider.toSpiderQuiet(
"CP", "_2@",
alignfile+"@",
)
#delete stack
mySpider.toSpiderQuiet(
"DE", "_2",
)
mySpider.close()
apDisplay.printColor("finished shifting particles in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def start(self):
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
# symmetry info
if self.params['sym'] == 'Icos':
self.params['symtype'] = 'I'
self.params['symfold'] = None
else:
self.params['symtype'] = self.params['sym'][0]
self.params['symfold'] = int(self.params['sym'][1:])
# eman "d" symmetry is spider "ci"
if self.params['symtype'].upper() == "D":
self.params['symtype'] = "CI"
# create symmetry doc file
sydoc = "sym.spi"
refine.symmetryDoc(self.params['symtype'], self.params['symfold'],
sydoc)
# convert incr to array of increments
ang_inc = self.params['incr'].split(',')
self.params['numiter'] = len(ang_inc)
self.params['increments'] = []
for i in range(0, self.params['numiter']):
self.params['increments'].append(int(ang_inc[i]))
# convert stack to spider stack
spiderstack = os.path.join(self.params['rundir'], 'start.spi')
operations.stackToSpiderStack(self.stack['file'],
spiderstack,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
numpart=self.params['numpart'])
# create filtered stack
spiderstackfilt = spiderstack
if (self.params['lowpass'] + self.params['highpass']) > 0:
spiderstackfilt = os.path.join(self.params['rundir'],
'start_filt.spi')
operations.stackToSpiderStack(self.stack['file'],
spiderstackfilt,
apix=self.stack['apix'],
boxsize=self.stack['boxsize'],
lp=self.params['lowpass'],
hp=self.params['highpass'],
numpart=self.params['numpart'])
# rescale initial model if necessary
outvol = os.path.join(self.params['rundir'], "vol000.spi")
apModel.rescaleModel(self.params['modelid'],
outvol,
self.stack['boxsize'],
self.stack['apix'],
spider=True)
self.params['itervol'] = outvol
for iter in range(1, self.params['numiter'] + 1):
# create projections for projection matching
apDisplay.printMsg("creating reference projections of volume: %s" %
self.params['itervol'])
projs, numprojs, ang, sel = refine.createProjections(
incr=self.params['increments'][iter - 1],
boxsz=self.stack['boxsize'],
symfold=self.params['symfold'],
invol=self.params['itervol'],
rad=self.params['rad'],
)
# run reference-based alignment
apDisplay.printMsg("running reference-based alignment (AP MQ)")
apmqfile = "apmq%03d.spi" % iter
outang = "angular%03d.spi" % iter
shf = "shifts%03d.spi" % iter
shiftedStack = "parts_shifted.spi"
apFile.removeFile(shf)
refine.spiderAPMQ(
projs=projs,
numprojs=numprojs,
tsearch=self.params['xysearch'],
tstep=self.params['xystep'],
firstRing=self.params['firstring'],
lastRing=self.params['lastring'],
stackfile=spiderstackfilt,
nump=self.params['numpart'],
ang=ang,
apmqfile=apmqfile,
outang=outang,
nproc=self.params['proc'],
)
# use cross-correlation to find the sub-pixel alignment
# of the particles,
# results will be saved in "peakfile.spi"
apFile.removeFile(shiftedStack)
# don't use MPI here - for some reason slower?
mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
apmqlist = refine.readDocFile(apmqfile)
avgccrot = 0
apDisplay.printMsg("creating shifted stack")
for p in range(0, self.params['numpart']):
ref = int(float(apmqlist[p][2]))
ccrot = float(apmqlist[p][3])
inplane = float(apmqlist[p][4])
avgccrot += ccrot
# invert the sign - ref projs will be rotated
inplane *= -1
# get corresponding projection
if (ref <= 0):
# mirror projection if necessary
ref *= -1
refimg = spyder.fileFilter(projs) + "@" + str(ref)
refine.mirrorImg(refimg, "_3", inMySpi=mySpi)
img = "_3"
else:
img = spyder.fileFilter(projs) + "@" + str(ref)
refine.rotAndShiftImg(img, "_2", inplane, inMySpi=mySpi)
refine.maskImg("_2",
"_3",
self.params['rad'],
"D",
"E",
center=int((self.stack['boxsize'] / 2) + 1),
inMySpi=mySpi)
# pad ref image & stack image to twice the size
refine.padImg("_3",
"_2",
2 * self.stack['boxsize'],
"N",
1,
1,
0,
inMySpi=mySpi)
stackimg = spyder.fileFilter(spiderstack) + "@" + str(p + 1)
refine.padImg(stackimg,
"_1",
2 * self.stack['boxsize'],
"B",
1,
1,
inMySpi=mySpi)
# calculate cross-correlation
refine.getCC("_1", "_2", "_1", inMySpi=mySpi)
# crop the correllation image to allowable shift amount
shift = int(self.params['allowedShift'] *
self.stack['boxsize'])
dim = 2 * shift + 1
topleftx = self.stack['boxsize'] - shift + 1
refine.windowImg("_1",
"_2",
dim,
topleftx,
topleftx,
inMySpi=mySpi)
# find the sub-pixel location of cc peak
mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17", "_2",
"0")
# create new stack of shifted particles
shpos = spyder.fileFilter(shiftedStack) + "@" + str(p + 1)
mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN")
mySpi.toSpiderQuiet("GP x17", "_2",
str(shift + 1) + "," + str(shift + 1))
refine.copyImg(stackimg, shpos, inMySpi=mySpi)
mySpi.toSpiderQuiet("ELSE")
#mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16")
mySpi.toSpiderQuiet("SH F", stackimg, shpos, "-x15,-x16")
mySpi.toSpiderQuiet("ENDIF")
# save shifts to file
mySpi.toSpiderQuiet("SD " + str(p + 1) + ",x15,x16,x17",
spyder.fileFilter(shf))
mySpi.toSpiderQuiet("SD E", spyder.fileFilter(shf))
mySpi.close()
# create class average images
refine.createClassAverages(
shiftedStack,
projs,
apmqfile,
numprojs,
self.stack['boxsize'],
shifted=True,
)
# rename class averages & variacnes for iteration
cmd = "/bin/mv classes.hed classes.%d.hed;" % iter
cmd += "/bin/mv classes.img classes.%d.img;" % iter
cmd += "/bin/mv variances.hed variances.%d.hed;" % iter
cmd += "/bin/mv variances.img variances.%d.img;" % iter
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
# calculate the stddev for the apmq cc's for throwing out particles
avgccrot /= self.params['numpart']
stdccrot = 0
for p in range(0, self.params['numpart']):
stdccrot += abs(float(apmqlist[p][3]) - avgccrot)
stdccrot /= self.params['numpart']
cccutoff = avgccrot + (stdccrot * self.params['keepsig'])
apDisplay.printMsg("average cc: %f" % avgccrot)
apDisplay.printMsg("setting cutoff to: %f" % cccutoff)
# create new selection file that only has particles with good cc's
selectfile = "select%03d.spi" % iter
apFile.removeFile(selectfile)
mySpi = spyder.SpiderSession(nproc=self.params['proc'],
dataext=".spi",
logo=False,
log=False)
i = 1
for p in range(0, self.params['numpart']):
ccrot = float(apmqlist[p][3])
if ccrot >= cccutoff:
mySpi.toSpiderQuiet("x11=%d" % (p + 1))
mySpi.toSpiderQuiet("SD %d,x11" % i,
spyder.fileFilter(selectfile))
i += 1
mySpi.close()
# calculate the new 3d structure using centered projections
# and the corrected angles from the angular doc file
apDisplay.printMsg("creating 3d volume")
out_rawvol = "vol_raw%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
selectfile,
rad=self.params['rad'],
ang=outang,
out=out_rawvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['bpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'],
)
else:
refine.backProjection(shiftedStack,
selectfile,
ang=outang,
out=out_rawvol,
sym=sydoc,
nproc=self.params['proc'])
# create even & odd select files
apDisplay.printMsg("creating even/odd volumes")
oddfile = "selectodd%03d.spi" % iter
evenfile = "selecteven%03d.spi" % iter
refine.docSplit(selectfile, oddfile, evenfile)
# get the even & odd volumesa
oddvol = "vol1%03d.spi" % iter
evenvol = "vol2%03d.spi" % iter
if self.params['voliter'] is not None:
refine.iterativeBackProjection(
shiftedStack,
oddfile,
rad=self.params['rad'],
ang=outang,
out=oddvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'])
refine.iterativeBackProjection(
shiftedStack,
evenfile,
rad=self.params['rad'],
ang=outang,
out=evenvol,
lam=self.params['lambda'],
iterlimit=self.params['voliter'],
mode=self.params['eobpmode'],
smoothfac=self.params['smoothfac'],
sym=sydoc,
nproc=self.params['proc'])
else:
refine.backProjection(
shiftedStack,
oddfile,
ang=outang,
out=oddvol,
sym=sydoc,
nproc=self.params['proc'],
)
refine.backProjection(
shiftedStack,
evenfile,
ang=outang,
out=evenvol,
sym=sydoc,
nproc=self.params['proc'],
)
# calculate the FSC
apDisplay.printMsg("calculating FSC")
fscfile = "fsc%03d.spi" % iter
emanfsc = "fsc.eotest.%d" % iter
refine.calcFSC(oddvol, evenvol, fscfile)
# convert to eman-style fscfile
refine.spiderFSCtoEMAN(fscfile, emanfsc)
# calculate the resolution at 0.5 FSC & write to file
res = apRecon.calcRes(emanfsc, self.stack['boxsize'],
self.stack['apix'])
restxt = "resolution.txt"
if iter == 1 and os.path.isfile(restxt):
os.remove(restxt)
resfile = open(restxt, "a")
resfile.write("iter %d:\t%.3f\n" % (iter, res))
resfile.close()
# filter & normalize the volume to be used as a reference in the next round
outvol = "vol%03d.spi" % iter
emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % (
out_rawvol, outvol, self.stack['apix'], res,
self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# create mrc files of volumes
emancmd = "proc3d %s %s apix=%.3f mask=%d norm" % (
out_rawvol, "threed.%da.mrc" % iter, self.stack['apix'],
self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.params['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % (
out_rawvol, "threed.%da.lp.mrc" % iter, self.stack['apix'],
res, self.params['rad'])
if self.params['imask'] is not None:
emancmd += " imask=%d" % self.param['imask']
apEMAN.executeEmanCmd(emancmd, verbose=True)
# set this model as start for next iteration, remove previous
os.remove(self.params['itervol'])
os.remove(out_rawvol)
self.params['itervol'] = outvol
# clean up directory
apDisplay.printMsg("cleaning up directory")
if os.path.isfile(oddvol):
os.remove(oddvol)
if os.path.isfile(evenvol):
os.remove(evenvol)
if os.path.isfile(ang):
os.remove(ang)
if os.path.isfile(sel):
os.remove(sel)
if os.path.isfile(projs):
os.remove(projs)
if os.path.isfile(oddfile):
os.remove(oddfile)
if os.path.isfile(evenfile):
os.remove(evenfile)
if os.path.isfile(emanfsc) and os.path.isfile(fscfile):
os.remove(fscfile)
if os.path.isfile(shiftedStack):
os.remove(shiftedStack)
os.remove(self.params['itervol'])
def refBasedAlignParticles(stackfile,
templatestack,
origstackfile,
xysearch,
xystep,
numpart,
numtemplate,
firstring=2,
lastring=100,
dataext=".spi",
iternum=1,
oldpartlist=None):
"""
inputs:
stack
template
search params
outputs:
aligned stack
rotation/shift params
"""
### setup
if dataext in templatestack:
templatestack = templatestack[:-4]
if dataext in stackfile:
stackfile = stackfile[:-4]
if dataext in origstackfile:
origstackfile = origstackfile[:-4]
t0 = time.time()
rundir = "alignments"
apParam.createDirectory(rundir)
nproc = apParam.getNumProcessors()
### remove previous iterations
apFile.removeFile(rundir + "/paramdoc%02d%s" % (iternum, dataext))
### perform alignment, should I use 'AP SH' instead?
mySpider = spyder.SpiderSession(dataext=dataext,
logo=True,
nproc=nproc,
log=False)
mySpider.toSpider(
"AP MQ",
spyder.fileFilter(templatestack) + "@**", # reference image series
"1-" + str(numtemplate), # enter number of templates of doc file
str(int(xysearch)) + "," +
str(int(xystep)), # translation search range, step size
str(int(firstring)) + "," +
str(int(lastring)), # first and last ring for rotational correlation
spyder.fileFilter(stackfile) + "@******", # unaligned image series
"1-" + str(numpart), # enter number of particles of doc file
rundir + ("/paramdoc%02d" % (iternum)), # output angles document file
)
mySpider.close()
"""
### perform alignment, should I use 'AP SH' instead?
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("AP SH",
spyder.fileFilter(templatestack)+"@**", # reference image series
"1-"+str(numtemplate), # enter number of templates of doc file
str(int(xysearch))+","+str(int(xystep)), # translation search range, step size
str(int(firstring))+","+str(int(lastring)), # first and last ring for rotational correlation
"*", # initial angles
spyder.fileFilter(stackfile)+"@******", # unaligned image series
"1-"+str(numpart), # enter number of particles of doc file
"*", # initial angles
"0.0", # angle search
"1", # check mirrors and shift/rotate input
rundir+("/paramdoc%02d" % (iternum)), # output angles document file
)
mySpider.close()
"""
### convert spider rotation, shift data to python
docfile = rundir + ("/paramdoc%02d" % (iternum)) + dataext
picklefile = rundir + ("/paramdoc%02d" % (iternum)) + ".pickle"
if oldpartlist is not None and iternum > 1:
apDisplay.printMsg("updating particle doc info")
partlist = updateRefBasedDocFile(oldpartlist, docfile, picklefile)
elif iternum == 1:
apDisplay.printMsg("reading initial particle doc info")
partlist = readRefBasedDocFile(docfile, picklefile)
else:
apDisplay.printError(
"reading (not updating) particle doc info on iteration " +
str(iternum))
### write aligned stack -- with python loop
alignedstack = rundir + ("/alignedstack%02d" % (iternum))
alignStack(origstackfile, alignedstack, partlist, dataext)
### average stack
emancmd = ("proc2d " + alignedstack + dataext + " " + rundir +
("/avgimg%02d" % (iternum)) + ".mrc " + " average")
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
td1 = time.time() - t0
apDisplay.printMsg("completed alignment of " + str(numpart) +
" particles in " + apDisplay.timeString(td1))
if numpart < 1:
apDisplay.printError("Failed to find any particles")
return alignedstack + dataext, partlist
def createClassAverages(stack,
projs,
apmq,
numprojs,
boxsz,
outclass="classes",
rotated=False,
shifted=False,
dataext=".spi"):
"""
creates EMAN-style class average file "classes.hed" & "classes.img"
and variance files "variances.hed" & "variances.img"
from spider classification
"""
apFile.removeFile(outclass)
outf = spyder.fileFilter(outclass)
outvf = "tmpvar"
apmqlist = readDocFile(apmq)
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
# create file containing the number of particles matched to each projection
mySpi.toSpiderQuiet(
"VO MQ",
"0.0",
spyder.fileFilter(apmq),
str(numprojs),
"cls*****",
"numinclass",
)
mySpi.close()
os.remove("numinclass%s" % dataext)
# create class average file
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
for i in range(0, numprojs):
clsfile = readDocFile("cls%05d%s" % (i + 1, dataext))
if clsfile == []:
apDisplay.printMsg("class %d has no particles" % (i + 1))
mySpi.toSpiderQuiet(
"BL",
"%s@%d" % (outf, (i + 1)),
"(%d,%d)" % (boxsz, boxsz),
"N",
"(0.0)",
)
mySpi.toSpiderQuiet(
"BL",
"%s@%d" % (outvf, (i + 1)),
"(%d,%d)" % (boxsz, boxsz),
"N",
"(0.0)",
)
else:
mySpi.toSpiderQuiet("DE", "_2@")
mySpi.toSpiderQuiet("MS", "_2@", "%d,%d,1" % (boxsz, boxsz),
str(len(clsfile)))
for p in range(0, len(clsfile)):
mySpi.toSpiderQuiet("DE", "_1")
# get the particle
part = int(float(clsfile[p][2])) - 1
pimg = spyder.fileFilter(stack) + "@%d" % (part + 1)
rot = float(apmqlist[part][4])
shx = float(apmqlist[part][5])
shy = float(apmqlist[part][6])
if shifted is True:
shx = 0
shy = 0
if rotated is True:
rot = 0
p_out = "_2@%d" % (p + 1)
rotAndShiftImg(pimg, "_1", rot, shx, shy, inMySpi=mySpi)
# mirror
if int(float(apmqlist[part][2])) < 0:
mirrorImg("_1", p_out, inMySpi=mySpi)
else:
mySpi.toSpiderQuiet("CP", "_1", p_out)
if len(clsfile) == 1:
mySpi.toSpiderQuiet("CP", "_2@1", "%s@%d" % (outf, (i + 1)))
mySpi.toSpiderQuiet(
"BL",
"%s@%d" % (outvf, (i + 1)),
"(%d,%d)" % (boxsz, boxsz),
"N",
"(0.0)",
)
else:
mySpi.toSpiderQuiet("AS R", "_2@*****", "1-%d" % len(clsfile),
"A", "%s@%d" % (outf, (i + 1)),
"%s@%d" % (outvf, i + 1))
mySpi.close()
# convert the class averages to EMAN class averages
# read classes & projections for EMAN classes output
if os.path.exists('variances.hed'):
os.remove('variances.hed')
if os.path.exists('variances.img'):
os.remove('variances.img')
if os.path.exists('classes.hed'):
os.remove('classes.hed')
if os.path.exists('classes.img'):
os.remove('classes.img')
### I would prefer to use apImagicFile.readImagic, writeImagic
variances = EMAN.readImages(outvf + dataext, -1, -1, 0)
averages = EMAN.readImages(outf + dataext, -1, -1, 0)
projections = EMAN.readImages(projs, -1, -1, 0)
for i in range(0, numprojs):
# copy projection to class average file
projections[i].writeImage('classes.hed')
projections[i].writeImage('variances.hed')
# get num of particles in class
clsfile = readDocFile("cls%05d%s" % (i + 1, dataext))
averages[i].setNImg(len(clsfile))
averages[i].writeImage('classes.hed', -1)
variances[i].setNImg(len(clsfile))
variances[i].writeImage('variances.hed', -1)
os.remove("cls%05d%s" % (i + 1, dataext))
os.remove(outf + dataext)
return
def refFreeAlignParticles(stackfile,
template,
numpart,
pixrad,
firstring=2,
lastring=100,
dataext=".spi",
rundir="alignment"):
"""
inputs:
stack
template
search params
outputs:
aligned stack
rotation/shift params
"""
### setup
if dataext in template:
template = template[:-4]
if dataext in stackfile:
stackfile = stackfile[:-4]
t0 = time.time()
apParam.createDirectory(rundir)
### remove previous iterations
numiter = 0
while os.path.isfile(rundir + "/avgimg%02d%s" % (numiter + 1, dataext)):
apFile.removeFile(rundir + "/avgimg%02d%s" % (numiter + 1, dataext))
pngfile = rundir + "/avgimg%02d%s" % (numiter + 1, ".png")
apFile.removeFile(pngfile)
numiter += 1
### perform alignment
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
apDisplay.printMsg("Performing particle alignment")
# copy template to memory
mySpider.toSpiderQuiet("CP", (template + "@1"), "_9")
mySpider.toSpider("AP SR",
spyder.fileFilter(stackfile) + "@******",
"1-" + str(numpart), str(int(pixrad)),
str(int(firstring)) + "," + str(int(lastring)), "_9",
rundir + "/avgimg**", rundir + "/paramdoc**")
mySpider.close()
### find number of iterations
numiter = 0
while os.path.isfile(rundir + "/avgimg%02d%s" % (numiter + 1, dataext)):
emancmd = ("proc2d " + " " + rundir + "/avgimg" +
("%02d%s" % (numiter + 1, dataext)) + " " + rundir +
"/avgimg" + ("%02d%s" % (numiter + 1, ".png")))
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
numiter += 1
if numiter == 0:
apDisplay.printError("alignment failed, no iterations were found")
emancmd = ("proc2d " + " " + rundir + "/avgimg" +
("%02d%s" % (numiter, dataext)) + " " + rundir + "/average.mrc")
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
apDisplay.printMsg(
str(numiter) + " alignment iterations were run by spider")
### convert spider rotation, shift data to python
docfile = rundir + ("/paramdoc%02d" % (numiter)) + dataext
picklefile = rundir + ("/paramdoc%02d" % (numiter)) + ".pickle"
partlist = readRefFreeDocFile(docfile, picklefile)
### write aligned stack -- with python loop
alignedstack = "alignedstack"
alignStack(stackfile, alignedstack, partlist, dataext)
td1 = time.time() - t0
apDisplay.printMsg("completed alignment of " + str(numpart) +
" particles in " + apDisplay.timeString(td1))
return ("alignedstack.spi", partlist)
def analyzeEigenFactors(alignedstack,
rundir,
numpart,
numfactors=8,
dataext=".spi"):
"""
inputs:
coran run data
outputs:
1. generate eigen images
2. collect eigenimage contribution percentage
3. 2D factor plot
Broken 4. 2D factor plot visualization
"""
### 1. generate eigen images
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
for fact in range(1, numfactors + 1):
mySpider.toSpiderQuiet(
#"CA SRE", rundir+"/corandata", str(fact),
#rundir+"/eigenstack@"+("%02d" % (fact)), )
"CA SRD",
rundir + "/corandata",
str(fact),
str(fact),
rundir + "/eigenstack@***",
)
mySpider.close()
### convert to nice individual eigen image pngs for webpage
eigenspistack = os.path.join(rundir, "eigenstack.spi")
if not os.path.isfile(eigenspistack):
apDisplay.printError("Failed to create Eigen images")
for fact in range(1, numfactors + 1):
pngfile = rundir + "/eigenimg" + ("%02d" % (fact)) + ".png"
apFile.removeFile(pngfile)
emancmd = ("proc2d " + eigenspistack + " " + pngfile + " " +
" first=" + str(fact - 1) + " last=" + str(fact - 1))
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
### convert eigen SPIDER stack to IMAGIC for stack viewer
eigenimagicstack = rundir + "/eigenstack.hed"
apFile.removeStack(eigenimagicstack)
emancmd = "proc2d " + eigenspistack + " " + eigenimagicstack
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
### 2. collect eigenimage contribution percentage
eigf = open(rundir + "/corandata_EIG" + dataext, "r")
count = 0
contriblist = []
for line in eigf:
bits = line.strip().split()
if len(contriblist) == numfactors:
break
if len(bits) < 3:
continue
contrib = float(bits[1])
cumm = float(bits[2])
eigval = float(bits[0])
if len(bits) == 3:
count += 1
contriblist.append(contrib)
print "Factor", count, contrib, "%\t", cumm, "%\t", eigval
### need to plot & insert this data
### hack to get 'CA VIS' to work: break up stack into individual particles
"""
### this is broken in SPIDER 13.0
apParam.createDirectory("unstacked")
mySpider = spyder.SpiderSession(dataext=dataext, logo=False)
mySpider.toSpiderQuiet(
"DO LB1 i=1,"+str(numpart),
" CP",
" "+alignedstack+"@{******x0}",
" unstacked/img{******x0}",
"LB1",
)
mySpider.close()
"""
### generate factor maps
apDisplay.printMsg("creating factor maps")
for f1 in range(1, min(numfactors, 2)):
for f2 in range(f1 + 1, min(3, numfactors + 1)):
sys.stderr.write(".")
try:
createFactorMap(f1, f2, rundir, dataext)
except:
sys.stderr.write("#")
pass
sys.stderr.write("\n")
return contriblist
def kmeansCluster(alignedstack,
numpart=None,
numclasses=40,
timestamp=None,
factorlist=range(1, 5),
corandata="coran/corandata",
dataext=".spi"):
"""
inputs:
outputs:
"""
if timestamp is None:
timestamp = apParam.makeTimestamp()
if alignedstack[-4:] == dataext:
alignedstack = alignedstack[:-4]
rundir = "cluster"
classavg = rundir + "/" + ("classavgstack_%s_%03d" %
(timestamp, numclasses))
classvar = rundir + "/" + ("classvarstack_%s_%03d" %
(timestamp, numclasses))
apParam.createDirectory(rundir)
for i in range(numclasses):
apFile.removeFile(rundir + ("/classdoc%04d" % (i + 1)) + dataext)
apFile.removeFile(rundir + ("/allclassesdoc%04d" % (numclasses)) + dataext)
### make list of factors
factorstr, factorkey = operations.intListToString(factorlist)
### do k-means clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
mySpider.toSpider(
"CL KM",
corandata + "_IMC", # path to coran data
str(numclasses), # num classes
factorstr, # factor string
)
## weight for each factor
for fact in factorlist:
mySpider.toSpiderQuiet("1.0")
randnum = (int(random.random() * 1000) + 1)
mySpider.toSpider(
str(randnum),
rundir + "/classdoc_" + timestamp + "_****", # class doc file
rundir + ("/allclassesdoc%04d" % (numclasses)), #clusterdoc file
)
mySpider.close()
### delete existing files
sys.stderr.write("delete existing files")
for dext in (".hed", ".img", dataext):
apFile.removeFile(classavg + dext)
apFile.removeFile(classvar + dext)
print ""
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
### create class averages
apDisplay.printMsg("Averaging particles into classes")
for i in range(numclasses):
classnum = i + 1
mySpider.toSpiderQuiet(
"AS R",
spyder.fileFilter(alignedstack) + "@******",
rundir + ("/classdoc_" + timestamp + "_%04d" % (classnum)),
"A",
(classavg + "@%04d" % (classnum)),
(classvar + "@%04d" % (classnum)),
)
if classnum % 10 == 0:
sys.stderr.write(".")
time.sleep(1)
mySpider.close()
### convert to IMAGIC
emancmd = "proc2d " + classavg + ".spi " + classavg + ".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
emancmd = "proc2d " + classvar + ".spi " + classvar + ".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
return classavg, classvar
def hierarchClusterClassify(alignedstack,
dendrogramfile,
numclasses=40,
timestamp=None,
rundir=".",
dataext=".spi"):
"""
inputs:
aligned particle stack
number of classes
timestamp
output directory
output:
class averages
class variances
dendrogram.png
"""
if timestamp is None:
timestamp = apParam.makeTimestamp()
classavg = rundir + "/" + ("classavgstack_%s_%03d" %
(timestamp, numclasses))
classvar = rundir + "/" + ("classvarstack_%s_%03d" %
(timestamp, numclasses))
thresh, classes = findThreshold(numclasses, dendrogramfile, rundir,
dataext)
### create class doc files
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
mySpider.toSpider(
"CL HE",
thresh,
spyder.fileFilter(dendrogramfile), # dendrogram doc file
rundir + "/classdoc_" + timestamp + "_****", # class doc file
)
### delete existing files
sys.stderr.write("delete existing files")
for dext in (".hed", ".img", dataext):
apFile.removeFile(classavg + dext)
apFile.removeFile(classvar + dext)
print ""
### create class averages
sys.stderr.write("create class averages")
for i in range(classes):
sys.stderr.write(".")
classnum = i + 1
mySpider.toSpiderQuiet(
"AS R",
spyder.fileFilter(alignedstack) + "@******",
rundir + ("/classdoc_" + timestamp + "_%04d" % (classnum)),
"A",
(classavg + "@%04d" % (classnum)),
(classvar + "@%04d" % (classnum)),
)
mySpider.close()
print ""
### convert to IMAGIC
emancmd = "proc2d " + classavg + ".spi " + classavg + ".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
emancmd = "proc2d " + classvar + ".spi " + classvar + ".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
return classavg, classvar
def alignStack(oldstack, alignedstack, partlist, dataext=".spi"):
"""
write aligned stack -- with python loop
inputs:
oldstack
newstack (empty)
list of particle dictionaries for operations
modifies:
newstack
output:
none
I tried this loop in both spider and python;
python was faster?!? -neil
"""
if not os.path.isfile(oldstack + dataext):
apDisplay.printError("Could not find original stack: " + oldstack +
dataext)
boxsize = apFile.getBoxSize(oldstack + dataext)
apDisplay.printMsg("applying alignment parameters to stack")
apFile.removeFile(alignedstack + dataext)
count = 0
t0 = time.time()
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext,
logo=True,
nproc=nproc,
log=False)
#create stack in core
numpart = len(partlist)
mySpider.toSpiderQuiet(
"MS I", #command
"_2@", #name
"%d,%d,%d" % (boxsize), #boxsize
str(numpart + 1), #num part to create in memory
str(numpart + 1), #max particle number
)
for partdict in partlist:
partnum = partdict['num']
#if partdict['num'] in [3,6,7]:
# print partdict['num'], partdict['template'], partdict['mirror'], round(partdict['rot'],3)
### Rotate and Shift operations
count += 1
#rotate/shift
mySpider.toSpiderQuiet(
"RT SQ",
spyder.fileFilter(oldstack) + "@" + ("%06d" % (partnum)),
"_1",
str(partdict['rot']),
str(partdict['xshift']) + "," + str(partdict['yshift']),
)
#mirror, if necessary
if 'mirror' in partdict and partdict['mirror'] is True:
mySpider.toSpiderQuiet(
"MR",
"_1",
"_2@" + ("%06d" % (partnum)),
"Y",
)
else:
mySpider.toSpiderQuiet(
"CP",
"_1",
"_2@" + ("%06d" % (partnum)),
)
### finish up
#save stack to file
mySpider.toSpiderQuiet(
"CP",
"_2@",
spyder.fileFilter(alignedstack) + "@",
)
#delete stack
mySpider.toSpiderQuiet(
"DE",
"_2",
)
mySpider.close()
apDisplay.printMsg("Completed transforming %d particles in %s" %
(count, apDisplay.timeString(time.time() - t0)))
if count < 1:
apDisplay.printError("Failed to transform any particles")
if not os.path.isfile(alignedstack + dataext):
apDisplay.printError("Failed to create stack " + alignedstack +
dataext)
return
