def projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext=".spi"):
"""
project 3D volumes using given Euler angles
"""
starttime = time.time()
volfile = spyder.fileFilter(volfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
projstackfile = spyder.fileFilter(projstackfile)
if not os.path.isfile(volfile+dataext):
apDisplay.printError("volume file not found: "+volfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(projstackfile)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("PJ 3Q",
volfile, #input vol file
str(pixrad), #pixel radius
"1-%d"%(numpart), #number of particles
eulerdocfile, #Euler DOC file
projstackfile+"@*****", #output projections
)
mySpider.close()
apDisplay.printColor("finished projections in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def backprojectCG(stackfile, eulerdocfile, volfile, numpart, pixrad, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
### setup
starttime = time.time()
stackfile = spyder.fileFilter(stackfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(stackfile+dataext):
apDisplay.printError("stack file not found: "+stackfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(volfile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("BP CG",
stackfile+"@*****", #stack file
"1-%d"%(numpart), #number of particles
str(pixrad), #particle radius
eulerdocfile, #angle doc file
"N", #has symmetry?, does not work
volfile, #filename for volume
"%.1e,%.1f" % (1.0e-5, 0.0), #error, chi^2 limits
"%d,%d" % (25,1), #iterations, 1st derivative mode
"2000", #lambda - higher=less sensitive to noise
)
mySpider.close()
apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def rotAndShiftVol(invol,outvol,rot=(0,0,0),center=(0,0,0),shift=(0.0,0.0,0.0),dataext=".spi",inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
else:
mySpi=inMySpi
vol1=spyder.fileFilter(invol)
tmpvol=spyder.fileFilter('temp')
vol3=spyder.fileFilter(outvol)
cleanlist = []
invol = vol1
if rot != (0,0,0):
outvol = tmpvol
rot = tuple(map((lambda x: float(x)), rot))
rotstr = '%.2f,%.2f,%.2f' % rot
center = tuple(map((lambda x: float(x)), center))
centerstr = '%.1f,%.1f,%.1f' % center
mySpi.toSpider("RT 3A",invol,outvol,rotstr,centerstr)
cleanlist.append(invol)
invol = tmpvol
if shift != (0,0,0):
outvol = vol3
shift = tuple(map((lambda x: float(x)), shift))
shiftstr = '%f.1,%f.1,%f.1' % shift
mySpi.toSpider("SH",invol,outvol,shiftstr)
cleanlist.append(invol)
invol = outvol
if outvol != vol3:
mySpi.toSpider("CP",invol,vol3)
if inMySpi is False:
mySpi.close()
for cleanfile in cleanlist:
os.remove(cleanfile+'.spi')
def rotshiftParticle(origstackfile, partnum, rotation, Xshift, Yshift, mirror, iternum, timestamp, classnum=None, dataext=".spi"):
origstack = spyder.fileFilter(origstackfile)
if classnum is not None:
apshstackfile = str(classnum)+"/"+"apshStack-%03d.spi"%(iternum)
else:
apshstackfile = "apshStack-%03d.spi"%(iternum)
apshstack = spyder.fileFilter(apshstackfile)
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
mySpider.toSpiderQuiet("RT SQ",
origstack+"@"+str(partnum), #stack and particle number
apshstack+"@"+str(partnum), #output stack and particle
"("+str(rotation)+")", #rotation
"("+str(Xshift)+","+str(Yshift)+")", #shift parameters
)
if mirror==-1:
mySpider.toSpiderQuiet("MR",
apshstack+"@"+str(partnum), #stack and particle number
apshstack+"@"+str(partnum), #output stack and particle
"Y",
)
mySpider.close()
return apshstackfile
def backProjection( stack, select, ang, out, sym=None, nproc=1, dataext=".spi", inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(nproc=nproc,dataext=dataext,logo=False,log=False) else: mySpi=inMySpi apFile.removeFile(out) if sym is None: sym="*" mySpi.toSpiderQuiet( "BP 3F", spyder.fileFilter(stack)+"@******", spyder.fileFilter(select), spyder.fileFilter(ang), spyder.fileFilter(sym), spyder.fileFilter(out) ) if inMySpi is False: mySpi.close()
def backproject3F(stackfile, eulerdocfile, volfile, numpart, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
### setup
starttime = time.time()
stackfile = spyder.fileFilter(stackfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(stackfile+dataext):
apDisplay.printError("stack file not found: "+stackfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(volfile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("BP 3F",
stackfile+"@*****", #stack file
"1-%d"%(numpart), #number of particles
eulerdocfile, #angle doc file
"*", #input symmetry file, '*' for skip
volfile, #filename for volume
)
mySpider.close()
apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"):
"""
inputs:
aligned particle stack
number of classes
timestamp
output directory
output:
class averages
class variances
dendrogram.png
"""
if timestamp is None:
timestamp = apParam.makeTimestamp()
classavg = rundir+"/"+("classavgstack_%s_%03d" % (timestamp, numclasses))
classvar = rundir+"/"+("classvarstack_%s_%03d" % (timestamp, numclasses))
thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext)
### create class doc files
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
mySpider.toSpider(
"CL HE",
thresh,
spyder.fileFilter(dendrogramfile), # dendrogram doc file
rundir+"/classdoc_"+timestamp+"_****", # class doc file
)
### delete existing files
sys.stderr.write("delete existing files")
for dext in (".hed", ".img", dataext):
apFile.removeFile(classavg+dext)
apFile.removeFile(classvar+dext)
print ""
### create class averages
sys.stderr.write("create class averages")
for i in range(classes):
sys.stderr.write(".")
classnum = i+1
mySpider.toSpiderQuiet(
"AS R",
spyder.fileFilter(alignedstack)+"@******",
rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)),
"A",
(classavg+"@%04d" % (classnum)),
(classvar+"@%04d" % (classnum)),
)
mySpider.close()
print ""
### convert to IMAGIC
emancmd = "proc2d "+classavg+".spi "+classavg+".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
emancmd = "proc2d "+classvar+".spi "+classvar+".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
return classavg,classvar
def addParticleToStack(partnum, partfile, stackfile, dataext=".spi"):
mySpider = spyder.SpiderSession(dataext=dataext, logo=False)
mySpider.toSpiderQuiet("CP",
spyder.fileFilter(partfile), #particle file
spyder.fileFilter(stackfile)+"@%06d"%(partnum), #stack file
)
mySpider.close()
return
def rotAndShiftImg(inimg,outimg,rot=0,shx=0,shy=0,dataext=".spi",inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
else:
mySpi=inMySpi
img1=spyder.fileFilter(inimg)
img2=spyder.fileFilter(outimg)
mySpi.toSpiderQuiet("RT SQ",img1,img2,rot,str(shx)+","+str(shy))
if inMySpi is False:
mySpi.close()
def mirrorImg(inimg,outimg,dataext=".spi",inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
else:
mySpi=inMySpi
img1=spyder.fileFilter(inimg)
img2=spyder.fileFilter(outimg)
mySpi.toSpiderQuiet("MR",img1,img2,"Y")
if inMySpi is False:
mySpi.close()
def getCC(image1,image2,outcc,dataext=".spi",inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext,logo=False,log=False)
else:
mySpi=inMySpi
img1=spyder.fileFilter(image1)
img2=spyder.fileFilter(image2)
outimg=spyder.fileFilter(outcc)
mySpi.toSpiderQuiet("CC N",img1,img2,outimg)
if inMySpi is False:
mySpi.close()
def averageStack(stackfile, numpart, avgfile, varfile, dataext=".spi"):
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
mySpider.toSpider("AS R",
spyder.fileFilter(stackfile)+"@******", #stack file
"1-%6d"%(numpart), #num of particles
"A", #use all particles
spyder.fileFilter(avgfile), #average file
spyder.fileFilter(varfile), #variance file
)
mySpider.close()
return
def iterativeBackProjection(
stack,
select,
rad,
ang,
out,
lam,
iterlimit,
mode,
smoothfac,
sym=None,
nproc=1,
dataext=".spi",
inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(nproc=nproc,dataext=dataext,logo=False,log=False)
else:
mySpi=inMySpi
if sym is None:
sym="*"
apFile.removeFile(out)
# calculate the smoothing factor
smooth=(1/(1+6*lam))*smoothfac
mySpi.toSpiderQuiet("x11=0")
mySpi.toSpiderQuiet("x47="+str(lam))
mySpi.toSpiderQuiet("x48="+str(smooth))
mySpi.toSpiderQuiet("DO LB11 i=1,%d" % (iterlimit*2))
mySpi.toSpiderQuiet("IF (x11.EQ.%d) GOTO LB11" % iterlimit)
mySpi.toSpiderQuiet(
"BP RP x11",
spyder.fileFilter(stack)+"@******",
spyder.fileFilter(select),
"("+str(rad)+")",
spyder.fileFilter(ang),
spyder.fileFilter(sym),
spyder.fileFilter(out),
"(x47,0)",
"("+str(iterlimit)+","+str(mode)+")",
"(0,0)",
"x48",
)
# check if BP RP finished the requested iterations,
# if not, modify lambda and smoothing constant and rerun
mySpi.toSpiderQuiet("IF (x11.LT.%d) THEN" % iterlimit)
mySpi.toSpiderQuiet("x47=x47/2")
mySpi.toSpiderQuiet("x48=(1/1+6*x47))*"+str(smoothfac))
mySpi.toSpiderQuiet("GOTO LB11")
mySpi.toSpiderQuiet("ENDIF")
mySpi.toSpiderQuiet("LB11")
if inMySpi is False:
mySpi.close()
def createAmpcorBatchFile(infile,params):
localinfile = spyder.fileFilter(infile)
appiondir = apParam.getAppionDirectory()
scriptfile = os.path.join(appiondir, "appionlib/data/enhance.bat")
pwscfile = os.path.join(appiondir, "appionlib/data/pwsc.bat")
if not os.path.isfile(scriptfile):
apDisplay.printError("could not find spider script: "+scriptfile)
if not os.path.isfile(pwscfile):
apDisplay.printError("could not find spider script: "+pwscfile)
inf = open(scriptfile, "r")
tmpfile = "out_"+os.path.basename(infile)
tmpfile = os.path.join(params['rundir'], tmpfile)
localtmpfile = spyder.fileFilter(tmpfile)
outfile = "enhance_edit.bat"
if os.path.isfile(outfile):
apDisplay.printWarning(outfile+" already exists; removing it")
time.sleep(2)
os.remove(outfile)
if os.path.isfile('pwsc.bat'):
apDisplay.printWarning(pwscfile+" already exists; replacing it")
time.sleep(2)
shutil.copyfile(pwscfile, 'pwsc.bat')
outf = open(outfile, "w")
notdone = True
for line in inf:
if notdone is False:
outf.write(line)
else:
thr = line[:3]
if thr == "x99":
outf.write(spiderline(99,params['box'],"box size in pixels"))
elif thr == "x98":
outf.write(spiderline(98,params['maxfilt'],"filter limit in angstroms"))
elif thr == "x80":
outf.write(spiderline(80,params['apix'],"pixel size"))
elif re.search("^\[vol\]",line):
outf.write(spiderline("vol",localinfile,"input density file"))
elif re.search("^\[outvol\]",line):
outf.write(spiderline("outvol",localtmpfile,"enhanced output file"))
elif re.search("^\[scatter\]",line):
outf.write(spiderline("scatter",params['ampfile'],"amplitude curve file"))
elif re.search("^\[pwsc\]",line):
outf.write(spiderline("pwsc","pwsc","scaling script"))
elif re.search("^\[applyfen\]",line):
outf.write(spiderline("applyfen",os.path.join(appiondir,"appionlib/data/applyfen"),"apply curve to data script"))
else:
outf.write(line)
return tmpfile
def docSplit(file1,file2,file3,dataext=".spi",inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext,logo=False,log=False)
else:
mySpi=inMySpi
apFile.removeFile(file2)
apFile.removeFile(file3)
f1=spyder.fileFilter(file1)
f2=spyder.fileFilter(file2)
f3=spyder.fileFilter(file3)
mySpi.toSpiderQuiet("DOC SPLIT",f1,f2,f3)
if inMySpi is False:
mySpi.close()
def centerVolume(volfile, outvolfile, dataext=".spi"):
volfile = spyder.fileFilter(volfile)
outvolfile = spyder.fileFilter(outvolfile)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
mySpider.toSpider("CG PH x11,x12,x13,x21,x22,x23",
volfile, #volume
)
mySpider.toSpider("SH F",
volfile, #volume
outvolfile, #output volume
"-x21,-x22,-x23", #shifts in x y z
)
mySpider.close()
return
def maskImg(inimg,outimg,outrad,cutoff,background="P",inrad="0",center="0",dataext=".spi",inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext,logo=False,log=False)
else:
mySpi=inMySpi
img1=spyder.fileFilter(inimg)
img2=spyder.fileFilter(outimg)
mySpi.toSpiderQuiet("MA",img1,img2,str(outrad)+","+str(inrad),cutoff,background)
if background=="E":
mySpi.toSpiderQuiet("0",str(center)+","+str(center))
if cutoff=="C" or cutoff=="G":
mySpi.toSpiderQuiet("3.5")
if inMySpi is False:
mySpi.close()
def windowImg(inimg,outimg,dim,topleftx,toplefty,dataext=".spi",inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext,logo=False,log=False) else: mySpi=inMySpi img1=spyder.fileFilter(inimg) img2=spyder.fileFilter(outimg) mySpi.toSpiderQuiet( "WI", img1, img2, str(dim)+","+str(dim), str(topleftx)+","+str(toplefty) ) if inMySpi is False: mySpi.close()
def normalizeVol(volfile, dataext=".spi"):
"""
inputs:
volume
outputs:
volume
"""
### setup
starttime = time.time()
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(volfile+dataext):
apDisplay.printError("volume file not found: "+volfile+dataext)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
### read out the statistics of the volume
mySpider.toSpider("FS x11,x12",
volfile, #filename for volume
)
mySpider.toSpider("IF(x12.LT.0.0)x12=-x12")
### set all values to positive
mySpider.toSpider("AR",
volfile, #filename for volume
"_1",
"(P1+x12)",
)
### save file
mySpider.toSpider("CP",
"_1",
volfile, #filename for volume
)
mySpider.close()
apDisplay.printColor("finished normalizing the volume to set all values to be positive"+apDisplay.timeString(time.time()-starttime), "cyan")
return
def createBoxMask(maskfile,boxsize,maskx,masky,falloff,imask=None,dataext=".spi"):
"""
creates a rectangular mask with soft falloff
"""
apDisplay.printMsg("Creating %i x %i box mask"%(boxsize,boxsize))
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, nproc=1, log=False)
# create blank image for mask
mySpi.toSpiderQuiet("BL","_1","%i,%i"%(boxsize,boxsize),"N","1")
# mask it in X
mySpi.toSpiderQuiet("MA X", "_1", "_2",
"%i"%maskx, "C", "E", "0",
"%i,%i"%(boxsize/2,boxsize/2),
"%.2f"%falloff)
# inner mask in X
if imask is not None:
mySpi.toSpiderQuiet("MA X","_2","_3",
"%i,%i"%(boxsize/2,imask),
"C","E","0",
"%i,%i"%(boxsize/2,boxsize/2),
"%.2f"%(falloff/4))
mySpi.toSpiderQuiet("CP","_3","_2")
# mask in Y
mySpi.toSpiderQuiet("MA Y","_2",
spyder.fileFilter(maskfile),
"%i"%masky, "C", "E", "0",
"%i,%i"%(boxsize/2,boxsize/2),
"%.2f"%falloff)
mySpi.close()
if not os.path.isfile(maskfile) or apFile.fileSize(maskfile) < 2:
apDisplay.printError("Failed to create mask file")
return
def symmetryDoc(symtype,symfold=None,outfile="sym.spi",dataext=".spi"):
mySpi=spyder.SpiderSession(dataext=dataext,logo=False,log=False)
apFile.removeFile(outfile)
mySpi.toSpiderQuiet("SY",spyder.fileFilter(outfile),symtype)
if symfold is not None:
mySpi.toSpiderQuiet(symfold)
mySpi.close()
def backprojectRP(stackfile, eulerdocfile, volfile, pixrad, classnum, lambDa, numpart=None, selfile=None, dataext=".spi"):
"""
inputs:
stack, in spider format
eulerdocfile
outputs:
volume
"""
### setup
starttime = time.time()
stackfile = spyder.fileFilter(stackfile)
eulerdocfile = spyder.fileFilter(eulerdocfile)
if selfile is not None:
selfile = spyder.fileFilter(selfile)
volfile = spyder.fileFilter(volfile)
counter = spyder.fileFilter("numiter")
if not os.path.isfile(stackfile+dataext):
apDisplay.printError("stack file not found: "+stackfile+dataext)
if not os.path.isfile(eulerdocfile+dataext):
apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
apFile.removeFile(volfile+dataext)
if (numpart is not None):
selection = "1-"+str(numpart)
elif (selfile is not None):
selection = selfile
else:
apDisplay.printError("Partilce selection is invalid for BP RP. Please make sure either numpart or selfile is specified in the function backprojectRP!")
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
mySpider.toSpider("BP RP x11",
stackfile+"@*****", #stack file
selection, #selection file or particle range
str(pixrad), #particle radius
eulerdocfile, #angle doc file
"*", #has symmetry?, does not work
volfile, #filename for volume
"%.1e,%.1f" % (lambDa, 0.0), #lambda, correction
"%d,%d" % (50,1), #iteration limit, mode
"%d,%d" % (0,0), #minimum, maximum
"%.5f" % ((1/(1+6*lambDa))*0.999), #smoothing constant = (1/(1+6*lambda))*0.999
)
mySpider.toSpider("SD 1,x11", str(classnum)+"/"+str(counter))
mySpider.close()
apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def hierarchClusterProcess(numpart=None, factorlist=range(1,5),
corandata="coran/corandata", rundir=".", dataext=".spi"):
"""
inputs:
coran data
number of particles
factor list
output directory
output:
dendrogram doc file
factorkey
"""
#apFile.removeFile(rundir+"/dendrogramdoc"+dataext)
factorstr, factorkey = operations.intListToString(factorlist)
dendrogramfile = rundir+"/dendrogramdoc"+factorkey+dataext
if os.path.isfile(dendrogramfile):
apDisplay.printMsg("Dendrogram file already exists, skipping processing "+dendrogramfile)
return dendrogramfile
apDisplay.printMsg("Creating dendrogram file: "+dendrogramfile)
### do hierarchical clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
mySpider.toSpider(
"CL HC",
spyder.fileFilter(corandata)+"_IMC", # path to coran data
factorstr, # factor string
)
## weight for each factor
for fact in factorlist:
mySpider.toSpiderQuiet("1.0")
minclasssize = "%.4f" % (numpart*0.0001+2.0)
mySpider.toSpider(
"5", #use Ward's method
"T", minclasssize, rundir+"/dendrogram.ps", #dendrogram image file
"Y", spyder.fileFilter(dendrogramfile), #dendrogram doc file
)
mySpider.close()
if not os.path.isfile(dendrogramfile):
apDisplay.printError("SPIDER dendrogram creation (CL HC) failed, too many particles??")
apImage.convertPostscriptToPng("cluster/dendrogram.ps", "dendrogram.png")
return dendrogramfile
def hierarchClusterProcess(numpart=None, factorlist=range(1,5),
corandata="coran/corandata", rundir=".", dataext=".spi"):
"""
inputs:
coran data
number of particles
factor list
output directory
output:
dendrogram doc file
factorkey
"""
#apFile.removeFile(rundir+"/dendrogramdoc"+dataext)
factorstr, factorkey = operations.intListToString(factorlist)
dendrogramfile = rundir+"/dendrogramdoc"+factorkey+dataext
if os.path.isfile(dendrogramfile):
apDisplay.printMsg("Dendrogram file already exists, skipping processing "+dendrogramfile)
return dendrogramfile
apDisplay.printMsg("Creating dendrogram file: "+dendrogramfile)
### do hierarchical clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True, nproc=1)
mySpider.toSpider(
"CL HC",
spyder.fileFilter(corandata)+"_IMC", # path to coran data
factorstr, # factor string
)
## weight for each factor
for fact in factorlist:
mySpider.toSpiderQuiet("1.0")
minclasssize = "%.4f" % (numpart*0.0001+2.0)
mySpider.toSpider(
"5", #use Ward's method
"T", minclasssize, rundir+"/dendrogram.ps", #dendrogram image file
"Y", spyder.fileFilter(dendrogramfile), #dendrogram doc file
)
mySpider.close()
if not os.path.isfile(dendrogramfile):
apDisplay.printError("SPIDER dendrogram creation (CL HC) failed, too many particles??")
apImage.convertPostscriptToPng("cluster/dendrogram.ps", "dendrogram.png")
return dendrogramfile
def windowImg(inimg,
outimg,
dim,
topleftx,
toplefty,
dataext=".spi",
inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
else:
mySpi = inMySpi
img1 = spyder.fileFilter(inimg)
img2 = spyder.fileFilter(outimg)
mySpi.toSpiderQuiet("WI", img1, img2,
str(dim) + "," + str(dim),
str(topleftx) + "," + str(toplefty))
if inMySpi is False:
mySpi.close()
def calcFSC(volfile1, volfile2, fscout, dataext=".spi"):
volfile1 = spyder.fileFilter(volfile1)
volfile2 = spyder.fileFilter(volfile2)
fscout = spyder.fileFilter(fscout)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
mySpider.toSpider("RF 3",
volfile1, #volume 1
volfile2, #volume 2
"0.5", # ring width for RF 3
"0.5,1.5", # scale factor - lower
#"1.5", # scale factor - upper
"C", # use 'C' for missing cone and 'W' for missing wedge
"90", # maximum tilt angle in data for OTR
"3", # factor for noise comparison ("sigma")
fscout, # fsc curve output
)
mySpider.close()
return
def pdb2vol(pdbfile, apix, box, outfile, dataext=".spi"):
outfile = spyder.fileFilter(outfile)
### create volume density file from PDB
if not os.path.isfile(pdbfile):
apDisplay.printError("Can not find PDB file for conversion: " +
pdbfile)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
### command request: infile, apix, center (y/n), atoms/temperature, boxsize, outfile
boxsize = "%i, %i, %i" % (box, box, box)
mySpider.toSpider("CP FROM PDB", spyder.fileFilter(pdbfile),
str(round(apix, 5)), "Y", "A", boxsize,
spyder.fileFilter(outfile))
mySpider.close()
if not os.path.isfile(outfile + dataext):
apDisplay.printError("SPIDER could not create density file: " +
outfile + dataext)
return
def pdb2vol(pdbfile, apix, box, outfile, dataext=".spi"):
outfile = spyder.fileFilter(outfile)
### create volume density file from PDB
if not os.path.isfile(pdbfile):
apDisplay.printError("Can not find PDB file for conversion: "+pdbfile)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
### command request: infile, apix, center (y/n), atoms/temperature, boxsize, outfile
boxsize = "%i, %i, %i" %(box, box, box)
mySpider.toSpider("CP FROM PDB",
spyder.fileFilter(pdbfile),
str(round(apix,5)),
"Y", "A",
boxsize,
spyder.fileFilter(outfile))
mySpider.close()
if not os.path.isfile(outfile+dataext):
apDisplay.printError("SPIDER could not create density file: "+outfile+dataext)
return
def padImg(inimg,outimg,dim,avg,topleftx,toplefty,const=1.2,dataext=".spi",inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext,logo=False,log=False) else: mySpi=inMySpi img1=spyder.fileFilter(inimg) img2=spyder.fileFilter(outimg) mySpi.toSpiderQuiet( "PD", img1, img2, str(dim)+","+str(dim), avg, ) if avg=="N" or avg=="n": mySpi.toSpiderQuiet(str(const)) mySpi.toSpiderQuiet(str(topleftx)+","+str(toplefty)) if inMySpi is False: mySpi.close()
def findRotation(self,avgimg):
# use radon transform in SPIDER to find rotation to orient the average image vertical
if os.path.isfile("angle.spi"):
os.remove("angle.spi")
box=self.alignstackdata['boxsize']
mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
# circular mask the average
mySpi.toSpider("MA",
spyder.fileFilter(avgimg)+"@1",
"_1",
"%i,0"%((box/2)-2),
"C",
"E",
"0",
"%i,%i"%((box/2)+1,(box/2)+1),
"3")
# get power spectrum
mySpi.toSpider("PW","_1","_2")
# Radon transform
mySpi.toSpider("RM 2DN",
"_2",
"1",
"_3",
"%i"%box,
"%i"%(box/2),
"0,0",
"N")
# mask in the X direction to only include equator
mySpi.toSpider("MA X",
"_3",
"_4",
"6,0",
"D",
"E",
"0",
"%i,%i"%((box/2),(box/2)))
# find peak
mySpi.toSpider("PK x20,x21,x22","_4","1,0")
# save the angles to a file
mySpi.toSpider("SD 1, x21","angle")
mySpi.toSpider("SD E","angle")
mySpi.close()
f = open("angle.spi")
for line in f:
d=line.strip().split()
if d[0][0]==";" or len(d) < 3:
continue
rot = float(d[2])
os.remove("angle.spi")
os.remove(avgimg)
return rot
def spiderVOEA(incr,ang,fold=1.0,dataext=".spi"): apFile.removeFile(ang) mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=False) mySpider.toSpiderQuiet( "VO EA", str(incr), "(0,90.0)", "(0,%.1f)" % ((360.0/fold)-0.1), spyder.fileFilter(ang), ) mySpider.close()
def rotshiftStack(origstackfile, rotShiftFile, timestamp, iternum, classnum=None, dataext=".spi"):
origstackfile = spyder.fileFilter(origstackfile)
if classnum is not None:
apshstackfile = str(classnum)+"/"+"apshstack%s.spi"%(timestamp)
else:
apshstackfile = "apshstack%s.spi"%(timestamp)
apshstackfile = spyder.fileFilter(apshstackfile)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
mySpider.toSpider("RT SQ",
origstackfile+"@*****", #stack and particle number
apshstackfile+"@*****", #output stack and particle
"("+str(rotation)+")", #rotation
"("+str(Xshift)+","+str(Yshift)+")", #shift parameters
)
mySpider.close()
return
def calcFSC(vol1,vol2,fscfile,dataext=".spi",inMySpi=False):
"""
Calculate the differential 3-D phase residual and the
Fourier Shell Correlation between two volumes.
The Differential Phase Residual over a shell with thickness
given by shell width and the Fourier Shell Correlation
between shells of specified widths are computed and stored
in the document file.
"""
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext,logo=False,log=False)
else:
mySpi=inMySpi
apFile.removeFile(fscfile)
v1=spyder.fileFilter(vol1)
v2=spyder.fileFilter(vol2)
fsc=spyder.fileFilter(fscfile)
mySpi.toSpiderQuiet("RF 3",v1,v2,"1","0.5,1.5","C","90","3",fsc)
if inMySpi is False:
mySpi.close()
def calcFSC(vol1, vol2, fscfile, dataext=".spi", inMySpi=False):
"""
Calculate the differential 3-D phase residual and the
Fourier Shell Correlation between two volumes.
The Differential Phase Residual over a shell with thickness
given by shell width and the Fourier Shell Correlation
between shells of specified widths are computed and stored
in the document file.
"""
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
else:
mySpi = inMySpi
apFile.removeFile(fscfile)
v1 = spyder.fileFilter(vol1)
v2 = spyder.fileFilter(vol2)
fsc = spyder.fileFilter(fscfile)
mySpi.toSpiderQuiet("RF 3", v1, v2, "1", "0.5,1.5", "C", "90", "3", fsc)
if inMySpi is False:
mySpi.close()
def spiderVOEA(incr, ang, fold=1.0, dataext=".spi"):
apFile.removeFile(ang)
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
mySpider.toSpiderQuiet(
"VO EA",
str(incr),
"(0,90.0)",
"(0,%.1f)" % ((360.0 / fold) - 0.1),
spyder.fileFilter(ang),
)
mySpider.close()
def createProjections(incr,
boxsz,
symfold,
invol,
rad,
sel="selvoea.spi",
ang="angvoea.spi",
projs="proj.spi",
nproc=1,
dataext=".spi"):
# create doc file containing euler angles
spiderVOEA(incr, ang, symfold)
projangles = readDocFile(ang)
numprojs = len(projangles)
mySpi = spyder.SpiderSession(nproc=nproc,
dataext=dataext,
logo=False,
log=True)
apFile.removeFile(sel)
mySpi.toSpiderQuiet(
"DOC CREATE",
spyder.fileFilter(sel),
"1",
"1-" + str(numprojs),
)
apFile.removeFile(projs)
mySpi.toSpiderQuiet(
"PJ 3Q",
spyder.fileFilter(invol),
str(rad),
spyder.fileFilter(sel),
spyder.fileFilter(ang),
spyder.fileFilter(projs) + "@*****",
)
mySpi.close()
return projs, numprojs, ang, sel
def rotAndShiftVol(invol,
outvol,
rot=(0, 0, 0),
center=(0, 0, 0),
shift=(0.0, 0.0, 0.0),
dataext=".spi",
inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
else:
mySpi = inMySpi
vol1 = spyder.fileFilter(invol)
tmpvol = spyder.fileFilter('temp')
vol3 = spyder.fileFilter(outvol)
cleanlist = []
invol = vol1
if rot != (0, 0, 0):
outvol = tmpvol
rot = tuple(map((lambda x: float(x)), rot))
rotstr = '%.2f,%.2f,%.2f' % rot
center = tuple(map((lambda x: float(x)), center))
centerstr = '%.1f,%.1f,%.1f' % center
mySpi.toSpider("RT 3A", invol, outvol, rotstr, centerstr)
cleanlist.append(invol)
invol = tmpvol
if shift != (0, 0, 0):
outvol = vol3
shift = tuple(map((lambda x: float(x)), shift))
shiftstr = '%f.1,%f.1,%f.1' % shift
mySpi.toSpider("SH", invol, outvol, shiftstr)
cleanlist.append(invol)
invol = outvol
if outvol != vol3:
mySpi.toSpider("CP", invol, vol3)
if inMySpi is False:
mySpi.close()
for cleanfile in cleanlist:
os.remove(cleanfile + '.spi')
def maskImg(inimg,
outimg,
outrad,
cutoff,
background="P",
inrad="0",
center="0",
dataext=".spi",
inMySpi=False):
if inMySpi is False:
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
else:
mySpi = inMySpi
img1 = spyder.fileFilter(inimg)
img2 = spyder.fileFilter(outimg)
mySpi.toSpiderQuiet("MA", img1, img2,
str(outrad) + "," + str(inrad), cutoff, background)
if background == "E":
mySpi.toSpiderQuiet("0", str(center) + "," + str(center))
if cutoff == "C" or cutoff == "G":
mySpi.toSpiderQuiet("3.5")
if inMySpi is False:
mySpi.close()
def findThreshold(numclasses, dendrogramdocfile, rundir, dataext):
if not os.path.isfile(dendrogramdocfile):
apDisplay.printError("dendrogram doc file does not exist")
### determining threshold cutoff for number of classes
minthresh = 0.0
maxthresh = 1.0
minclass = 0.0
maxclass = 1.0
classes = 0
count = 0
### changes for later versions of SPIDER:
sp = spyder.SpiderSession()
if sp.version() >= 18.03:
maxthresh = 100.0
numclasses += 1
sp.close()
sys.stderr.write("finding threshold")
while (classes != numclasses and count < 50):
count += 1
if count % 70 == 0:
sys.stderr.write("\n[" + str(minclass) + "->" + str(minclass) +
"]")
thresh = (maxthresh - minthresh) / 3.0 + minthresh
classfile = rundir + "/classes"
apFile.removeFile(classfile + dataext)
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
mySpider.toSpiderQuiet(
"CL HD",
thresh, #threshold
spyder.fileFilter(dendrogramdocfile), # dendrogram doc file
classfile)
mySpider.close()
claf = open(classfile + dataext, "r")
classes = len(claf.readlines()) - 1
claf.close()
if classes > numclasses:
minthresh = thresh
maxclass = classes
sys.stderr.write(">")
elif classes < numclasses:
maxthresh = thresh
minclass = classes
sys.stderr.write("<")
#print " ",count, classes, thresh, maxthresh, minthresh
print count, "rounds for", classes, "classes"
return thresh, classes
def mirrorParticles(self, partdatas, spiderstack):
partnum = 0
mySpider = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
for stackpartdata in partdatas:
partnum += 1
inplane, mirror = self.getParticleInPlaneRotation(stackpartdata)
if mirror is True:
sys.stderr.write("m")
mySpider.toSpiderQuiet(
"MR",
spyder.fileFilter(spiderstack) + ("@%05d" % (partnum)),
"_9",
"Y",
)
mySpider.toSpiderQuiet(
"CP",
"_9",
spyder.fileFilter(spiderstack) + ("@%05d" % (partnum)),
)
else:
sys.stderr.write(".")
sys.stderr.write("\n")
mySpider.close()
def butterworthFscLP(volfile, fscout, dataext=".spi"):
"""
inputs:
volume
fsc output file
outputs:
volume
"""
### setup
starttime = time.time()
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(volfile+dataext):
apDisplay.printError("volume file not found: "+volfile+dataext)
if not os.path.isfile(fscout):
apDisplay.printError("fsc output file not found: "+fscout+dataext)
fscfile = open(fscout, "r")
prevValue = None
for line in fscfile.readlines():
value = line.split()
try:
int(value[0])
except:
#apDisplay.printMsg(line)
continue
if float(value[4]) < 0.5:
if prevValue is None:
passband = float(value[2])
stopband = float(value[2])+0.1
else:
passband = prevValue
stopband = prevValue + 0.1
break
prevValue = float(value[2])
fscfile.close()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
mySpider.toSpider("FQ",
volfile, #filename for volume
volfile+"_filtered", #filename for output volume
"(7)",
"(%.5f,%.5f)" % (passband, stopband), #pass-band and stop-band
)
mySpider.close()
apDisplay.printColor("finished filtering the volume "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def createMask(maskfile, maskrad, boxsize, dataext=".spi"):
"""
We should use imagefun.filled_circle() instead
"""
apDisplay.printMsg("Creating mask with diameter %.1f and boxsize %d"%(maskrad*2.0,boxsize))
mySpider = spyder.SpiderSession(dataext=dataext, logo=False)
mySpider.toSpiderQuiet("MO",
spyder.fileFilter(maskfile),
"%d,%d" % (boxsize, boxsize),
"C",
str(maskrad),
)
mySpider.close()
if not os.path.isfile(maskfile) or apFile.fileSize(maskfile) < 2:
apDisplay.printError("Failed to create mask file")
return
def createGaussianSphere(self, volfile, boxsize):
stdev = boxsize/5.0
randdev = boxsize/20.0
halfbox = boxsize/2.0
#width of Gaussian
gaussstr = ("%.3f,%.3f,%.3f"
%(stdev+randdev*random.random(), stdev+randdev*random.random(), stdev+randdev*random.random()))
apDisplay.printMsg("Creating Gaussian volume with stdev="+gaussstr)
mySpider = spyder.SpiderSession(logo=False)
mySpider.toSpiderQuiet("MO 3",
spyder.fileFilter(volfile),
"%d,%d,%d"%(boxsize,boxsize,boxsize),
"G", #G for Gaussian
"%.3f,%.3f,%.3f"%(halfbox,halfbox,halfbox), #center of Gaussian
gaussstr,
)
mySpider.close()
return
def correspondenceAnalysis(alignedstack,
boxsize,
maskpixrad,
numpart,
numfactors=8,
dataext=".spi"):
"""
inputs:
aligned stack
search params
outputs:
eigen images
eigen vectors
coran parameters
"""
### setup
if dataext in alignedstack:
alignedstack = alignedstack[:-4]
t0 = time.time()
rundir = "coran"
apParam.createDirectory(rundir)
### make template in memory
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
mySpider.toSpiderQuiet("MO", "_9", "%d,%d" % (boxsize, boxsize), "C",
str(maskpixrad * 2.0))
### performing correspondence analysis
apDisplay.printMsg("Performing correspondence analysis (long wait)")
mySpider.toSpider("CA S",
spyder.fileFilter(alignedstack) + "@******",
"1-" + str(numpart), "_9", str(numfactors), "C", "10",
rundir + "/corandata")
mySpider.close()
contriblist = analyzeEigenFactors(alignedstack, rundir, numpart,
numfactors, dataext)
td1 = time.time() - t0
apDisplay.printMsg("completed correspondence analysis of " + str(numpart) +
" particles in " + apDisplay.timeString(td1))
return contriblist
def findRotation(self, avgimg):
# use radon transform in SPIDER to find rotation to orient the average image vertical
if os.path.isfile("angle.spi"):
os.remove("angle.spi")
box = self.alignstackdata['boxsize']
mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
# circular mask the average
mySpi.toSpider("MA",
spyder.fileFilter(avgimg) + "@1", "_1",
"%i,0" % ((box / 2) - 2), "C", "E", "0",
"%i,%i" % ((box / 2) + 1, (box / 2) + 1), "3")
# get power spectrum
mySpi.toSpider("PW", "_1", "_2")
# Radon transform
mySpi.toSpider("RM 2DN", "_2", "1", "_3", "%i" % box, "%i" % (box / 2),
"0,0", "N")
# mask in the X direction to only include equator
mySpi.toSpider("MA X", "_3", "_4", "6,0", "D", "E", "0",
"%i,%i" % ((box / 2), (box / 2)))
# find peak
mySpi.toSpider("PK x20,x21,x22", "_4", "1,0")
# save the angles to a file
mySpi.toSpider("SD 1, x21", "angle")
mySpi.toSpider("SD E", "angle")
mySpi.close()
f = open("angle.spi")
for line in f:
d = line.strip().split()
if d[0][0] == ";" or len(d) < 3:
continue
rot = float(d[2])
os.remove("angle.spi")
os.remove(avgimg)
return rot
def createBoxMask(maskfile,
boxsize,
maskx,
masky,
falloff,
imask=None,
dataext=".spi"):
"""
creates a rectangular mask with soft falloff
"""
apDisplay.printMsg("Creating %i x %i box mask" % (boxsize, boxsize))
mySpi = spyder.SpiderSession(dataext=dataext,
logo=False,
nproc=1,
log=False)
# create blank image for mask
mySpi.toSpiderQuiet("BL", "_1", "%i,%i" % (boxsize, boxsize), "N", "1")
# mask it in X
mySpi.toSpiderQuiet("MA X", "_1", "_2", "%i" % maskx, "C", "E", "0",
"%i,%i" % (boxsize / 2, boxsize / 2), "%.2f" % falloff)
# inner mask in X
if imask is not None:
mySpi.toSpiderQuiet("MA X", "_2", "_3", "%i,%i" % (boxsize / 2, imask),
"C", "E", "0",
"%i,%i" % (boxsize / 2, boxsize / 2),
"%.2f" % (falloff / 4))
mySpi.toSpiderQuiet("CP", "_3", "_2")
# mask in Y
mySpi.toSpiderQuiet("MA Y", "_2", spyder.fileFilter(maskfile),
"%i" % masky, "C", "E", "0",
"%i,%i" % (boxsize / 2, boxsize / 2), "%.2f" % falloff)
mySpi.close()
if not os.path.isfile(maskfile) or apFile.fileSize(maskfile) < 2:
apDisplay.printError("Failed to create mask file")
return
def butterworthLP(volfile, pixelsize, dataext=".spi"):
"""
inputs:
volume
outputs:
volume
"""
### setup
starttime = time.time()
volfile = spyder.fileFilter(volfile)
if not os.path.isfile(volfile+dataext):
apDisplay.printError("volume file not found: "+volfile+dataext)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
mySpider.toSpider("FQ",
volfile, #filename for volume
"_1",
# volfile+"_filtered", #filename for output volume
"(7)",
"(%.5f,%.5f)" % (pixelsize/25,pixelsize/15), #pass-band and stop-band
)
mySpider.close()
apDisplay.printColor("finished filtering the volume "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def spiderAPMQ(projs,
numprojs,
tsearch,
tstep,
lastRing,
stackfile,
nump,
ang,
firstRing=1,
startp=1,
apmqfile="apmq.spi",
nproc=1,
outang="angular.spi",
dataext=".spi"):
apFile.removeFile(apmqfile)
mySpi = spyder.SpiderSession(nproc=nproc,
dataext=dataext,
logo=False,
log=True)
mySpi.toSpiderQuiet(
"AP MQ",
spyder.fileFilter(projs) + "@*****",
"1-" + str(numprojs),
str(tsearch) + "," + str(tstep),
str(firstRing) + "," + str(lastRing),
spyder.fileFilter(stackfile) + "@******",
str(startp) + "-" + str(nump),
spyder.fileFilter(apmqfile),
)
apFile.removeFile(outang)
mySpi.toSpiderQuiet(
"VO MD",
spyder.fileFilter(ang),
spyder.fileFilter(apmqfile),
spyder.fileFilter(outang),
)
mySpi.close()
return outang
def boxMask(self, infile, outfile, spirots=None):
from appionlib.apSpider import operations
# boxmask the particles
apDisplay.printMsg("masking the particles with a rectangular box")
nump = apStack.getNumberStackParticlesFromId(self.params['stackid'])
box = self.stackdata['boxsize']
apix = self.stackdata['pixelsize'] * 1e10
if self.params['mask'] is None:
mask = box / 2 - 2
else:
mask = int(self.params['mask'] / apix)
imask = int(self.params['imask'] / apix)
length = int(self.params['length'] / apix)
falloff = self.params['falloff'] / apix
mask -= falloff / 2
length = (length / 2) - (falloff / 2)
# create blank image for mask
maskfile = "boxmask.spi"
operations.createBoxMask(maskfile, box, mask, length, falloff, imask)
mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False)
mySpi.toSpiderQuiet("CP", spyder.fileFilter(maskfile), "_4")
mySpi.toSpider("do x10=1,%i" % nump)
if self.params['vertical'] is not True:
mySpi.toSpider("UD IC x10,x30", spyder.fileFilter(spirots),
"x30 = -1*x30", "RT B", "_4", "_9", "(x30)", "(0)",
"MU",
spyder.fileFilter(infile) + "@{******x10}", "_9")
else:
mySpi.toSpider("MU",
spyder.fileFilter(infile) + "@{******x10}", "_4")
mySpi.toSpider(
spyder.fileFilter(outfile) + "@{******x10}", "*", "enddo")
if self.params['vertical'] is not True:
mySpi.toSpider("UD ICE", spyder.fileFilter(spirots))
mySpi.close()
def hierarchClusterClassify(alignedstack,
dendrogramfile,
numclasses=40,
timestamp=None,
rundir=".",
dataext=".spi"):
"""
inputs:
aligned particle stack
number of classes
timestamp
output directory
output:
class averages
class variances
dendrogram.png
"""
if timestamp is None:
timestamp = apParam.makeTimestamp()
classavg = rundir + "/" + ("classavgstack_%s_%03d" %
(timestamp, numclasses))
classvar = rundir + "/" + ("classvarstack_%s_%03d" %
(timestamp, numclasses))
thresh, classes = findThreshold(numclasses, dendrogramfile, rundir,
dataext)
### create class doc files
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True)
mySpider.toSpider(
"CL HE",
thresh,
spyder.fileFilter(dendrogramfile), # dendrogram doc file
rundir + "/classdoc_" + timestamp + "_****", # class doc file
)
### delete existing files
sys.stderr.write("delete existing files")
for dext in (".hed", ".img", dataext):
apFile.removeFile(classavg + dext)
apFile.removeFile(classvar + dext)
print ""
### create class averages
sys.stderr.write("create class averages")
for i in range(classes):
sys.stderr.write(".")
classnum = i + 1
mySpider.toSpiderQuiet(
"AS R",
spyder.fileFilter(alignedstack) + "@******",
rundir + ("/classdoc_" + timestamp + "_%04d" % (classnum)),
"A",
(classavg + "@%04d" % (classnum)),
(classvar + "@%04d" % (classnum)),
)
mySpider.close()
print ""
### convert to IMAGIC
emancmd = "proc2d " + classavg + ".spi " + classavg + ".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
emancmd = "proc2d " + classvar + ".spi " + classvar + ".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
return classavg, classvar
def createAmpcorBatchFile(infile, params):
localinfile = spyder.fileFilter(infile)
appiondir = apParam.getAppionDirectory()
scriptfile = os.path.join(appiondir, "appionlib/data/enhance.bat")
pwscfile = os.path.join(appiondir, "appionlib/data/pwsc.bat")
if not os.path.isfile(scriptfile):
apDisplay.printError("could not find spider script: " + scriptfile)
if not os.path.isfile(pwscfile):
apDisplay.printError("could not find spider script: " + pwscfile)
inf = open(scriptfile, "r")
tmpfile = "out_" + os.path.basename(infile)
tmpfile = os.path.join(params['rundir'], tmpfile)
localtmpfile = spyder.fileFilter(tmpfile)
outfile = "enhance_edit.bat"
if os.path.isfile(outfile):
apDisplay.printWarning(outfile + " already exists; removing it")
time.sleep(2)
os.remove(outfile)
if os.path.isfile('pwsc.bat'):
apDisplay.printWarning(pwscfile + " already exists; replacing it")
time.sleep(2)
shutil.copyfile(pwscfile, 'pwsc.bat')
outf = open(outfile, "w")
notdone = True
for line in inf:
if notdone is False:
outf.write(line)
else:
thr = line[:3]
if thr == "x99":
outf.write(spiderline(99, params['box'], "box size in pixels"))
elif thr == "x98":
outf.write(
spiderline(98, params['maxfilt'],
"filter limit in angstroms"))
elif thr == "x80":
outf.write(spiderline(80, params['apix'], "pixel size"))
elif re.search("^\[vol\]", line):
outf.write(spiderline("vol", localinfile,
"input density file"))
elif re.search("^\[outvol\]", line):
outf.write(
spiderline("outvol", localtmpfile, "enhanced output file"))
elif re.search("^\[scatter\]", line):
outf.write(
spiderline("scatter", params['ampfile'],
"amplitude curve file"))
elif re.search("^\[pwsc\]", line):
outf.write(spiderline("pwsc", "pwsc", "scaling script"))
elif re.search("^\[applyfen\]", line):
outf.write(
spiderline(
"applyfen",
os.path.join(appiondir, "appionlib/data/applyfen"),
"apply curve to data script"))
else:
outf.write(line)
return tmpfile
def setupBatchFile(self, spiderstack, templatestack, orientref):
"""
sets up Ed's batch script to run
"""
### write particle selection file
partsel = os.path.join(self.params['rundir'], "partlist.spi")
f = open(partsel, "w")
for i in range(self.params['numpart']):
f.write("%06d 1 %06d \n" % (i + 1, i + 1))
f.close()
refsel = os.path.join(self.params['rundir'], "reflist.spi")
f = open(refsel, "w")
for i in range(self.params['numtemplate']):
f.write("%03d 1 %03d \n" % (i + 1, i + 1))
f.close()
### read / write batch file
globalbatch = os.path.join(apParam.getAppionDirectory(),
"spiderbatch/IterativeClassifyAlign.spi")
localbatch = os.path.join(self.params['rundir'],
"IterativeClassifyAlign.spi")
gf = open(globalbatch, "r")
lf = open(localbatch, "w")
modify = True
for line in gf:
if modify is True: #flag for line is in header
### Input files ###
if re.match("\<pcltmpl\>", line):
### template for spider particle names
lf.write("<pcltmpl>" + spyder.fileFilter(spiderstack) +
"@****** \n")
elif re.match("\<pcllist\>", line):
### list of particle numbers
lf.write("<pcllist>" + spyder.fileFilter(partsel) + "\n")
elif re.match("\<reftmpl\>", line):
### template for reference image names
lf.write("<reftmpl>" + spyder.fileFilter(templatestack) +
"@*** \n")
elif re.match("\<reflist\>", line):
### list of reference numbers
lf.write("<reflist>" + spyder.fileFilter(refsel) + "\n")
elif re.match("\<ref\>", line):
### orientation reference
lf.write("<ref>" + spyder.fileFilter(orientref) + "\n")
### Parameters ###
elif re.match("23 ;pixels", line):
### particle radius in pixels
pixrad = int(self.params['partrad'] / self.stack['apix'] /
self.params['bin'])
lf.write("%d ;pixels\n" % (pixrad))
elif re.match("10 ;iterations", line):
### number of ref-based iterations
lf.write("%d ;iterations\n" % (self.params['numiter']))
#elif re.match("1 ;free-align rounds", line):
### number of ref-free alignments per class
#lf.write("%d ;free-align rounds\n"%(self.params['freealigns']))
elif re.match("0 ;processors", line):
### number of processors
lf.write("%d ;processors\n" % (self.params['nproc']))
elif re.match("\<dir\>", line):
### sub-directory, we use "."
lf.write("<dir>.\n")
elif re.search("- END BATCH HEADER -", line):
### reached last line of batch header ###
modify = False
else:
lf.write(line)
else:
lf.write(line)
return localbatch
def crossCorrelateAndShift(infile, reffile, alignfile, ccdocfile, numpart, dataext=".spi"):
### rewriten to do the whole thing in memory in SPIDER, it should be faster
starttime = time.time()
infile = spyder.fileFilter(infile)
reffile = spyder.fileFilter(reffile)
alignfile = spyder.fileFilter(alignfile)
partimg = "_4"
ccmap = "_5"
windccmap = "_6"
boxsize = apFile.getBoxSize(infile+dataext)
if not os.path.isfile(infile+dataext):
apDisplay.printError("input stack file not found: "+infile+dataext)
if not os.path.isfile(reffile+dataext):
apDisplay.printError("reference stack file not found: "+reffile+dataext)
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
### Allocate empty stack
mySpider.toSpiderQuiet(
"MS I", #command
"_2@", #name
"%d,%d,%d"%(boxsize), #boxsize
str(numpart+1), #num part to create in memory
str(numpart+1), #max particle number
)
partnum = 0
while partnum < numpart:
partnum+=1
mySpider.toSpiderQuiet("CP",
infile+("@%05d"%(partnum)), #picture
partimg,
)
### cross correlate images; reversed order to avoid -1*shift
mySpider.toSpiderQuiet("CC N",
reffile+("@%05d"%(partnum)), #reference
partimg, #picture
ccmap, #output file
)
### cannot shift more the 1/4 size of the image
mySpider.toSpiderQuiet("FI x52", partimg, "12" )
mySpider.toSpiderQuiet("x54=int(x52/2)") #window size
mySpider.toSpiderQuiet("x55=int(x52/4)") #window topleft
mySpider.toSpiderQuiet("WI",
ccmap, #input file
windccmap, #output file
"x54,x54", #window size
"x55,x55", #window origin
)
### find the cross-correlation peak
mySpider.toSpiderQuiet("x56=int(x52/4)+1") #center of window
mySpider.toSpiderQuiet("PK M x11,x12,x13,x14",
windccmap, #input ccmap file
"x56,x56", #origin coordinates
)
### save info to doc file
mySpider.toSpiderQuiet("SD %d,x13,x14"%(partnum),
ccdocfile, #input ccmap file
)
### shift the images images
mySpider.toSpiderQuiet("SH",
partimg, #old stack
("_2@%05d"%(partnum)), #new stack
"x13,x14", #shift value file
)
### finish up
#save stack to file
mySpider.toSpiderQuiet(
"CP", "_2@",
alignfile+"@",
)
#delete stack
mySpider.toSpiderQuiet(
"DE", "_2",
)
mySpider.close()
apDisplay.printColor("finished shifting particles in "+apDisplay.timeString(time.time()-starttime), "cyan")
return
def createClassAverages(stack,
projs,
apmq,
numprojs,
boxsz,
outclass="classes",
rotated=False,
shifted=False,
dataext=".spi"):
"""
creates EMAN-style class average file "classes.hed" & "classes.img"
and variance files "variances.hed" & "variances.img"
from spider classification
"""
apFile.removeFile(outclass)
outf = spyder.fileFilter(outclass)
outvf = "tmpvar"
apmqlist = readDocFile(apmq)
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
# create file containing the number of particles matched to each projection
mySpi.toSpiderQuiet(
"VO MQ",
"0.0",
spyder.fileFilter(apmq),
str(numprojs),
"cls*****",
"numinclass",
)
mySpi.close()
os.remove("numinclass%s" % dataext)
# create class average file
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
for i in range(0, numprojs):
clsfile = readDocFile("cls%05d%s" % (i + 1, dataext))
if clsfile == []:
apDisplay.printMsg("class %d has no particles" % (i + 1))
mySpi.toSpiderQuiet(
"BL",
"%s@%d" % (outf, (i + 1)),
"(%d,%d)" % (boxsz, boxsz),
"N",
"(0.0)",
)
mySpi.toSpiderQuiet(
"BL",
"%s@%d" % (outvf, (i + 1)),
"(%d,%d)" % (boxsz, boxsz),
"N",
"(0.0)",
)
else:
mySpi.toSpiderQuiet("DE", "_2@")
mySpi.toSpiderQuiet("MS", "_2@", "%d,%d,1" % (boxsz, boxsz),
str(len(clsfile)))
for p in range(0, len(clsfile)):
mySpi.toSpiderQuiet("DE", "_1")
# get the particle
part = int(float(clsfile[p][2])) - 1
pimg = spyder.fileFilter(stack) + "@%d" % (part + 1)
rot = float(apmqlist[part][4])
shx = float(apmqlist[part][5])
shy = float(apmqlist[part][6])
if shifted is True:
shx = 0
shy = 0
if rotated is True:
rot = 0
p_out = "_2@%d" % (p + 1)
rotAndShiftImg(pimg, "_1", rot, shx, shy, inMySpi=mySpi)
# mirror
if int(float(apmqlist[part][2])) < 0:
mirrorImg("_1", p_out, inMySpi=mySpi)
else:
mySpi.toSpiderQuiet("CP", "_1", p_out)
if len(clsfile) == 1:
mySpi.toSpiderQuiet("CP", "_2@1", "%s@%d" % (outf, (i + 1)))
mySpi.toSpiderQuiet(
"BL",
"%s@%d" % (outvf, (i + 1)),
"(%d,%d)" % (boxsz, boxsz),
"N",
"(0.0)",
)
else:
mySpi.toSpiderQuiet("AS R", "_2@*****", "1-%d" % len(clsfile),
"A", "%s@%d" % (outf, (i + 1)),
"%s@%d" % (outvf, i + 1))
mySpi.close()
# convert the class averages to EMAN class averages
# read classes & projections for EMAN classes output
if os.path.exists('variances.hed'):
os.remove('variances.hed')
if os.path.exists('variances.img'):
os.remove('variances.img')
if os.path.exists('classes.hed'):
os.remove('classes.hed')
if os.path.exists('classes.img'):
os.remove('classes.img')
### I would prefer to use apImagicFile.readImagic, writeImagic
variances = EMAN.readImages(outvf + dataext, -1, -1, 0)
averages = EMAN.readImages(outf + dataext, -1, -1, 0)
projections = EMAN.readImages(projs, -1, -1, 0)
for i in range(0, numprojs):
# copy projection to class average file
projections[i].writeImage('classes.hed')
projections[i].writeImage('variances.hed')
# get num of particles in class
clsfile = readDocFile("cls%05d%s" % (i + 1, dataext))
averages[i].setNImg(len(clsfile))
averages[i].writeImage('classes.hed', -1)
variances[i].setNImg(len(clsfile))
variances[i].writeImage('variances.hed', -1)
os.remove("cls%05d%s" % (i + 1, dataext))
os.remove(outf + dataext)
return
def alignStack(oldstack, alignedstack, partlist, dataext=".spi"):
"""
write aligned stack -- with python loop
inputs:
oldstack
newstack (empty)
list of particle dictionaries for operations
modifies:
newstack
output:
none
I tried this loop in both spider and python;
python was faster?!? -neil
"""
if not os.path.isfile(oldstack + dataext):
apDisplay.printError("Could not find original stack: " + oldstack +
dataext)
boxsize = apFile.getBoxSize(oldstack + dataext)
apDisplay.printMsg("applying alignment parameters to stack")
apFile.removeFile(alignedstack + dataext)
count = 0
t0 = time.time()
nproc = apParam.getNumProcessors()
mySpider = spyder.SpiderSession(dataext=dataext,
logo=True,
nproc=nproc,
log=False)
#create stack in core
numpart = len(partlist)
mySpider.toSpiderQuiet(
"MS I", #command
"_2@", #name
"%d,%d,%d" % (boxsize), #boxsize
str(numpart + 1), #num part to create in memory
str(numpart + 1), #max particle number
)
for partdict in partlist:
partnum = partdict['num']
#if partdict['num'] in [3,6,7]:
# print partdict['num'], partdict['template'], partdict['mirror'], round(partdict['rot'],3)
### Rotate and Shift operations
count += 1
#rotate/shift
mySpider.toSpiderQuiet(
"RT SQ",
spyder.fileFilter(oldstack) + "@" + ("%06d" % (partnum)),
"_1",
str(partdict['rot']),
str(partdict['xshift']) + "," + str(partdict['yshift']),
)
#mirror, if necessary
if 'mirror' in partdict and partdict['mirror'] is True:
mySpider.toSpiderQuiet(
"MR",
"_1",
"_2@" + ("%06d" % (partnum)),
"Y",
)
else:
mySpider.toSpiderQuiet(
"CP",
"_1",
"_2@" + ("%06d" % (partnum)),
)
### finish up
#save stack to file
mySpider.toSpiderQuiet(
"CP",
"_2@",
spyder.fileFilter(alignedstack) + "@",
)
#delete stack
mySpider.toSpiderQuiet(
"DE",
"_2",
)
mySpider.close()
apDisplay.printMsg("Completed transforming %d particles in %s" %
(count, apDisplay.timeString(time.time() - t0)))
if count < 1:
apDisplay.printError("Failed to transform any particles")
if not os.path.isfile(alignedstack + dataext):
apDisplay.printError("Failed to create stack " + alignedstack +
dataext)
return
def kmeansCluster(alignedstack,
numpart=None,
numclasses=40,
timestamp=None,
factorlist=range(1, 5),
corandata="coran/corandata",
dataext=".spi"):
"""
inputs:
outputs:
"""
if timestamp is None:
timestamp = apParam.makeTimestamp()
if alignedstack[-4:] == dataext:
alignedstack = alignedstack[:-4]
rundir = "cluster"
classavg = rundir + "/" + ("classavgstack_%s_%03d" %
(timestamp, numclasses))
classvar = rundir + "/" + ("classvarstack_%s_%03d" %
(timestamp, numclasses))
apParam.createDirectory(rundir)
for i in range(numclasses):
apFile.removeFile(rundir + ("/classdoc%04d" % (i + 1)) + dataext)
apFile.removeFile(rundir + ("/allclassesdoc%04d" % (numclasses)) + dataext)
### make list of factors
factorstr, factorkey = operations.intListToString(factorlist)
### do k-means clustering
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
mySpider.toSpider(
"CL KM",
corandata + "_IMC", # path to coran data
str(numclasses), # num classes
factorstr, # factor string
)
## weight for each factor
for fact in factorlist:
mySpider.toSpiderQuiet("1.0")
randnum = (int(random.random() * 1000) + 1)
mySpider.toSpider(
str(randnum),
rundir + "/classdoc_" + timestamp + "_****", # class doc file
rundir + ("/allclassesdoc%04d" % (numclasses)), #clusterdoc file
)
mySpider.close()
### delete existing files
sys.stderr.write("delete existing files")
for dext in (".hed", ".img", dataext):
apFile.removeFile(classavg + dext)
apFile.removeFile(classvar + dext)
print ""
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
### create class averages
apDisplay.printMsg("Averaging particles into classes")
for i in range(numclasses):
classnum = i + 1
mySpider.toSpiderQuiet(
"AS R",
spyder.fileFilter(alignedstack) + "@******",
rundir + ("/classdoc_" + timestamp + "_%04d" % (classnum)),
"A",
(classavg + "@%04d" % (classnum)),
(classvar + "@%04d" % (classnum)),
)
if classnum % 10 == 0:
sys.stderr.write(".")
time.sleep(1)
mySpider.close()
### convert to IMAGIC
emancmd = "proc2d " + classavg + ".spi " + classavg + ".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
emancmd = "proc2d " + classvar + ".spi " + classvar + ".hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
return classavg, classvar
def alignAPSH(volfile, origstackfile, eulerdocfile, classnum, boxsize, numpart, pixrad, timestamp, iternum, dataext=".spi"):
apshAngularFile = str(classnum)+"/"+"apshAngularFile-%03d.spi"%(iternum)
if os.path.isfile(apshAngularFile):
os.remove(apshAngularFile)
apDisplay.printColor("File exist! Removing file "+apshAngularFile, "cyan")
apshAngularFile = spyder.fileFilter(apshAngularFile)
apshListFile = str(classnum)+"/"+"apshListFile-%03d.spi"%(iternum)
if os.path.isfile(apshListFile):
os.remove(apshListFile)
apDisplay.printColor("File exist! Removing file "+apshListFile, "cyan")
apshListFile = spyder.fileFilter(apshListFile)
apshOutFile = str(classnum)+"/"+"apshOut-%03d.spi"%(iternum)
if os.path.isfile(apshOutFile):
os.remove(apshOutFile)
apDisplay.printColor("File exist! Removing file "+apshOutFile, "cyan")
apshOut = spyder.fileFilter(apshOutFile)
apshProjStack = str(classnum)+"/"+"apshProjStack-%03d.spi"%(iternum)
if os.path.isfile(apshProjStack):
os.remove(apshProjStack)
apDisplay.printColor("File exist! Removing file "+apshProjStack, "cyan")
apshProjStack = spyder.fileFilter(apshProjStack)
origstackfile = spyder.fileFilter(origstackfile)
volfile = spyder.fileFilter(volfile)
origeulerdocfile = spyder.fileFilter(eulerdocfile)
mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
mySpider.toSpider("VO EA,x53",
"10", #angular increment
"0,90.0", #Range of Theta
"0,359.9", #Range of Phi
apshAngularFile, #Angular file name
)
mySpider.toSpider("DOC CREATE",
apshListFile, #List file name
"1",
"1-x53"
)
#mySpider.toSpider("MS",
# "_9@", #reference projections
# str(boxsize),str(boxsize),"1", #boxsize
# "x53",
#)
mySpider.toSpider("PJ 3Q",
volfile, #input vol file
str(pixrad), #pixel radius
apshListFile, #number of particles
apshAngularFile, #Euler DOC file
apshProjStack+"@*****", #output projections
)
mySpider.toSpider("AP SH",
apshProjStack+"@*****", #reference projections
"1-x53", #reference numbers
"6,2", #translational search range, step size
"1,%d"%(boxsize/3), #first and last ring
apshAngularFile,
origstackfile+"@*****",
"1-%d"%(numpart),
origeulerdocfile,
"20", #"(0.0)", # no restrictions for alignment
"1", # mirror check?
apshOut, # output
)
mySpider.close()
return apshOutFile
