def projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext=".spi"): """ project 3D volumes using given Euler angles """ starttime = time.time() volfile = spyder.fileFilter(volfile) eulerdocfile = spyder.fileFilter(eulerdocfile) projstackfile = spyder.fileFilter(projstackfile) if not os.path.isfile(volfile+dataext): apDisplay.printError("volume file not found: "+volfile+dataext) if not os.path.isfile(eulerdocfile+dataext): apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext) apFile.removeFile(projstackfile) nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("PJ 3Q", volfile, #input vol file str(pixrad), #pixel radius "1-%d"%(numpart), #number of particles eulerdocfile, #Euler DOC file projstackfile+"@*****", #output projections ) mySpider.close() apDisplay.printColor("finished projections in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def backprojectCG(stackfile, eulerdocfile, volfile, numpart, pixrad, dataext=".spi"): """ inputs: stack, in spider format eulerdocfile outputs: volume """ ### setup starttime = time.time() stackfile = spyder.fileFilter(stackfile) eulerdocfile = spyder.fileFilter(eulerdocfile) volfile = spyder.fileFilter(volfile) if not os.path.isfile(stackfile+dataext): apDisplay.printError("stack file not found: "+stackfile+dataext) if not os.path.isfile(eulerdocfile+dataext): apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext) apFile.removeFile(volfile+dataext) nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("BP CG", stackfile+"@*****", #stack file "1-%d"%(numpart), #number of particles str(pixrad), #particle radius eulerdocfile, #angle doc file "N", #has symmetry?, does not work volfile, #filename for volume "%.1e,%.1f" % (1.0e-5, 0.0), #error, chi^2 limits "%d,%d" % (25,1), #iterations, 1st derivative mode "2000", #lambda - higher=less sensitive to noise ) mySpider.close() apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def rotAndShiftVol(invol,outvol,rot=(0,0,0),center=(0,0,0),shift=(0.0,0.0,0.0),dataext=".spi",inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=True) else: mySpi=inMySpi vol1=spyder.fileFilter(invol) tmpvol=spyder.fileFilter('temp') vol3=spyder.fileFilter(outvol) cleanlist = [] invol = vol1 if rot != (0,0,0): outvol = tmpvol rot = tuple(map((lambda x: float(x)), rot)) rotstr = '%.2f,%.2f,%.2f' % rot center = tuple(map((lambda x: float(x)), center)) centerstr = '%.1f,%.1f,%.1f' % center mySpi.toSpider("RT 3A",invol,outvol,rotstr,centerstr) cleanlist.append(invol) invol = tmpvol if shift != (0,0,0): outvol = vol3 shift = tuple(map((lambda x: float(x)), shift)) shiftstr = '%f.1,%f.1,%f.1' % shift mySpi.toSpider("SH",invol,outvol,shiftstr) cleanlist.append(invol) invol = outvol if outvol != vol3: mySpi.toSpider("CP",invol,vol3) if inMySpi is False: mySpi.close() for cleanfile in cleanlist: os.remove(cleanfile+'.spi')
def rotshiftParticle(origstackfile, partnum, rotation, Xshift, Yshift, mirror, iternum, timestamp, classnum=None, dataext=".spi"): origstack = spyder.fileFilter(origstackfile) if classnum is not None: apshstackfile = str(classnum)+"/"+"apshStack-%03d.spi"%(iternum) else: apshstackfile = "apshStack-%03d.spi"%(iternum) apshstack = spyder.fileFilter(apshstackfile) mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=False) mySpider.toSpiderQuiet("RT SQ", origstack+"@"+str(partnum), #stack and particle number apshstack+"@"+str(partnum), #output stack and particle "("+str(rotation)+")", #rotation "("+str(Xshift)+","+str(Yshift)+")", #shift parameters ) if mirror==-1: mySpider.toSpiderQuiet("MR", apshstack+"@"+str(partnum), #stack and particle number apshstack+"@"+str(partnum), #output stack and particle "Y", ) mySpider.close() return apshstackfile
def backProjection( stack, select, ang, out, sym=None, nproc=1, dataext=".spi", inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(nproc=nproc,dataext=dataext,logo=False,log=False) else: mySpi=inMySpi apFile.removeFile(out) if sym is None: sym="*" mySpi.toSpiderQuiet( "BP 3F", spyder.fileFilter(stack)+"@******", spyder.fileFilter(select), spyder.fileFilter(ang), spyder.fileFilter(sym), spyder.fileFilter(out) ) if inMySpi is False: mySpi.close()
def backproject3F(stackfile, eulerdocfile, volfile, numpart, dataext=".spi"): """ inputs: stack, in spider format eulerdocfile outputs: volume """ ### setup starttime = time.time() stackfile = spyder.fileFilter(stackfile) eulerdocfile = spyder.fileFilter(eulerdocfile) volfile = spyder.fileFilter(volfile) if not os.path.isfile(stackfile+dataext): apDisplay.printError("stack file not found: "+stackfile+dataext) if not os.path.isfile(eulerdocfile+dataext): apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext) apFile.removeFile(volfile+dataext) nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("BP 3F", stackfile+"@*****", #stack file "1-%d"%(numpart), #number of particles eulerdocfile, #angle doc file "*", #input symmetry file, '*' for skip volfile, #filename for volume ) mySpider.close() apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"): """ inputs: aligned particle stack number of classes timestamp output directory output: class averages class variances dendrogram.png """ if timestamp is None: timestamp = apParam.makeTimestamp() classavg = rundir+"/"+("classavgstack_%s_%03d" % (timestamp, numclasses)) classvar = rundir+"/"+("classvarstack_%s_%03d" % (timestamp, numclasses)) thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext) ### create class doc files mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True) mySpider.toSpider( "CL HE", thresh, spyder.fileFilter(dendrogramfile), # dendrogram doc file rundir+"/classdoc_"+timestamp+"_****", # class doc file ) ### delete existing files sys.stderr.write("delete existing files") for dext in (".hed", ".img", dataext): apFile.removeFile(classavg+dext) apFile.removeFile(classvar+dext) print "" ### create class averages sys.stderr.write("create class averages") for i in range(classes): sys.stderr.write(".") classnum = i+1 mySpider.toSpiderQuiet( "AS R", spyder.fileFilter(alignedstack)+"@******", rundir+("/classdoc_"+timestamp+"_%04d" % (classnum)), "A", (classavg+"@%04d" % (classnum)), (classvar+"@%04d" % (classnum)), ) mySpider.close() print "" ### convert to IMAGIC emancmd = "proc2d "+classavg+".spi "+classavg+".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) emancmd = "proc2d "+classvar+".spi "+classvar+".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) return classavg,classvar
def addParticleToStack(partnum, partfile, stackfile, dataext=".spi"): mySpider = spyder.SpiderSession(dataext=dataext, logo=False) mySpider.toSpiderQuiet("CP", spyder.fileFilter(partfile), #particle file spyder.fileFilter(stackfile)+"@%06d"%(partnum), #stack file ) mySpider.close() return
def rotAndShiftImg(inimg,outimg,rot=0,shx=0,shy=0,dataext=".spi",inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False) else: mySpi=inMySpi img1=spyder.fileFilter(inimg) img2=spyder.fileFilter(outimg) mySpi.toSpiderQuiet("RT SQ",img1,img2,rot,str(shx)+","+str(shy)) if inMySpi is False: mySpi.close()
def mirrorImg(inimg,outimg,dataext=".spi",inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False) else: mySpi=inMySpi img1=spyder.fileFilter(inimg) img2=spyder.fileFilter(outimg) mySpi.toSpiderQuiet("MR",img1,img2,"Y") if inMySpi is False: mySpi.close()
def getCC(image1,image2,outcc,dataext=".spi",inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext,logo=False,log=False) else: mySpi=inMySpi img1=spyder.fileFilter(image1) img2=spyder.fileFilter(image2) outimg=spyder.fileFilter(outcc) mySpi.toSpiderQuiet("CC N",img1,img2,outimg) if inMySpi is False: mySpi.close()
def averageStack(stackfile, numpart, avgfile, varfile, dataext=".spi"): mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) mySpider.toSpider("AS R", spyder.fileFilter(stackfile)+"@******", #stack file "1-%6d"%(numpart), #num of particles "A", #use all particles spyder.fileFilter(avgfile), #average file spyder.fileFilter(varfile), #variance file ) mySpider.close() return
def iterativeBackProjection( stack, select, rad, ang, out, lam, iterlimit, mode, smoothfac, sym=None, nproc=1, dataext=".spi", inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(nproc=nproc,dataext=dataext,logo=False,log=False) else: mySpi=inMySpi if sym is None: sym="*" apFile.removeFile(out) # calculate the smoothing factor smooth=(1/(1+6*lam))*smoothfac mySpi.toSpiderQuiet("x11=0") mySpi.toSpiderQuiet("x47="+str(lam)) mySpi.toSpiderQuiet("x48="+str(smooth)) mySpi.toSpiderQuiet("DO LB11 i=1,%d" % (iterlimit*2)) mySpi.toSpiderQuiet("IF (x11.EQ.%d) GOTO LB11" % iterlimit) mySpi.toSpiderQuiet( "BP RP x11", spyder.fileFilter(stack)+"@******", spyder.fileFilter(select), "("+str(rad)+")", spyder.fileFilter(ang), spyder.fileFilter(sym), spyder.fileFilter(out), "(x47,0)", "("+str(iterlimit)+","+str(mode)+")", "(0,0)", "x48", ) # check if BP RP finished the requested iterations, # if not, modify lambda and smoothing constant and rerun mySpi.toSpiderQuiet("IF (x11.LT.%d) THEN" % iterlimit) mySpi.toSpiderQuiet("x47=x47/2") mySpi.toSpiderQuiet("x48=(1/1+6*x47))*"+str(smoothfac)) mySpi.toSpiderQuiet("GOTO LB11") mySpi.toSpiderQuiet("ENDIF") mySpi.toSpiderQuiet("LB11") if inMySpi is False: mySpi.close()
def createAmpcorBatchFile(infile,params): localinfile = spyder.fileFilter(infile) appiondir = apParam.getAppionDirectory() scriptfile = os.path.join(appiondir, "appionlib/data/enhance.bat") pwscfile = os.path.join(appiondir, "appionlib/data/pwsc.bat") if not os.path.isfile(scriptfile): apDisplay.printError("could not find spider script: "+scriptfile) if not os.path.isfile(pwscfile): apDisplay.printError("could not find spider script: "+pwscfile) inf = open(scriptfile, "r") tmpfile = "out_"+os.path.basename(infile) tmpfile = os.path.join(params['rundir'], tmpfile) localtmpfile = spyder.fileFilter(tmpfile) outfile = "enhance_edit.bat" if os.path.isfile(outfile): apDisplay.printWarning(outfile+" already exists; removing it") time.sleep(2) os.remove(outfile) if os.path.isfile('pwsc.bat'): apDisplay.printWarning(pwscfile+" already exists; replacing it") time.sleep(2) shutil.copyfile(pwscfile, 'pwsc.bat') outf = open(outfile, "w") notdone = True for line in inf: if notdone is False: outf.write(line) else: thr = line[:3] if thr == "x99": outf.write(spiderline(99,params['box'],"box size in pixels")) elif thr == "x98": outf.write(spiderline(98,params['maxfilt'],"filter limit in angstroms")) elif thr == "x80": outf.write(spiderline(80,params['apix'],"pixel size")) elif re.search("^\[vol\]",line): outf.write(spiderline("vol",localinfile,"input density file")) elif re.search("^\[outvol\]",line): outf.write(spiderline("outvol",localtmpfile,"enhanced output file")) elif re.search("^\[scatter\]",line): outf.write(spiderline("scatter",params['ampfile'],"amplitude curve file")) elif re.search("^\[pwsc\]",line): outf.write(spiderline("pwsc","pwsc","scaling script")) elif re.search("^\[applyfen\]",line): outf.write(spiderline("applyfen",os.path.join(appiondir,"appionlib/data/applyfen"),"apply curve to data script")) else: outf.write(line) return tmpfile
def docSplit(file1,file2,file3,dataext=".spi",inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext,logo=False,log=False) else: mySpi=inMySpi apFile.removeFile(file2) apFile.removeFile(file3) f1=spyder.fileFilter(file1) f2=spyder.fileFilter(file2) f3=spyder.fileFilter(file3) mySpi.toSpiderQuiet("DOC SPLIT",f1,f2,f3) if inMySpi is False: mySpi.close()
def centerVolume(volfile, outvolfile, dataext=".spi"): volfile = spyder.fileFilter(volfile) outvolfile = spyder.fileFilter(outvolfile) mySpider = spyder.SpiderSession(dataext=dataext, logo=True) mySpider.toSpider("CG PH x11,x12,x13,x21,x22,x23", volfile, #volume ) mySpider.toSpider("SH F", volfile, #volume outvolfile, #output volume "-x21,-x22,-x23", #shifts in x y z ) mySpider.close() return
def maskImg(inimg,outimg,outrad,cutoff,background="P",inrad="0",center="0",dataext=".spi",inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext,logo=False,log=False) else: mySpi=inMySpi img1=spyder.fileFilter(inimg) img2=spyder.fileFilter(outimg) mySpi.toSpiderQuiet("MA",img1,img2,str(outrad)+","+str(inrad),cutoff,background) if background=="E": mySpi.toSpiderQuiet("0",str(center)+","+str(center)) if cutoff=="C" or cutoff=="G": mySpi.toSpiderQuiet("3.5") if inMySpi is False: mySpi.close()
def windowImg(inimg,outimg,dim,topleftx,toplefty,dataext=".spi",inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext,logo=False,log=False) else: mySpi=inMySpi img1=spyder.fileFilter(inimg) img2=spyder.fileFilter(outimg) mySpi.toSpiderQuiet( "WI", img1, img2, str(dim)+","+str(dim), str(topleftx)+","+str(toplefty) ) if inMySpi is False: mySpi.close()
def normalizeVol(volfile, dataext=".spi"): """ inputs: volume outputs: volume """ ### setup starttime = time.time() volfile = spyder.fileFilter(volfile) if not os.path.isfile(volfile+dataext): apDisplay.printError("volume file not found: "+volfile+dataext) mySpider = spyder.SpiderSession(dataext=dataext, logo=True) ### read out the statistics of the volume mySpider.toSpider("FS x11,x12", volfile, #filename for volume ) mySpider.toSpider("IF(x12.LT.0.0)x12=-x12") ### set all values to positive mySpider.toSpider("AR", volfile, #filename for volume "_1", "(P1+x12)", ) ### save file mySpider.toSpider("CP", "_1", volfile, #filename for volume ) mySpider.close() apDisplay.printColor("finished normalizing the volume to set all values to be positive"+apDisplay.timeString(time.time()-starttime), "cyan") return
def createBoxMask(maskfile,boxsize,maskx,masky,falloff,imask=None,dataext=".spi"): """ creates a rectangular mask with soft falloff """ apDisplay.printMsg("Creating %i x %i box mask"%(boxsize,boxsize)) mySpi = spyder.SpiderSession(dataext=dataext, logo=False, nproc=1, log=False) # create blank image for mask mySpi.toSpiderQuiet("BL","_1","%i,%i"%(boxsize,boxsize),"N","1") # mask it in X mySpi.toSpiderQuiet("MA X", "_1", "_2", "%i"%maskx, "C", "E", "0", "%i,%i"%(boxsize/2,boxsize/2), "%.2f"%falloff) # inner mask in X if imask is not None: mySpi.toSpiderQuiet("MA X","_2","_3", "%i,%i"%(boxsize/2,imask), "C","E","0", "%i,%i"%(boxsize/2,boxsize/2), "%.2f"%(falloff/4)) mySpi.toSpiderQuiet("CP","_3","_2") # mask in Y mySpi.toSpiderQuiet("MA Y","_2", spyder.fileFilter(maskfile), "%i"%masky, "C", "E", "0", "%i,%i"%(boxsize/2,boxsize/2), "%.2f"%falloff) mySpi.close() if not os.path.isfile(maskfile) or apFile.fileSize(maskfile) < 2: apDisplay.printError("Failed to create mask file") return
def symmetryDoc(symtype,symfold=None,outfile="sym.spi",dataext=".spi"): mySpi=spyder.SpiderSession(dataext=dataext,logo=False,log=False) apFile.removeFile(outfile) mySpi.toSpiderQuiet("SY",spyder.fileFilter(outfile),symtype) if symfold is not None: mySpi.toSpiderQuiet(symfold) mySpi.close()
def backprojectRP(stackfile, eulerdocfile, volfile, pixrad, classnum, lambDa, numpart=None, selfile=None, dataext=".spi"): """ inputs: stack, in spider format eulerdocfile outputs: volume """ ### setup starttime = time.time() stackfile = spyder.fileFilter(stackfile) eulerdocfile = spyder.fileFilter(eulerdocfile) if selfile is not None: selfile = spyder.fileFilter(selfile) volfile = spyder.fileFilter(volfile) counter = spyder.fileFilter("numiter") if not os.path.isfile(stackfile+dataext): apDisplay.printError("stack file not found: "+stackfile+dataext) if not os.path.isfile(eulerdocfile+dataext): apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext) apFile.removeFile(volfile+dataext) if (numpart is not None): selection = "1-"+str(numpart) elif (selfile is not None): selection = selfile else: apDisplay.printError("Partilce selection is invalid for BP RP. Please make sure either numpart or selfile is specified in the function backprojectRP!") nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("BP RP x11", stackfile+"@*****", #stack file selection, #selection file or particle range str(pixrad), #particle radius eulerdocfile, #angle doc file "*", #has symmetry?, does not work volfile, #filename for volume "%.1e,%.1f" % (lambDa, 0.0), #lambda, correction "%d,%d" % (50,1), #iteration limit, mode "%d,%d" % (0,0), #minimum, maximum "%.5f" % ((1/(1+6*lambDa))*0.999), #smoothing constant = (1/(1+6*lambda))*0.999 ) mySpider.toSpider("SD 1,x11", str(classnum)+"/"+str(counter)) mySpider.close() apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def hierarchClusterProcess(numpart=None, factorlist=range(1,5), corandata="coran/corandata", rundir=".", dataext=".spi"): """ inputs: coran data number of particles factor list output directory output: dendrogram doc file factorkey """ #apFile.removeFile(rundir+"/dendrogramdoc"+dataext) factorstr, factorkey = operations.intListToString(factorlist) dendrogramfile = rundir+"/dendrogramdoc"+factorkey+dataext if os.path.isfile(dendrogramfile): apDisplay.printMsg("Dendrogram file already exists, skipping processing "+dendrogramfile) return dendrogramfile apDisplay.printMsg("Creating dendrogram file: "+dendrogramfile) ### do hierarchical clustering mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True) mySpider.toSpider( "CL HC", spyder.fileFilter(corandata)+"_IMC", # path to coran data factorstr, # factor string ) ## weight for each factor for fact in factorlist: mySpider.toSpiderQuiet("1.0") minclasssize = "%.4f" % (numpart*0.0001+2.0) mySpider.toSpider( "5", #use Ward's method "T", minclasssize, rundir+"/dendrogram.ps", #dendrogram image file "Y", spyder.fileFilter(dendrogramfile), #dendrogram doc file ) mySpider.close() if not os.path.isfile(dendrogramfile): apDisplay.printError("SPIDER dendrogram creation (CL HC) failed, too many particles??") apImage.convertPostscriptToPng("cluster/dendrogram.ps", "dendrogram.png") return dendrogramfile
def hierarchClusterProcess(numpart=None, factorlist=range(1,5), corandata="coran/corandata", rundir=".", dataext=".spi"): """ inputs: coran data number of particles factor list output directory output: dendrogram doc file factorkey """ #apFile.removeFile(rundir+"/dendrogramdoc"+dataext) factorstr, factorkey = operations.intListToString(factorlist) dendrogramfile = rundir+"/dendrogramdoc"+factorkey+dataext if os.path.isfile(dendrogramfile): apDisplay.printMsg("Dendrogram file already exists, skipping processing "+dendrogramfile) return dendrogramfile apDisplay.printMsg("Creating dendrogram file: "+dendrogramfile) ### do hierarchical clustering mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True, nproc=1) mySpider.toSpider( "CL HC", spyder.fileFilter(corandata)+"_IMC", # path to coran data factorstr, # factor string ) ## weight for each factor for fact in factorlist: mySpider.toSpiderQuiet("1.0") minclasssize = "%.4f" % (numpart*0.0001+2.0) mySpider.toSpider( "5", #use Ward's method "T", minclasssize, rundir+"/dendrogram.ps", #dendrogram image file "Y", spyder.fileFilter(dendrogramfile), #dendrogram doc file ) mySpider.close() if not os.path.isfile(dendrogramfile): apDisplay.printError("SPIDER dendrogram creation (CL HC) failed, too many particles??") apImage.convertPostscriptToPng("cluster/dendrogram.ps", "dendrogram.png") return dendrogramfile
def windowImg(inimg, outimg, dim, topleftx, toplefty, dataext=".spi", inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False) else: mySpi = inMySpi img1 = spyder.fileFilter(inimg) img2 = spyder.fileFilter(outimg) mySpi.toSpiderQuiet("WI", img1, img2, str(dim) + "," + str(dim), str(topleftx) + "," + str(toplefty)) if inMySpi is False: mySpi.close()
def calcFSC(volfile1, volfile2, fscout, dataext=".spi"): volfile1 = spyder.fileFilter(volfile1) volfile2 = spyder.fileFilter(volfile2) fscout = spyder.fileFilter(fscout) mySpider = spyder.SpiderSession(dataext=dataext, logo=True) mySpider.toSpider("RF 3", volfile1, #volume 1 volfile2, #volume 2 "0.5", # ring width for RF 3 "0.5,1.5", # scale factor - lower #"1.5", # scale factor - upper "C", # use 'C' for missing cone and 'W' for missing wedge "90", # maximum tilt angle in data for OTR "3", # factor for noise comparison ("sigma") fscout, # fsc curve output ) mySpider.close() return
def pdb2vol(pdbfile, apix, box, outfile, dataext=".spi"): outfile = spyder.fileFilter(outfile) ### create volume density file from PDB if not os.path.isfile(pdbfile): apDisplay.printError("Can not find PDB file for conversion: " + pdbfile) mySpider = spyder.SpiderSession(dataext=dataext, logo=True) ### command request: infile, apix, center (y/n), atoms/temperature, boxsize, outfile boxsize = "%i, %i, %i" % (box, box, box) mySpider.toSpider("CP FROM PDB", spyder.fileFilter(pdbfile), str(round(apix, 5)), "Y", "A", boxsize, spyder.fileFilter(outfile)) mySpider.close() if not os.path.isfile(outfile + dataext): apDisplay.printError("SPIDER could not create density file: " + outfile + dataext) return
def pdb2vol(pdbfile, apix, box, outfile, dataext=".spi"): outfile = spyder.fileFilter(outfile) ### create volume density file from PDB if not os.path.isfile(pdbfile): apDisplay.printError("Can not find PDB file for conversion: "+pdbfile) mySpider = spyder.SpiderSession(dataext=dataext, logo=True) ### command request: infile, apix, center (y/n), atoms/temperature, boxsize, outfile boxsize = "%i, %i, %i" %(box, box, box) mySpider.toSpider("CP FROM PDB", spyder.fileFilter(pdbfile), str(round(apix,5)), "Y", "A", boxsize, spyder.fileFilter(outfile)) mySpider.close() if not os.path.isfile(outfile+dataext): apDisplay.printError("SPIDER could not create density file: "+outfile+dataext) return
def padImg(inimg,outimg,dim,avg,topleftx,toplefty,const=1.2,dataext=".spi",inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext,logo=False,log=False) else: mySpi=inMySpi img1=spyder.fileFilter(inimg) img2=spyder.fileFilter(outimg) mySpi.toSpiderQuiet( "PD", img1, img2, str(dim)+","+str(dim), avg, ) if avg=="N" or avg=="n": mySpi.toSpiderQuiet(str(const)) mySpi.toSpiderQuiet(str(topleftx)+","+str(toplefty)) if inMySpi is False: mySpi.close()
def findRotation(self,avgimg): # use radon transform in SPIDER to find rotation to orient the average image vertical if os.path.isfile("angle.spi"): os.remove("angle.spi") box=self.alignstackdata['boxsize'] mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False) # circular mask the average mySpi.toSpider("MA", spyder.fileFilter(avgimg)+"@1", "_1", "%i,0"%((box/2)-2), "C", "E", "0", "%i,%i"%((box/2)+1,(box/2)+1), "3") # get power spectrum mySpi.toSpider("PW","_1","_2") # Radon transform mySpi.toSpider("RM 2DN", "_2", "1", "_3", "%i"%box, "%i"%(box/2), "0,0", "N") # mask in the X direction to only include equator mySpi.toSpider("MA X", "_3", "_4", "6,0", "D", "E", "0", "%i,%i"%((box/2),(box/2))) # find peak mySpi.toSpider("PK x20,x21,x22","_4","1,0") # save the angles to a file mySpi.toSpider("SD 1, x21","angle") mySpi.toSpider("SD E","angle") mySpi.close() f = open("angle.spi") for line in f: d=line.strip().split() if d[0][0]==";" or len(d) < 3: continue rot = float(d[2]) os.remove("angle.spi") os.remove(avgimg) return rot
def spiderVOEA(incr,ang,fold=1.0,dataext=".spi"): apFile.removeFile(ang) mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=False) mySpider.toSpiderQuiet( "VO EA", str(incr), "(0,90.0)", "(0,%.1f)" % ((360.0/fold)-0.1), spyder.fileFilter(ang), ) mySpider.close()
def rotshiftStack(origstackfile, rotShiftFile, timestamp, iternum, classnum=None, dataext=".spi"): origstackfile = spyder.fileFilter(origstackfile) if classnum is not None: apshstackfile = str(classnum)+"/"+"apshstack%s.spi"%(timestamp) else: apshstackfile = "apshstack%s.spi"%(timestamp) apshstackfile = spyder.fileFilter(apshstackfile) mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) mySpider.toSpider("RT SQ", origstackfile+"@*****", #stack and particle number apshstackfile+"@*****", #output stack and particle "("+str(rotation)+")", #rotation "("+str(Xshift)+","+str(Yshift)+")", #shift parameters ) mySpider.close() return
def calcFSC(vol1,vol2,fscfile,dataext=".spi",inMySpi=False): """ Calculate the differential 3-D phase residual and the Fourier Shell Correlation between two volumes. The Differential Phase Residual over a shell with thickness given by shell width and the Fourier Shell Correlation between shells of specified widths are computed and stored in the document file. """ if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext,logo=False,log=False) else: mySpi=inMySpi apFile.removeFile(fscfile) v1=spyder.fileFilter(vol1) v2=spyder.fileFilter(vol2) fsc=spyder.fileFilter(fscfile) mySpi.toSpiderQuiet("RF 3",v1,v2,"1","0.5,1.5","C","90","3",fsc) if inMySpi is False: mySpi.close()
def calcFSC(vol1, vol2, fscfile, dataext=".spi", inMySpi=False): """ Calculate the differential 3-D phase residual and the Fourier Shell Correlation between two volumes. The Differential Phase Residual over a shell with thickness given by shell width and the Fourier Shell Correlation between shells of specified widths are computed and stored in the document file. """ if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False) else: mySpi = inMySpi apFile.removeFile(fscfile) v1 = spyder.fileFilter(vol1) v2 = spyder.fileFilter(vol2) fsc = spyder.fileFilter(fscfile) mySpi.toSpiderQuiet("RF 3", v1, v2, "1", "0.5,1.5", "C", "90", "3", fsc) if inMySpi is False: mySpi.close()
def spiderVOEA(incr, ang, fold=1.0, dataext=".spi"): apFile.removeFile(ang) mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=False) mySpider.toSpiderQuiet( "VO EA", str(incr), "(0,90.0)", "(0,%.1f)" % ((360.0 / fold) - 0.1), spyder.fileFilter(ang), ) mySpider.close()
def createProjections(incr, boxsz, symfold, invol, rad, sel="selvoea.spi", ang="angvoea.spi", projs="proj.spi", nproc=1, dataext=".spi"): # create doc file containing euler angles spiderVOEA(incr, ang, symfold) projangles = readDocFile(ang) numprojs = len(projangles) mySpi = spyder.SpiderSession(nproc=nproc, dataext=dataext, logo=False, log=True) apFile.removeFile(sel) mySpi.toSpiderQuiet( "DOC CREATE", spyder.fileFilter(sel), "1", "1-" + str(numprojs), ) apFile.removeFile(projs) mySpi.toSpiderQuiet( "PJ 3Q", spyder.fileFilter(invol), str(rad), spyder.fileFilter(sel), spyder.fileFilter(ang), spyder.fileFilter(projs) + "@*****", ) mySpi.close() return projs, numprojs, ang, sel
def rotAndShiftVol(invol, outvol, rot=(0, 0, 0), center=(0, 0, 0), shift=(0.0, 0.0, 0.0), dataext=".spi", inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=True) else: mySpi = inMySpi vol1 = spyder.fileFilter(invol) tmpvol = spyder.fileFilter('temp') vol3 = spyder.fileFilter(outvol) cleanlist = [] invol = vol1 if rot != (0, 0, 0): outvol = tmpvol rot = tuple(map((lambda x: float(x)), rot)) rotstr = '%.2f,%.2f,%.2f' % rot center = tuple(map((lambda x: float(x)), center)) centerstr = '%.1f,%.1f,%.1f' % center mySpi.toSpider("RT 3A", invol, outvol, rotstr, centerstr) cleanlist.append(invol) invol = tmpvol if shift != (0, 0, 0): outvol = vol3 shift = tuple(map((lambda x: float(x)), shift)) shiftstr = '%f.1,%f.1,%f.1' % shift mySpi.toSpider("SH", invol, outvol, shiftstr) cleanlist.append(invol) invol = outvol if outvol != vol3: mySpi.toSpider("CP", invol, vol3) if inMySpi is False: mySpi.close() for cleanfile in cleanlist: os.remove(cleanfile + '.spi')
def maskImg(inimg, outimg, outrad, cutoff, background="P", inrad="0", center="0", dataext=".spi", inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False) else: mySpi = inMySpi img1 = spyder.fileFilter(inimg) img2 = spyder.fileFilter(outimg) mySpi.toSpiderQuiet("MA", img1, img2, str(outrad) + "," + str(inrad), cutoff, background) if background == "E": mySpi.toSpiderQuiet("0", str(center) + "," + str(center)) if cutoff == "C" or cutoff == "G": mySpi.toSpiderQuiet("3.5") if inMySpi is False: mySpi.close()
def findThreshold(numclasses, dendrogramdocfile, rundir, dataext): if not os.path.isfile(dendrogramdocfile): apDisplay.printError("dendrogram doc file does not exist") ### determining threshold cutoff for number of classes minthresh = 0.0 maxthresh = 1.0 minclass = 0.0 maxclass = 1.0 classes = 0 count = 0 ### changes for later versions of SPIDER: sp = spyder.SpiderSession() if sp.version() >= 18.03: maxthresh = 100.0 numclasses += 1 sp.close() sys.stderr.write("finding threshold") while (classes != numclasses and count < 50): count += 1 if count % 70 == 0: sys.stderr.write("\n[" + str(minclass) + "->" + str(minclass) + "]") thresh = (maxthresh - minthresh) / 3.0 + minthresh classfile = rundir + "/classes" apFile.removeFile(classfile + dataext) mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True) mySpider.toSpiderQuiet( "CL HD", thresh, #threshold spyder.fileFilter(dendrogramdocfile), # dendrogram doc file classfile) mySpider.close() claf = open(classfile + dataext, "r") classes = len(claf.readlines()) - 1 claf.close() if classes > numclasses: minthresh = thresh maxclass = classes sys.stderr.write(">") elif classes < numclasses: maxthresh = thresh minclass = classes sys.stderr.write("<") #print " ",count, classes, thresh, maxthresh, minthresh print count, "rounds for", classes, "classes" return thresh, classes
def mirrorParticles(self, partdatas, spiderstack): partnum = 0 mySpider = spyder.SpiderSession(dataext=".spi", logo=False, log=False) for stackpartdata in partdatas: partnum += 1 inplane, mirror = self.getParticleInPlaneRotation(stackpartdata) if mirror is True: sys.stderr.write("m") mySpider.toSpiderQuiet( "MR", spyder.fileFilter(spiderstack) + ("@%05d" % (partnum)), "_9", "Y", ) mySpider.toSpiderQuiet( "CP", "_9", spyder.fileFilter(spiderstack) + ("@%05d" % (partnum)), ) else: sys.stderr.write(".") sys.stderr.write("\n") mySpider.close()
def butterworthFscLP(volfile, fscout, dataext=".spi"): """ inputs: volume fsc output file outputs: volume """ ### setup starttime = time.time() volfile = spyder.fileFilter(volfile) if not os.path.isfile(volfile+dataext): apDisplay.printError("volume file not found: "+volfile+dataext) if not os.path.isfile(fscout): apDisplay.printError("fsc output file not found: "+fscout+dataext) fscfile = open(fscout, "r") prevValue = None for line in fscfile.readlines(): value = line.split() try: int(value[0]) except: #apDisplay.printMsg(line) continue if float(value[4]) < 0.5: if prevValue is None: passband = float(value[2]) stopband = float(value[2])+0.1 else: passband = prevValue stopband = prevValue + 0.1 break prevValue = float(value[2]) fscfile.close() mySpider = spyder.SpiderSession(dataext=dataext, logo=True) mySpider.toSpider("FQ", volfile, #filename for volume volfile+"_filtered", #filename for output volume "(7)", "(%.5f,%.5f)" % (passband, stopband), #pass-band and stop-band ) mySpider.close() apDisplay.printColor("finished filtering the volume "+apDisplay.timeString(time.time()-starttime), "cyan") return
def createMask(maskfile, maskrad, boxsize, dataext=".spi"): """ We should use imagefun.filled_circle() instead """ apDisplay.printMsg("Creating mask with diameter %.1f and boxsize %d"%(maskrad*2.0,boxsize)) mySpider = spyder.SpiderSession(dataext=dataext, logo=False) mySpider.toSpiderQuiet("MO", spyder.fileFilter(maskfile), "%d,%d" % (boxsize, boxsize), "C", str(maskrad), ) mySpider.close() if not os.path.isfile(maskfile) or apFile.fileSize(maskfile) < 2: apDisplay.printError("Failed to create mask file") return
def createGaussianSphere(self, volfile, boxsize): stdev = boxsize/5.0 randdev = boxsize/20.0 halfbox = boxsize/2.0 #width of Gaussian gaussstr = ("%.3f,%.3f,%.3f" %(stdev+randdev*random.random(), stdev+randdev*random.random(), stdev+randdev*random.random())) apDisplay.printMsg("Creating Gaussian volume with stdev="+gaussstr) mySpider = spyder.SpiderSession(logo=False) mySpider.toSpiderQuiet("MO 3", spyder.fileFilter(volfile), "%d,%d,%d"%(boxsize,boxsize,boxsize), "G", #G for Gaussian "%.3f,%.3f,%.3f"%(halfbox,halfbox,halfbox), #center of Gaussian gaussstr, ) mySpider.close() return
def correspondenceAnalysis(alignedstack, boxsize, maskpixrad, numpart, numfactors=8, dataext=".spi"): """ inputs: aligned stack search params outputs: eigen images eigen vectors coran parameters """ ### setup if dataext in alignedstack: alignedstack = alignedstack[:-4] t0 = time.time() rundir = "coran" apParam.createDirectory(rundir) ### make template in memory mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) mySpider.toSpiderQuiet("MO", "_9", "%d,%d" % (boxsize, boxsize), "C", str(maskpixrad * 2.0)) ### performing correspondence analysis apDisplay.printMsg("Performing correspondence analysis (long wait)") mySpider.toSpider("CA S", spyder.fileFilter(alignedstack) + "@******", "1-" + str(numpart), "_9", str(numfactors), "C", "10", rundir + "/corandata") mySpider.close() contriblist = analyzeEigenFactors(alignedstack, rundir, numpart, numfactors, dataext) td1 = time.time() - t0 apDisplay.printMsg("completed correspondence analysis of " + str(numpart) + " particles in " + apDisplay.timeString(td1)) return contriblist
def findRotation(self, avgimg): # use radon transform in SPIDER to find rotation to orient the average image vertical if os.path.isfile("angle.spi"): os.remove("angle.spi") box = self.alignstackdata['boxsize'] mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False) # circular mask the average mySpi.toSpider("MA", spyder.fileFilter(avgimg) + "@1", "_1", "%i,0" % ((box / 2) - 2), "C", "E", "0", "%i,%i" % ((box / 2) + 1, (box / 2) + 1), "3") # get power spectrum mySpi.toSpider("PW", "_1", "_2") # Radon transform mySpi.toSpider("RM 2DN", "_2", "1", "_3", "%i" % box, "%i" % (box / 2), "0,0", "N") # mask in the X direction to only include equator mySpi.toSpider("MA X", "_3", "_4", "6,0", "D", "E", "0", "%i,%i" % ((box / 2), (box / 2))) # find peak mySpi.toSpider("PK x20,x21,x22", "_4", "1,0") # save the angles to a file mySpi.toSpider("SD 1, x21", "angle") mySpi.toSpider("SD E", "angle") mySpi.close() f = open("angle.spi") for line in f: d = line.strip().split() if d[0][0] == ";" or len(d) < 3: continue rot = float(d[2]) os.remove("angle.spi") os.remove(avgimg) return rot
def createBoxMask(maskfile, boxsize, maskx, masky, falloff, imask=None, dataext=".spi"): """ creates a rectangular mask with soft falloff """ apDisplay.printMsg("Creating %i x %i box mask" % (boxsize, boxsize)) mySpi = spyder.SpiderSession(dataext=dataext, logo=False, nproc=1, log=False) # create blank image for mask mySpi.toSpiderQuiet("BL", "_1", "%i,%i" % (boxsize, boxsize), "N", "1") # mask it in X mySpi.toSpiderQuiet("MA X", "_1", "_2", "%i" % maskx, "C", "E", "0", "%i,%i" % (boxsize / 2, boxsize / 2), "%.2f" % falloff) # inner mask in X if imask is not None: mySpi.toSpiderQuiet("MA X", "_2", "_3", "%i,%i" % (boxsize / 2, imask), "C", "E", "0", "%i,%i" % (boxsize / 2, boxsize / 2), "%.2f" % (falloff / 4)) mySpi.toSpiderQuiet("CP", "_3", "_2") # mask in Y mySpi.toSpiderQuiet("MA Y", "_2", spyder.fileFilter(maskfile), "%i" % masky, "C", "E", "0", "%i,%i" % (boxsize / 2, boxsize / 2), "%.2f" % falloff) mySpi.close() if not os.path.isfile(maskfile) or apFile.fileSize(maskfile) < 2: apDisplay.printError("Failed to create mask file") return
def butterworthLP(volfile, pixelsize, dataext=".spi"): """ inputs: volume outputs: volume """ ### setup starttime = time.time() volfile = spyder.fileFilter(volfile) if not os.path.isfile(volfile+dataext): apDisplay.printError("volume file not found: "+volfile+dataext) mySpider = spyder.SpiderSession(dataext=dataext, logo=True) mySpider.toSpider("FQ", volfile, #filename for volume "_1", # volfile+"_filtered", #filename for output volume "(7)", "(%.5f,%.5f)" % (pixelsize/25,pixelsize/15), #pass-band and stop-band ) mySpider.close() apDisplay.printColor("finished filtering the volume "+apDisplay.timeString(time.time()-starttime), "cyan") return
def spiderAPMQ(projs, numprojs, tsearch, tstep, lastRing, stackfile, nump, ang, firstRing=1, startp=1, apmqfile="apmq.spi", nproc=1, outang="angular.spi", dataext=".spi"): apFile.removeFile(apmqfile) mySpi = spyder.SpiderSession(nproc=nproc, dataext=dataext, logo=False, log=True) mySpi.toSpiderQuiet( "AP MQ", spyder.fileFilter(projs) + "@*****", "1-" + str(numprojs), str(tsearch) + "," + str(tstep), str(firstRing) + "," + str(lastRing), spyder.fileFilter(stackfile) + "@******", str(startp) + "-" + str(nump), spyder.fileFilter(apmqfile), ) apFile.removeFile(outang) mySpi.toSpiderQuiet( "VO MD", spyder.fileFilter(ang), spyder.fileFilter(apmqfile), spyder.fileFilter(outang), ) mySpi.close() return outang
def boxMask(self, infile, outfile, spirots=None): from appionlib.apSpider import operations # boxmask the particles apDisplay.printMsg("masking the particles with a rectangular box") nump = apStack.getNumberStackParticlesFromId(self.params['stackid']) box = self.stackdata['boxsize'] apix = self.stackdata['pixelsize'] * 1e10 if self.params['mask'] is None: mask = box / 2 - 2 else: mask = int(self.params['mask'] / apix) imask = int(self.params['imask'] / apix) length = int(self.params['length'] / apix) falloff = self.params['falloff'] / apix mask -= falloff / 2 length = (length / 2) - (falloff / 2) # create blank image for mask maskfile = "boxmask.spi" operations.createBoxMask(maskfile, box, mask, length, falloff, imask) mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False) mySpi.toSpiderQuiet("CP", spyder.fileFilter(maskfile), "_4") mySpi.toSpider("do x10=1,%i" % nump) if self.params['vertical'] is not True: mySpi.toSpider("UD IC x10,x30", spyder.fileFilter(spirots), "x30 = -1*x30", "RT B", "_4", "_9", "(x30)", "(0)", "MU", spyder.fileFilter(infile) + "@{******x10}", "_9") else: mySpi.toSpider("MU", spyder.fileFilter(infile) + "@{******x10}", "_4") mySpi.toSpider( spyder.fileFilter(outfile) + "@{******x10}", "*", "enddo") if self.params['vertical'] is not True: mySpi.toSpider("UD ICE", spyder.fileFilter(spirots)) mySpi.close()
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"): """ inputs: aligned particle stack number of classes timestamp output directory output: class averages class variances dendrogram.png """ if timestamp is None: timestamp = apParam.makeTimestamp() classavg = rundir + "/" + ("classavgstack_%s_%03d" % (timestamp, numclasses)) classvar = rundir + "/" + ("classvarstack_%s_%03d" % (timestamp, numclasses)) thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext) ### create class doc files mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True) mySpider.toSpider( "CL HE", thresh, spyder.fileFilter(dendrogramfile), # dendrogram doc file rundir + "/classdoc_" + timestamp + "_****", # class doc file ) ### delete existing files sys.stderr.write("delete existing files") for dext in (".hed", ".img", dataext): apFile.removeFile(classavg + dext) apFile.removeFile(classvar + dext) print "" ### create class averages sys.stderr.write("create class averages") for i in range(classes): sys.stderr.write(".") classnum = i + 1 mySpider.toSpiderQuiet( "AS R", spyder.fileFilter(alignedstack) + "@******", rundir + ("/classdoc_" + timestamp + "_%04d" % (classnum)), "A", (classavg + "@%04d" % (classnum)), (classvar + "@%04d" % (classnum)), ) mySpider.close() print "" ### convert to IMAGIC emancmd = "proc2d " + classavg + ".spi " + classavg + ".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) emancmd = "proc2d " + classvar + ".spi " + classvar + ".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) return classavg, classvar
def createAmpcorBatchFile(infile, params): localinfile = spyder.fileFilter(infile) appiondir = apParam.getAppionDirectory() scriptfile = os.path.join(appiondir, "appionlib/data/enhance.bat") pwscfile = os.path.join(appiondir, "appionlib/data/pwsc.bat") if not os.path.isfile(scriptfile): apDisplay.printError("could not find spider script: " + scriptfile) if not os.path.isfile(pwscfile): apDisplay.printError("could not find spider script: " + pwscfile) inf = open(scriptfile, "r") tmpfile = "out_" + os.path.basename(infile) tmpfile = os.path.join(params['rundir'], tmpfile) localtmpfile = spyder.fileFilter(tmpfile) outfile = "enhance_edit.bat" if os.path.isfile(outfile): apDisplay.printWarning(outfile + " already exists; removing it") time.sleep(2) os.remove(outfile) if os.path.isfile('pwsc.bat'): apDisplay.printWarning(pwscfile + " already exists; replacing it") time.sleep(2) shutil.copyfile(pwscfile, 'pwsc.bat') outf = open(outfile, "w") notdone = True for line in inf: if notdone is False: outf.write(line) else: thr = line[:3] if thr == "x99": outf.write(spiderline(99, params['box'], "box size in pixels")) elif thr == "x98": outf.write( spiderline(98, params['maxfilt'], "filter limit in angstroms")) elif thr == "x80": outf.write(spiderline(80, params['apix'], "pixel size")) elif re.search("^\[vol\]", line): outf.write(spiderline("vol", localinfile, "input density file")) elif re.search("^\[outvol\]", line): outf.write( spiderline("outvol", localtmpfile, "enhanced output file")) elif re.search("^\[scatter\]", line): outf.write( spiderline("scatter", params['ampfile'], "amplitude curve file")) elif re.search("^\[pwsc\]", line): outf.write(spiderline("pwsc", "pwsc", "scaling script")) elif re.search("^\[applyfen\]", line): outf.write( spiderline( "applyfen", os.path.join(appiondir, "appionlib/data/applyfen"), "apply curve to data script")) else: outf.write(line) return tmpfile
def setupBatchFile(self, spiderstack, templatestack, orientref): """ sets up Ed's batch script to run """ ### write particle selection file partsel = os.path.join(self.params['rundir'], "partlist.spi") f = open(partsel, "w") for i in range(self.params['numpart']): f.write("%06d 1 %06d \n" % (i + 1, i + 1)) f.close() refsel = os.path.join(self.params['rundir'], "reflist.spi") f = open(refsel, "w") for i in range(self.params['numtemplate']): f.write("%03d 1 %03d \n" % (i + 1, i + 1)) f.close() ### read / write batch file globalbatch = os.path.join(apParam.getAppionDirectory(), "spiderbatch/IterativeClassifyAlign.spi") localbatch = os.path.join(self.params['rundir'], "IterativeClassifyAlign.spi") gf = open(globalbatch, "r") lf = open(localbatch, "w") modify = True for line in gf: if modify is True: #flag for line is in header ### Input files ### if re.match("\<pcltmpl\>", line): ### template for spider particle names lf.write("<pcltmpl>" + spyder.fileFilter(spiderstack) + "@****** \n") elif re.match("\<pcllist\>", line): ### list of particle numbers lf.write("<pcllist>" + spyder.fileFilter(partsel) + "\n") elif re.match("\<reftmpl\>", line): ### template for reference image names lf.write("<reftmpl>" + spyder.fileFilter(templatestack) + "@*** \n") elif re.match("\<reflist\>", line): ### list of reference numbers lf.write("<reflist>" + spyder.fileFilter(refsel) + "\n") elif re.match("\<ref\>", line): ### orientation reference lf.write("<ref>" + spyder.fileFilter(orientref) + "\n") ### Parameters ### elif re.match("23 ;pixels", line): ### particle radius in pixels pixrad = int(self.params['partrad'] / self.stack['apix'] / self.params['bin']) lf.write("%d ;pixels\n" % (pixrad)) elif re.match("10 ;iterations", line): ### number of ref-based iterations lf.write("%d ;iterations\n" % (self.params['numiter'])) #elif re.match("1 ;free-align rounds", line): ### number of ref-free alignments per class #lf.write("%d ;free-align rounds\n"%(self.params['freealigns'])) elif re.match("0 ;processors", line): ### number of processors lf.write("%d ;processors\n" % (self.params['nproc'])) elif re.match("\<dir\>", line): ### sub-directory, we use "." lf.write("<dir>.\n") elif re.search("- END BATCH HEADER -", line): ### reached last line of batch header ### modify = False else: lf.write(line) else: lf.write(line) return localbatch
def crossCorrelateAndShift(infile, reffile, alignfile, ccdocfile, numpart, dataext=".spi"): ### rewriten to do the whole thing in memory in SPIDER, it should be faster starttime = time.time() infile = spyder.fileFilter(infile) reffile = spyder.fileFilter(reffile) alignfile = spyder.fileFilter(alignfile) partimg = "_4" ccmap = "_5" windccmap = "_6" boxsize = apFile.getBoxSize(infile+dataext) if not os.path.isfile(infile+dataext): apDisplay.printError("input stack file not found: "+infile+dataext) if not os.path.isfile(reffile+dataext): apDisplay.printError("reference stack file not found: "+reffile+dataext) nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) ### Allocate empty stack mySpider.toSpiderQuiet( "MS I", #command "_2@", #name "%d,%d,%d"%(boxsize), #boxsize str(numpart+1), #num part to create in memory str(numpart+1), #max particle number ) partnum = 0 while partnum < numpart: partnum+=1 mySpider.toSpiderQuiet("CP", infile+("@%05d"%(partnum)), #picture partimg, ) ### cross correlate images; reversed order to avoid -1*shift mySpider.toSpiderQuiet("CC N", reffile+("@%05d"%(partnum)), #reference partimg, #picture ccmap, #output file ) ### cannot shift more the 1/4 size of the image mySpider.toSpiderQuiet("FI x52", partimg, "12" ) mySpider.toSpiderQuiet("x54=int(x52/2)") #window size mySpider.toSpiderQuiet("x55=int(x52/4)") #window topleft mySpider.toSpiderQuiet("WI", ccmap, #input file windccmap, #output file "x54,x54", #window size "x55,x55", #window origin ) ### find the cross-correlation peak mySpider.toSpiderQuiet("x56=int(x52/4)+1") #center of window mySpider.toSpiderQuiet("PK M x11,x12,x13,x14", windccmap, #input ccmap file "x56,x56", #origin coordinates ) ### save info to doc file mySpider.toSpiderQuiet("SD %d,x13,x14"%(partnum), ccdocfile, #input ccmap file ) ### shift the images images mySpider.toSpiderQuiet("SH", partimg, #old stack ("_2@%05d"%(partnum)), #new stack "x13,x14", #shift value file ) ### finish up #save stack to file mySpider.toSpiderQuiet( "CP", "_2@", alignfile+"@", ) #delete stack mySpider.toSpiderQuiet( "DE", "_2", ) mySpider.close() apDisplay.printColor("finished shifting particles in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def createClassAverages(stack, projs, apmq, numprojs, boxsz, outclass="classes", rotated=False, shifted=False, dataext=".spi"): """ creates EMAN-style class average file "classes.hed" & "classes.img" and variance files "variances.hed" & "variances.img" from spider classification """ apFile.removeFile(outclass) outf = spyder.fileFilter(outclass) outvf = "tmpvar" apmqlist = readDocFile(apmq) mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False) # create file containing the number of particles matched to each projection mySpi.toSpiderQuiet( "VO MQ", "0.0", spyder.fileFilter(apmq), str(numprojs), "cls*****", "numinclass", ) mySpi.close() os.remove("numinclass%s" % dataext) # create class average file mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False) for i in range(0, numprojs): clsfile = readDocFile("cls%05d%s" % (i + 1, dataext)) if clsfile == []: apDisplay.printMsg("class %d has no particles" % (i + 1)) mySpi.toSpiderQuiet( "BL", "%s@%d" % (outf, (i + 1)), "(%d,%d)" % (boxsz, boxsz), "N", "(0.0)", ) mySpi.toSpiderQuiet( "BL", "%s@%d" % (outvf, (i + 1)), "(%d,%d)" % (boxsz, boxsz), "N", "(0.0)", ) else: mySpi.toSpiderQuiet("DE", "_2@") mySpi.toSpiderQuiet("MS", "_2@", "%d,%d,1" % (boxsz, boxsz), str(len(clsfile))) for p in range(0, len(clsfile)): mySpi.toSpiderQuiet("DE", "_1") # get the particle part = int(float(clsfile[p][2])) - 1 pimg = spyder.fileFilter(stack) + "@%d" % (part + 1) rot = float(apmqlist[part][4]) shx = float(apmqlist[part][5]) shy = float(apmqlist[part][6]) if shifted is True: shx = 0 shy = 0 if rotated is True: rot = 0 p_out = "_2@%d" % (p + 1) rotAndShiftImg(pimg, "_1", rot, shx, shy, inMySpi=mySpi) # mirror if int(float(apmqlist[part][2])) < 0: mirrorImg("_1", p_out, inMySpi=mySpi) else: mySpi.toSpiderQuiet("CP", "_1", p_out) if len(clsfile) == 1: mySpi.toSpiderQuiet("CP", "_2@1", "%s@%d" % (outf, (i + 1))) mySpi.toSpiderQuiet( "BL", "%s@%d" % (outvf, (i + 1)), "(%d,%d)" % (boxsz, boxsz), "N", "(0.0)", ) else: mySpi.toSpiderQuiet("AS R", "_2@*****", "1-%d" % len(clsfile), "A", "%s@%d" % (outf, (i + 1)), "%s@%d" % (outvf, i + 1)) mySpi.close() # convert the class averages to EMAN class averages # read classes & projections for EMAN classes output if os.path.exists('variances.hed'): os.remove('variances.hed') if os.path.exists('variances.img'): os.remove('variances.img') if os.path.exists('classes.hed'): os.remove('classes.hed') if os.path.exists('classes.img'): os.remove('classes.img') ### I would prefer to use apImagicFile.readImagic, writeImagic variances = EMAN.readImages(outvf + dataext, -1, -1, 0) averages = EMAN.readImages(outf + dataext, -1, -1, 0) projections = EMAN.readImages(projs, -1, -1, 0) for i in range(0, numprojs): # copy projection to class average file projections[i].writeImage('classes.hed') projections[i].writeImage('variances.hed') # get num of particles in class clsfile = readDocFile("cls%05d%s" % (i + 1, dataext)) averages[i].setNImg(len(clsfile)) averages[i].writeImage('classes.hed', -1) variances[i].setNImg(len(clsfile)) variances[i].writeImage('variances.hed', -1) os.remove("cls%05d%s" % (i + 1, dataext)) os.remove(outf + dataext) return
def alignStack(oldstack, alignedstack, partlist, dataext=".spi"): """ write aligned stack -- with python loop inputs: oldstack newstack (empty) list of particle dictionaries for operations modifies: newstack output: none I tried this loop in both spider and python; python was faster?!? -neil """ if not os.path.isfile(oldstack + dataext): apDisplay.printError("Could not find original stack: " + oldstack + dataext) boxsize = apFile.getBoxSize(oldstack + dataext) apDisplay.printMsg("applying alignment parameters to stack") apFile.removeFile(alignedstack + dataext) count = 0 t0 = time.time() nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) #create stack in core numpart = len(partlist) mySpider.toSpiderQuiet( "MS I", #command "_2@", #name "%d,%d,%d" % (boxsize), #boxsize str(numpart + 1), #num part to create in memory str(numpart + 1), #max particle number ) for partdict in partlist: partnum = partdict['num'] #if partdict['num'] in [3,6,7]: # print partdict['num'], partdict['template'], partdict['mirror'], round(partdict['rot'],3) ### Rotate and Shift operations count += 1 #rotate/shift mySpider.toSpiderQuiet( "RT SQ", spyder.fileFilter(oldstack) + "@" + ("%06d" % (partnum)), "_1", str(partdict['rot']), str(partdict['xshift']) + "," + str(partdict['yshift']), ) #mirror, if necessary if 'mirror' in partdict and partdict['mirror'] is True: mySpider.toSpiderQuiet( "MR", "_1", "_2@" + ("%06d" % (partnum)), "Y", ) else: mySpider.toSpiderQuiet( "CP", "_1", "_2@" + ("%06d" % (partnum)), ) ### finish up #save stack to file mySpider.toSpiderQuiet( "CP", "_2@", spyder.fileFilter(alignedstack) + "@", ) #delete stack mySpider.toSpiderQuiet( "DE", "_2", ) mySpider.close() apDisplay.printMsg("Completed transforming %d particles in %s" % (count, apDisplay.timeString(time.time() - t0))) if count < 1: apDisplay.printError("Failed to transform any particles") if not os.path.isfile(alignedstack + dataext): apDisplay.printError("Failed to create stack " + alignedstack + dataext) return
def kmeansCluster(alignedstack, numpart=None, numclasses=40, timestamp=None, factorlist=range(1, 5), corandata="coran/corandata", dataext=".spi"): """ inputs: outputs: """ if timestamp is None: timestamp = apParam.makeTimestamp() if alignedstack[-4:] == dataext: alignedstack = alignedstack[:-4] rundir = "cluster" classavg = rundir + "/" + ("classavgstack_%s_%03d" % (timestamp, numclasses)) classvar = rundir + "/" + ("classvarstack_%s_%03d" % (timestamp, numclasses)) apParam.createDirectory(rundir) for i in range(numclasses): apFile.removeFile(rundir + ("/classdoc%04d" % (i + 1)) + dataext) apFile.removeFile(rundir + ("/allclassesdoc%04d" % (numclasses)) + dataext) ### make list of factors factorstr, factorkey = operations.intListToString(factorlist) ### do k-means clustering mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) mySpider.toSpider( "CL KM", corandata + "_IMC", # path to coran data str(numclasses), # num classes factorstr, # factor string ) ## weight for each factor for fact in factorlist: mySpider.toSpiderQuiet("1.0") randnum = (int(random.random() * 1000) + 1) mySpider.toSpider( str(randnum), rundir + "/classdoc_" + timestamp + "_****", # class doc file rundir + ("/allclassesdoc%04d" % (numclasses)), #clusterdoc file ) mySpider.close() ### delete existing files sys.stderr.write("delete existing files") for dext in (".hed", ".img", dataext): apFile.removeFile(classavg + dext) apFile.removeFile(classvar + dext) print "" mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) ### create class averages apDisplay.printMsg("Averaging particles into classes") for i in range(numclasses): classnum = i + 1 mySpider.toSpiderQuiet( "AS R", spyder.fileFilter(alignedstack) + "@******", rundir + ("/classdoc_" + timestamp + "_%04d" % (classnum)), "A", (classavg + "@%04d" % (classnum)), (classvar + "@%04d" % (classnum)), ) if classnum % 10 == 0: sys.stderr.write(".") time.sleep(1) mySpider.close() ### convert to IMAGIC emancmd = "proc2d " + classavg + ".spi " + classavg + ".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) emancmd = "proc2d " + classvar + ".spi " + classvar + ".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) return classavg, classvar
def alignAPSH(volfile, origstackfile, eulerdocfile, classnum, boxsize, numpart, pixrad, timestamp, iternum, dataext=".spi"): apshAngularFile = str(classnum)+"/"+"apshAngularFile-%03d.spi"%(iternum) if os.path.isfile(apshAngularFile): os.remove(apshAngularFile) apDisplay.printColor("File exist! Removing file "+apshAngularFile, "cyan") apshAngularFile = spyder.fileFilter(apshAngularFile) apshListFile = str(classnum)+"/"+"apshListFile-%03d.spi"%(iternum) if os.path.isfile(apshListFile): os.remove(apshListFile) apDisplay.printColor("File exist! Removing file "+apshListFile, "cyan") apshListFile = spyder.fileFilter(apshListFile) apshOutFile = str(classnum)+"/"+"apshOut-%03d.spi"%(iternum) if os.path.isfile(apshOutFile): os.remove(apshOutFile) apDisplay.printColor("File exist! Removing file "+apshOutFile, "cyan") apshOut = spyder.fileFilter(apshOutFile) apshProjStack = str(classnum)+"/"+"apshProjStack-%03d.spi"%(iternum) if os.path.isfile(apshProjStack): os.remove(apshProjStack) apDisplay.printColor("File exist! Removing file "+apshProjStack, "cyan") apshProjStack = spyder.fileFilter(apshProjStack) origstackfile = spyder.fileFilter(origstackfile) volfile = spyder.fileFilter(volfile) origeulerdocfile = spyder.fileFilter(eulerdocfile) mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) mySpider.toSpider("VO EA,x53", "10", #angular increment "0,90.0", #Range of Theta "0,359.9", #Range of Phi apshAngularFile, #Angular file name ) mySpider.toSpider("DOC CREATE", apshListFile, #List file name "1", "1-x53" ) #mySpider.toSpider("MS", # "_9@", #reference projections # str(boxsize),str(boxsize),"1", #boxsize # "x53", #) mySpider.toSpider("PJ 3Q", volfile, #input vol file str(pixrad), #pixel radius apshListFile, #number of particles apshAngularFile, #Euler DOC file apshProjStack+"@*****", #output projections ) mySpider.toSpider("AP SH", apshProjStack+"@*****", #reference projections "1-x53", #reference numbers "6,2", #translational search range, step size "1,%d"%(boxsize/3), #first and last ring apshAngularFile, origstackfile+"@*****", "1-%d"%(numpart), origeulerdocfile, "20", #"(0.0)", # no restrictions for alignment "1", # mirror check? apshOut, # output ) mySpider.close() return apshOutFile