def calcFSC(vol1, vol2, fscfile, dataext=".spi", inMySpi=False): """ Calculate the differential 3-D phase residual and the Fourier Shell Correlation between two volumes. The Differential Phase Residual over a shell with thickness given by shell width and the Fourier Shell Correlation between shells of specified widths are computed and stored in the document file. """ if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False) else: mySpi = inMySpi apFile.removeFile(fscfile) v1 = spyder.fileFilter(vol1) v2 = spyder.fileFilter(vol2) fsc = spyder.fileFilter(fscfile) mySpi.toSpiderQuiet("RF 3", v1, v2, "1", "0.5,1.5", "C", "90", "3", fsc) if inMySpi is False: mySpi.close()
def rotshiftStack(origstackfile, rotShiftFile, timestamp, iternum, classnum=None, dataext=".spi"): origstackfile = spyder.fileFilter(origstackfile) if classnum is not None: apshstackfile = str(classnum)+"/"+"apshstack%s.spi"%(timestamp) else: apshstackfile = "apshstack%s.spi"%(timestamp) apshstackfile = spyder.fileFilter(apshstackfile) mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) mySpider.toSpider("RT SQ", origstackfile+"@*****", #stack and particle number apshstackfile+"@*****", #output stack and particle "("+str(rotation)+")", #rotation "("+str(Xshift)+","+str(Yshift)+")", #shift parameters ) mySpider.close() return
def spiderAPMQ(projs, numprojs, tsearch, tstep, lastRing, stackfile, nump, ang, firstRing=1, startp=1, apmqfile="apmq.spi", nproc=1, outang="angular.spi", dataext=".spi"): apFile.removeFile(apmqfile) mySpi = spyder.SpiderSession(nproc=nproc, dataext=dataext, logo=False, log=True) mySpi.toSpiderQuiet( "AP MQ", spyder.fileFilter(projs) + "@*****", "1-" + str(numprojs), str(tsearch) + "," + str(tstep), str(firstRing) + "," + str(lastRing), spyder.fileFilter(stackfile) + "@******", str(startp) + "-" + str(nump), spyder.fileFilter(apmqfile), ) apFile.removeFile(outang) mySpi.toSpiderQuiet( "VO MD", spyder.fileFilter(ang), spyder.fileFilter(apmqfile), spyder.fileFilter(outang), ) mySpi.close() return outang
def createProjections(incr, boxsz, symfold, invol, rad, sel="selvoea.spi", ang="angvoea.spi", projs="proj.spi", nproc=1, dataext=".spi"): # create doc file containing euler angles spiderVOEA(incr, ang, symfold) projangles = readDocFile(ang) numprojs = len(projangles) mySpi = spyder.SpiderSession(nproc=nproc, dataext=dataext, logo=False, log=True) apFile.removeFile(sel) mySpi.toSpiderQuiet( "DOC CREATE", spyder.fileFilter(sel), "1", "1-" + str(numprojs), ) apFile.removeFile(projs) mySpi.toSpiderQuiet( "PJ 3Q", spyder.fileFilter(invol), str(rad), spyder.fileFilter(sel), spyder.fileFilter(ang), spyder.fileFilter(projs) + "@*****", ) mySpi.close() return projs, numprojs, ang, sel
def commonLines(stackfile, maskdiam=None, minfreq=0, maxfreq=0.25, ang=5.0, numiter=200, outdocfile=None, numpart=None, dataext=".spi"): """ performs common lines on a input spider stack """ if numpart is None or numpart < 3: apDisplay.printError("undefined number of particles") if maskdiam is None: apDisplay.printError("undefined mask diameter") starttime = time.time() if dataext in stackfile: stackfile = stackfile[:-4] randdocfile = generateRandomAngles(numpart) mySpider = spyder.SpiderSession(dataext=dataext, logo=True) mySpider.toSpider( "OP", stackfile + "@*****", #stack file "1-%d" % (numpart), #number of particles str(maskdiam), #mark diameter "%.3f,%.3f" % (minfreq, maxfreq), #frequency range for line projections str(ang), #angular increment or accuracy str(numiter), #number of iterations randdocfile, #random angles doc file outdocfile, #output angles doc file ) mySpider.close() apDisplay.printColor( "finished common lines in " + apDisplay.timeString(time.time() - starttime), "cyan") return
def createGaussianSphere(self, volfile, boxsize): stdev = boxsize / 5.0 randdev = boxsize / 20.0 halfbox = boxsize / 2.0 #width of Gaussian gaussstr = ( "%.3f,%.3f,%.3f" % (stdev + randdev * random.random(), stdev + randdev * random.random(), stdev + randdev * random.random())) apDisplay.printMsg("Creating Gaussian volume with stdev=" + gaussstr) mySpider = spyder.SpiderSession(logo=False) mySpider.toSpiderQuiet( "MO 3", spyder.fileFilter(volfile), "%d,%d,%d" % (boxsize, boxsize, boxsize), "G", #G for Gaussian "%.3f,%.3f,%.3f" % (halfbox, halfbox, halfbox), #center of Gaussian gaussstr, ) mySpider.close() return
def rotAndShiftVol(invol, outvol, rot=(0, 0, 0), center=(0, 0, 0), shift=(0.0, 0.0, 0.0), dataext=".spi", inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=True) else: mySpi = inMySpi vol1 = spyder.fileFilter(invol) tmpvol = spyder.fileFilter('temp') vol3 = spyder.fileFilter(outvol) cleanlist = [] invol = vol1 if rot != (0, 0, 0): outvol = tmpvol rot = tuple(map((lambda x: float(x)), rot)) rotstr = '%.2f,%.2f,%.2f' % rot center = tuple(map((lambda x: float(x)), center)) centerstr = '%.1f,%.1f,%.1f' % center mySpi.toSpider("RT 3A", invol, outvol, rotstr, centerstr) cleanlist.append(invol) invol = tmpvol if shift != (0, 0, 0): outvol = vol3 shift = tuple(map((lambda x: float(x)), shift)) shiftstr = '%f.1,%f.1,%f.1' % shift mySpi.toSpider("SH", invol, outvol, shiftstr) cleanlist.append(invol) invol = outvol if outvol != vol3: mySpi.toSpider("CP", invol, vol3) if inMySpi is False: mySpi.close() for cleanfile in cleanlist: os.remove(cleanfile + '.spi')
def boxMask(self, infile, outfile, spirots=None): from appionlib.apSpider import operations # boxmask the particles apDisplay.printMsg("masking the particles with a rectangular box") nump = apStack.getNumberStackParticlesFromId(self.params['stackid']) box = self.stackdata['boxsize'] apix = self.stackdata['pixelsize'] * 1e10 if self.params['mask'] is None: mask = box / 2 - 2 else: mask = int(self.params['mask'] / apix) imask = int(self.params['imask'] / apix) length = int(self.params['length'] / apix) falloff = self.params['falloff'] / apix mask -= falloff / 2 length = (length / 2) - (falloff / 2) # create blank image for mask maskfile = "boxmask.spi" operations.createBoxMask(maskfile, box, mask, length, falloff, imask) mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False) mySpi.toSpiderQuiet("CP", spyder.fileFilter(maskfile), "_4") mySpi.toSpider("do x10=1,%i" % nump) if self.params['vertical'] is not True: mySpi.toSpider("UD IC x10,x30", spyder.fileFilter(spirots), "x30 = -1*x30", "RT B", "_4", "_9", "(x30)", "(0)", "MU", spyder.fileFilter(infile) + "@{******x10}", "_9") else: mySpi.toSpider("MU", spyder.fileFilter(infile) + "@{******x10}", "_4") mySpi.toSpider( spyder.fileFilter(outfile) + "@{******x10}", "*", "enddo") if self.params['vertical'] is not True: mySpi.toSpider("UD ICE", spyder.fileFilter(spirots)) mySpi.close()
def ClCla(alignedstack, numpart=None, numclasses=40, factorlist=range(1, 5), corandata="coran/corandata", dataext=".spi"): """ this doesn't work """ if alignedstack[-4:] == dataext: alignedstack = alignedstack[:-4] rundir = "cluster" classavg = rundir + "/" + ("classavgstack%03d" % numclasses) classvar = rundir + "/" + ("classvarstack%03d" % numclasses) apParam.createDirectory(rundir) for i in range(numclasses): apFile.removeFile(rundir + ("/classdoc%04d" % (i + 1)) + dataext) apFile.removeFile(rundir + "/clusterdoc" + dataext) factorstr, factorkey = operations.intListToString(factorlist) ### do hierarchical clustering mySpider = spyder.SpiderSession(dataext=dataext, logo=True) mySpider.toSpider( "CL CLA", corandata, # path to coran data rundir + "/clusterdoc", #clusterdoc file factorstr, #factor numbers "5,8", "4", "2", # minimum number of particles per class "Y", rundir + "/dendrogram.ps", "Y", rundir + "/dendrogramdoc", ) mySpider.close()
def mirrorParticles(self, partdatas, spiderstack): partnum = 0 mySpider = spyder.SpiderSession(dataext=".spi", logo=False, log=False) for stackpartdata in partdatas: partnum += 1 inplane, mirror = self.getParticleInPlaneRotation(stackpartdata) if mirror is True: sys.stderr.write("m") mySpider.toSpiderQuiet( "MR", spyder.fileFilter(spiderstack) + ("@%05d" % (partnum)), "_9", "Y", ) mySpider.toSpiderQuiet( "CP", "_9", spyder.fileFilter(spiderstack) + ("@%05d" % (partnum)), ) else: sys.stderr.write(".") sys.stderr.write("\n") mySpider.close()
def maskImg(inimg, outimg, outrad, cutoff, background="P", inrad="0", center="0", dataext=".spi", inMySpi=False): if inMySpi is False: mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False) else: mySpi = inMySpi img1 = spyder.fileFilter(inimg) img2 = spyder.fileFilter(outimg) mySpi.toSpiderQuiet("MA", img1, img2, str(outrad) + "," + str(inrad), cutoff, background) if background == "E": mySpi.toSpiderQuiet("0", str(center) + "," + str(center)) if cutoff == "C" or cutoff == "G": mySpi.toSpiderQuiet("3.5") if inMySpi is False: mySpi.close()
def correspondenceAnalysis(alignedstack, boxsize, maskpixrad, numpart, numfactors=8, dataext=".spi", nproc=1): """ inputs: aligned stack search params outputs: eigen images eigen vectors coran parameters """ ### setup if dataext in alignedstack: alignedstack = alignedstack[:-4] t0 = time.time() rundir = "coran" apParam.createDirectory(rundir) ### make template in memory mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False, nproc=nproc) mySpider.toSpiderQuiet("MO", "_9", "%d,%d" % (boxsize, boxsize), "C", str(maskpixrad*2.0)) ### performing correspondence analysis apDisplay.printMsg("Performing correspondence analysis (long wait)") mySpider.toSpider( "CA S", spyder.fileFilter(alignedstack)+"@******", "1-"+str(numpart), "_9", str(numfactors), "C", "100", rundir+"/corandata") mySpider.close() contriblist = analyzeEigenFactors(alignedstack, rundir, numpart, numfactors, dataext) td1 = time.time()-t0 apDisplay.printMsg("completed correspondence analysis of "+str(numpart) +" particles in "+apDisplay.timeString(td1)) return contriblist
def findRotation(self, avgimg): # use radon transform in SPIDER to find rotation to orient the average image vertical if os.path.isfile("angle.spi"): os.remove("angle.spi") box = self.alignstackdata['boxsize'] mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False) # circular mask the average mySpi.toSpider("MA", spyder.fileFilter(avgimg) + "@1", "_1", "%i,0" % ((box / 2) - 2), "C", "E", "0", "%i,%i" % ((box / 2) + 1, (box / 2) + 1), "3") # get power spectrum mySpi.toSpider("PW", "_1", "_2") # Radon transform mySpi.toSpider("RM 2DN", "_2", "1", "_3", "%i" % box, "%i" % (box / 2), "0,0", "N") # mask in the X direction to only include equator mySpi.toSpider("MA X", "_3", "_4", "6,0", "D", "E", "0", "%i,%i" % ((box / 2), (box / 2))) # find peak mySpi.toSpider("PK x20,x21,x22", "_4", "1,0") # save the angles to a file mySpi.toSpider("SD 1, x21", "angle") mySpi.toSpider("SD E", "angle") mySpi.close() f = open("angle.spi") for line in f: d = line.strip().split() if d[0][0] == ";" or len(d) < 3: continue rot = float(d[2]) os.remove("angle.spi") os.remove(avgimg) return rot
def createBoxMask(maskfile, boxsize, maskx, masky, falloff, imask=None, dataext=".spi"): """ creates a rectangular mask with soft falloff """ apDisplay.printMsg("Creating %i x %i box mask" % (boxsize, boxsize)) mySpi = spyder.SpiderSession(dataext=dataext, logo=False, nproc=1, log=False) # create blank image for mask mySpi.toSpiderQuiet("BL", "_1", "%i,%i" % (boxsize, boxsize), "N", "1") # mask it in X mySpi.toSpiderQuiet("MA X", "_1", "_2", "%i" % maskx, "C", "E", "0", "%i,%i" % (boxsize / 2, boxsize / 2), "%.2f" % falloff) # inner mask in X if imask is not None: mySpi.toSpiderQuiet("MA X", "_2", "_3", "%i,%i" % (boxsize / 2, imask), "C", "E", "0", "%i,%i" % (boxsize / 2, boxsize / 2), "%.2f" % (falloff / 4)) mySpi.toSpiderQuiet("CP", "_3", "_2") # mask in Y mySpi.toSpiderQuiet("MA Y", "_2", spyder.fileFilter(maskfile), "%i" % masky, "C", "E", "0", "%i,%i" % (boxsize / 2, boxsize / 2), "%.2f" % falloff) mySpi.close() if not os.path.isfile(maskfile) or apFile.fileSize(maskfile) < 2: apDisplay.printError("Failed to create mask file") return
def butterworthLP(volfile, pixelsize, dataext=".spi"): """ inputs: volume outputs: volume """ ### setup starttime = time.time() volfile = spyder.fileFilter(volfile) if not os.path.isfile(volfile+dataext): apDisplay.printError("volume file not found: "+volfile+dataext) mySpider = spyder.SpiderSession(dataext=dataext, logo=True) mySpider.toSpider("FQ", volfile, #filename for volume "_1", # volfile+"_filtered", #filename for output volume "(7)", "(%.5f,%.5f)" % (pixelsize/25,pixelsize/15), #pass-band and stop-band ) mySpider.close() apDisplay.printColor("finished filtering the volume "+apDisplay.timeString(time.time()-starttime), "cyan") return
def makeSpiderCoranBatch(params, filename, clsdir): nfacts = 20 if params['nptcls'] < 21: nfacts = params['nptcls'] - 1 f = open(os.path.join(clsdir, filename), 'w') f.write('MD ; verbose off in spider log file\n') f.write('VB OFF\n') f.write('\n') f.write('x99=%d ; number of particles in stack\n' % params['nptcls']) f.write('x98=%d ; box size\n' % params['boxsize']) f.write('x94=%d ; mask radius\n' % params['coranmask']) haccut = params['haccut'] sp = spyder.SpiderSession() if sp.version() >= 18.03: haccut *= 100 sp.close() f.write('x93=%f ; cutoff for hierarchical clustering\n' % haccut) f.write('x92=20 ; additive constant for hierarchical clustering\n') f.write('\n') alignstack = 'aligned' if params['hp'] is not None: f.write('FR G ; aligned hp stack file\n') f.write('[alignedhp]alignedhp\n') alignstack = 'alignedhp' f.write('FR G ; aligned stack file\n') f.write('[aligned]aligned\n') f.write('\n') f.write(';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n') f.write('\n') f.write('FR G ; where to write class lists\n') f.write('[clhc_cls]classes/clhc_cls\n') f.write('\n') f.write('FR G ; where to write alignment data\n') f.write('[ali]alignment/\n') f.write('\n') f.write('VM\n') f.write('mkdir alignment\n') f.write('\n') f.write( ';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n' ) f.write( ';; create the sequential file and then use that file and do a hierarchical ;;\n' ) f.write( ';; clustering. Run clhd and clhe to classify the particles into different ;;\n' ) f.write( ';; groups. ;;\n' ) f.write( ';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n' ) f.write('\n') f.write('VM\n') f.write('echo Performing multivariate statistical analysis\n') f.write('VM\n') f.write('echo " making template file"\n') f.write('\n') f.write('MO ; make mask template\n') f.write('_9 ; save template in memory\n') f.write('x98,x98 ; box size\n') f.write('c ; circle\n') f.write('x94 ; radius of mask\n') f.write('\n') f.write('VM\n') f.write('echo " doing correspondence analysis"\n') f.write('\n') f.write('CA S ; do correspondence analysis\n') f.write('[%s]@***** ; aligned stack\n' % alignstack) f.write('1-x99 ; particles to use\n') f.write('_9 ; mask file\n') f.write('%d ; number of factors to be used\n' % nfacts) f.write('C ; Coran analysis\n') f.write( 'x92 ; additive constant (since coran cannot have negative values)\n' ) f.write('[ali]coran ; output file prefix\n') f.write('\n') f.write('\n') f.write('DO LB14 x11=1,%d\n' % nfacts) f.write('CA SRE\n') f.write('[ali]coran\n') f.write('x11\n') f.write('[ali]sre@{***x11}\n') f.write('LB14\n') f.write('\n') #f.write('VM\n') #f.write('eigendoc.py alignment/coran_EIG.spi alignment/eigendoc.out 30\n') #f.write('\n') f.write('VM\n') f.write('echo " clustering..."\n') f.write('\n') f.write('CL HC ; do hierarchical clustering\n') f.write('[ali]coran_IMC ; coran image factor coordinate file\n') f.write('1-3\n') f.write( '1.00 ; factor numbers to be included in clustering algorithm\n' ) f.write('1.00 ; factor weights\n') f.write('1.00 ; for each factor number\n') f.write('5 ; use Wards method\n') f.write('Y ; make a postscript of dendogram\n') f.write('[ali]clhc.ps ; dendogram image file\n') f.write('Y ; save dendogram doc file\n') f.write('[ali]clhc_doc ; dendogram doc file\n') f.write('\n') f.write('\n') f.write(';;;determine number of classes for given threshold\n') f.write('CL HD\n') f.write('x93\n') f.write('[ali]clhc_doc\n') f.write('clhc_classes\n') f.write('\n') f.write('UD N,x12\n') f.write('clhc_classes\n') f.write('\n') f.write('VM\n') f.write('mkdir classes\n') f.write('\n') f.write('VM\n') f.write( 'echo "Creating {%F5.1%x12} classes using a threshold of {%F7.5%x93}"\n' ) f.write( 'CL HE ; generate doc files containing particle numbers for classes\n' ) f.write('x93 ; threshold (closer to 0=more classes)\n') f.write('[ali]clhc_doc ; dendogram doc file\n') f.write( '[clhc_cls]**** ; selection doc file that will contain # of objects for classes\n' ) f.write('\n') f.write(';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n') f.write(';; average aligned particles together ;;\n') f.write(';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n') f.write('\n') f.write('VM\n') f.write('echo Averaging particles into classes\n') f.write('\n') f.write('DO LB20 x81=1,x12\n') f.write('AS R\n') f.write('[aligned]@*****\n') f.write('[clhc_cls]{****x81}\n') f.write('A\n') f.write('classes_avg@{****x81}\n') f.write('classes_var@{****x81}\n') f.write('LB20\n') f.write('\n') f.write('EN D\n')
def alignAPSH(volfile, origstackfile, eulerdocfile, classnum, boxsize, numpart, pixrad, timestamp, iternum, dataext=".spi"): apshAngularFile = str(classnum)+"/"+"apshAngularFile-%03d.spi"%(iternum) if os.path.isfile(apshAngularFile): os.remove(apshAngularFile) apDisplay.printColor("File exist! Removing file "+apshAngularFile, "cyan") apshAngularFile = spyder.fileFilter(apshAngularFile) apshListFile = str(classnum)+"/"+"apshListFile-%03d.spi"%(iternum) if os.path.isfile(apshListFile): os.remove(apshListFile) apDisplay.printColor("File exist! Removing file "+apshListFile, "cyan") apshListFile = spyder.fileFilter(apshListFile) apshOutFile = str(classnum)+"/"+"apshOut-%03d.spi"%(iternum) if os.path.isfile(apshOutFile): os.remove(apshOutFile) apDisplay.printColor("File exist! Removing file "+apshOutFile, "cyan") apshOut = spyder.fileFilter(apshOutFile) apshProjStack = str(classnum)+"/"+"apshProjStack-%03d.spi"%(iternum) if os.path.isfile(apshProjStack): os.remove(apshProjStack) apDisplay.printColor("File exist! Removing file "+apshProjStack, "cyan") apshProjStack = spyder.fileFilter(apshProjStack) origstackfile = spyder.fileFilter(origstackfile) volfile = spyder.fileFilter(volfile) origeulerdocfile = spyder.fileFilter(eulerdocfile) mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) mySpider.toSpider("VO EA,x53", "10", #angular increment "0,90.0", #Range of Theta "0,359.9", #Range of Phi apshAngularFile, #Angular file name ) mySpider.toSpider("DOC CREATE", apshListFile, #List file name "1", "1-x53" ) #mySpider.toSpider("MS", # "_9@", #reference projections # str(boxsize),str(boxsize),"1", #boxsize # "x53", #) mySpider.toSpider("PJ 3Q", volfile, #input vol file str(pixrad), #pixel radius apshListFile, #number of particles apshAngularFile, #Euler DOC file apshProjStack+"@*****", #output projections ) mySpider.toSpider("AP SH", apshProjStack+"@*****", #reference projections "1-x53", #reference numbers "6,2", #translational search range, step size "1,%d"%(boxsize/3), #first and last ring apshAngularFile, origstackfile+"@*****", "1-%d"%(numpart), origeulerdocfile, "20", #"(0.0)", # no restrictions for alignment "1", # mirror check? apshOut, # output ) mySpider.close() return apshOutFile
def crossCorrelateAndShift(infile, reffile, alignfile, ccdocfile, numpart, dataext=".spi"): ### rewriten to do the whole thing in memory in SPIDER, it should be faster starttime = time.time() infile = spyder.fileFilter(infile) reffile = spyder.fileFilter(reffile) alignfile = spyder.fileFilter(alignfile) partimg = "_4" ccmap = "_5" windccmap = "_6" boxsize = apFile.getBoxSize(infile+dataext) if not os.path.isfile(infile+dataext): apDisplay.printError("input stack file not found: "+infile+dataext) if not os.path.isfile(reffile+dataext): apDisplay.printError("reference stack file not found: "+reffile+dataext) nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) ### Allocate empty stack mySpider.toSpiderQuiet( "MS I", #command "_2@", #name "%d,%d,%d"%(boxsize), #boxsize str(numpart+1), #num part to create in memory str(numpart+1), #max particle number ) partnum = 0 while partnum < numpart: partnum+=1 mySpider.toSpiderQuiet("CP", infile+("@%05d"%(partnum)), #picture partimg, ) ### cross correlate images; reversed order to avoid -1*shift mySpider.toSpiderQuiet("CC N", reffile+("@%05d"%(partnum)), #reference partimg, #picture ccmap, #output file ) ### cannot shift more the 1/4 size of the image mySpider.toSpiderQuiet("FI x52", partimg, "12" ) mySpider.toSpiderQuiet("x54=int(x52/2)") #window size mySpider.toSpiderQuiet("x55=int(x52/4)") #window topleft mySpider.toSpiderQuiet("WI", ccmap, #input file windccmap, #output file "x54,x54", #window size "x55,x55", #window origin ) ### find the cross-correlation peak mySpider.toSpiderQuiet("x56=int(x52/4)+1") #center of window mySpider.toSpiderQuiet("PK M x11,x12,x13,x14", windccmap, #input ccmap file "x56,x56", #origin coordinates ) ### save info to doc file mySpider.toSpiderQuiet("SD %d,x13,x14"%(partnum), ccdocfile, #input ccmap file ) ### shift the images images mySpider.toSpiderQuiet("SH", partimg, #old stack ("_2@%05d"%(partnum)), #new stack "x13,x14", #shift value file ) ### finish up #save stack to file mySpider.toSpiderQuiet( "CP", "_2@", alignfile+"@", ) #delete stack mySpider.toSpiderQuiet( "DE", "_2", ) mySpider.close() apDisplay.printColor("finished shifting particles in "+apDisplay.timeString(time.time()-starttime), "cyan") return
def start(self): self.stack = {} self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) self.stack['part'] = apStack.getOneParticleFromStackId( self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) # symmetry info if self.params['sym'] == 'Icos': self.params['symtype'] = 'I' self.params['symfold'] = None else: self.params['symtype'] = self.params['sym'][0] self.params['symfold'] = int(self.params['sym'][1:]) # eman "d" symmetry is spider "ci" if self.params['symtype'].upper() == "D": self.params['symtype'] = "CI" # create symmetry doc file sydoc = "sym.spi" refine.symmetryDoc(self.params['symtype'], self.params['symfold'], sydoc) # convert incr to array of increments ang_inc = self.params['incr'].split(',') self.params['numiter'] = len(ang_inc) self.params['increments'] = [] for i in range(0, self.params['numiter']): self.params['increments'].append(int(ang_inc[i])) # convert stack to spider stack spiderstack = os.path.join(self.params['rundir'], 'start.spi') operations.stackToSpiderStack(self.stack['file'], spiderstack, apix=self.stack['apix'], boxsize=self.stack['boxsize'], numpart=self.params['numpart']) # create filtered stack spiderstackfilt = spiderstack if (self.params['lowpass'] + self.params['highpass']) > 0: spiderstackfilt = os.path.join(self.params['rundir'], 'start_filt.spi') operations.stackToSpiderStack(self.stack['file'], spiderstackfilt, apix=self.stack['apix'], boxsize=self.stack['boxsize'], lp=self.params['lowpass'], hp=self.params['highpass'], numpart=self.params['numpart']) # rescale initial model if necessary outvol = os.path.join(self.params['rundir'], "vol000.spi") apModel.rescaleModel(self.params['modelid'], outvol, self.stack['boxsize'], self.stack['apix'], spider=True) self.params['itervol'] = outvol for iter in range(1, self.params['numiter'] + 1): # create projections for projection matching apDisplay.printMsg("creating reference projections of volume: %s" % self.params['itervol']) projs, numprojs, ang, sel = refine.createProjections( incr=self.params['increments'][iter - 1], boxsz=self.stack['boxsize'], symfold=self.params['symfold'], invol=self.params['itervol'], rad=self.params['rad'], ) # run reference-based alignment apDisplay.printMsg("running reference-based alignment (AP MQ)") apmqfile = "apmq%03d.spi" % iter outang = "angular%03d.spi" % iter shf = "shifts%03d.spi" % iter shiftedStack = "parts_shifted.spi" apFile.removeFile(shf) refine.spiderAPMQ( projs=projs, numprojs=numprojs, tsearch=self.params['xysearch'], tstep=self.params['xystep'], firstRing=self.params['firstring'], lastRing=self.params['lastring'], stackfile=spiderstackfilt, nump=self.params['numpart'], ang=ang, apmqfile=apmqfile, outang=outang, nproc=self.params['proc'], ) # use cross-correlation to find the sub-pixel alignment # of the particles, # results will be saved in "peakfile.spi" apFile.removeFile(shiftedStack) # don't use MPI here - for some reason slower? mySpi = spyder.SpiderSession(dataext=".spi", logo=False, log=False) apmqlist = refine.readDocFile(apmqfile) avgccrot = 0 apDisplay.printMsg("creating shifted stack") for p in range(0, self.params['numpart']): ref = int(float(apmqlist[p][2])) ccrot = float(apmqlist[p][3]) inplane = float(apmqlist[p][4]) avgccrot += ccrot # invert the sign - ref projs will be rotated inplane *= -1 # get corresponding projection if (ref <= 0): # mirror projection if necessary ref *= -1 refimg = spyder.fileFilter(projs) + "@" + str(ref) refine.mirrorImg(refimg, "_3", inMySpi=mySpi) img = "_3" else: img = spyder.fileFilter(projs) + "@" + str(ref) refine.rotAndShiftImg(img, "_2", inplane, inMySpi=mySpi) refine.maskImg("_2", "_3", self.params['rad'], "D", "E", center=int((self.stack['boxsize'] / 2) + 1), inMySpi=mySpi) # pad ref image & stack image to twice the size refine.padImg("_3", "_2", 2 * self.stack['boxsize'], "N", 1, 1, 0, inMySpi=mySpi) stackimg = spyder.fileFilter(spiderstack) + "@" + str(p + 1) refine.padImg(stackimg, "_1", 2 * self.stack['boxsize'], "B", 1, 1, inMySpi=mySpi) # calculate cross-correlation refine.getCC("_1", "_2", "_1", inMySpi=mySpi) # crop the correllation image to allowable shift amount shift = int(self.params['allowedShift'] * self.stack['boxsize']) dim = 2 * shift + 1 topleftx = self.stack['boxsize'] - shift + 1 refine.windowImg("_1", "_2", dim, topleftx, topleftx, inMySpi=mySpi) # find the sub-pixel location of cc peak mySpi.toSpiderQuiet("PK x11,x12,x13,x14,x15,x16,x17", "_2", "0") # create new stack of shifted particles shpos = spyder.fileFilter(shiftedStack) + "@" + str(p + 1) mySpi.toSpiderQuiet("IF(x17.EQ.0.0) THEN") mySpi.toSpiderQuiet("GP x17", "_2", str(shift + 1) + "," + str(shift + 1)) refine.copyImg(stackimg, shpos, inMySpi=mySpi) mySpi.toSpiderQuiet("ELSE") #mySpi.toSpiderQuiet("RT SQ",stackimg,shpos,inplane*-1,"-x15,-x16") mySpi.toSpiderQuiet("SH F", stackimg, shpos, "-x15,-x16") mySpi.toSpiderQuiet("ENDIF") # save shifts to file mySpi.toSpiderQuiet("SD " + str(p + 1) + ",x15,x16,x17", spyder.fileFilter(shf)) mySpi.toSpiderQuiet("SD E", spyder.fileFilter(shf)) mySpi.close() # create class average images refine.createClassAverages( shiftedStack, projs, apmqfile, numprojs, self.stack['boxsize'], shifted=True, ) # rename class averages & variacnes for iteration cmd = "/bin/mv classes.hed classes.%d.hed;" % iter cmd += "/bin/mv classes.img classes.%d.img;" % iter cmd += "/bin/mv variances.hed variances.%d.hed;" % iter cmd += "/bin/mv variances.img variances.%d.img;" % iter proc = subprocess.Popen(cmd, shell=True) proc.wait() # calculate the stddev for the apmq cc's for throwing out particles avgccrot /= self.params['numpart'] stdccrot = 0 for p in range(0, self.params['numpart']): stdccrot += abs(float(apmqlist[p][3]) - avgccrot) stdccrot /= self.params['numpart'] cccutoff = avgccrot + (stdccrot * self.params['keepsig']) apDisplay.printMsg("average cc: %f" % avgccrot) apDisplay.printMsg("setting cutoff to: %f" % cccutoff) # create new selection file that only has particles with good cc's selectfile = "select%03d.spi" % iter apFile.removeFile(selectfile) mySpi = spyder.SpiderSession(nproc=self.params['proc'], dataext=".spi", logo=False, log=False) i = 1 for p in range(0, self.params['numpart']): ccrot = float(apmqlist[p][3]) if ccrot >= cccutoff: mySpi.toSpiderQuiet("x11=%d" % (p + 1)) mySpi.toSpiderQuiet("SD %d,x11" % i, spyder.fileFilter(selectfile)) i += 1 mySpi.close() # calculate the new 3d structure using centered projections # and the corrected angles from the angular doc file apDisplay.printMsg("creating 3d volume") out_rawvol = "vol_raw%03d.spi" % iter if self.params['voliter'] is not None: refine.iterativeBackProjection( shiftedStack, selectfile, rad=self.params['rad'], ang=outang, out=out_rawvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['bpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc'], ) else: refine.backProjection(shiftedStack, selectfile, ang=outang, out=out_rawvol, sym=sydoc, nproc=self.params['proc']) # create even & odd select files apDisplay.printMsg("creating even/odd volumes") oddfile = "selectodd%03d.spi" % iter evenfile = "selecteven%03d.spi" % iter refine.docSplit(selectfile, oddfile, evenfile) # get the even & odd volumesa oddvol = "vol1%03d.spi" % iter evenvol = "vol2%03d.spi" % iter if self.params['voliter'] is not None: refine.iterativeBackProjection( shiftedStack, oddfile, rad=self.params['rad'], ang=outang, out=oddvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['eobpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc']) refine.iterativeBackProjection( shiftedStack, evenfile, rad=self.params['rad'], ang=outang, out=evenvol, lam=self.params['lambda'], iterlimit=self.params['voliter'], mode=self.params['eobpmode'], smoothfac=self.params['smoothfac'], sym=sydoc, nproc=self.params['proc']) else: refine.backProjection( shiftedStack, oddfile, ang=outang, out=oddvol, sym=sydoc, nproc=self.params['proc'], ) refine.backProjection( shiftedStack, evenfile, ang=outang, out=evenvol, sym=sydoc, nproc=self.params['proc'], ) # calculate the FSC apDisplay.printMsg("calculating FSC") fscfile = "fsc%03d.spi" % iter emanfsc = "fsc.eotest.%d" % iter refine.calcFSC(oddvol, evenvol, fscfile) # convert to eman-style fscfile refine.spiderFSCtoEMAN(fscfile, emanfsc) # calculate the resolution at 0.5 FSC & write to file res = apRecon.calcRes(emanfsc, self.stack['boxsize'], self.stack['apix']) restxt = "resolution.txt" if iter == 1 and os.path.isfile(restxt): os.remove(restxt) resfile = open(restxt, "a") resfile.write("iter %d:\t%.3f\n" % (iter, res)) resfile.close() # filter & normalize the volume to be used as a reference in the next round outvol = "vol%03d.spi" % iter emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm spidersingle" % ( out_rawvol, outvol, self.stack['apix'], res, self.params['rad']) if self.params['imask'] is not None: emancmd += " imask=%d" % self.params['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) # create mrc files of volumes emancmd = "proc3d %s %s apix=%.3f mask=%d norm" % ( out_rawvol, "threed.%da.mrc" % iter, self.stack['apix'], self.params['rad']) if self.params['imask'] is not None: emancmd += " imask=%d" % self.params['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) emancmd = "proc3d %s %s apix=%.3f lp=%.3f mask=%d norm" % ( out_rawvol, "threed.%da.lp.mrc" % iter, self.stack['apix'], res, self.params['rad']) if self.params['imask'] is not None: emancmd += " imask=%d" % self.param['imask'] apEMAN.executeEmanCmd(emancmd, verbose=True) # set this model as start for next iteration, remove previous os.remove(self.params['itervol']) os.remove(out_rawvol) self.params['itervol'] = outvol # clean up directory apDisplay.printMsg("cleaning up directory") if os.path.isfile(oddvol): os.remove(oddvol) if os.path.isfile(evenvol): os.remove(evenvol) if os.path.isfile(ang): os.remove(ang) if os.path.isfile(sel): os.remove(sel) if os.path.isfile(projs): os.remove(projs) if os.path.isfile(oddfile): os.remove(oddfile) if os.path.isfile(evenfile): os.remove(evenfile) if os.path.isfile(emanfsc) and os.path.isfile(fscfile): os.remove(fscfile) if os.path.isfile(shiftedStack): os.remove(shiftedStack) os.remove(self.params['itervol'])
def refBasedAlignParticles(stackfile, templatestack, origstackfile, xysearch, xystep, numpart, numtemplate, firstring=2, lastring=100, dataext=".spi", iternum=1, oldpartlist=None): """ inputs: stack template search params outputs: aligned stack rotation/shift params """ ### setup if dataext in templatestack: templatestack = templatestack[:-4] if dataext in stackfile: stackfile = stackfile[:-4] if dataext in origstackfile: origstackfile = origstackfile[:-4] t0 = time.time() rundir = "alignments" apParam.createDirectory(rundir) nproc = apParam.getNumProcessors() ### remove previous iterations apFile.removeFile(rundir + "/paramdoc%02d%s" % (iternum, dataext)) ### perform alignment, should I use 'AP SH' instead? mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider( "AP MQ", spyder.fileFilter(templatestack) + "@**", # reference image series "1-" + str(numtemplate), # enter number of templates of doc file str(int(xysearch)) + "," + str(int(xystep)), # translation search range, step size str(int(firstring)) + "," + str(int(lastring)), # first and last ring for rotational correlation spyder.fileFilter(stackfile) + "@******", # unaligned image series "1-" + str(numpart), # enter number of particles of doc file rundir + ("/paramdoc%02d" % (iternum)), # output angles document file ) mySpider.close() """ ### perform alignment, should I use 'AP SH' instead? mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) mySpider.toSpider("AP SH", spyder.fileFilter(templatestack)+"@**", # reference image series "1-"+str(numtemplate), # enter number of templates of doc file str(int(xysearch))+","+str(int(xystep)), # translation search range, step size str(int(firstring))+","+str(int(lastring)), # first and last ring for rotational correlation "*", # initial angles spyder.fileFilter(stackfile)+"@******", # unaligned image series "1-"+str(numpart), # enter number of particles of doc file "*", # initial angles "0.0", # angle search "1", # check mirrors and shift/rotate input rundir+("/paramdoc%02d" % (iternum)), # output angles document file ) mySpider.close() """ ### convert spider rotation, shift data to python docfile = rundir + ("/paramdoc%02d" % (iternum)) + dataext picklefile = rundir + ("/paramdoc%02d" % (iternum)) + ".pickle" if oldpartlist is not None and iternum > 1: apDisplay.printMsg("updating particle doc info") partlist = updateRefBasedDocFile(oldpartlist, docfile, picklefile) elif iternum == 1: apDisplay.printMsg("reading initial particle doc info") partlist = readRefBasedDocFile(docfile, picklefile) else: apDisplay.printError( "reading (not updating) particle doc info on iteration " + str(iternum)) ### write aligned stack -- with python loop alignedstack = rundir + ("/alignedstack%02d" % (iternum)) alignStack(origstackfile, alignedstack, partlist, dataext) ### average stack emancmd = ("proc2d " + alignedstack + dataext + " " + rundir + ("/avgimg%02d" % (iternum)) + ".mrc " + " average") apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) td1 = time.time() - t0 apDisplay.printMsg("completed alignment of " + str(numpart) + " particles in " + apDisplay.timeString(td1)) if numpart < 1: apDisplay.printError("Failed to find any particles") return alignedstack + dataext, partlist
def createClassAverages(stack, projs, apmq, numprojs, boxsz, outclass="classes", rotated=False, shifted=False, dataext=".spi"): """ creates EMAN-style class average file "classes.hed" & "classes.img" and variance files "variances.hed" & "variances.img" from spider classification """ apFile.removeFile(outclass) outf = spyder.fileFilter(outclass) outvf = "tmpvar" apmqlist = readDocFile(apmq) mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False) # create file containing the number of particles matched to each projection mySpi.toSpiderQuiet( "VO MQ", "0.0", spyder.fileFilter(apmq), str(numprojs), "cls*****", "numinclass", ) mySpi.close() os.remove("numinclass%s" % dataext) # create class average file mySpi = spyder.SpiderSession(dataext=dataext, logo=False, log=False) for i in range(0, numprojs): clsfile = readDocFile("cls%05d%s" % (i + 1, dataext)) if clsfile == []: apDisplay.printMsg("class %d has no particles" % (i + 1)) mySpi.toSpiderQuiet( "BL", "%s@%d" % (outf, (i + 1)), "(%d,%d)" % (boxsz, boxsz), "N", "(0.0)", ) mySpi.toSpiderQuiet( "BL", "%s@%d" % (outvf, (i + 1)), "(%d,%d)" % (boxsz, boxsz), "N", "(0.0)", ) else: mySpi.toSpiderQuiet("DE", "_2@") mySpi.toSpiderQuiet("MS", "_2@", "%d,%d,1" % (boxsz, boxsz), str(len(clsfile))) for p in range(0, len(clsfile)): mySpi.toSpiderQuiet("DE", "_1") # get the particle part = int(float(clsfile[p][2])) - 1 pimg = spyder.fileFilter(stack) + "@%d" % (part + 1) rot = float(apmqlist[part][4]) shx = float(apmqlist[part][5]) shy = float(apmqlist[part][6]) if shifted is True: shx = 0 shy = 0 if rotated is True: rot = 0 p_out = "_2@%d" % (p + 1) rotAndShiftImg(pimg, "_1", rot, shx, shy, inMySpi=mySpi) # mirror if int(float(apmqlist[part][2])) < 0: mirrorImg("_1", p_out, inMySpi=mySpi) else: mySpi.toSpiderQuiet("CP", "_1", p_out) if len(clsfile) == 1: mySpi.toSpiderQuiet("CP", "_2@1", "%s@%d" % (outf, (i + 1))) mySpi.toSpiderQuiet( "BL", "%s@%d" % (outvf, (i + 1)), "(%d,%d)" % (boxsz, boxsz), "N", "(0.0)", ) else: mySpi.toSpiderQuiet("AS R", "_2@*****", "1-%d" % len(clsfile), "A", "%s@%d" % (outf, (i + 1)), "%s@%d" % (outvf, i + 1)) mySpi.close() # convert the class averages to EMAN class averages # read classes & projections for EMAN classes output if os.path.exists('variances.hed'): os.remove('variances.hed') if os.path.exists('variances.img'): os.remove('variances.img') if os.path.exists('classes.hed'): os.remove('classes.hed') if os.path.exists('classes.img'): os.remove('classes.img') ### I would prefer to use apImagicFile.readImagic, writeImagic variances = EMAN.readImages(outvf + dataext, -1, -1, 0) averages = EMAN.readImages(outf + dataext, -1, -1, 0) projections = EMAN.readImages(projs, -1, -1, 0) for i in range(0, numprojs): # copy projection to class average file projections[i].writeImage('classes.hed') projections[i].writeImage('variances.hed') # get num of particles in class clsfile = readDocFile("cls%05d%s" % (i + 1, dataext)) averages[i].setNImg(len(clsfile)) averages[i].writeImage('classes.hed', -1) variances[i].setNImg(len(clsfile)) variances[i].writeImage('variances.hed', -1) os.remove("cls%05d%s" % (i + 1, dataext)) os.remove(outf + dataext) return
def refFreeAlignParticles(stackfile, template, numpart, pixrad, firstring=2, lastring=100, dataext=".spi", rundir="alignment"): """ inputs: stack template search params outputs: aligned stack rotation/shift params """ ### setup if dataext in template: template = template[:-4] if dataext in stackfile: stackfile = stackfile[:-4] t0 = time.time() apParam.createDirectory(rundir) ### remove previous iterations numiter = 0 while os.path.isfile(rundir + "/avgimg%02d%s" % (numiter + 1, dataext)): apFile.removeFile(rundir + "/avgimg%02d%s" % (numiter + 1, dataext)) pngfile = rundir + "/avgimg%02d%s" % (numiter + 1, ".png") apFile.removeFile(pngfile) numiter += 1 ### perform alignment mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) apDisplay.printMsg("Performing particle alignment") # copy template to memory mySpider.toSpiderQuiet("CP", (template + "@1"), "_9") mySpider.toSpider("AP SR", spyder.fileFilter(stackfile) + "@******", "1-" + str(numpart), str(int(pixrad)), str(int(firstring)) + "," + str(int(lastring)), "_9", rundir + "/avgimg**", rundir + "/paramdoc**") mySpider.close() ### find number of iterations numiter = 0 while os.path.isfile(rundir + "/avgimg%02d%s" % (numiter + 1, dataext)): emancmd = ("proc2d " + " " + rundir + "/avgimg" + ("%02d%s" % (numiter + 1, dataext)) + " " + rundir + "/avgimg" + ("%02d%s" % (numiter + 1, ".png"))) apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False) numiter += 1 if numiter == 0: apDisplay.printError("alignment failed, no iterations were found") emancmd = ("proc2d " + " " + rundir + "/avgimg" + ("%02d%s" % (numiter, dataext)) + " " + rundir + "/average.mrc") apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False) apDisplay.printMsg( str(numiter) + " alignment iterations were run by spider") ### convert spider rotation, shift data to python docfile = rundir + ("/paramdoc%02d" % (numiter)) + dataext picklefile = rundir + ("/paramdoc%02d" % (numiter)) + ".pickle" partlist = readRefFreeDocFile(docfile, picklefile) ### write aligned stack -- with python loop alignedstack = "alignedstack" alignStack(stackfile, alignedstack, partlist, dataext) td1 = time.time() - t0 apDisplay.printMsg("completed alignment of " + str(numpart) + " particles in " + apDisplay.timeString(td1)) return ("alignedstack.spi", partlist)
def analyzeEigenFactors(alignedstack, rundir, numpart, numfactors=8, dataext=".spi"): """ inputs: coran run data outputs: 1. generate eigen images 2. collect eigenimage contribution percentage 3. 2D factor plot Broken 4. 2D factor plot visualization """ ### 1. generate eigen images mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=False) for fact in range(1, numfactors + 1): mySpider.toSpiderQuiet( #"CA SRE", rundir+"/corandata", str(fact), #rundir+"/eigenstack@"+("%02d" % (fact)), ) "CA SRD", rundir + "/corandata", str(fact), str(fact), rundir + "/eigenstack@***", ) mySpider.close() ### convert to nice individual eigen image pngs for webpage eigenspistack = os.path.join(rundir, "eigenstack.spi") if not os.path.isfile(eigenspistack): apDisplay.printError("Failed to create Eigen images") for fact in range(1, numfactors + 1): pngfile = rundir + "/eigenimg" + ("%02d" % (fact)) + ".png" apFile.removeFile(pngfile) emancmd = ("proc2d " + eigenspistack + " " + pngfile + " " + " first=" + str(fact - 1) + " last=" + str(fact - 1)) apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False) ### convert eigen SPIDER stack to IMAGIC for stack viewer eigenimagicstack = rundir + "/eigenstack.hed" apFile.removeStack(eigenimagicstack) emancmd = "proc2d " + eigenspistack + " " + eigenimagicstack apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) ### 2. collect eigenimage contribution percentage eigf = open(rundir + "/corandata_EIG" + dataext, "r") count = 0 contriblist = [] for line in eigf: bits = line.strip().split() if len(contriblist) == numfactors: break if len(bits) < 3: continue contrib = float(bits[1]) cumm = float(bits[2]) eigval = float(bits[0]) if len(bits) == 3: count += 1 contriblist.append(contrib) print "Factor", count, contrib, "%\t", cumm, "%\t", eigval ### need to plot & insert this data ### hack to get 'CA VIS' to work: break up stack into individual particles """ ### this is broken in SPIDER 13.0 apParam.createDirectory("unstacked") mySpider = spyder.SpiderSession(dataext=dataext, logo=False) mySpider.toSpiderQuiet( "DO LB1 i=1,"+str(numpart), " CP", " "+alignedstack+"@{******x0}", " unstacked/img{******x0}", "LB1", ) mySpider.close() """ ### generate factor maps apDisplay.printMsg("creating factor maps") for f1 in range(1, min(numfactors, 2)): for f2 in range(f1 + 1, min(3, numfactors + 1)): sys.stderr.write(".") try: createFactorMap(f1, f2, rundir, dataext) except: sys.stderr.write("#") pass sys.stderr.write("\n") return contriblist
def kmeansCluster(alignedstack, numpart=None, numclasses=40, timestamp=None, factorlist=range(1, 5), corandata="coran/corandata", dataext=".spi"): """ inputs: outputs: """ if timestamp is None: timestamp = apParam.makeTimestamp() if alignedstack[-4:] == dataext: alignedstack = alignedstack[:-4] rundir = "cluster" classavg = rundir + "/" + ("classavgstack_%s_%03d" % (timestamp, numclasses)) classvar = rundir + "/" + ("classvarstack_%s_%03d" % (timestamp, numclasses)) apParam.createDirectory(rundir) for i in range(numclasses): apFile.removeFile(rundir + ("/classdoc%04d" % (i + 1)) + dataext) apFile.removeFile(rundir + ("/allclassesdoc%04d" % (numclasses)) + dataext) ### make list of factors factorstr, factorkey = operations.intListToString(factorlist) ### do k-means clustering mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) mySpider.toSpider( "CL KM", corandata + "_IMC", # path to coran data str(numclasses), # num classes factorstr, # factor string ) ## weight for each factor for fact in factorlist: mySpider.toSpiderQuiet("1.0") randnum = (int(random.random() * 1000) + 1) mySpider.toSpider( str(randnum), rundir + "/classdoc_" + timestamp + "_****", # class doc file rundir + ("/allclassesdoc%04d" % (numclasses)), #clusterdoc file ) mySpider.close() ### delete existing files sys.stderr.write("delete existing files") for dext in (".hed", ".img", dataext): apFile.removeFile(classavg + dext) apFile.removeFile(classvar + dext) print "" mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False) ### create class averages apDisplay.printMsg("Averaging particles into classes") for i in range(numclasses): classnum = i + 1 mySpider.toSpiderQuiet( "AS R", spyder.fileFilter(alignedstack) + "@******", rundir + ("/classdoc_" + timestamp + "_%04d" % (classnum)), "A", (classavg + "@%04d" % (classnum)), (classvar + "@%04d" % (classnum)), ) if classnum % 10 == 0: sys.stderr.write(".") time.sleep(1) mySpider.close() ### convert to IMAGIC emancmd = "proc2d " + classavg + ".spi " + classavg + ".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) emancmd = "proc2d " + classvar + ".spi " + classvar + ".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) return classavg, classvar
def hierarchClusterClassify(alignedstack, dendrogramfile, numclasses=40, timestamp=None, rundir=".", dataext=".spi"): """ inputs: aligned particle stack number of classes timestamp output directory output: class averages class variances dendrogram.png """ if timestamp is None: timestamp = apParam.makeTimestamp() classavg = rundir + "/" + ("classavgstack_%s_%03d" % (timestamp, numclasses)) classvar = rundir + "/" + ("classvarstack_%s_%03d" % (timestamp, numclasses)) thresh, classes = findThreshold(numclasses, dendrogramfile, rundir, dataext) ### create class doc files mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=True) mySpider.toSpider( "CL HE", thresh, spyder.fileFilter(dendrogramfile), # dendrogram doc file rundir + "/classdoc_" + timestamp + "_****", # class doc file ) ### delete existing files sys.stderr.write("delete existing files") for dext in (".hed", ".img", dataext): apFile.removeFile(classavg + dext) apFile.removeFile(classvar + dext) print "" ### create class averages sys.stderr.write("create class averages") for i in range(classes): sys.stderr.write(".") classnum = i + 1 mySpider.toSpiderQuiet( "AS R", spyder.fileFilter(alignedstack) + "@******", rundir + ("/classdoc_" + timestamp + "_%04d" % (classnum)), "A", (classavg + "@%04d" % (classnum)), (classvar + "@%04d" % (classnum)), ) mySpider.close() print "" ### convert to IMAGIC emancmd = "proc2d " + classavg + ".spi " + classavg + ".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) emancmd = "proc2d " + classvar + ".spi " + classvar + ".hed" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True) return classavg, classvar
def alignStack(oldstack, alignedstack, partlist, dataext=".spi"): """ write aligned stack -- with python loop inputs: oldstack newstack (empty) list of particle dictionaries for operations modifies: newstack output: none I tried this loop in both spider and python; python was faster?!? -neil """ if not os.path.isfile(oldstack + dataext): apDisplay.printError("Could not find original stack: " + oldstack + dataext) boxsize = apFile.getBoxSize(oldstack + dataext) apDisplay.printMsg("applying alignment parameters to stack") apFile.removeFile(alignedstack + dataext) count = 0 t0 = time.time() nproc = apParam.getNumProcessors() mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False) #create stack in core numpart = len(partlist) mySpider.toSpiderQuiet( "MS I", #command "_2@", #name "%d,%d,%d" % (boxsize), #boxsize str(numpart + 1), #num part to create in memory str(numpart + 1), #max particle number ) for partdict in partlist: partnum = partdict['num'] #if partdict['num'] in [3,6,7]: # print partdict['num'], partdict['template'], partdict['mirror'], round(partdict['rot'],3) ### Rotate and Shift operations count += 1 #rotate/shift mySpider.toSpiderQuiet( "RT SQ", spyder.fileFilter(oldstack) + "@" + ("%06d" % (partnum)), "_1", str(partdict['rot']), str(partdict['xshift']) + "," + str(partdict['yshift']), ) #mirror, if necessary if 'mirror' in partdict and partdict['mirror'] is True: mySpider.toSpiderQuiet( "MR", "_1", "_2@" + ("%06d" % (partnum)), "Y", ) else: mySpider.toSpiderQuiet( "CP", "_1", "_2@" + ("%06d" % (partnum)), ) ### finish up #save stack to file mySpider.toSpiderQuiet( "CP", "_2@", spyder.fileFilter(alignedstack) + "@", ) #delete stack mySpider.toSpiderQuiet( "DE", "_2", ) mySpider.close() apDisplay.printMsg("Completed transforming %d particles in %s" % (count, apDisplay.timeString(time.time() - t0))) if count < 1: apDisplay.printError("Failed to transform any particles") if not os.path.isfile(alignedstack + dataext): apDisplay.printError("Failed to create stack " + alignedstack + dataext) return