def runAffinityPropagation(self, alignedstack): ### Get initial correlation values ### this is really, really slow similarfile, simlist = self.fillSimilarityMatrix(alignedstack) ### Preference value stats preffile = self.setPreferences(simlist) ### run apcluster.exe program outfile = "clusters.out" apDisplay.printMsg("Run apcluster.exe program") apclusterexe = os.path.join(apParam.getAppionDirectory(), "bin/apcluster.exe") apFile.removeFile(outfile) clustercmd = apclusterexe+" "+similarfile+" "+preffile+" "+outfile clusttime = time.time() proc = subprocess.Popen(clustercmd, shell=True) proc.wait() apDisplay.printMsg("apCluster time: "+apDisplay.timeString(time.time()-clusttime)) if not os.path.isfile(outfile): apDisplay.printError("apCluster did not run") ### Parse apcluster output file: clusters.out apDisplay.printMsg("Parse apcluster output file: "+outfile) clustf = open(outfile, "r") ### each line is the particle and the number is the class partnum = 0 classes = {} for line in clustf: sline = line.strip() if sline: partnum += 1 classnum = int(sline) if not classnum in classes: classes[classnum] = [partnum,] else: classes[classnum].append(partnum) clustf.close() apDisplay.printMsg("Found %d classes"%(len(classes.keys()))) ### Create class averages classavgdata = [] classnames = classes.keys() classnames.sort() for classnum in classnames: apDisplay.printMsg("Class %d, %d members"%(classnum, len(classes[classnum]))) #clsf = open('subcls%04d.lst'%(classnum), 'w') #for partnum in classes[classnum]: # clsf.write("%d\n"%(partnum)) #clsf.close() classdatalist = apImagicFile.readParticleListFromStack(alignedstack, classes[classnum], msg=False) classdatarray = numpy.asarray(classdatalist) classavgarray = classdatarray.mean(0) #mrc.write(classavgarray, 'subcls%04d.mrc'%(classnum)) classavgdata.append(classavgarray) apFile.removeStack("classaverage-"+self.timestamp+".hed") apImagicFile.writeImagic(classavgdata, "classaverage-"+self.timestamp+".hed") return classes
def averageSubStack(partlist, stackfile, bin=1): if len(partlist) > 300: partlist = partlist[:300] boxsize = apImagicFile.getBoxsize(stackfile) if len(partlist) == 0: binboxsize = boxsize/bin blank = numpy.ones((binboxsize, binboxsize), dtype=numpy.float32) return blank if not os.path.isfile(stackfile): apDisplay.printWarning("could not find stack, "+stackfile) return False partdatalist = apImagicFile.readParticleListFromStack(stackfile, partlist, boxsize, msg=False) partdataarray = numpy.asarray(partdatalist) finaldata = partdataarray.mean(0) if bin > 1: finaldata = apImage.binImg(finaldata, bin) return finaldata
def createMontageInMemory(self, apix): self.cluster_resolution = [] apDisplay.printMsg("Converting files") ### Set binning of images boxsize = apImagicFile.getBoxsize(self.instack) bin = 1 while boxsize/bin > 200: bin+=1 binboxsize = boxsize/bin ### create averages files = glob.glob(self.timestamp+".[0-9]*") files.sort(self.sortFile) montage = [] montagepngs = [] i = 0 for listname in files: i += 1 apDisplay.printMsg("%d of %d classes"%(i,len(files))) pngfile = listname+".png" if not os.path.isfile(listname) or apFile.fileSize(listname) < 1: ### create a ghost particle sys.stderr.write("skipping "+listname+"\n") blank = numpy.ones((binboxsize, binboxsize), dtype=numpy.float32) ### add to montage stack montage.append(blank) self.cluster_resolution.append(None) ### create png apImage.arrayToPng(blank, pngfile) else: ### read particle list partlist = self.readListFile(listname) ### average particles partdatalist = apImagicFile.readParticleListFromStack(self.instack, partlist, boxsize, msg=False) partdataarray = numpy.asarray(partdatalist) finaldata = partdataarray.mean(0) if bin > 1: finaldata = apImage.binImg(finaldata, bin) ### add to montage stack montage.append(finaldata) res = apFourier.spectralSNR(partdatalist, apix) self.cluster_resolution.append(res) ### create png apImage.arrayToPng(finaldata, pngfile) ### check for png file if os.path.isfile(pngfile): montagepngs.append(pngfile) else: apDisplay.printError("failed to create montage") stackname = "kerdenstack"+self.timestamp+".hed" apImagicFile.writeImagic(montage, stackname) ### create montage montagecmd = ("montage -geometry +4+4 -tile %dx%d "%(self.params['xdim'], self.params['ydim'])) for monpng in montagepngs: montagecmd += monpng+" " montagecmd += "montage.png" apEMAN.executeEmanCmd(montagecmd, showcmd=True, verbose=False) time.sleep(1) apFile.removeFilePattern(self.timestamp+".*.png") return bin
def start(): ### backup old cls files classfile = os.path.join(params['emandir'], "cls.%d.tar"%(params['iter'])) oldclassfile = os.path.join(params['emandir'], "cls.%d.old.tar"%(params['iter'])) shutil.move(classfile, oldclassfile) projhed = os.path.join(params['emandir'], 'proj.hed') projhed = os.path.join(params['emandir'], 'proj.img') numproj = apFile.numImagesInStack(projhed) ### extract cls files tar = tarfile.open(oldclassfile) tar.extractall(path=params['rundir']) tar.close() clslist = glob.glob("cls*.lst") if numproj != len(clslist): apDisplay.printError("array length mismatch") ### loop through classes clsnum = 0 goodavg = [] for clsfile in clslist: clsnum += 1 clsf = open(clsfile, 'r') partlist = clsf.readlines() clsf.close() ### write the projection??? #e=projections[clsNum].getEuler() #projections[clsNum].setNImg(-1) #projections[clsNum].writeImage('goodavgs.hed', -1) if len(partlist) < params['minpart']: ### not enough particles skip to next projection #origaverage = goodavg.append() #emanClsAvgs[(clsNum+1)*2 - 1].writeImage('goodavgs.hed',-1) continue ### make aligned stack if params['eotest'] is False: command='clstoaligned.py -c' + cls elif params['eotest']=='odd': fw=open(cls,'r') Ptcls = fw.readlines() fw.close() fw = open('cls_odd.lst', 'w') fw.writelines(Ptcls[0]) fw.writelines(Ptcls[1]) for i1 in range(2,len(Ptcls)): if i1%2==0: fw.writelines(Ptcls[i1]) fw.close() command='clstoaligned.py -c cls_odd.lst' elif params['eotest']=='even': fw=open(cls,'r') Ptcls = fw.readlines() fw.close() fw = open('cls_even.lst', 'w') fw.writelines(Ptcls[0]) fw.writelines(Ptcls[1]) for i1 in range(2,len(Ptcls)): if i1%2==1: fw.writelines(Ptcls[i1]) fw.close() command='clstoaligned.py -c cls_even.lst' apDisplay.printMsg(command) proc = subprocess.Popen(command, shell=True) proc.wait() #set up cls dir clsdir=cls.split('.')[0]+'.dir' os.mkdir(clsdir) os.rename('aligned.spi',os.path.join(clsdir,'aligned.spi')) alignedImgsName = os.path.join(clsdir,'aligned.spi') #alignedImgs = EMAN.readImages(alignedImgsName,-1,-1,0) #N = len(alignedImgs) apDisplay.printMsg("Starting cluster process for "+clsdir) ### fill similarity matrix with CC values similarfile, simlist = fillSimilarityMatrix(alignedImgsName) ### set preferences preffile = setPreferences(simlist, params['preftype']) ### run apcluster.exe program outfile = "clusters.out" apDisplay.printMsg("Run apcluster.exe program") apclusterexe = os.path.join("apcluster.exe") if os.path.isfile(outfile): os.remove(outfile) clustercmd = apclusterexe+" "+similarfile+" "+preffile+" "+outfile proc = subprocess.Popen(clustercmd, shell=True) proc.wait() if not os.path.isfile(outfile): apDisplay.printError("affinity propagration cluster program did not run") ### Parse apcluster output file: clusters.out apDisplay.printMsg("Parse apcluster output file: "+outfile) clustf = open(outfile, "r") ### each line is the particle and the number is the class partnum = 0 classes = {} for line in clustf: sline = line.strip() if sline: partnum += 1 classnum = int(sline) if not classnum in classes: classes[classnum] = [partnum,] else: classes[classnum].append(partnum) clustf.close() apDisplay.printMsg("Found %d classes"%(len(classes.keys()))) ### Create class averages classavgdata = [] classnames = classes.keys() classnames.sort() for classnum in classnames: apDisplay.printMsg("Class %d, %d members"%(classnum, len(classes[classnum]))) clsf = open('subcls%03d.lst'%(classnum), 'w') for partnum in classes[classnum]: clsf.write("%d\n"%(partnum)) clsf.close() classdatalist = apImagicFile.readParticleListFromStack(stackfile, classes[classnum], msg=False) classdatarray = numpy.asarray(classdatalist) classavgarray = classdatarray.mean(0) classavgdata.append(classavgarray) apFile.removeStack("classaverage.hed") apImagicFile.writeImagic(classavgdata, "classaverage.hed") k=0 for i in range(0,len(E)): if len(E[i])==0: continue else: f1=open('%s/subcls%02d.lst' % (str1,k), 'w') for j in range(0,len(E[i])): f1.write('%d aligned.spi clusterCenterImgNum%d\n' % (E[i][j], i)) f1.close() proc = subprocess.Popen('proc2d aligned.spi tempClsAvg.hed list=%s/subcls%02d.lst mask=%d average edgenorm' % (str1,k,params['mask']), shell=True) proc.wait() k=k+1 clsAvgs = EMAN.readImages('tempClsAvg.hed',-1,-1,0) j=0 for i in range(0,len(E)): if len(E[i])==0: continue else: clsAvgs[j].setNImg(len(E[i])) clsAvgs[j].writeImage('subclasses_avg.hed',-1) j=j+1 os.chdir('../') ### Determine best averages proc = subprocess.Popen('/bin/rm tempClsAvg.*', shell=True) proc.wait() proc = subprocess.Popen('proc2d %s/aligned.spi tempClsAvg.hed mask=%d average edgenorm' % (clsdir, params['mask']), shell=True) proc.wait() class_avg = EMAN.readImages('tempClsAvg.hed',-1,-1,0) avgname=os.path.join(clsdir,'subclasses_avg.hed') averages=EMAN.readImages(avgname,-1,-1,0) cclist=[] for avg in averages: cclist.append(cc(projections[clsNum],avg)) f1 = open('%s/CCValues.txt'%(clsdir), 'w') for i in range(len(cclist)): f1.write(str(cclist[i])+'\n') f1.close() ### Merge top best subclasses ccListSort = cclist ccListSort.sort() Ptcls = [] for i in range(0,len(ccListSort)): cci = ccListSort[len(ccListSort)-i-1] if cci>=params['corCutOff']: bestclass_i=cclist.index(cci) classname_i=clslist[clsNum].split('.')[0]+'.dir/subClassAvgs/subcls'+string.zfill(bestclass_i,2)+'.lst' f1=open(classname_i,'r') Ptcls_i = f1.readlines() f1.close() Ptcls.extend(Ptcls_i) else: print "Not included - ", cci pass if len(Ptcls)>0: fw=open('mergeClasses.lst', 'w') fw.writelines(Ptcls) fw.close() proc = subprocess.Popen('/bin/rm mergedClsAvg.spi', shell=True) proc.wait() proc = subprocess.Popen('proc2d %s/aligned.spi mergedClsAvg.spi list=mergeClasses.lst mask=%d average' % (clsdir, params['mask']), shell=True) proc.wait() mergedavg=EMAN.readImages('mergedClsAvg.spi',-1,-1,0) mergedavg[0].setNImg(len(Ptcls)) mergedavg[0].setRAlign(e) mergedavg[0].writeImage('goodavgs.hed',-1) else: pass writeNewClsfile(cls,pretext,Ptext,Ptcls) #Create list of cc values for cls in range(0,len(clslist)): clsdir=clslist[cls].split('.')[0]+'.dir' apDisplay.printMsg("Starting class number %d" %(cls)) #break pad=params['boxsize']*1.25 if pad%2: pad=pad+1 if params['sym']==None: make3dcommand='make3d goodavgs.hed out=threed.%d.mrc mask=%d pad=%d mode=2 hard=%d' % (params['iter'], params['mask'], pad, params['hard']) else: make3dcommand='make3d goodavgs.hed out=threed.%d.mrc mask=%d sym=%s pad=%d mode=2 hard=%d' % (params['iter'], params['mask'], params['sym'], pad, params['hard']) apDisplay.printMsg(make3dcommand) proc = subprocess.Popen(make3dcommand, shell=True) proc.wait() proc3dcommand='proc3d threed.%d.mrc threed.%da.mrc mask=%d norm' % (params['iter'],params['iter'],params['mask']) apDisplay.printMsg(proc3dcommand) proc = subprocess.Popen(proc3dcommand, shell=True) proc.wait() if params['eotest'] is False: #copy the resulting class average images to the main recon directory proc = subprocess.Popen('/bin/cp threed.%da.mrc ../.'%(params['iter']), shell=True) proc.wait() proc = subprocess.Popen('/bin/cp goodavgs.hed ../classes_msgp.%d.hed' %(params['iter']), shell=True) proc.wait() proc = subprocess.Popen('/bin/cp goodavgs.img ../classes_msgp.%d.img' %(params['iter']), shell=True) proc.wait() #link msgp result as the final result for this iteration rmcommand='/bin/rm -f ../classes.%d.hed ../classes.%d.img' % (params['iter'], params['iter']) proc = subprocess.Popen(rmcommand, shell=True) proc.wait() lncommand='ln -s classes_msgp.%d.hed ../classes.%d.hed' % (params['iter'], params['iter']) proc = subprocess.Popen(lncommand, shell=True) proc.wait() lncommand='ln -s classes_msgp.%d.img ../classes.%d.img' % (params['iter'], params['iter']) proc = subprocess.Popen(lncommand, shell=True) proc.wait() elif params['eotest']=='odd': proc = subprocess.Popen('/bin/cp threed.%da.mrc ../threed.%da.o.mrc' %(params['iter'], params['iter']), shell=True) proc.wait() elif params['eotest']=='even': proc = subprocess.Popen('/bin/cp threed.%da.mrc ../threed.%da.e.mrc' %(params['iter'], params['iter']), shell=True) proc.wait() proc = subprocess.Popen('proc3d threed.%da.mrc ../threed.%da.o.mrc fsc=../corEO%d.fsc.dat' %(params['iter'], params['iter'], params['iter']), shell=True) proc.wait() #replace the old cls*.lst with the new extended one proc = subprocess.Popen('tar cvzf %s %s' % (newclassfile,"cls*.lst.new"), shell=True) proc.wait() proc = subprocess.Popen('/bin/cp %s ../%s' %(newclassfile,classfile), shell=True) proc.wait() apDisplay.printMsg("Done!")