def runAffinityPropagation(self, alignedstack):
### Get initial correlation values
### this is really, really slow
similarfile, simlist = self.fillSimilarityMatrix(alignedstack)
### Preference value stats
preffile = self.setPreferences(simlist)
### run apcluster.exe program
outfile = "clusters.out"
apDisplay.printMsg("Run apcluster.exe program")
apclusterexe = os.path.join(apParam.getAppionDirectory(), "bin/apcluster.exe")
apFile.removeFile(outfile)
clustercmd = apclusterexe+" "+similarfile+" "+preffile+" "+outfile
clusttime = time.time()
proc = subprocess.Popen(clustercmd, shell=True)
proc.wait()
apDisplay.printMsg("apCluster time: "+apDisplay.timeString(time.time()-clusttime))
if not os.path.isfile(outfile):
apDisplay.printError("apCluster did not run")
### Parse apcluster output file: clusters.out
apDisplay.printMsg("Parse apcluster output file: "+outfile)
clustf = open(outfile, "r")
### each line is the particle and the number is the class
partnum = 0
classes = {}
for line in clustf:
sline = line.strip()
if sline:
partnum += 1
classnum = int(sline)
if not classnum in classes:
classes[classnum] = [partnum,]
else:
classes[classnum].append(partnum)
clustf.close()
apDisplay.printMsg("Found %d classes"%(len(classes.keys())))
### Create class averages
classavgdata = []
classnames = classes.keys()
classnames.sort()
for classnum in classnames:
apDisplay.printMsg("Class %d, %d members"%(classnum, len(classes[classnum])))
#clsf = open('subcls%04d.lst'%(classnum), 'w')
#for partnum in classes[classnum]:
# clsf.write("%d\n"%(partnum))
#clsf.close()
classdatalist = apImagicFile.readParticleListFromStack(alignedstack, classes[classnum], msg=False)
classdatarray = numpy.asarray(classdatalist)
classavgarray = classdatarray.mean(0)
#mrc.write(classavgarray, 'subcls%04d.mrc'%(classnum))
classavgdata.append(classavgarray)
apFile.removeStack("classaverage-"+self.timestamp+".hed")
apImagicFile.writeImagic(classavgdata, "classaverage-"+self.timestamp+".hed")
return classes
def averageSubStack(partlist, stackfile, bin=1):
if len(partlist) > 300:
partlist = partlist[:300]
boxsize = apImagicFile.getBoxsize(stackfile)
if len(partlist) == 0:
binboxsize = boxsize/bin
blank = numpy.ones((binboxsize, binboxsize), dtype=numpy.float32)
return blank
if not os.path.isfile(stackfile):
apDisplay.printWarning("could not find stack, "+stackfile)
return False
partdatalist = apImagicFile.readParticleListFromStack(stackfile, partlist, boxsize, msg=False)
partdataarray = numpy.asarray(partdatalist)
finaldata = partdataarray.mean(0)
if bin > 1:
finaldata = apImage.binImg(finaldata, bin)
return finaldata
def createMontageInMemory(self, apix):
self.cluster_resolution = []
apDisplay.printMsg("Converting files")
### Set binning of images
boxsize = apImagicFile.getBoxsize(self.instack)
bin = 1
while boxsize/bin > 200:
bin+=1
binboxsize = boxsize/bin
### create averages
files = glob.glob(self.timestamp+".[0-9]*")
files.sort(self.sortFile)
montage = []
montagepngs = []
i = 0
for listname in files:
i += 1
apDisplay.printMsg("%d of %d classes"%(i,len(files)))
pngfile = listname+".png"
if not os.path.isfile(listname) or apFile.fileSize(listname) < 1:
### create a ghost particle
sys.stderr.write("skipping "+listname+"\n")
blank = numpy.ones((binboxsize, binboxsize), dtype=numpy.float32)
### add to montage stack
montage.append(blank)
self.cluster_resolution.append(None)
### create png
apImage.arrayToPng(blank, pngfile)
else:
### read particle list
partlist = self.readListFile(listname)
### average particles
partdatalist = apImagicFile.readParticleListFromStack(self.instack, partlist, boxsize, msg=False)
partdataarray = numpy.asarray(partdatalist)
finaldata = partdataarray.mean(0)
if bin > 1:
finaldata = apImage.binImg(finaldata, bin)
### add to montage stack
montage.append(finaldata)
res = apFourier.spectralSNR(partdatalist, apix)
self.cluster_resolution.append(res)
### create png
apImage.arrayToPng(finaldata, pngfile)
### check for png file
if os.path.isfile(pngfile):
montagepngs.append(pngfile)
else:
apDisplay.printError("failed to create montage")
stackname = "kerdenstack"+self.timestamp+".hed"
apImagicFile.writeImagic(montage, stackname)
### create montage
montagecmd = ("montage -geometry +4+4 -tile %dx%d "%(self.params['xdim'], self.params['ydim']))
for monpng in montagepngs:
montagecmd += monpng+" "
montagecmd += "montage.png"
apEMAN.executeEmanCmd(montagecmd, showcmd=True, verbose=False)
time.sleep(1)
apFile.removeFilePattern(self.timestamp+".*.png")
return bin
def start():
### backup old cls files
classfile = os.path.join(params['emandir'], "cls.%d.tar"%(params['iter']))
oldclassfile = os.path.join(params['emandir'], "cls.%d.old.tar"%(params['iter']))
shutil.move(classfile, oldclassfile)
projhed = os.path.join(params['emandir'], 'proj.hed')
projhed = os.path.join(params['emandir'], 'proj.img')
numproj = apFile.numImagesInStack(projhed)
### extract cls files
tar = tarfile.open(oldclassfile)
tar.extractall(path=params['rundir'])
tar.close()
clslist = glob.glob("cls*.lst")
if numproj != len(clslist):
apDisplay.printError("array length mismatch")
### loop through classes
clsnum = 0
goodavg = []
for clsfile in clslist:
clsnum += 1
clsf = open(clsfile, 'r')
partlist = clsf.readlines()
clsf.close()
### write the projection???
#e=projections[clsNum].getEuler()
#projections[clsNum].setNImg(-1)
#projections[clsNum].writeImage('goodavgs.hed', -1)
if len(partlist) < params['minpart']:
### not enough particles skip to next projection
#origaverage =
goodavg.append()
#emanClsAvgs[(clsNum+1)*2 - 1].writeImage('goodavgs.hed',-1)
continue
### make aligned stack
if params['eotest'] is False:
command='clstoaligned.py -c' + cls
elif params['eotest']=='odd':
fw=open(cls,'r')
Ptcls = fw.readlines()
fw.close()
fw = open('cls_odd.lst', 'w')
fw.writelines(Ptcls[0])
fw.writelines(Ptcls[1])
for i1 in range(2,len(Ptcls)):
if i1%2==0:
fw.writelines(Ptcls[i1])
fw.close()
command='clstoaligned.py -c cls_odd.lst'
elif params['eotest']=='even':
fw=open(cls,'r')
Ptcls = fw.readlines()
fw.close()
fw = open('cls_even.lst', 'w')
fw.writelines(Ptcls[0])
fw.writelines(Ptcls[1])
for i1 in range(2,len(Ptcls)):
if i1%2==1:
fw.writelines(Ptcls[i1])
fw.close()
command='clstoaligned.py -c cls_even.lst'
apDisplay.printMsg(command)
proc = subprocess.Popen(command, shell=True)
proc.wait()
#set up cls dir
clsdir=cls.split('.')[0]+'.dir'
os.mkdir(clsdir)
os.rename('aligned.spi',os.path.join(clsdir,'aligned.spi'))
alignedImgsName = os.path.join(clsdir,'aligned.spi')
#alignedImgs = EMAN.readImages(alignedImgsName,-1,-1,0)
#N = len(alignedImgs)
apDisplay.printMsg("Starting cluster process for "+clsdir)
### fill similarity matrix with CC values
similarfile, simlist = fillSimilarityMatrix(alignedImgsName)
### set preferences
preffile = setPreferences(simlist, params['preftype'])
### run apcluster.exe program
outfile = "clusters.out"
apDisplay.printMsg("Run apcluster.exe program")
apclusterexe = os.path.join("apcluster.exe")
if os.path.isfile(outfile):
os.remove(outfile)
clustercmd = apclusterexe+" "+similarfile+" "+preffile+" "+outfile
proc = subprocess.Popen(clustercmd, shell=True)
proc.wait()
if not os.path.isfile(outfile):
apDisplay.printError("affinity propagration cluster program did not run")
### Parse apcluster output file: clusters.out
apDisplay.printMsg("Parse apcluster output file: "+outfile)
clustf = open(outfile, "r")
### each line is the particle and the number is the class
partnum = 0
classes = {}
for line in clustf:
sline = line.strip()
if sline:
partnum += 1
classnum = int(sline)
if not classnum in classes:
classes[classnum] = [partnum,]
else:
classes[classnum].append(partnum)
clustf.close()
apDisplay.printMsg("Found %d classes"%(len(classes.keys())))
### Create class averages
classavgdata = []
classnames = classes.keys()
classnames.sort()
for classnum in classnames:
apDisplay.printMsg("Class %d, %d members"%(classnum, len(classes[classnum])))
clsf = open('subcls%03d.lst'%(classnum), 'w')
for partnum in classes[classnum]:
clsf.write("%d\n"%(partnum))
clsf.close()
classdatalist = apImagicFile.readParticleListFromStack(stackfile, classes[classnum], msg=False)
classdatarray = numpy.asarray(classdatalist)
classavgarray = classdatarray.mean(0)
classavgdata.append(classavgarray)
apFile.removeStack("classaverage.hed")
apImagicFile.writeImagic(classavgdata, "classaverage.hed")
k=0
for i in range(0,len(E)):
if len(E[i])==0:
continue
else:
f1=open('%s/subcls%02d.lst' % (str1,k), 'w')
for j in range(0,len(E[i])):
f1.write('%d aligned.spi clusterCenterImgNum%d\n' % (E[i][j], i))
f1.close()
proc = subprocess.Popen('proc2d aligned.spi tempClsAvg.hed list=%s/subcls%02d.lst mask=%d average edgenorm' % (str1,k,params['mask']), shell=True)
proc.wait()
k=k+1
clsAvgs = EMAN.readImages('tempClsAvg.hed',-1,-1,0)
j=0
for i in range(0,len(E)):
if len(E[i])==0:
continue
else:
clsAvgs[j].setNImg(len(E[i]))
clsAvgs[j].writeImage('subclasses_avg.hed',-1)
j=j+1
os.chdir('../')
### Determine best averages
proc = subprocess.Popen('/bin/rm tempClsAvg.*', shell=True)
proc.wait()
proc = subprocess.Popen('proc2d %s/aligned.spi tempClsAvg.hed mask=%d average edgenorm' % (clsdir, params['mask']), shell=True)
proc.wait()
class_avg = EMAN.readImages('tempClsAvg.hed',-1,-1,0)
avgname=os.path.join(clsdir,'subclasses_avg.hed')
averages=EMAN.readImages(avgname,-1,-1,0)
cclist=[]
for avg in averages:
cclist.append(cc(projections[clsNum],avg))
f1 = open('%s/CCValues.txt'%(clsdir), 'w')
for i in range(len(cclist)):
f1.write(str(cclist[i])+'\n')
f1.close()
### Merge top best subclasses
ccListSort = cclist
ccListSort.sort()
Ptcls = []
for i in range(0,len(ccListSort)):
cci = ccListSort[len(ccListSort)-i-1]
if cci>=params['corCutOff']:
bestclass_i=cclist.index(cci)
classname_i=clslist[clsNum].split('.')[0]+'.dir/subClassAvgs/subcls'+string.zfill(bestclass_i,2)+'.lst'
f1=open(classname_i,'r')
Ptcls_i = f1.readlines()
f1.close()
Ptcls.extend(Ptcls_i)
else:
print "Not included - ", cci
pass
if len(Ptcls)>0:
fw=open('mergeClasses.lst', 'w')
fw.writelines(Ptcls)
fw.close()
proc = subprocess.Popen('/bin/rm mergedClsAvg.spi', shell=True)
proc.wait()
proc = subprocess.Popen('proc2d %s/aligned.spi mergedClsAvg.spi list=mergeClasses.lst mask=%d average' % (clsdir, params['mask']), shell=True)
proc.wait()
mergedavg=EMAN.readImages('mergedClsAvg.spi',-1,-1,0)
mergedavg[0].setNImg(len(Ptcls))
mergedavg[0].setRAlign(e)
mergedavg[0].writeImage('goodavgs.hed',-1)
else:
pass
writeNewClsfile(cls,pretext,Ptext,Ptcls)
#Create list of cc values
for cls in range(0,len(clslist)):
clsdir=clslist[cls].split('.')[0]+'.dir'
apDisplay.printMsg("Starting class number %d" %(cls))
#break
pad=params['boxsize']*1.25
if pad%2:
pad=pad+1
if params['sym']==None:
make3dcommand='make3d goodavgs.hed out=threed.%d.mrc mask=%d pad=%d mode=2 hard=%d' % (params['iter'], params['mask'], pad, params['hard'])
else:
make3dcommand='make3d goodavgs.hed out=threed.%d.mrc mask=%d sym=%s pad=%d mode=2 hard=%d' % (params['iter'], params['mask'], params['sym'], pad, params['hard'])
apDisplay.printMsg(make3dcommand)
proc = subprocess.Popen(make3dcommand, shell=True)
proc.wait()
proc3dcommand='proc3d threed.%d.mrc threed.%da.mrc mask=%d norm' % (params['iter'],params['iter'],params['mask'])
apDisplay.printMsg(proc3dcommand)
proc = subprocess.Popen(proc3dcommand, shell=True)
proc.wait()
if params['eotest'] is False:
#copy the resulting class average images to the main recon directory
proc = subprocess.Popen('/bin/cp threed.%da.mrc ../.'%(params['iter']), shell=True)
proc.wait()
proc = subprocess.Popen('/bin/cp goodavgs.hed ../classes_msgp.%d.hed' %(params['iter']), shell=True)
proc.wait()
proc = subprocess.Popen('/bin/cp goodavgs.img ../classes_msgp.%d.img' %(params['iter']), shell=True)
proc.wait()
#link msgp result as the final result for this iteration
rmcommand='/bin/rm -f ../classes.%d.hed ../classes.%d.img' % (params['iter'], params['iter'])
proc = subprocess.Popen(rmcommand, shell=True)
proc.wait()
lncommand='ln -s classes_msgp.%d.hed ../classes.%d.hed' % (params['iter'], params['iter'])
proc = subprocess.Popen(lncommand, shell=True)
proc.wait()
lncommand='ln -s classes_msgp.%d.img ../classes.%d.img' % (params['iter'], params['iter'])
proc = subprocess.Popen(lncommand, shell=True)
proc.wait()
elif params['eotest']=='odd':
proc = subprocess.Popen('/bin/cp threed.%da.mrc ../threed.%da.o.mrc' %(params['iter'], params['iter']), shell=True)
proc.wait()
elif params['eotest']=='even':
proc = subprocess.Popen('/bin/cp threed.%da.mrc ../threed.%da.e.mrc' %(params['iter'], params['iter']), shell=True)
proc.wait()
proc = subprocess.Popen('proc3d threed.%da.mrc ../threed.%da.o.mrc fsc=../corEO%d.fsc.dat' %(params['iter'], params['iter'], params['iter']), shell=True)
proc.wait()
#replace the old cls*.lst with the new extended one
proc = subprocess.Popen('tar cvzf %s %s' % (newclassfile,"cls*.lst.new"), shell=True)
proc.wait()
proc = subprocess.Popen('/bin/cp %s ../%s' %(newclassfile,classfile), shell=True)
proc.wait()
apDisplay.printMsg("Done!")
