def createAlignedReferenceStack(self):
searchstr = "part"+self.params['timestamp']+"_ref0*.xmp"
files = glob.glob(searchstr)
files.sort()
stack = []
reflist = self.readRefDocFile()
for i in range(len(files)):
fname = files[i]
refdict = reflist[i]
if refdict['partnum'] != i+1:
print i, refdict['partnum']
apDisplay.printError("sorting error in reflist, see neil")
refarray = spider.read(fname)
xyshift = (refdict['xshift'], refdict['yshift'])
alignrefarray = apImage.xmippTransform(refarray, rot=refdict['inplane'],
shift=xyshift, mirror=refdict['mirror'])
stack.append(alignrefarray)
stackarray = numpy.asarray(stack, dtype=numpy.float32)
#print stackarray.shape
avgstack = "part"+self.params['timestamp']+"_average.hed"
apFile.removeStack(avgstack, warn=False)
apImagicFile.writeImagic(stackarray, avgstack)
### create a average mrc
avgdata = stackarray.mean(0)
apImage.arrayToMrc(avgdata, "average.mrc")
return
def spiderToArray(filename, msg=True):
"""
takes a numpy and writes a SPIDER image
"""
numer = spider.read(filename)
if msg is True:
apDisplay.printMsg("reading SPIDER image: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
return numer
def spiderToArray(filename, msg=True):
"""
takes a numpy and writes a SPIDER image
"""
numer = spider.read(filename)
if msg is True:
apDisplay.printMsg("reading SPIDER image: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
return numer
def createReferenceStack(self):
# Create a stack for the class averages at each level
Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.sel"))
for level in range(Nlevels):
stack=[]
for f in glob.glob("part"+self.params['timestamp']+"_level_%02d_[0-9]*.xmp"%level):
stack.append(spider.read(f))
apImagicFile.writeImagic(stack, "part"+self.params['timestamp']+"_level_%02d_.hed"%level)
if self.params['align']:
apXmipp.gatherSingleFilesIntoStack("partlist.sel","alignedStack.hed")
return
def openFile(self):
if not os.path.isfile(self.params['filename']):
apDisplay.printError("Cound not find file "+self.params['filename'])
if self.params['filename'][-4:].lower() == ".mrc":
imgarray = mrc.read(self.params['filename'])
else:
try:
### assume file is of type spider
imgarray = spider.read(self.params['filename'])
except:
apDisplay.printError("Cound not read file "+self.params['filename'])
return imgarray
"""
def createReferenceStack(self):
# Create a stack for the class averages at each level
Nlevels = len(
glob.glob("part" + self.params['timestamp'] + "_level_??_.sel"))
for level in range(Nlevels):
stack = []
for f in glob.glob("part" + self.params['timestamp'] +
"_level_%02d_[0-9]*.xmp" % level):
stack.append(spider.read(f))
apImagicFile.writeImagic(
stack,
"part" + self.params['timestamp'] + "_level_%02d_.hed" % level)
if self.params['align']:
apXmipp.gatherSingleFilesIntoStack("partlist.sel",
"alignedStack.hed")
return
def openFile(self):
if not os.path.isfile(self.params['filename']):
apDisplay.printError("Cound not find file " +
self.params['filename'])
if self.params['filename'][-4:].lower() == ".mrc":
imgarray = mrc.read(self.params['filename'])
else:
try:
### assume file is of type spider
imgarray = spider.read(self.params['filename'])
except:
apDisplay.printError("Cound not read file " +
self.params['filename'])
return imgarray
"""
def fermiLowPassFilter(imgarray, pixrad=2.0, dataext="spi", nproc=None):
if dataext[0] == '.': dataext = dataext[1:]
if nproc is None:
nproc = apParam.getNumProcessors(msg=False)
### save array to spider file
spider.write(imgarray, "rawimg."+dataext)
### run the filter
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, nproc=nproc)
### filter request: infile, outfile, filter-type, inv-radius, temperature
mySpider.toSpiderQuiet("FQ", "rawimg", "filtimg", "5", str(1.0/pixrad), "0.04")
mySpider.close()
### read array from spider file
filtarray = spider.read("filtimg."+dataext)
### clean up
apFile.removeFile("rawimg."+dataext)
apFile.removeFile("filtimg."+dataext)
return filtarray
def fermiLowPassFilter(imgarray, pixrad=2.0, dataext="spi", nproc=None):
if dataext[0] == '.': dataext = dataext[1:]
if nproc is None:
nproc = apParam.getNumProcessors(msg=False)
### save array to spider file
spider.write(imgarray, "rawimg." + dataext)
### run the filter
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, nproc=nproc)
### filter request: infile, outfile, filter-type, inv-radius, temperature
mySpider.toSpiderQuiet("FQ", "rawimg", "filtimg", "5", str(1.0 / pixrad),
"0.04")
mySpider.close()
### read array from spider file
filtarray = spider.read("filtimg." + dataext)
### clean up
apFile.removeFile("rawimg." + dataext)
apFile.removeFile("filtimg." + dataext)
return filtarray
def createReferenceStack(self):
avgstack = "part"+self.timestamp+"_average.hed"
apFile.removeStack(avgstack, warn=False)
searchstr = "part"+self.timestamp+"_ref0*.xmp"
files = glob.glob(searchstr)
if len(files) == 0:
apDisplay.printError("Xmipp did not run")
files.sort()
stack = []
for i in range(len(files)):
fname = files[i]
refarray = spider.read(fname)
stack.append(refarray)
apImagicFile.writeImagic(stack, avgstack)
### create a average mrc
stackarray = numpy.asarray(stack, dtype=numpy.float32)
avgdata = stackarray.mean(0)
apImage.arrayToMrc(avgdata, "average.mrc")
return
def stackToEmanSpi(stackfile, swap=False):
numpart = apFile.numImagesInStack(stackfile)
partlist = []
for i in range(numpart):
spifile = "eman%d.spi"%(i)
emanspifile = spifile
if i > 0:
### hack for eman auto-naming
emanspifile = "eman"
emancmd = "proc2d %s %s first=%d last=%d"%(stackfile, emanspifile, i, i)
if swap is True:
emancmd += " spiderswap-single"
else:
emancmd += " spider-single"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
if not os.path.isfile(spifile):
print "Failed to create SPIDER file"
sys.exit(1)
part = spider.read(spifile)
apFile.removeFile(spifile)
partlist.append(part)
return partlist
def compute_stack_of_class_averages_and_reprojections(dir, selfile, refvolume, docfile, boxsize, resultspath, timestamp, iteration, reference_number=1, extract=False):
''' takes Xmipp single files, doc and sel files in routine, creates a stack of class averages in the results directory '''
workingdir = os.getcwd()
os.chdir(dir)
if dir.endswith("/"):
dir = dir[:-1]
head, tail = os.path.split(dir)
### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile
f = open(selfile, "r")
lines = f.readlines()
newlines = [re.sub(str(tail)+"/", "", line) for line in lines]
f.close()
f = open(selfile[:-4]+"_new.sel", "w")
f.writelines(newlines)
f.close()
if extract is True:
extractcmd = "xmipp_header_extract -i %s.sel -o %s.doc" % (selfile[:-4], docfile[:-4])
apParam.runCmd(extractcmd, "Xmipp")
### create a projection params file and project the volume along identical Euler angles
f = open("paramfile.descr", "w")
f.write("%s\n" % refvolume)
f.write("tmpproj 1 xmp\n")
f.write("%d %d\n" % (boxsize, boxsize))
f.write("%s rot tilt psi\n" % docfile)
f.write("NULL\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.close()
projectcmd = "xmipp_project -i paramfile.descr"
apParam.runCmd(projectcmd, "Xmipp")
### get order of projections in docfile
d = open(docfile, "r")
lines = d.readlines()[1:]
d.close()
projfile_sequence = []
for i, l in enumerate(lines):
if i % 2 == 0:
filename = os.path.basename(l.split()[1])
projfile_sequence.append(filename)
else: pass
### create stack of projections and class averages
projections = glob.glob("tmpproj**xmp")
projections.sort()
if len(projections) != len(projfile_sequence):
apDisplay.printWarning("number of projections does not match number of classes")
stackarray = []
stackname = os.path.join(resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (timestamp, iteration, reference_number))
for i in range(len(projections)):
stackarray.append(spider.read(projections[i]))
stackarray.append(spider.read(projfile_sequence[i]))
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### remove unnecessary files
for file in glob.glob("tmpproj*"):
apFile.removeFile(file)
os.chdir(workingdir)
return
def compute_stack_of_class_averages_and_reprojections(self, iteration, reference_number):
''' takes Xmipp single files, doc and sel files associated with ML3D, creates a stack of class averages in the results directory '''
'''
### make projections, and put them back into resultspath
selfile = "ml3d_it%.6d_vol%.6d.sel" % (iteration, reference_number)
refvolume = "ml3d_it%.6d_vol%.6d.vol\n" % (iteration, reference_number)
docfile = "ml3d_it%.6d_vol%.6d.doc" % (iteration, reference_number)
apXmipp.compute_stack_of_class_averages_and_reprojections(self.ml3dpath, selfile, refvolume, docfile, \
self.runparams['boxsize'], self.resultspath, self.params['timestamp'], iteration, reference_number, extract=True)
return
'''
os.chdir(self.ml3dpath)
### remove "RunML3D/" from selfile (created by ML3D program), then extract header information to docfile
selfile = "ml3d_it%.6d_vol%.6d.sel" % (iteration, reference_number)
f = open(selfile, "r")
lines = f.readlines()
newlines = [re.sub("RunML3D/", "", line) for line in lines]
f.close()
f = open(selfile, "w")
f.writelines(newlines)
f.close()
extractcmd = "xmipp_header_extract -i ml3d_it%.6d_vol%.6d.sel -o ml3d_it%.6d_vol%.6d.doc" \
% (iteration, reference_number, iteration, reference_number)
apParam.runCmd(extractcmd, "Xmipp")
### create a projection params file and project the volume along identical Euler angles
f = open("paramfile.descr", "w")
f.write("ml3d_it%.6d_vol%.6d.vol\n" % (iteration, reference_number))
f.write("tmpproj 1 xmp\n")
f.write("%d %d\n" % (self.runparams['boxsize'], self.runparams['boxsize']))
f.write("ml3d_it%.6d_vol%.6d.doc rot tilt psi\n" % (iteration, reference_number))
f.write("NULL\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.close()
projectcmd = "xmipp_project -i paramfile.descr"
apParam.runCmd(projectcmd, "Xmipp")
### get order of projections in docfile
docfile = "ml3d_it%.6d_vol%.6d.doc" % (iteration, reference_number)
d = open(docfile, "r")
lines = d.readlines()[1:]
d.close()
projfile_sequence = []
for i, l in enumerate(lines):
if i % 2 == 0:
filename = os.path.basename(l.split()[1])
projfile_sequence.append(filename)
else: pass
### create stack of projections and class averages
projections = glob.glob("tmpproj**xmp")
projections.sort()
if len(projections) != len(projfile_sequence):
apDisplay.printWarning("number of projections does not match number of classes for model %d, iteration %d")
stackarray = []
stackname = os.path.join(self.resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (self.params['timestamp'], iteration, reference_number))
for i in range(len(projections)):
stackarray.append(spider.read(projections[i]))
stackarray.append(spider.read(projfile_sequence[i]))
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### remove unnecessary files
for file in glob.glob("tmpproj*"):
apFile.removeFile(file)
os.chdir(self.params['rundir'])
return
def compute_stack_of_class_averages_and_reprojections(dir,
selfile,
refvolume,
docfile,
boxsize,
resultspath,
timestamp,
iteration,
reference_number=1,
extract=False):
''' takes Xmipp single files, doc and sel files in routine, creates a stack of class averages in the results directory '''
workingdir = os.getcwd()
os.chdir(dir)
if dir.endswith("/"):
dir = dir[:-1]
head, tail = os.path.split(dir)
### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile
f = open(selfile, "r")
lines = f.readlines()
newlines = [re.sub(str(tail) + "/", "", line) for line in lines]
f.close()
f = open(selfile[:-4] + "_new.sel", "w")
f.writelines(newlines)
f.close()
if extract is True:
extractcmd = "xmipp_header_extract -i %s.sel -o %s.doc" % (
selfile[:-4], docfile[:-4])
apParam.runCmd(extractcmd, "Xmipp")
### create a projection params file and project the volume along identical Euler angles
f = open("paramfile.descr", "w")
f.write("%s\n" % refvolume)
f.write("tmpproj 1 xmp\n")
f.write("%d %d\n" % (boxsize, boxsize))
f.write("%s rot tilt psi\n" % docfile)
f.write("NULL\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.close()
projectcmd = "xmipp_project -i paramfile.descr"
apParam.runCmd(projectcmd, "Xmipp")
### get order of projections in docfile
d = open(docfile, "r")
lines = d.readlines()[1:]
d.close()
projfile_sequence = []
for i, l in enumerate(lines):
if i % 2 == 0:
filename = os.path.basename(l.split()[1])
projfile_sequence.append(filename)
else:
pass
### create stack of projections and class averages
projections = glob.glob("tmpproj**xmp")
projections.sort()
if len(projections) != len(projfile_sequence):
apDisplay.printWarning(
"number of projections does not match number of classes")
stackarray = []
stackname = os.path.join(
resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" %
(timestamp, iteration, reference_number))
for i in range(len(projections)):
stackarray.append(spider.read(projections[i]))
stackarray.append(spider.read(projfile_sequence[i]))
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### remove unnecessary files
for file in glob.glob("tmpproj*"):
apFile.removeFile(file)
os.chdir(workingdir)
return
def gatherSingleFilesIntoStack(selfile, stackfile, filetype="spider"):
"""
takes a selfile and creates an EMAN stack
"""
selfile = os.path.abspath(selfile)
stackfile = os.path.abspath(stackfile)
if stackfile[-4:] != ".hed":
apDisplay.printWarning("Stack file does not end in .hed")
stackfile = stackfile[:-4] + ".hed"
apDisplay.printColor("Merging files into a stack, this can take a while",
"cyan")
starttime = time.time()
if not os.path.isfile(selfile):
apDisplay.printError("selfile does not exist: " + selfile)
### Process selfile
fh = open(selfile, 'r')
filelist = []
for line in fh:
sline = line.strip()
if sline:
args = sline.split()
if (len(args) > 1):
filename = args[0].strip()
filelist.append(filename)
fh.close()
### Set variables
boxsize = apFile.getBoxSize(filelist[0])
partperiter = int(1e9 / (boxsize[0]**2) / 16.)
if partperiter > 4096:
partperiter = 4096
apDisplay.printMsg("Using %d particle per iteration" % (partperiter))
numpart = len(filelist)
if numpart < partperiter:
partperiter = numpart
### Process images
imgnum = 0
stacklist = []
stackroot = stackfile[:-4]
### get memory in kB
startmem = mem.active()
while imgnum < len(filelist):
filename = filelist[imgnum]
index = imgnum % partperiter
if imgnum % 100 == 0:
sys.stderr.write(".")
#sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index))
if mem.active() - startmem > 2e6:
apDisplay.printWarning("Out of memory")
if index < 1:
#print "img num", imgnum
### deal with large stacks, reset loop
if imgnum > 0:
sys.stderr.write("\n")
stackname = "%s-%d.hed" % (stackroot, imgnum)
apDisplay.printMsg("writing single particles to file " +
stackname)
stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
perpart = (time.time() - starttime) / imgnum
apDisplay.printColor(
"part %d of %d :: %.1fM mem :: %s/part :: %s remain" %
(imgnum + 1, numpart, (mem.active() - startmem) / 1024.,
apDisplay.timeString(perpart),
apDisplay.timeString(perpart * (numpart - imgnum))),
"blue")
stackarray = []
### merge particles
if filetype == "mrc":
partimg = mrc.read(filename)
else:
partimg = spider.read(filename)
stackarray.append(partimg)
imgnum += 1
### write remaining particles to file
sys.stderr.write("\n")
stackname = "%s-%d.hed" % (stackroot, imgnum)
apDisplay.printMsg("writing particles to file " + stackname)
stacklist.append(stackname)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### merge stacks
apFile.removeStack(stackfile, warn=False)
apImagicFile.mergeStacks(stacklist, stackfile)
print stackfile
filepart = apFile.numImagesInStack(stackfile)
if filepart != numpart:
apDisplay.printError(
"number merged particles (%d) not equal number expected particles (%d)"
% (filepart, numpart))
for stackname in stacklist:
apFile.removeStack(stackname, warn=False)
### summarize
apDisplay.printColor(
"merged %d particles in %s" %
(imgnum, apDisplay.timeString(time.time() - starttime)), "cyan")
def start(self):
### start the outfile name
fileroot = os.path.splitext(self.params['filename'])[0]
fileroot += "-"+self.timestamp
filename = self.params['file'].split('/')[-1]
filepath = self.params['file'].strip(filename)
self.params['box'] = apVolume.getModelDimensions(self.params['file'])
if self.params['ampfile'] is not None:
### run amplitude correction
spifile = apVolume.MRCtoSPI(self.params['file'], self.params['rundir'])
tmpfile = apAmpCorrect.createAmpcorBatchFile(spifile, self.params)
apAmpCorrect.runAmpcor()
### check if spider was successful
fileroot += ".amp"
if not os.path.isfile(tmpfile) :
apDisplay.printError("amplitude correction failed")
### convert amplitude corrected file back to mrc
fileroot += ".amp"
a = spider.read(tmpfile)
outfile = os.path.join( self.params['rundir'], "ampl-fix.mrc" )
mrc.write(a, outfile)
curfile = outfile
elif self.params['bfactor'] is True:
outfile = os.path.join( self.params['rundir'], "bfactor-fix.mrc" )
outfile = apAmpCorrect.applyBfactor(self.params['file'], fscfile=self.params['fscfile'], apix=self.params['apix'], mass=self.params['mass'], outfile=outfile)
curfile = outfile
elif self.params['lrdw'] is True:
lrdwcmd = "s_diffmap2 %s %s %s %s"%(self.params['filepath'],self.params['filename'],self.params['ampx'],self.params['resol'])
proc = subprocess.Popen(lrdwcmd)
print lrdwcmd
else :
### just run proc3d
curfile = self.params['file']
emancmd = "proc3d "+self.params['file']+" "
emancmd = "proc3d "+curfile+" "
if self.params['median'] is not None:
data = mrc.read(curfile)
data = ndimage.median_filter(data, size=self.params['median'])
mrc.write(data, curfile)
emancmd+="apix=%s " %self.params['apix']
if self.params['lp'] is not None:
fileroot += (".lp%d" % ( int(self.params['lp']), ))
emancmd += "lp=%d " %self.params['lp']
if self.params['yflip'] is True:
fileroot += ".yflip"
emancmd +="yflip "
if self.params['invert'] is True:
fileroot += ".inv"
emancmd +="mult=-1 "
if self.params['viper'] is True:
fileroot += ".vip"
emancmd +="icos5fTo2f "
if self.params['mask'] is not None:
# convert ang to pixels
maskpix=int(self.params['mask']/self.params['apix'])
fileroot += (".m%d" % ( int(self.params['mask']), ))
emancmd += "mask=%d " %maskpix
self.params['mask'] = maskpix
if self.params['imask'] is not None:
# convert ang to pixels
maskpix=int(self.params['imask']/self.params['apix'])
fileroot += (".im%d" % ( int(self.params['imask']), ))
emancmd += "imask=%d " %maskpix
self.params['imask'] = maskpix
if self.params['norm'] is True:
fileroot += ".norm"
emancmd += "norm=0,1 "
### add output filename to emancmd string
fileroot += ".mrc"
self.params['name'] = fileroot
outfile = os.path.join(self.params['rundir'], fileroot)
emancmd = re.sub(" apix=",(" %s apix=" % outfile), emancmd)
apEMAN.executeEmanCmd(emancmd)
if self.params['description'] is None:
self.params['description'] = "Volume from recon with amplitude adjustment"
### clean up files created during amp correction
if self.params['ampfile'] is not None:
apFile.removeFile(spifile)
apFile.removeFile(tmpfile)
if self.params['commit'] is True:
symdata = apSymmetry.findSymmetry(self.params['sym'])
print symdata
symmetry = symdata['eman_name']
self.params['reconid'] = self.params['reconiterid']
self.insert3dDensity()
### render chimera images of model
apChimera.filterAndChimera(outfile, res=self.params['res'], apix=self.params['apix'], box=self.params['box'],
chimtype='snapshot', mass=self.params['mass'], contour=self.params['contour'],
zoom=self.params['zoom'], sym=symmetry)
def compute_stack_of_class_averages_and_reprojections(self, iteration):
''' takes Xmipp single files, doc and sel files in projection-matching, creates a stack of class averages in the results directory '''
if bool(self.runparams['package_params']['CleanUpFiles']) is False:
os.chdir(
os.path.join(self.projmatchpath, "Iter_%d" % iteration,
"ProjMatchClasses"))
### make projections, and put them back into resultspath
selfile = "proj_match_classes.sel"
refvolume = "../Iter_%d_reconstruction.vol" % iteration
docfile = "proj_match_classes.doc"
# apXmipp.compute_stack_of_class_averages_and_reprojections(d, selfile, refvolume, docfile, \
# self.runparams['boxsize'], self.resultspath, self.params['timestamp'], iteration)
### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile
f = open(selfile, "r")
lines = f.readlines()
newlines = [
re.sub("ProjMatchClasses/", "", line) for line in lines
]
f.close()
f = open(selfile[:-4] + "_new.sel", "w")
f.writelines(newlines)
f.close()
### create a projection params file and project the volume along identical Euler angles
f = open("paramfile.descr", "w")
f.write("%s\n" % refvolume)
f.write("tmpproj 1 xmp\n")
f.write("%d %d\n" %
(self.runparams['boxsize'], self.runparams['boxsize']))
f.write("%s rot tilt psi\n" % docfile)
f.write("NULL\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.close()
projectcmd = "xmipp_project -i paramfile.descr"
apParam.runCmd(projectcmd, "Xmipp")
### get order of projections in docfile
d = open(docfile, "r")
lines = d.readlines()[1:]
d.close()
projfile_sequence = []
for i, l in enumerate(lines):
if i % 2 == 0:
filename = os.path.basename(l.split()[1])
projfile_sequence.append(filename)
else:
pass
### create stack of projections and class averages
projections = glob.glob("tmpproj**xmp")
projections.sort()
if len(projections) != len(projfile_sequence):
apDisplay.printWarning(
"number of projections does not match number of classes")
stackarray = []
stackname = os.path.join(
self.resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" %
(self.params['timestamp'], iteration, 1))
for i in range(len(projections)):
stackarray.append(spider.read(projections[i]))
stackarray.append(spider.read(projfile_sequence[i]))
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### remove unnecessary files
for file in glob.glob("tmpproj*"):
apFile.removeFile(file)
os.chdir(self.params['rundir'])
else:
apDisplay.printWarning(
"all projection-matching files were cleaned up ... NOT creating class-average / re-projection stack"
)
return
def compute_stack_of_class_averages_and_reprojections(self, iteration):
''' takes Xmipp single files, doc and sel files in projection-matching, creates a stack of class averages in the results directory '''
if bool(self.runparams['package_params']['CleanUpFiles']) is False:
os.chdir(os.path.join(self.projmatchpath, "Iter_%d" % iteration, "ProjMatchClasses"))
### make projections, and put them back into resultspath
selfile = "proj_match_classes.sel"
refvolume = "../Iter_%d_reconstruction.vol" % iteration
docfile = "proj_match_classes.doc"
# apXmipp.compute_stack_of_class_averages_and_reprojections(d, selfile, refvolume, docfile, \
# self.runparams['boxsize'], self.resultspath, self.params['timestamp'], iteration)
### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile
f = open(selfile, "r")
lines = f.readlines()
newlines = [re.sub("ProjMatchClasses/", "", line) for line in lines]
f.close()
f = open(selfile[:-4]+"_new.sel", "w")
f.writelines(newlines)
f.close()
### create a projection params file and project the volume along identical Euler angles
f = open("paramfile.descr", "w")
f.write("%s\n" % refvolume)
f.write("tmpproj 1 xmp\n")
f.write("%d %d\n" % (self.runparams['boxsize'], self.runparams['boxsize']))
f.write("%s rot tilt psi\n" % docfile)
f.write("NULL\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.close()
projectcmd = "xmipp_project -i paramfile.descr"
apParam.runCmd(projectcmd, "Xmipp")
### get order of projections in docfile
d = open(docfile, "r")
lines = d.readlines()[1:]
d.close()
projfile_sequence = []
for i, l in enumerate(lines):
if i % 2 == 0:
filename = os.path.basename(l.split()[1])
projfile_sequence.append(filename)
else: pass
### create stack of projections and class averages
projections = glob.glob("tmpproj**xmp")
projections.sort()
if len(projections) != len(projfile_sequence):
apDisplay.printWarning("number of projections does not match number of classes")
stackarray = []
stackname = os.path.join(self.resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (self.params['timestamp'], iteration, 1))
for i in range(len(projections)):
stackarray.append(spider.read(projections[i]))
stackarray.append(spider.read(projfile_sequence[i]))
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### remove unnecessary files
for file in glob.glob("tmpproj*"):
apFile.removeFile(file)
os.chdir(self.params['rundir'])
else:
apDisplay.printWarning("all projection-matching files were cleaned up ... NOT creating class-average / re-projection stack")
return
def gatherSingleFilesIntoStack(selfile, stackfile, filetype="spider"):
"""
takes a selfile and creates an EMAN stack
"""
selfile = os.path.abspath(selfile)
stackfile = os.path.abspath(stackfile)
if stackfile[-4:] != ".hed":
apDisplay.printWarning("Stack file does not end in .hed")
stackfile = stackfile[:-4]+".hed"
apDisplay.printColor("Merging files into a stack, this can take a while", "cyan")
starttime = time.time()
if not os.path.isfile(selfile):
apDisplay.printError("selfile does not exist: "+selfile)
### Process selfile
fh = open(selfile, 'r')
filelist = []
for line in fh:
sline = line.strip()
if sline:
args=sline.split()
if (len(args)>1):
filename = args[0].strip()
filelist.append(filename)
fh.close()
### Set variables
boxsize = apFile.getBoxSize(filelist[0])
partperiter = int(1e9/(boxsize[0]**2)/16.)
if partperiter > 4096:
partperiter = 4096
apDisplay.printMsg("Using %d particle per iteration"%(partperiter))
numpart = len(filelist)
if numpart < partperiter:
partperiter = numpart
### Process images
imgnum = 0
stacklist = []
stackroot = stackfile[:-4]
### get memory in kB
startmem = mem.active()
while imgnum < len(filelist):
filename = filelist[imgnum]
index = imgnum % partperiter
if imgnum % 100 == 0:
sys.stderr.write(".")
#sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index))
if mem.active()-startmem > 2e6:
apDisplay.printWarning("Out of memory")
if index < 1:
#print "img num", imgnum
### deal with large stacks, reset loop
if imgnum > 0:
sys.stderr.write("\n")
stackname = "%s-%d.hed"%(stackroot, imgnum)
apDisplay.printMsg("writing single particles to file "+stackname)
stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
perpart = (time.time()-starttime)/imgnum
apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"%
(imgnum+1, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart),
apDisplay.timeString(perpart*(numpart-imgnum))), "blue")
stackarray = []
### merge particles
if filetype == "mrc":
partimg = mrc.read(filename)
else:
partimg = spider.read(filename)
stackarray.append(partimg)
imgnum += 1
### write remaining particles to file
sys.stderr.write("\n")
stackname = "%s-%d.hed"%(stackroot, imgnum)
apDisplay.printMsg("writing particles to file "+stackname)
stacklist.append(stackname)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### merge stacks
apFile.removeStack(stackfile, warn=False)
apImagicFile.mergeStacks(stacklist, stackfile)
print stackfile
filepart = apFile.numImagesInStack(stackfile)
if filepart != numpart:
apDisplay.printError("number merged particles (%d) not equal number expected particles (%d)"%
(filepart, numpart))
for stackname in stacklist:
apFile.removeStack(stackname, warn=False)
### summarize
apDisplay.printColor("merged %d particles in %s"%(imgnum, apDisplay.timeString(time.time()-starttime)), "cyan")
