def createAlignedReferenceStack(self): searchstr = "part"+self.params['timestamp']+"_ref0*.xmp" files = glob.glob(searchstr) files.sort() stack = [] reflist = self.readRefDocFile() for i in range(len(files)): fname = files[i] refdict = reflist[i] if refdict['partnum'] != i+1: print i, refdict['partnum'] apDisplay.printError("sorting error in reflist, see neil") refarray = spider.read(fname) xyshift = (refdict['xshift'], refdict['yshift']) alignrefarray = apImage.xmippTransform(refarray, rot=refdict['inplane'], shift=xyshift, mirror=refdict['mirror']) stack.append(alignrefarray) stackarray = numpy.asarray(stack, dtype=numpy.float32) #print stackarray.shape avgstack = "part"+self.params['timestamp']+"_average.hed" apFile.removeStack(avgstack, warn=False) apImagicFile.writeImagic(stackarray, avgstack) ### create a average mrc avgdata = stackarray.mean(0) apImage.arrayToMrc(avgdata, "average.mrc") return
def spiderToArray(filename, msg=True): """ takes a numpy and writes a SPIDER image """ numer = spider.read(filename) if msg is True: apDisplay.printMsg("reading SPIDER image: "+apDisplay.short(filename)+\ " size:"+str(numer.shape)+" dtype:"+str(numer.dtype)) return numer
def createReferenceStack(self): # Create a stack for the class averages at each level Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.sel")) for level in range(Nlevels): stack=[] for f in glob.glob("part"+self.params['timestamp']+"_level_%02d_[0-9]*.xmp"%level): stack.append(spider.read(f)) apImagicFile.writeImagic(stack, "part"+self.params['timestamp']+"_level_%02d_.hed"%level) if self.params['align']: apXmipp.gatherSingleFilesIntoStack("partlist.sel","alignedStack.hed") return
def openFile(self): if not os.path.isfile(self.params['filename']): apDisplay.printError("Cound not find file "+self.params['filename']) if self.params['filename'][-4:].lower() == ".mrc": imgarray = mrc.read(self.params['filename']) else: try: ### assume file is of type spider imgarray = spider.read(self.params['filename']) except: apDisplay.printError("Cound not read file "+self.params['filename']) return imgarray """
def createReferenceStack(self): # Create a stack for the class averages at each level Nlevels = len( glob.glob("part" + self.params['timestamp'] + "_level_??_.sel")) for level in range(Nlevels): stack = [] for f in glob.glob("part" + self.params['timestamp'] + "_level_%02d_[0-9]*.xmp" % level): stack.append(spider.read(f)) apImagicFile.writeImagic( stack, "part" + self.params['timestamp'] + "_level_%02d_.hed" % level) if self.params['align']: apXmipp.gatherSingleFilesIntoStack("partlist.sel", "alignedStack.hed") return
def openFile(self): if not os.path.isfile(self.params['filename']): apDisplay.printError("Cound not find file " + self.params['filename']) if self.params['filename'][-4:].lower() == ".mrc": imgarray = mrc.read(self.params['filename']) else: try: ### assume file is of type spider imgarray = spider.read(self.params['filename']) except: apDisplay.printError("Cound not read file " + self.params['filename']) return imgarray """
def fermiLowPassFilter(imgarray, pixrad=2.0, dataext="spi", nproc=None): if dataext[0] == '.': dataext = dataext[1:] if nproc is None: nproc = apParam.getNumProcessors(msg=False) ### save array to spider file spider.write(imgarray, "rawimg."+dataext) ### run the filter mySpider = spyder.SpiderSession(dataext=dataext, logo=False, nproc=nproc) ### filter request: infile, outfile, filter-type, inv-radius, temperature mySpider.toSpiderQuiet("FQ", "rawimg", "filtimg", "5", str(1.0/pixrad), "0.04") mySpider.close() ### read array from spider file filtarray = spider.read("filtimg."+dataext) ### clean up apFile.removeFile("rawimg."+dataext) apFile.removeFile("filtimg."+dataext) return filtarray
def fermiLowPassFilter(imgarray, pixrad=2.0, dataext="spi", nproc=None): if dataext[0] == '.': dataext = dataext[1:] if nproc is None: nproc = apParam.getNumProcessors(msg=False) ### save array to spider file spider.write(imgarray, "rawimg." + dataext) ### run the filter mySpider = spyder.SpiderSession(dataext=dataext, logo=False, nproc=nproc) ### filter request: infile, outfile, filter-type, inv-radius, temperature mySpider.toSpiderQuiet("FQ", "rawimg", "filtimg", "5", str(1.0 / pixrad), "0.04") mySpider.close() ### read array from spider file filtarray = spider.read("filtimg." + dataext) ### clean up apFile.removeFile("rawimg." + dataext) apFile.removeFile("filtimg." + dataext) return filtarray
def createReferenceStack(self): avgstack = "part"+self.timestamp+"_average.hed" apFile.removeStack(avgstack, warn=False) searchstr = "part"+self.timestamp+"_ref0*.xmp" files = glob.glob(searchstr) if len(files) == 0: apDisplay.printError("Xmipp did not run") files.sort() stack = [] for i in range(len(files)): fname = files[i] refarray = spider.read(fname) stack.append(refarray) apImagicFile.writeImagic(stack, avgstack) ### create a average mrc stackarray = numpy.asarray(stack, dtype=numpy.float32) avgdata = stackarray.mean(0) apImage.arrayToMrc(avgdata, "average.mrc") return
def stackToEmanSpi(stackfile, swap=False): numpart = apFile.numImagesInStack(stackfile) partlist = [] for i in range(numpart): spifile = "eman%d.spi"%(i) emanspifile = spifile if i > 0: ### hack for eman auto-naming emanspifile = "eman" emancmd = "proc2d %s %s first=%d last=%d"%(stackfile, emanspifile, i, i) if swap is True: emancmd += " spiderswap-single" else: emancmd += " spider-single" apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False) if not os.path.isfile(spifile): print "Failed to create SPIDER file" sys.exit(1) part = spider.read(spifile) apFile.removeFile(spifile) partlist.append(part) return partlist
def compute_stack_of_class_averages_and_reprojections(dir, selfile, refvolume, docfile, boxsize, resultspath, timestamp, iteration, reference_number=1, extract=False): ''' takes Xmipp single files, doc and sel files in routine, creates a stack of class averages in the results directory ''' workingdir = os.getcwd() os.chdir(dir) if dir.endswith("/"): dir = dir[:-1] head, tail = os.path.split(dir) ### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile f = open(selfile, "r") lines = f.readlines() newlines = [re.sub(str(tail)+"/", "", line) for line in lines] f.close() f = open(selfile[:-4]+"_new.sel", "w") f.writelines(newlines) f.close() if extract is True: extractcmd = "xmipp_header_extract -i %s.sel -o %s.doc" % (selfile[:-4], docfile[:-4]) apParam.runCmd(extractcmd, "Xmipp") ### create a projection params file and project the volume along identical Euler angles f = open("paramfile.descr", "w") f.write("%s\n" % refvolume) f.write("tmpproj 1 xmp\n") f.write("%d %d\n" % (boxsize, boxsize)) f.write("%s rot tilt psi\n" % docfile) f.write("NULL\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.close() projectcmd = "xmipp_project -i paramfile.descr" apParam.runCmd(projectcmd, "Xmipp") ### get order of projections in docfile d = open(docfile, "r") lines = d.readlines()[1:] d.close() projfile_sequence = [] for i, l in enumerate(lines): if i % 2 == 0: filename = os.path.basename(l.split()[1]) projfile_sequence.append(filename) else: pass ### create stack of projections and class averages projections = glob.glob("tmpproj**xmp") projections.sort() if len(projections) != len(projfile_sequence): apDisplay.printWarning("number of projections does not match number of classes") stackarray = [] stackname = os.path.join(resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (timestamp, iteration, reference_number)) for i in range(len(projections)): stackarray.append(spider.read(projections[i])) stackarray.append(spider.read(projfile_sequence[i])) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### remove unnecessary files for file in glob.glob("tmpproj*"): apFile.removeFile(file) os.chdir(workingdir) return
def compute_stack_of_class_averages_and_reprojections(self, iteration, reference_number): ''' takes Xmipp single files, doc and sel files associated with ML3D, creates a stack of class averages in the results directory ''' ''' ### make projections, and put them back into resultspath selfile = "ml3d_it%.6d_vol%.6d.sel" % (iteration, reference_number) refvolume = "ml3d_it%.6d_vol%.6d.vol\n" % (iteration, reference_number) docfile = "ml3d_it%.6d_vol%.6d.doc" % (iteration, reference_number) apXmipp.compute_stack_of_class_averages_and_reprojections(self.ml3dpath, selfile, refvolume, docfile, \ self.runparams['boxsize'], self.resultspath, self.params['timestamp'], iteration, reference_number, extract=True) return ''' os.chdir(self.ml3dpath) ### remove "RunML3D/" from selfile (created by ML3D program), then extract header information to docfile selfile = "ml3d_it%.6d_vol%.6d.sel" % (iteration, reference_number) f = open(selfile, "r") lines = f.readlines() newlines = [re.sub("RunML3D/", "", line) for line in lines] f.close() f = open(selfile, "w") f.writelines(newlines) f.close() extractcmd = "xmipp_header_extract -i ml3d_it%.6d_vol%.6d.sel -o ml3d_it%.6d_vol%.6d.doc" \ % (iteration, reference_number, iteration, reference_number) apParam.runCmd(extractcmd, "Xmipp") ### create a projection params file and project the volume along identical Euler angles f = open("paramfile.descr", "w") f.write("ml3d_it%.6d_vol%.6d.vol\n" % (iteration, reference_number)) f.write("tmpproj 1 xmp\n") f.write("%d %d\n" % (self.runparams['boxsize'], self.runparams['boxsize'])) f.write("ml3d_it%.6d_vol%.6d.doc rot tilt psi\n" % (iteration, reference_number)) f.write("NULL\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.close() projectcmd = "xmipp_project -i paramfile.descr" apParam.runCmd(projectcmd, "Xmipp") ### get order of projections in docfile docfile = "ml3d_it%.6d_vol%.6d.doc" % (iteration, reference_number) d = open(docfile, "r") lines = d.readlines()[1:] d.close() projfile_sequence = [] for i, l in enumerate(lines): if i % 2 == 0: filename = os.path.basename(l.split()[1]) projfile_sequence.append(filename) else: pass ### create stack of projections and class averages projections = glob.glob("tmpproj**xmp") projections.sort() if len(projections) != len(projfile_sequence): apDisplay.printWarning("number of projections does not match number of classes for model %d, iteration %d") stackarray = [] stackname = os.path.join(self.resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (self.params['timestamp'], iteration, reference_number)) for i in range(len(projections)): stackarray.append(spider.read(projections[i])) stackarray.append(spider.read(projfile_sequence[i])) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### remove unnecessary files for file in glob.glob("tmpproj*"): apFile.removeFile(file) os.chdir(self.params['rundir']) return
def compute_stack_of_class_averages_and_reprojections(dir, selfile, refvolume, docfile, boxsize, resultspath, timestamp, iteration, reference_number=1, extract=False): ''' takes Xmipp single files, doc and sel files in routine, creates a stack of class averages in the results directory ''' workingdir = os.getcwd() os.chdir(dir) if dir.endswith("/"): dir = dir[:-1] head, tail = os.path.split(dir) ### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile f = open(selfile, "r") lines = f.readlines() newlines = [re.sub(str(tail) + "/", "", line) for line in lines] f.close() f = open(selfile[:-4] + "_new.sel", "w") f.writelines(newlines) f.close() if extract is True: extractcmd = "xmipp_header_extract -i %s.sel -o %s.doc" % ( selfile[:-4], docfile[:-4]) apParam.runCmd(extractcmd, "Xmipp") ### create a projection params file and project the volume along identical Euler angles f = open("paramfile.descr", "w") f.write("%s\n" % refvolume) f.write("tmpproj 1 xmp\n") f.write("%d %d\n" % (boxsize, boxsize)) f.write("%s rot tilt psi\n" % docfile) f.write("NULL\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.close() projectcmd = "xmipp_project -i paramfile.descr" apParam.runCmd(projectcmd, "Xmipp") ### get order of projections in docfile d = open(docfile, "r") lines = d.readlines()[1:] d.close() projfile_sequence = [] for i, l in enumerate(lines): if i % 2 == 0: filename = os.path.basename(l.split()[1]) projfile_sequence.append(filename) else: pass ### create stack of projections and class averages projections = glob.glob("tmpproj**xmp") projections.sort() if len(projections) != len(projfile_sequence): apDisplay.printWarning( "number of projections does not match number of classes") stackarray = [] stackname = os.path.join( resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (timestamp, iteration, reference_number)) for i in range(len(projections)): stackarray.append(spider.read(projections[i])) stackarray.append(spider.read(projfile_sequence[i])) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### remove unnecessary files for file in glob.glob("tmpproj*"): apFile.removeFile(file) os.chdir(workingdir) return
def gatherSingleFilesIntoStack(selfile, stackfile, filetype="spider"): """ takes a selfile and creates an EMAN stack """ selfile = os.path.abspath(selfile) stackfile = os.path.abspath(stackfile) if stackfile[-4:] != ".hed": apDisplay.printWarning("Stack file does not end in .hed") stackfile = stackfile[:-4] + ".hed" apDisplay.printColor("Merging files into a stack, this can take a while", "cyan") starttime = time.time() if not os.path.isfile(selfile): apDisplay.printError("selfile does not exist: " + selfile) ### Process selfile fh = open(selfile, 'r') filelist = [] for line in fh: sline = line.strip() if sline: args = sline.split() if (len(args) > 1): filename = args[0].strip() filelist.append(filename) fh.close() ### Set variables boxsize = apFile.getBoxSize(filelist[0]) partperiter = int(1e9 / (boxsize[0]**2) / 16.) if partperiter > 4096: partperiter = 4096 apDisplay.printMsg("Using %d particle per iteration" % (partperiter)) numpart = len(filelist) if numpart < partperiter: partperiter = numpart ### Process images imgnum = 0 stacklist = [] stackroot = stackfile[:-4] ### get memory in kB startmem = mem.active() while imgnum < len(filelist): filename = filelist[imgnum] index = imgnum % partperiter if imgnum % 100 == 0: sys.stderr.write(".") #sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index)) if mem.active() - startmem > 2e6: apDisplay.printWarning("Out of memory") if index < 1: #print "img num", imgnum ### deal with large stacks, reset loop if imgnum > 0: sys.stderr.write("\n") stackname = "%s-%d.hed" % (stackroot, imgnum) apDisplay.printMsg("writing single particles to file " + stackname) stacklist.append(stackname) apFile.removeStack(stackname, warn=False) apImagicFile.writeImagic(stackarray, stackname, msg=False) perpart = (time.time() - starttime) / imgnum apDisplay.printColor( "part %d of %d :: %.1fM mem :: %s/part :: %s remain" % (imgnum + 1, numpart, (mem.active() - startmem) / 1024., apDisplay.timeString(perpart), apDisplay.timeString(perpart * (numpart - imgnum))), "blue") stackarray = [] ### merge particles if filetype == "mrc": partimg = mrc.read(filename) else: partimg = spider.read(filename) stackarray.append(partimg) imgnum += 1 ### write remaining particles to file sys.stderr.write("\n") stackname = "%s-%d.hed" % (stackroot, imgnum) apDisplay.printMsg("writing particles to file " + stackname) stacklist.append(stackname) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### merge stacks apFile.removeStack(stackfile, warn=False) apImagicFile.mergeStacks(stacklist, stackfile) print stackfile filepart = apFile.numImagesInStack(stackfile) if filepart != numpart: apDisplay.printError( "number merged particles (%d) not equal number expected particles (%d)" % (filepart, numpart)) for stackname in stacklist: apFile.removeStack(stackname, warn=False) ### summarize apDisplay.printColor( "merged %d particles in %s" % (imgnum, apDisplay.timeString(time.time() - starttime)), "cyan")
def start(self): ### start the outfile name fileroot = os.path.splitext(self.params['filename'])[0] fileroot += "-"+self.timestamp filename = self.params['file'].split('/')[-1] filepath = self.params['file'].strip(filename) self.params['box'] = apVolume.getModelDimensions(self.params['file']) if self.params['ampfile'] is not None: ### run amplitude correction spifile = apVolume.MRCtoSPI(self.params['file'], self.params['rundir']) tmpfile = apAmpCorrect.createAmpcorBatchFile(spifile, self.params) apAmpCorrect.runAmpcor() ### check if spider was successful fileroot += ".amp" if not os.path.isfile(tmpfile) : apDisplay.printError("amplitude correction failed") ### convert amplitude corrected file back to mrc fileroot += ".amp" a = spider.read(tmpfile) outfile = os.path.join( self.params['rundir'], "ampl-fix.mrc" ) mrc.write(a, outfile) curfile = outfile elif self.params['bfactor'] is True: outfile = os.path.join( self.params['rundir'], "bfactor-fix.mrc" ) outfile = apAmpCorrect.applyBfactor(self.params['file'], fscfile=self.params['fscfile'], apix=self.params['apix'], mass=self.params['mass'], outfile=outfile) curfile = outfile elif self.params['lrdw'] is True: lrdwcmd = "s_diffmap2 %s %s %s %s"%(self.params['filepath'],self.params['filename'],self.params['ampx'],self.params['resol']) proc = subprocess.Popen(lrdwcmd) print lrdwcmd else : ### just run proc3d curfile = self.params['file'] emancmd = "proc3d "+self.params['file']+" " emancmd = "proc3d "+curfile+" " if self.params['median'] is not None: data = mrc.read(curfile) data = ndimage.median_filter(data, size=self.params['median']) mrc.write(data, curfile) emancmd+="apix=%s " %self.params['apix'] if self.params['lp'] is not None: fileroot += (".lp%d" % ( int(self.params['lp']), )) emancmd += "lp=%d " %self.params['lp'] if self.params['yflip'] is True: fileroot += ".yflip" emancmd +="yflip " if self.params['invert'] is True: fileroot += ".inv" emancmd +="mult=-1 " if self.params['viper'] is True: fileroot += ".vip" emancmd +="icos5fTo2f " if self.params['mask'] is not None: # convert ang to pixels maskpix=int(self.params['mask']/self.params['apix']) fileroot += (".m%d" % ( int(self.params['mask']), )) emancmd += "mask=%d " %maskpix self.params['mask'] = maskpix if self.params['imask'] is not None: # convert ang to pixels maskpix=int(self.params['imask']/self.params['apix']) fileroot += (".im%d" % ( int(self.params['imask']), )) emancmd += "imask=%d " %maskpix self.params['imask'] = maskpix if self.params['norm'] is True: fileroot += ".norm" emancmd += "norm=0,1 " ### add output filename to emancmd string fileroot += ".mrc" self.params['name'] = fileroot outfile = os.path.join(self.params['rundir'], fileroot) emancmd = re.sub(" apix=",(" %s apix=" % outfile), emancmd) apEMAN.executeEmanCmd(emancmd) if self.params['description'] is None: self.params['description'] = "Volume from recon with amplitude adjustment" ### clean up files created during amp correction if self.params['ampfile'] is not None: apFile.removeFile(spifile) apFile.removeFile(tmpfile) if self.params['commit'] is True: symdata = apSymmetry.findSymmetry(self.params['sym']) print symdata symmetry = symdata['eman_name'] self.params['reconid'] = self.params['reconiterid'] self.insert3dDensity() ### render chimera images of model apChimera.filterAndChimera(outfile, res=self.params['res'], apix=self.params['apix'], box=self.params['box'], chimtype='snapshot', mass=self.params['mass'], contour=self.params['contour'], zoom=self.params['zoom'], sym=symmetry)
def compute_stack_of_class_averages_and_reprojections(self, iteration): ''' takes Xmipp single files, doc and sel files in projection-matching, creates a stack of class averages in the results directory ''' if bool(self.runparams['package_params']['CleanUpFiles']) is False: os.chdir( os.path.join(self.projmatchpath, "Iter_%d" % iteration, "ProjMatchClasses")) ### make projections, and put them back into resultspath selfile = "proj_match_classes.sel" refvolume = "../Iter_%d_reconstruction.vol" % iteration docfile = "proj_match_classes.doc" # apXmipp.compute_stack_of_class_averages_and_reprojections(d, selfile, refvolume, docfile, \ # self.runparams['boxsize'], self.resultspath, self.params['timestamp'], iteration) ### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile f = open(selfile, "r") lines = f.readlines() newlines = [ re.sub("ProjMatchClasses/", "", line) for line in lines ] f.close() f = open(selfile[:-4] + "_new.sel", "w") f.writelines(newlines) f.close() ### create a projection params file and project the volume along identical Euler angles f = open("paramfile.descr", "w") f.write("%s\n" % refvolume) f.write("tmpproj 1 xmp\n") f.write("%d %d\n" % (self.runparams['boxsize'], self.runparams['boxsize'])) f.write("%s rot tilt psi\n" % docfile) f.write("NULL\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.close() projectcmd = "xmipp_project -i paramfile.descr" apParam.runCmd(projectcmd, "Xmipp") ### get order of projections in docfile d = open(docfile, "r") lines = d.readlines()[1:] d.close() projfile_sequence = [] for i, l in enumerate(lines): if i % 2 == 0: filename = os.path.basename(l.split()[1]) projfile_sequence.append(filename) else: pass ### create stack of projections and class averages projections = glob.glob("tmpproj**xmp") projections.sort() if len(projections) != len(projfile_sequence): apDisplay.printWarning( "number of projections does not match number of classes") stackarray = [] stackname = os.path.join( self.resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (self.params['timestamp'], iteration, 1)) for i in range(len(projections)): stackarray.append(spider.read(projections[i])) stackarray.append(spider.read(projfile_sequence[i])) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### remove unnecessary files for file in glob.glob("tmpproj*"): apFile.removeFile(file) os.chdir(self.params['rundir']) else: apDisplay.printWarning( "all projection-matching files were cleaned up ... NOT creating class-average / re-projection stack" ) return
def compute_stack_of_class_averages_and_reprojections(self, iteration): ''' takes Xmipp single files, doc and sel files in projection-matching, creates a stack of class averages in the results directory ''' if bool(self.runparams['package_params']['CleanUpFiles']) is False: os.chdir(os.path.join(self.projmatchpath, "Iter_%d" % iteration, "ProjMatchClasses")) ### make projections, and put them back into resultspath selfile = "proj_match_classes.sel" refvolume = "../Iter_%d_reconstruction.vol" % iteration docfile = "proj_match_classes.doc" # apXmipp.compute_stack_of_class_averages_and_reprojections(d, selfile, refvolume, docfile, \ # self.runparams['boxsize'], self.resultspath, self.params['timestamp'], iteration) ### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile f = open(selfile, "r") lines = f.readlines() newlines = [re.sub("ProjMatchClasses/", "", line) for line in lines] f.close() f = open(selfile[:-4]+"_new.sel", "w") f.writelines(newlines) f.close() ### create a projection params file and project the volume along identical Euler angles f = open("paramfile.descr", "w") f.write("%s\n" % refvolume) f.write("tmpproj 1 xmp\n") f.write("%d %d\n" % (self.runparams['boxsize'], self.runparams['boxsize'])) f.write("%s rot tilt psi\n" % docfile) f.write("NULL\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.write("0 0\n") f.close() projectcmd = "xmipp_project -i paramfile.descr" apParam.runCmd(projectcmd, "Xmipp") ### get order of projections in docfile d = open(docfile, "r") lines = d.readlines()[1:] d.close() projfile_sequence = [] for i, l in enumerate(lines): if i % 2 == 0: filename = os.path.basename(l.split()[1]) projfile_sequence.append(filename) else: pass ### create stack of projections and class averages projections = glob.glob("tmpproj**xmp") projections.sort() if len(projections) != len(projfile_sequence): apDisplay.printWarning("number of projections does not match number of classes") stackarray = [] stackname = os.path.join(self.resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (self.params['timestamp'], iteration, 1)) for i in range(len(projections)): stackarray.append(spider.read(projections[i])) stackarray.append(spider.read(projfile_sequence[i])) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### remove unnecessary files for file in glob.glob("tmpproj*"): apFile.removeFile(file) os.chdir(self.params['rundir']) else: apDisplay.printWarning("all projection-matching files were cleaned up ... NOT creating class-average / re-projection stack") return
def gatherSingleFilesIntoStack(selfile, stackfile, filetype="spider"): """ takes a selfile and creates an EMAN stack """ selfile = os.path.abspath(selfile) stackfile = os.path.abspath(stackfile) if stackfile[-4:] != ".hed": apDisplay.printWarning("Stack file does not end in .hed") stackfile = stackfile[:-4]+".hed" apDisplay.printColor("Merging files into a stack, this can take a while", "cyan") starttime = time.time() if not os.path.isfile(selfile): apDisplay.printError("selfile does not exist: "+selfile) ### Process selfile fh = open(selfile, 'r') filelist = [] for line in fh: sline = line.strip() if sline: args=sline.split() if (len(args)>1): filename = args[0].strip() filelist.append(filename) fh.close() ### Set variables boxsize = apFile.getBoxSize(filelist[0]) partperiter = int(1e9/(boxsize[0]**2)/16.) if partperiter > 4096: partperiter = 4096 apDisplay.printMsg("Using %d particle per iteration"%(partperiter)) numpart = len(filelist) if numpart < partperiter: partperiter = numpart ### Process images imgnum = 0 stacklist = [] stackroot = stackfile[:-4] ### get memory in kB startmem = mem.active() while imgnum < len(filelist): filename = filelist[imgnum] index = imgnum % partperiter if imgnum % 100 == 0: sys.stderr.write(".") #sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index)) if mem.active()-startmem > 2e6: apDisplay.printWarning("Out of memory") if index < 1: #print "img num", imgnum ### deal with large stacks, reset loop if imgnum > 0: sys.stderr.write("\n") stackname = "%s-%d.hed"%(stackroot, imgnum) apDisplay.printMsg("writing single particles to file "+stackname) stacklist.append(stackname) apFile.removeStack(stackname, warn=False) apImagicFile.writeImagic(stackarray, stackname, msg=False) perpart = (time.time()-starttime)/imgnum apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"% (imgnum+1, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart), apDisplay.timeString(perpart*(numpart-imgnum))), "blue") stackarray = [] ### merge particles if filetype == "mrc": partimg = mrc.read(filename) else: partimg = spider.read(filename) stackarray.append(partimg) imgnum += 1 ### write remaining particles to file sys.stderr.write("\n") stackname = "%s-%d.hed"%(stackroot, imgnum) apDisplay.printMsg("writing particles to file "+stackname) stacklist.append(stackname) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### merge stacks apFile.removeStack(stackfile, warn=False) apImagicFile.mergeStacks(stacklist, stackfile) print stackfile filepart = apFile.numImagesInStack(stackfile) if filepart != numpart: apDisplay.printError("number merged particles (%d) not equal number expected particles (%d)"% (filepart, numpart)) for stackname in stacklist: apFile.removeStack(stackname, warn=False) ### summarize apDisplay.printColor("merged %d particles in %s"%(imgnum, apDisplay.timeString(time.time()-starttime)), "cyan")