def postLoop(self): if len(self.stackarray) > 0: stackroot = self.outstackfile[:-4] stackname = "%s-%d.hed"%(stackroot, self.imgnum) apDisplay.printMsg("writing single particles to file "+stackname) self.stacklist.append(stackname) apFile.removeStack(stackname, warn=False) apImagicFile.writeImagic(self.stackarray, stackname, msg=False) ### merge stacks apFile.removeStack(self.outstackfile, warn=False) apImagicFile.mergeStacks(self.stacklist, self.outstackfile) filepart = apFile.numImagesInStack(self.outstackfile) if filepart != self.imgnum: apDisplay.printError("number merged particles (%d) not equal number expected particles (%d)"% (filepart, numpart)) for stackname in self.stacklist: apFile.removeStack(stackname, warn=False) ### summarize apDisplay.printColor("merged %d particles in %s"%(self.imgnum, apDisplay.timeString(time.time()-self.starttime)), "cyan")
def postLoop(self): if len(self.stackarray) > 0: stackroot = self.outstackfile[:-4] stackname = "%s-%d.hed" % (stackroot, self.imgnum) apDisplay.printMsg("writing single particles to file " + stackname) self.stacklist.append(stackname) apFile.removeStack(stackname, warn=False) apImagicFile.writeImagic(self.stackarray, stackname, msg=False) ### merge stacks apFile.removeStack(self.outstackfile, warn=False) apImagicFile.mergeStacks(self.stacklist, self.outstackfile) filepart = apFile.numImagesInStack(self.outstackfile) if filepart != self.imgnum: apDisplay.printError( "number merged particles (%d) not equal number expected particles (%d)" % (filepart, numpart)) for stackname in self.stacklist: apFile.removeStack(stackname, warn=False) ### summarize apDisplay.printColor( "merged %d particles in %s" % (self.imgnum, apDisplay.timeString(time.time() - self.starttime)), "cyan")
def gatherSingleFilesIntoStack(selfile, stackfile, filetype="spider"): """ takes a selfile and creates an EMAN stack """ selfile = os.path.abspath(selfile) stackfile = os.path.abspath(stackfile) if stackfile[-4:] != ".hed": apDisplay.printWarning("Stack file does not end in .hed") stackfile = stackfile[:-4] + ".hed" apDisplay.printColor("Merging files into a stack, this can take a while", "cyan") starttime = time.time() if not os.path.isfile(selfile): apDisplay.printError("selfile does not exist: " + selfile) ### Process selfile fh = open(selfile, 'r') filelist = [] for line in fh: sline = line.strip() if sline: args = sline.split() if (len(args) > 1): filename = args[0].strip() filelist.append(filename) fh.close() ### Set variables boxsize = apFile.getBoxSize(filelist[0]) partperiter = int(1e9 / (boxsize[0]**2) / 16.) if partperiter > 4096: partperiter = 4096 apDisplay.printMsg("Using %d particle per iteration" % (partperiter)) numpart = len(filelist) if numpart < partperiter: partperiter = numpart ### Process images imgnum = 0 stacklist = [] stackroot = stackfile[:-4] ### get memory in kB startmem = mem.active() while imgnum < len(filelist): filename = filelist[imgnum] index = imgnum % partperiter if imgnum % 100 == 0: sys.stderr.write(".") #sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index)) if mem.active() - startmem > 2e6: apDisplay.printWarning("Out of memory") if index < 1: #print "img num", imgnum ### deal with large stacks, reset loop if imgnum > 0: sys.stderr.write("\n") stackname = "%s-%d.hed" % (stackroot, imgnum) apDisplay.printMsg("writing single particles to file " + stackname) stacklist.append(stackname) apFile.removeStack(stackname, warn=False) apImagicFile.writeImagic(stackarray, stackname, msg=False) perpart = (time.time() - starttime) / imgnum apDisplay.printColor( "part %d of %d :: %.1fM mem :: %s/part :: %s remain" % (imgnum + 1, numpart, (mem.active() - startmem) / 1024., apDisplay.timeString(perpart), apDisplay.timeString(perpart * (numpart - imgnum))), "blue") stackarray = [] ### merge particles if filetype == "mrc": partimg = mrc.read(filename) else: partimg = spider.read(filename) stackarray.append(partimg) imgnum += 1 ### write remaining particles to file sys.stderr.write("\n") stackname = "%s-%d.hed" % (stackroot, imgnum) apDisplay.printMsg("writing particles to file " + stackname) stacklist.append(stackname) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### merge stacks apFile.removeStack(stackfile, warn=False) apImagicFile.mergeStacks(stacklist, stackfile) print stackfile filepart = apFile.numImagesInStack(stackfile) if filepart != numpart: apDisplay.printError( "number merged particles (%d) not equal number expected particles (%d)" % (filepart, numpart)) for stackname in stacklist: apFile.removeStack(stackname, warn=False) ### summarize apDisplay.printColor( "merged %d particles in %s" % (imgnum, apDisplay.timeString(time.time() - starttime)), "cyan")
def makeNewStacks(self, parttree): ### untilted stack self.notstackdata = apStack.getOnlyStackData(self.params['notstackid']) notstackfile = os.path.join(self.notstackdata['path']['path'], self.notstackdata['name']) ### tilted stack if not self.tiltstackdata: self.tiltstackdata = apStack.getOnlyStackData(self.params['tiltstackid']) tiltstackfile = os.path.join(self.tiltstackdata['path']['path'], self.tiltstackdata['name']) ### make doc file of Euler angles #eulerfile = self.makeEulerDoc(parttree) eulerfile = os.path.join(self.params['rundir'], "eulersdoc"+self.timestamp+".spi") if os.path.isfile(eulerfile): apFile.removeFile(eulerfile) count = 0 notstacklist = [] tiltstacklist = [] sizelimit = 2048 notbox = apImagicFile.getBoxsize(notstackfile) tiltbox = apImagicFile.getBoxsize(tiltstackfile) tiltstacks = [] notstacks = [] t0 = time.time() for partdict in parttree: ### print friendly message if count % 100 == 0: backs = "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b" sys.stderr.write(backs+backs+backs+backs) if count > sizelimit: esttime = (len(parttree)/float(count)-1.0)*(time.time()-t0) sys.stderr.write(str(count)+" particles of "+str(len(parttree)) +", "+apDisplay.timeString(esttime)+" remaining") else: sys.stderr.write(str(count)+" particles of "+str(len(parttree))) ### save stacks to file to save memory if count%sizelimit == 0: if count > 1: apDisplay.printMsg("Writing stacks to file") t1 = time.time() tiltname = os.path.join(self.params['rundir'], "tiltstack%d.hed"%(count)) apFile.removeStack(tiltname) apImagicFile.writeImagic(tiltstacklist, tiltname, msg=False) tiltstacks.append(tiltname) apDisplay.printMsg("finished tilted stack in "+apDisplay.timeString(time.time()-t1)) t1 = time.time() notname = os.path.join(self.params['rundir'], "notstack%d.hed"%(count)) apFile.removeStack(notname) apImagicFile.writeImagic(notstacklist, notname, msg=False) notstacks.append(notname) apDisplay.printMsg("finished untilted stack in "+apDisplay.timeString(time.time()-t1)) ### reset stacks apDisplay.printMsg("Reset stacks in memory") notstacklist = [] tiltstacklist = [] ### increment count count += 1 ### write to Euler doc self.appendEulerDoc(eulerfile, partdict['tilt'], count) ### untilted stack notpartarray = apImagicFile.readSingleParticleFromStack(notstackfile, partdict['not'], notbox, False) notstacklist.append(notpartarray) ### tilted stack tiltpartarray = apImagicFile.readSingleParticleFromStack(tiltstackfile, partdict['tilt'], tiltbox, False) tiltstacklist.append(tiltpartarray) ### write remaining particles to stack if len(notstacklist) > 0: apDisplay.printMsg("Writing stacks to file") t1 = time.time() tiltname = os.path.join(self.params['rundir'], "tiltstack%d.hed"%(count)) apFile.removeStack(tiltname) apImagicFile.writeImagic(tiltstacklist, tiltname, msg=False) tiltstacks.append(tiltname) apDisplay.printMsg("finished tilted stack in "+apDisplay.timeString(time.time()-t1)) t1 = time.time() notname = os.path.join(self.params['rundir'], "notstack%d.hed"%(count)) apFile.removeStack(notname) apImagicFile.writeImagic(notstacklist, notname, msg=False) notstacks.append(notname) apDisplay.printMsg("finished untilted stack in "+apDisplay.timeString(time.time()-t1)) ### merge NOT stack notname = os.path.join(self.params['rundir'], "notstack.hed") apImagicFile.mergeStacks(notstacks, notname) for stackname in notstacks: apFile.removeStack(stackname, warn=False) ### merge TILT stack tiltname = os.path.join(self.params['rundir'], "tiltstack.hed") apImagicFile.mergeStacks(tiltstacks, tiltname) for stackname in tiltstacks: apFile.removeStack(stackname, warn=False) ### upload results if self.params['commit'] is True: self.uploadResults()
def gatherSingleFilesIntoStack(selfile, stackfile, filetype="spider"): """ takes a selfile and creates an EMAN stack """ selfile = os.path.abspath(selfile) stackfile = os.path.abspath(stackfile) if stackfile[-4:] != ".hed": apDisplay.printWarning("Stack file does not end in .hed") stackfile = stackfile[:-4]+".hed" apDisplay.printColor("Merging files into a stack, this can take a while", "cyan") starttime = time.time() if not os.path.isfile(selfile): apDisplay.printError("selfile does not exist: "+selfile) ### Process selfile fh = open(selfile, 'r') filelist = [] for line in fh: sline = line.strip() if sline: args=sline.split() if (len(args)>1): filename = args[0].strip() filelist.append(filename) fh.close() ### Set variables boxsize = apFile.getBoxSize(filelist[0]) partperiter = int(1e9/(boxsize[0]**2)/16.) if partperiter > 4096: partperiter = 4096 apDisplay.printMsg("Using %d particle per iteration"%(partperiter)) numpart = len(filelist) if numpart < partperiter: partperiter = numpart ### Process images imgnum = 0 stacklist = [] stackroot = stackfile[:-4] ### get memory in kB startmem = mem.active() while imgnum < len(filelist): filename = filelist[imgnum] index = imgnum % partperiter if imgnum % 100 == 0: sys.stderr.write(".") #sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index)) if mem.active()-startmem > 2e6: apDisplay.printWarning("Out of memory") if index < 1: #print "img num", imgnum ### deal with large stacks, reset loop if imgnum > 0: sys.stderr.write("\n") stackname = "%s-%d.hed"%(stackroot, imgnum) apDisplay.printMsg("writing single particles to file "+stackname) stacklist.append(stackname) apFile.removeStack(stackname, warn=False) apImagicFile.writeImagic(stackarray, stackname, msg=False) perpart = (time.time()-starttime)/imgnum apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"% (imgnum+1, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart), apDisplay.timeString(perpart*(numpart-imgnum))), "blue") stackarray = [] ### merge particles if filetype == "mrc": partimg = mrc.read(filename) else: partimg = spider.read(filename) stackarray.append(partimg) imgnum += 1 ### write remaining particles to file sys.stderr.write("\n") stackname = "%s-%d.hed"%(stackroot, imgnum) apDisplay.printMsg("writing particles to file "+stackname) stacklist.append(stackname) apImagicFile.writeImagic(stackarray, stackname, msg=False) ### merge stacks apFile.removeStack(stackfile, warn=False) apImagicFile.mergeStacks(stacklist, stackfile) print stackfile filepart = apFile.numImagesInStack(stackfile) if filepart != numpart: apDisplay.printError("number merged particles (%d) not equal number expected particles (%d)"% (filepart, numpart)) for stackname in stacklist: apFile.removeStack(stackname, warn=False) ### summarize apDisplay.printColor("merged %d particles in %s"%(imgnum, apDisplay.timeString(time.time()-starttime)), "cyan")
def createAlignedStacks(self, partlist, origstackfile): partperiter = min(4096,apImagicFile.getPartSegmentLimit(origstackfile)) numpart = len(partlist) if numpart < partperiter: partperiter = numpart t0 = time.time() imgnum = 0 stacklist = [] apDisplay.printMsg("rotating and shifting particles at "+time.asctime()) while imgnum < len(partlist): index = imgnum % partperiter if imgnum % 100 == 0: sys.stderr.write(".") if index == 0: ### deal with large stacks if imgnum > 0: sys.stderr.write("\n") stackname = "alignstack%d.hed"%(imgnum) apDisplay.printMsg("writing aligned particles to file "+stackname) stacklist.append(stackname) apFile.removeStack(stackname, warn=False) apImagicFile.writeImagic(alignstack, stackname, msg=False) perpart = (time.time()-t0)/imgnum apDisplay.printColor("particle %d of %d :: %s per part :: %s remain"% (imgnum+1, numpart, apDisplay.timeString(perpart), apDisplay.timeString(perpart*(numpart-imgnum))), "blue") alignstack = [] imagesdict = apImagicFile.readImagic(origstackfile, first=imgnum+1, last=imgnum+partperiter, msg=False) ### align particles partimg = imagesdict['images'][index] partdict = partlist[imgnum] partnum = imgnum+1 if partdict['partnum'] != partnum: apDisplay.printError("particle shifting "+str(partnum)+" != "+str(partdict)) xyshift = (partdict['xshift'], partdict['yshift']) alignpartimg = apImage.xmippTransform(partimg, rot=partdict['inplane'], shift=xyshift, mirror=partdict['mirror']) alignstack.append(alignpartimg) imgnum += 1 ### write remaining particle to file sys.stderr.write("\n") stackname = "alignstack%d.hed"%(imgnum) apDisplay.printMsg("writing aligned particles to file "+stackname) stacklist.append(stackname) apImagicFile.writeImagic(alignstack, stackname, msg=False) ### merge stacks alignimagicfile = "alignstack.hed" apFile.removeStack(alignimagicfile, warn=False) apImagicFile.mergeStacks(stacklist, alignimagicfile) #for stackname in stacklist: # emancmd = "proc2d %s %s"%(stackname, alignimagicfile) # apEMAN.executeEmanCmd(emancmd, verbose=False) filepart = apFile.numImagesInStack(alignimagicfile) if filepart != numpart: apDisplay.printError("number aligned particles (%d) not equal number expected particles (%d)"% (filepart, numpart)) for stackname in stacklist: apFile.removeStack(stackname, warn=False) ### summarize apDisplay.printMsg("rotated and shifted %d particles in %s"%(imgnum, apDisplay.timeString(time.time()-t0))) return alignimagicfile