def postLoop(self):
if len(self.stackarray) > 0:
stackroot = self.outstackfile[:-4]
stackname = "%s-%d.hed"%(stackroot, self.imgnum)
apDisplay.printMsg("writing single particles to file "+stackname)
self.stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(self.stackarray, stackname, msg=False)
### merge stacks
apFile.removeStack(self.outstackfile, warn=False)
apImagicFile.mergeStacks(self.stacklist, self.outstackfile)
filepart = apFile.numImagesInStack(self.outstackfile)
if filepart != self.imgnum:
apDisplay.printError("number merged particles (%d) not equal number expected particles (%d)"%
(filepart, numpart))
for stackname in self.stacklist:
apFile.removeStack(stackname, warn=False)
### summarize
apDisplay.printColor("merged %d particles in %s"%(self.imgnum, apDisplay.timeString(time.time()-self.starttime)), "cyan")
def postLoop(self):
if len(self.stackarray) > 0:
stackroot = self.outstackfile[:-4]
stackname = "%s-%d.hed" % (stackroot, self.imgnum)
apDisplay.printMsg("writing single particles to file " + stackname)
self.stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(self.stackarray, stackname, msg=False)
### merge stacks
apFile.removeStack(self.outstackfile, warn=False)
apImagicFile.mergeStacks(self.stacklist, self.outstackfile)
filepart = apFile.numImagesInStack(self.outstackfile)
if filepart != self.imgnum:
apDisplay.printError(
"number merged particles (%d) not equal number expected particles (%d)"
% (filepart, numpart))
for stackname in self.stacklist:
apFile.removeStack(stackname, warn=False)
### summarize
apDisplay.printColor(
"merged %d particles in %s" %
(self.imgnum, apDisplay.timeString(time.time() - self.starttime)),
"cyan")
def gatherSingleFilesIntoStack(selfile, stackfile, filetype="spider"):
"""
takes a selfile and creates an EMAN stack
"""
selfile = os.path.abspath(selfile)
stackfile = os.path.abspath(stackfile)
if stackfile[-4:] != ".hed":
apDisplay.printWarning("Stack file does not end in .hed")
stackfile = stackfile[:-4] + ".hed"
apDisplay.printColor("Merging files into a stack, this can take a while",
"cyan")
starttime = time.time()
if not os.path.isfile(selfile):
apDisplay.printError("selfile does not exist: " + selfile)
### Process selfile
fh = open(selfile, 'r')
filelist = []
for line in fh:
sline = line.strip()
if sline:
args = sline.split()
if (len(args) > 1):
filename = args[0].strip()
filelist.append(filename)
fh.close()
### Set variables
boxsize = apFile.getBoxSize(filelist[0])
partperiter = int(1e9 / (boxsize[0]**2) / 16.)
if partperiter > 4096:
partperiter = 4096
apDisplay.printMsg("Using %d particle per iteration" % (partperiter))
numpart = len(filelist)
if numpart < partperiter:
partperiter = numpart
### Process images
imgnum = 0
stacklist = []
stackroot = stackfile[:-4]
### get memory in kB
startmem = mem.active()
while imgnum < len(filelist):
filename = filelist[imgnum]
index = imgnum % partperiter
if imgnum % 100 == 0:
sys.stderr.write(".")
#sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index))
if mem.active() - startmem > 2e6:
apDisplay.printWarning("Out of memory")
if index < 1:
#print "img num", imgnum
### deal with large stacks, reset loop
if imgnum > 0:
sys.stderr.write("\n")
stackname = "%s-%d.hed" % (stackroot, imgnum)
apDisplay.printMsg("writing single particles to file " +
stackname)
stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
perpart = (time.time() - starttime) / imgnum
apDisplay.printColor(
"part %d of %d :: %.1fM mem :: %s/part :: %s remain" %
(imgnum + 1, numpart, (mem.active() - startmem) / 1024.,
apDisplay.timeString(perpart),
apDisplay.timeString(perpart * (numpart - imgnum))),
"blue")
stackarray = []
### merge particles
if filetype == "mrc":
partimg = mrc.read(filename)
else:
partimg = spider.read(filename)
stackarray.append(partimg)
imgnum += 1
### write remaining particles to file
sys.stderr.write("\n")
stackname = "%s-%d.hed" % (stackroot, imgnum)
apDisplay.printMsg("writing particles to file " + stackname)
stacklist.append(stackname)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### merge stacks
apFile.removeStack(stackfile, warn=False)
apImagicFile.mergeStacks(stacklist, stackfile)
print stackfile
filepart = apFile.numImagesInStack(stackfile)
if filepart != numpart:
apDisplay.printError(
"number merged particles (%d) not equal number expected particles (%d)"
% (filepart, numpart))
for stackname in stacklist:
apFile.removeStack(stackname, warn=False)
### summarize
apDisplay.printColor(
"merged %d particles in %s" %
(imgnum, apDisplay.timeString(time.time() - starttime)), "cyan")
def makeNewStacks(self, parttree):
### untilted stack
self.notstackdata = apStack.getOnlyStackData(self.params['notstackid'])
notstackfile = os.path.join(self.notstackdata['path']['path'], self.notstackdata['name'])
### tilted stack
if not self.tiltstackdata:
self.tiltstackdata = apStack.getOnlyStackData(self.params['tiltstackid'])
tiltstackfile = os.path.join(self.tiltstackdata['path']['path'], self.tiltstackdata['name'])
### make doc file of Euler angles
#eulerfile = self.makeEulerDoc(parttree)
eulerfile = os.path.join(self.params['rundir'], "eulersdoc"+self.timestamp+".spi")
if os.path.isfile(eulerfile):
apFile.removeFile(eulerfile)
count = 0
notstacklist = []
tiltstacklist = []
sizelimit = 2048
notbox = apImagicFile.getBoxsize(notstackfile)
tiltbox = apImagicFile.getBoxsize(tiltstackfile)
tiltstacks = []
notstacks = []
t0 = time.time()
for partdict in parttree:
### print friendly message
if count % 100 == 0:
backs = "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b"
sys.stderr.write(backs+backs+backs+backs)
if count > sizelimit:
esttime = (len(parttree)/float(count)-1.0)*(time.time()-t0)
sys.stderr.write(str(count)+" particles of "+str(len(parttree))
+", "+apDisplay.timeString(esttime)+" remaining")
else:
sys.stderr.write(str(count)+" particles of "+str(len(parttree)))
### save stacks to file to save memory
if count%sizelimit == 0:
if count > 1:
apDisplay.printMsg("Writing stacks to file")
t1 = time.time()
tiltname = os.path.join(self.params['rundir'], "tiltstack%d.hed"%(count))
apFile.removeStack(tiltname)
apImagicFile.writeImagic(tiltstacklist, tiltname, msg=False)
tiltstacks.append(tiltname)
apDisplay.printMsg("finished tilted stack in "+apDisplay.timeString(time.time()-t1))
t1 = time.time()
notname = os.path.join(self.params['rundir'], "notstack%d.hed"%(count))
apFile.removeStack(notname)
apImagicFile.writeImagic(notstacklist, notname, msg=False)
notstacks.append(notname)
apDisplay.printMsg("finished untilted stack in "+apDisplay.timeString(time.time()-t1))
### reset stacks
apDisplay.printMsg("Reset stacks in memory")
notstacklist = []
tiltstacklist = []
### increment count
count += 1
### write to Euler doc
self.appendEulerDoc(eulerfile, partdict['tilt'], count)
### untilted stack
notpartarray = apImagicFile.readSingleParticleFromStack(notstackfile, partdict['not'], notbox, False)
notstacklist.append(notpartarray)
### tilted stack
tiltpartarray = apImagicFile.readSingleParticleFromStack(tiltstackfile, partdict['tilt'], tiltbox, False)
tiltstacklist.append(tiltpartarray)
### write remaining particles to stack
if len(notstacklist) > 0:
apDisplay.printMsg("Writing stacks to file")
t1 = time.time()
tiltname = os.path.join(self.params['rundir'], "tiltstack%d.hed"%(count))
apFile.removeStack(tiltname)
apImagicFile.writeImagic(tiltstacklist, tiltname, msg=False)
tiltstacks.append(tiltname)
apDisplay.printMsg("finished tilted stack in "+apDisplay.timeString(time.time()-t1))
t1 = time.time()
notname = os.path.join(self.params['rundir'], "notstack%d.hed"%(count))
apFile.removeStack(notname)
apImagicFile.writeImagic(notstacklist, notname, msg=False)
notstacks.append(notname)
apDisplay.printMsg("finished untilted stack in "+apDisplay.timeString(time.time()-t1))
### merge NOT stack
notname = os.path.join(self.params['rundir'], "notstack.hed")
apImagicFile.mergeStacks(notstacks, notname)
for stackname in notstacks:
apFile.removeStack(stackname, warn=False)
### merge TILT stack
tiltname = os.path.join(self.params['rundir'], "tiltstack.hed")
apImagicFile.mergeStacks(tiltstacks, tiltname)
for stackname in tiltstacks:
apFile.removeStack(stackname, warn=False)
### upload results
if self.params['commit'] is True:
self.uploadResults()
def gatherSingleFilesIntoStack(selfile, stackfile, filetype="spider"):
"""
takes a selfile and creates an EMAN stack
"""
selfile = os.path.abspath(selfile)
stackfile = os.path.abspath(stackfile)
if stackfile[-4:] != ".hed":
apDisplay.printWarning("Stack file does not end in .hed")
stackfile = stackfile[:-4]+".hed"
apDisplay.printColor("Merging files into a stack, this can take a while", "cyan")
starttime = time.time()
if not os.path.isfile(selfile):
apDisplay.printError("selfile does not exist: "+selfile)
### Process selfile
fh = open(selfile, 'r')
filelist = []
for line in fh:
sline = line.strip()
if sline:
args=sline.split()
if (len(args)>1):
filename = args[0].strip()
filelist.append(filename)
fh.close()
### Set variables
boxsize = apFile.getBoxSize(filelist[0])
partperiter = int(1e9/(boxsize[0]**2)/16.)
if partperiter > 4096:
partperiter = 4096
apDisplay.printMsg("Using %d particle per iteration"%(partperiter))
numpart = len(filelist)
if numpart < partperiter:
partperiter = numpart
### Process images
imgnum = 0
stacklist = []
stackroot = stackfile[:-4]
### get memory in kB
startmem = mem.active()
while imgnum < len(filelist):
filename = filelist[imgnum]
index = imgnum % partperiter
if imgnum % 100 == 0:
sys.stderr.write(".")
#sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index))
if mem.active()-startmem > 2e6:
apDisplay.printWarning("Out of memory")
if index < 1:
#print "img num", imgnum
### deal with large stacks, reset loop
if imgnum > 0:
sys.stderr.write("\n")
stackname = "%s-%d.hed"%(stackroot, imgnum)
apDisplay.printMsg("writing single particles to file "+stackname)
stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
perpart = (time.time()-starttime)/imgnum
apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"%
(imgnum+1, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart),
apDisplay.timeString(perpart*(numpart-imgnum))), "blue")
stackarray = []
### merge particles
if filetype == "mrc":
partimg = mrc.read(filename)
else:
partimg = spider.read(filename)
stackarray.append(partimg)
imgnum += 1
### write remaining particles to file
sys.stderr.write("\n")
stackname = "%s-%d.hed"%(stackroot, imgnum)
apDisplay.printMsg("writing particles to file "+stackname)
stacklist.append(stackname)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### merge stacks
apFile.removeStack(stackfile, warn=False)
apImagicFile.mergeStacks(stacklist, stackfile)
print stackfile
filepart = apFile.numImagesInStack(stackfile)
if filepart != numpart:
apDisplay.printError("number merged particles (%d) not equal number expected particles (%d)"%
(filepart, numpart))
for stackname in stacklist:
apFile.removeStack(stackname, warn=False)
### summarize
apDisplay.printColor("merged %d particles in %s"%(imgnum, apDisplay.timeString(time.time()-starttime)), "cyan")
def createAlignedStacks(self, partlist, origstackfile):
partperiter = min(4096,apImagicFile.getPartSegmentLimit(origstackfile))
numpart = len(partlist)
if numpart < partperiter:
partperiter = numpart
t0 = time.time()
imgnum = 0
stacklist = []
apDisplay.printMsg("rotating and shifting particles at "+time.asctime())
while imgnum < len(partlist):
index = imgnum % partperiter
if imgnum % 100 == 0:
sys.stderr.write(".")
if index == 0:
### deal with large stacks
if imgnum > 0:
sys.stderr.write("\n")
stackname = "alignstack%d.hed"%(imgnum)
apDisplay.printMsg("writing aligned particles to file "+stackname)
stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(alignstack, stackname, msg=False)
perpart = (time.time()-t0)/imgnum
apDisplay.printColor("particle %d of %d :: %s per part :: %s remain"%
(imgnum+1, numpart, apDisplay.timeString(perpart),
apDisplay.timeString(perpart*(numpart-imgnum))), "blue")
alignstack = []
imagesdict = apImagicFile.readImagic(origstackfile, first=imgnum+1, last=imgnum+partperiter, msg=False)
### align particles
partimg = imagesdict['images'][index]
partdict = partlist[imgnum]
partnum = imgnum+1
if partdict['partnum'] != partnum:
apDisplay.printError("particle shifting "+str(partnum)+" != "+str(partdict))
xyshift = (partdict['xshift'], partdict['yshift'])
alignpartimg = apImage.xmippTransform(partimg, rot=partdict['inplane'],
shift=xyshift, mirror=partdict['mirror'])
alignstack.append(alignpartimg)
imgnum += 1
### write remaining particle to file
sys.stderr.write("\n")
stackname = "alignstack%d.hed"%(imgnum)
apDisplay.printMsg("writing aligned particles to file "+stackname)
stacklist.append(stackname)
apImagicFile.writeImagic(alignstack, stackname, msg=False)
### merge stacks
alignimagicfile = "alignstack.hed"
apFile.removeStack(alignimagicfile, warn=False)
apImagicFile.mergeStacks(stacklist, alignimagicfile)
#for stackname in stacklist:
# emancmd = "proc2d %s %s"%(stackname, alignimagicfile)
# apEMAN.executeEmanCmd(emancmd, verbose=False)
filepart = apFile.numImagesInStack(alignimagicfile)
if filepart != numpart:
apDisplay.printError("number aligned particles (%d) not equal number expected particles (%d)"%
(filepart, numpart))
for stackname in stacklist:
apFile.removeStack(stackname, warn=False)
### summarize
apDisplay.printMsg("rotated and shifted %d particles in %s"%(imgnum, apDisplay.timeString(time.time()-t0)))
return alignimagicfile