def readPartDocFile(self, reflist):
partlist = []
docfile = "part"+self.params['timestamp']+".doc"
if not os.path.isfile(docfile) or apFile.fileSize(docfile) < 50:
apDisplay.printWarning("Could not find doc file "+docfile+" to read particle angles")
lastdocfile = "iter%03d/part%s_it%06d.doc"%(self.lastiter, self.params['timestamp'], self.lastiter)
if not os.path.isfile(lastdocfile) or apFile.fileSize(lastdocfile) < 50:
apDisplay.printError("Could not find backup doc file")
apDisplay.printColor("Found a backup copy of docfile", "green")
shutil.copy(lastdocfile, docfile)
f = open(docfile, "r")
mininplane = 360.0
for line in f:
if line[:2] == ' ;':
continue
spidict = operations.spiderInLine(line)
origpartdict = self.spidict2partdict(spidict)
partdict = self.adjustPartDict(origpartdict, reflist)
if partdict['inplane'] < mininplane:
mininplane = partdict['inplane']
partlist.append(partdict)
apDisplay.printMsg("minimum inplane: "+str(mininplane))
for partdict in partlist:
partdict['inplane'] = partdict['inplane']-mininplane
apDisplay.printMsg("read rotation and shift parameters for "+str(len(partlist))+" particles")
if len(partlist) < 1:
apDisplay.printError("Did not find any particles in doc file: "+docfile)
return partlist
def createSplitStacks(self):
self.splitStacks = {}
self.splitStacks['left'] = "particles.left.mrc"
self.splitStacks['right'] = "particles.right.mrc"
if not os.path.isfile(self.splitStacks['left']):
pass
elif not os.path.isfile(self.splitStacks['right']):
pass
elif apFile.fileSize(self.splitStacks['left']) < 100:
pass
elif apFile.fileSize(self.splitStacks['right']) < 100:
pass
else:
apDisplay.printMsg("split stacks already exist, skipping this step")
return
apDisplay.printColor("Step 1A: Splitting input stack %s into two parts"
%(self.params['inputstack']), "purple")
oddfile, evenfile = apImagicFile.splitStackEvenOdd(self.params['inputstack'],
rundir=self.params['rundir'], msg=True)
apDisplay.printColor("Step 1B: Converting odd particles from %s into Left MRC stack %s"
%(oddfile, self.splitStacks['left']), "purple")
imagic2mrc.imagic_to_mrc(oddfile, self.splitStacks['left'])
apFile.removeStack(oddfile)
apDisplay.printColor("Step 1C: Converting even particles from %s into Right MRC stack %s"
%(evenfile, self.splitStacks['right']), "purple")
imagic2mrc.imagic_to_mrc(evenfile, self.splitStacks['right'])
apFile.removeStack(evenfile)
return
def convertStackToXmippData(instack, outdata, maskpixrad, boxsize, numpart=None):
"""
From http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Img2Data
This program applies a mask to a set of images.
This set is given by a selfile.
After applying the mask the result is storaged as a vector in the following format:
The first line indicates the dimension of the vectors and the number of vectors.
The rest of the lines are the feature vectors.
Each line is a vector and each column is a vectors' component (pixels values inside the mask).
"""
apDisplay.printMsg("Convert stack file to Xmipp data file")
maskfile = "circlemask.spi"
operations.createMask(maskfile, maskpixrad, boxsize)
partlistdocfile = breakupStackIntoSingleFiles(instack, numpart=numpart)
convertcmd = "xmipp_convert_img2data -i %s -mask %s -o %s"%(partlistdocfile, maskfile, outdata)
proc = subprocess.Popen(convertcmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,)
proc.wait()
outfilesize = apFile.fileSize(outdata)
partfilesize = apFile.fileSize(partlistdocfile)
if outfilesize < 2*partfilesize:
apDisplay.printError("Outdata conversion did not work, data file smaller than docfile, %s < %s"
%(apDisplay.bytes(outfilesize), apDisplay.bytes(partfilesize)))
apFile.removeFilePattern("partfiles/*")
return outdata
def downloadImagesFromAMI(self):
imglist = []
from urllib import urlretrieve
imgdict = {
110: '06jul12a_00022gr_00037sq_00025hl_00005en.mrc',
111: '06jul12a_00035gr_00063sq_00012hl_00004en.mrc',
112: '06jul12a_00015gr_00028sq_00004hl_00002en.mrc',
113: '06jul12a_00015gr_00028sq_00023hl_00002en.mrc',
114: '06jul12a_00015gr_00028sq_00023hl_00004en.mrc',
115: '06jul12a_00022gr_00013sq_00002hl_00004en.mrc',
116: '06jul12a_00022gr_00013sq_00003hl_00005en.mrc',
}
for key in imgdict.keys():
imgfile = os.path.join(self.params['rundir'], imgdict[key])
url = (
"http://ami.scripps.edu/redmine/attachments/download/%d/%s" %
(key, imgdict[key]))
apDisplay.printMsg("Downloading image '%s'" % (imgdict[key]))
if os.path.isfile(imgfile) and apFile.fileSize(
imgfile) == 67109888:
apDisplay.printMsg("complete image file exists: %s" %
(imgfile))
continue
urlretrieve(url, imgfile)
if not os.path.isfile(imgfile):
apDisplay.printError("could not download file: %s" % (url))
if apFile.fileSize(imgfile) < 30e6:
apDisplay.printError("error in downloaded file: %s" % (url))
imglist.append(imgfile)
return imglist
def downloadImagesFromAMI(self):
imglist = []
from urllib import urlretrieve
imgdict = {
110: '06jul12a_00022gr_00037sq_00025hl_00005en.mrc',
111: '06jul12a_00035gr_00063sq_00012hl_00004en.mrc',
112: '06jul12a_00015gr_00028sq_00004hl_00002en.mrc',
113: '06jul12a_00015gr_00028sq_00023hl_00002en.mrc',
114: '06jul12a_00015gr_00028sq_00023hl_00004en.mrc',
115: '06jul12a_00022gr_00013sq_00002hl_00004en.mrc',
116: '06jul12a_00022gr_00013sq_00003hl_00005en.mrc',
}
for key in imgdict.keys():
imgfile = os.path.join(self.params['rundir'], imgdict[key])
url = ("http://emg.nysbc.org/redmine/attachments/download/%d/%s"
%(key, imgdict[key]))
apDisplay.printMsg("Downloading image '%s'"%(imgdict[key]))
apDisplay.printMsg("Downloading image '%s'"%(url))
if os.path.isfile(imgfile) and apFile.fileSize(imgfile) == 67109888:
apDisplay.printMsg("complete image file exists: %s"%(imgfile))
continue
urlretrieve(url, imgfile)
if not os.path.isfile(imgfile):
apDisplay.printError("could not download file: %s"%(url))
if apFile.fileSize(imgfile) < 30e6:
apDisplay.printError("error in downloaded file: %s"%(url))
imglist.append(imgfile)
return imglist
def appendParticleListToStackFile(partlist, mergestackfile, msg=True):
"""
takes a list of 2D numpy arrays and add to stack file
due to hack, we must re-number the stack later
"""
### initialization
t0 = time.time()
root = os.path.splitext(mergestackfile)[0]
mergeheaderfile = root + ".hed"
mergedatafile = root + ".img"
### merge data files
premergesize = apFile.fileSize(mergedatafile)
mergedata = file(mergedatafile, 'ab')
for partarray in partlist:
part32bit = numpy.asarray(partarray, dtype=numpy.float32)
mergedata.write(part32bit.tostring())
mergedata.close()
finalsize = apFile.fileSize(mergedatafile)
addsize = len(part32bit.tostring() * len(partlist))
if finalsize != addsize + premergesize:
apDisplay.printError(
"size mismatch %s vs. %s + %s = %s" %
(apDisplay.bytes(finalsize), apDisplay.bytes(addsize),
apDisplay.bytes(premergesize),
apDisplay.bytes(premergesize + addsize)))
elif msg is True:
apDisplay.printMsg(
"size match %s vs. %s + %s = %s" %
(apDisplay.bytes(finalsize), apDisplay.bytes(addsize),
apDisplay.bytes(premergesize),
apDisplay.bytes(premergesize + addsize)))
### merge header files
premergenumpart = apFile.numImagesInStack(mergeheaderfile)
mergehead = open(mergeheaderfile, 'ab')
count = 0
for partarray in partlist:
count += 1
headerstr = makeHeaderStrFromArray(premergenumpart + count, partarray)
mergehead.write(headerstr)
mergehead.close()
numberStackFile(mergeheaderfile, msg=msg)
finalnumpart = apFile.numImagesInStack(mergeheaderfile)
addpart = len(partlist)
if finalnumpart != addpart + premergenumpart:
apDisplay.printError("size mismatch %d vs. %d + %d = %d" %
(finalnumpart, addpart, premergenumpart,
addpart + premergenumpart))
elif msg is True:
apDisplay.printMsg("size match %d vs. %d + %d = %d" %
(finalnumpart, addpart, premergenumpart,
addpart + premergenumpart))
return True
def createBoxMask(maskfile,boxsize,maskx,masky,falloff,imask=None,dataext=".spi"):
"""
creates a rectangular mask with soft falloff
"""
apDisplay.printMsg("Creating %i x %i box mask"%(boxsize,boxsize))
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, nproc=1, log=False)
# create blank image for mask
mySpi.toSpiderQuiet("BL","_1","%i,%i"%(boxsize,boxsize),"N","1")
# mask it in X
mySpi.toSpiderQuiet("MA X", "_1", "_2",
"%i"%maskx, "C", "E", "0",
"%i,%i"%(boxsize/2,boxsize/2),
"%.2f"%falloff)
# inner mask in X
if imask is not None:
mySpi.toSpiderQuiet("MA X","_2","_3",
"%i,%i"%(boxsize/2,imask),
"C","E","0",
"%i,%i"%(boxsize/2,boxsize/2),
"%.2f"%(falloff/4))
mySpi.toSpiderQuiet("CP","_3","_2")
# mask in Y
mySpi.toSpiderQuiet("MA Y","_2",
spyder.fileFilter(maskfile),
"%i"%masky, "C", "E", "0",
"%i,%i"%(boxsize/2,boxsize/2),
"%.2f"%falloff)
mySpi.close()
if not os.path.isfile(maskfile) or apFile.fileSize(maskfile) < 2:
apDisplay.printError("Failed to create mask file")
return
def processImage(self, imgdata):
stackedname = os.path.join(self.params['rundir'],
imgdata['filename'] + "power.jpg")
if os.path.isfile(stackedname) and apFile.fileSize(stackedname) > 100:
return
### make the power spectra
powerspectra = imagefun.power(imgdata['image'],
mask_radius=1.0,
thresh=4)
binpowerspectra = imagefun.bin2(powerspectra, self.params['bin'])
del powerspectra
if self.params['hp'] is True:
binpowerspectra = apImage.fermiHighPassFilter(binpowerspectra,
apix=4.0,
radius=2000.0)
binpowerspectra = apImage.normStdevMed(binpowerspectra, size=5)
binpowerspectra = apImage.pixelLimitFilter(binpowerspectra, pixlimit=4)
binpowerspectra = apImage.normRange(binpowerspectra)
### filter the image
imagedata = imagefun.bin2(imgdata['image'], self.params['bin'])
del imgdata['image']
imagedata = apImage.normStdevMed(imagedata, size=5)
imagedata = apImage.pixelLimitFilter(imagedata, pixlimit=2)
imagedata = apImage.normRange(imagedata)
### write to file
stacked = numpy.hstack([imagedata, binpowerspectra])
del imagedata, binpowerspectra
apImage.arrayToJpeg(stacked, stackedname)
def createBoxMask(maskfile,boxsize,maskx,masky,falloff,imask=None,dataext=".spi"):
"""
creates a rectangular mask with soft falloff
"""
apDisplay.printMsg("Creating %i x %i box mask"%(boxsize,boxsize))
mySpi = spyder.SpiderSession(dataext=dataext, logo=False, nproc=1, log=False)
# create blank image for mask
mySpi.toSpiderQuiet("BL","_1","%i,%i"%(boxsize,boxsize),"N","1")
# mask it in X
mySpi.toSpiderQuiet("MA X", "_1", "_2",
"%i"%maskx, "C", "E", "0",
"%i,%i"%(boxsize/2,boxsize/2),
"%.2f"%falloff)
# inner mask in X
if imask is not None:
mySpi.toSpiderQuiet("MA X","_2","_3",
"%i,%i"%(boxsize/2,imask),
"C","E","0",
"%i,%i"%(boxsize/2,boxsize/2),
"%.2f"%(falloff/4))
mySpi.toSpiderQuiet("CP","_3","_2")
# mask in Y
mySpi.toSpiderQuiet("MA Y","_2",
spyder.fileFilter(maskfile),
"%i"%masky, "C", "E", "0",
"%i,%i"%(boxsize/2,boxsize/2),
"%.2f"%falloff)
mySpi.close()
if not os.path.isfile(maskfile) or apFile.fileSize(maskfile) < 2:
apDisplay.printError("Failed to create mask file")
return
def processImage(self, imgdata): stackedname = os.path.join(self.params['rundir'], imgdata['filename']+"power.jpg") if os.path.isfile(stackedname) and apFile.fileSize(stackedname) > 100: return ### make the power spectra powerspectra = imagefun.power(imgdata['image'], mask_radius=1.0, thresh=4) binpowerspectra = imagefun.bin2(powerspectra, self.params['bin']) del powerspectra if self.params['hp'] is True: binpowerspectra = apImage.fermiHighPassFilter(binpowerspectra, apix=4.0, radius=2000.0) binpowerspectra = apImage.normStdevMed(binpowerspectra, size=5) binpowerspectra = apImage.pixelLimitFilter(binpowerspectra, pixlimit=4) binpowerspectra = apImage.normRange(binpowerspectra) ### filter the image imagedata = imagefun.bin2(imgdata['image'], self.params['bin']) del imgdata['image'] imagedata = apImage.normStdevMed(imagedata, size=5) imagedata = apImage.pixelLimitFilter(imagedata, pixlimit=2) imagedata = apImage.normRange(imagedata) ### write to file stacked = numpy.hstack([imagedata, binpowerspectra]) del imagedata, binpowerspectra apImage.arrayToJpeg(stacked, stackedname)
def appendStackFileToStackFile(stackfile, mergestackfile, msg=True):
"""
takes two stack files and merges them into second file
"""
### initialization
t0 = time.time()
root = os.path.splitext(mergestackfile)[0]
mergeheaderfile = root + ".hed"
mergedatafile = root + ".img"
root = os.path.splitext(stackfile)[0]
stackheaderfile = root + ".hed"
stackdatafile = root + ".img"
### merge data files
addnumpart = apFile.numImagesInStack(stackheaderfile)
addsize = apFile.fileSize(stackdatafile)
premergenumpart = apFile.numImagesInStack(mergeheaderfile)
premergesize = apFile.fileSize(mergedatafile)
fout = file(mergedatafile, 'ab')
fin = file(stackdatafile, 'rb')
shutil.copyfileobj(fin, fout, 65536)
fin.close()
fout.close()
finalsize = apFile.fileSize(mergedatafile)
if finalsize != addsize + premergesize:
apDisplay.printError(
"size mismatch %s vs. %s + %s = %s" %
(apDisplay.bytes(finalsize), apDisplay.bytes(addsize),
apDisplay.bytes(premergesize),
apDisplay.bytes(premergesize + addsize)))
### merge header files
fout = file(mergeheaderfile, 'ab')
fin = file(stackheaderfile, 'rb')
shutil.copyfileobj(fin, fout, 65536)
fin.close()
fout.close()
numberStackFile(mergeheaderfile, msg=msg)
finalnumpart = apFile.numImagesInStack(mergeheaderfile)
if finalnumpart != addnumpart + premergenumpart:
apDisplay.printError("size mismatch %d vs. %d + %d = %d" %
(finalnumpart, addnumpart, premergenumpart,
addnumpart + premergenumpart))
def createMontageByEMAN(self):
self.cluster_resolution = []
apDisplay.printMsg("Converting files")
### create crappy files
emancmd = ("proc2d " + self.instack +
" crap.mrc first=0 last=0 mask=1")
apEMAN.executeEmanCmd(emancmd, showcmd=False, verbose=False)
emancmd = ("proc2d crap.mrc crap.png")
apEMAN.executeEmanCmd(emancmd, showcmd=False, verbose=False)
files = glob.glob(self.timestamp + ".[0-9]*")
files.sort(self.sortFile)
montagecmd = ("montage -geometry +4+4 -tile %dx%d " %
(self.params['xdim'], self.params['ydim']))
stackname = "kerdenstack" + self.timestamp + ".hed"
count = 0
numclass = self.params['xdim'] * self.params['ydim']
i = 0
for listname in files:
i += 1
apDisplay.printMsg("%d of %d classes" % (i, len(files)))
#listname = self.timestamp+str(i)
if not os.path.isfile(listname) or apFile.fileSize(listname) < 1:
### create a ghost particle
emancmd = ("proc2d crap.mrc " + stackname + " ")
sys.stderr.write("skipping " + listname + "\n")
apEMAN.executeEmanCmd(emancmd, showcmd=False, verbose=False)
### create png
shutil.copy("crap.png", listname + ".png")
else:
### average particles
emancmd = ("proc2d %s %s list=%s average" %
(self.instack, stackname, listname))
apEMAN.executeEmanCmd(emancmd, showcmd=True, verbose=False)
### create mrc
emancmd = ("proc2d %s %s first=%d last=%d" %
(stackname, listname + ".mrc", count, count))
apEMAN.executeEmanCmd(emancmd, showcmd=False, verbose=False)
### create png
emancmd = ("proc2d %s %s" %
(listname + ".mrc", listname + ".png"))
apEMAN.executeEmanCmd(emancmd, showcmd=False, verbose=False)
### FIX ME: for now fill self.clsuter_sersolution with None, although it
### should be possible to calculate it if particle list exists like in createMontageInMemory
self.cluster_resolution.append(None)
montagecmd += listname + ".png "
count += 1
montagecmd += "montage.png"
apEMAN.executeEmanCmd(montagecmd, showcmd=True, verbose=False)
time.sleep(1)
apFile.removeFile("crap.mrc")
apFile.removeFile("crap.png")
apFile.removeFilePattern(self.timestamp + ".*.png")
def readSingleParticleFromStack(filename, partnum=1, boxsize=None, msg=True):
"""
reads a single particle from imagic stack
particle number starts at 1
assumes particles have squares boxes
"""
t0 = time.time()
if partnum < 1:
apDisplay.printError("particle numbering starts at 1")
root = os.path.splitext(filename)[0]
headerfilename = root + ".hed"
datafilename = root + ".img"
if msg is True:
apDisplay.printMsg("reading particle %d from stack %s into memory" %
(partnum, os.path.basename(filename)))
### determine boxsize
if boxsize is None:
headerdict = readImagicHeader(headerfilename)
boxsize = headerdict['rows']
if partnum > headerdict['nimg']:
apDisplay.printError(
"requested particle %d from stack of length %d" %
(partnum, headerdict['nimg']))
filesize = apFile.fileSize(datafilename)
### calculate number of bytes per particle
partbytes = boxsize**2 * 4
if partbytes * partnum > filesize:
apDisplay.printError("requested particle %d from stack of length %d" %
(partnum, filesize / partbytes))
### open file
f = open(datafilename, 'rb')
### skip ahead to desired particle
f.seek(partbytes * (partnum - 1))
### read particle image
data = f.read(partbytes)
f.close()
shape = (boxsize, boxsize)
partimg = numpy.fromstring(data, dtype=numpy.float32)
try:
partimg = partimg.reshape(boxsize, boxsize)
partimg = numpy.fliplr(partimg)
except:
print partimg
print boxsize, boxsize * boxsize, partimg.shape
apDisplay.printError("could not read particle from stack")
return partimg
def readImagic(filename, first=1, last=None, msg=True):
"""
Rudimentary Imagic stack reader
Could be improved with more sophisticated error testing and header parsing
Currently only reads image data as floats
Currently reads header information for only first image in stack
"""
t0 = time.time()
if first < 1:
apDisplay.printError("particle numbering starts at 1")
if last is not None and first > last:
apDisplay.printError("requested first particle %d is greater than last particle %d"%(first,last))
if msg is True:
apDisplay.printMsg("reading stack from disk into memory: "+os.path.basename(filename))
if last is not None:
apDisplay.printMsg("particles %d through %d"%(first, last))
root=os.path.splitext(filename)[0]
headerfilename=root + ".hed"
datafilename=root + ".img"
### check file size, no more than 2 GB is possible
### it takes double memory on machine to read stack
filesize = apFile.fileSize(datafilename)
if first is None and last is None and filesize > bytelimit:
apDisplay.printError("Stack is too large to read %s"%(apDisplay.bytes(filesize)))
### read stack header
headerdict = readImagicHeader(headerfilename)
### determine amount of memory needed
partbytes = 4*headerdict['rows']*headerdict['lines']
if last is None:
last = headerdict['nimg']
elif last > headerdict['nimg']:
apDisplay.printWarning("requested particle %d from stack of length %d"%(last, headerdict['nimg']))
last = headerdict['nimg']
numpart = last - first + 1
if partbytes*numpart > filesize:
apDisplay.printError("requested particle %d from stack of length %d"%(last, filesize/partbytes))
if partbytes*numpart > bytelimit:
apDisplay.printError("Stack is too large to read %d particles, requesting %s"
%(numpart, apDisplay.bytes(partbytes*numpart)))
### read stack images
images = readImagicData(datafilename, headerdict, first, numpart)
stack = {'header': headerdict, 'images': images}
if msg is True:
apDisplay.printMsg("read %d particles equaling %s in size"%(numpart, apDisplay.bytes(partbytes*numpart)))
apDisplay.printMsg("finished in "+apDisplay.timeString(time.time()-t0))
return stack
def createMontageByEMAN(self):
self.cluster_resolution = []
apDisplay.printMsg("Converting files")
### create crappy files
emancmd = ( "proc2d "+self.instack+" crap.mrc first=0 last=0 mask=1" )
apEMAN.executeEmanCmd(emancmd, showcmd=False, verbose=False)
emancmd = ( "proc2d crap.mrc crap.png" )
apEMAN.executeEmanCmd(emancmd, showcmd=False, verbose=False)
files = glob.glob(self.timestamp+".[0-9]*")
files.sort(self.sortFile)
montagecmd = ("montage -geometry +4+4 -tile %dx%d "%(self.params['xdim'], self.params['ydim']))
stackname = "kerdenstack"+self.timestamp+".hed"
count = 0
numclass = self.params['xdim']*self.params['ydim']
i = 0
for listname in files:
i += 1
apDisplay.printMsg("%d of %d classes"%(i,len(files)))
#listname = self.timestamp+str(i)
if not os.path.isfile(listname) or apFile.fileSize(listname) < 1:
### create a ghost particle
emancmd = ( "proc2d crap.mrc "+stackname+" " )
sys.stderr.write("skipping "+listname+"\n")
apEMAN.executeEmanCmd(emancmd, showcmd=False, verbose=False)
### create png
shutil.copy("crap.png", listname+".png")
else:
### average particles
emancmd = ("proc2d %s %s list=%s average"%
(self.instack, stackname, listname))
apEMAN.executeEmanCmd(emancmd, showcmd=True, verbose=False)
### create mrc
emancmd = ("proc2d %s %s first=%d last=%d"%
(stackname, listname+".mrc", count, count))
apEMAN.executeEmanCmd(emancmd, showcmd=False, verbose=False)
### create png
emancmd = ("proc2d %s %s"%
(listname+".mrc", listname+".png"))
apEMAN.executeEmanCmd(emancmd, showcmd=False, verbose=False)
### FIX ME: for now fill self.clsuter_sersolution with None, although it
### should be possible to calculate it if particle list exists like in createMontageInMemory
self.cluster_resolution.append(None)
montagecmd += listname+".png "
count +=1
montagecmd += "montage.png"
apEMAN.executeEmanCmd(montagecmd, showcmd=True, verbose=False)
time.sleep(1)
apFile.removeFile("crap.mrc")
apFile.removeFile("crap.png")
apFile.removeFilePattern(self.timestamp+".*.png")
def setImagic4DHeader(oldhedfile,machineonly=False):
### set IMAGIC header values for 2D stack:
### IDAT(61) - int - # sections in 3D volume (1)
### IDAT(62) - int - # objects (number of particles)
### IDAT(69) - machine stamp (33686019 for Linux)
if oldhedfile[-4:] != ".hed":
oldhedfile = os.path.splitext(oldhedfile)[0]+".hed"
if checkImagic4DHeader(oldhedfile,machineonly) is True:
return
newhedfile = oldhedfile+".temp"
numimg = int(os.stat(oldhedfile)[6]/1024)
of = open(oldhedfile, "rb")
nf = open(newhedfile, "wb")
for i in range(numimg):
data = of.read(1024)
headerstr = data[0:60*4]
if machineonly is not True:
headerstr += intToFourByte(1)
headerstr += intToFourByte(numimg)
else:
headerstr += data[60*4:62*4]
headerstr += data[62*4:68*4]
headerstr += intToFourByte(33686018)
headerstr += data[69*4:]
nf.write(headerstr)
of.close()
nf.close()
if not os.path.isfile(newhedfile):
apDisplay.printError("failed to imagic header in file %s"%oldhedfile)
oldsize = apFile.fileSize(oldhedfile)
newsize = apFile.fileSize(newhedfile)
if oldsize != newsize:
apDisplay.printError("failed to imagic header in file %s"%oldhedfile)
shutil.move(newhedfile, oldhedfile)
return
def setImagic4DHeader(oldhedfile, machineonly=False):
### set IMAGIC header values for 2D stack:
### IDAT(61) - int - # sections in 3D volume (1)
### IDAT(62) - int - # objects (number of particles)
### IDAT(69) - machine stamp (33686019 for Linux)
if oldhedfile[-4:] != ".hed":
oldhedfile = os.path.splitext(oldhedfile)[0] + ".hed"
if checkImagic4DHeader(oldhedfile, machineonly) is True:
return
newhedfile = oldhedfile + ".temp"
numimg = int(os.stat(oldhedfile)[6] / 1024)
of = open(oldhedfile, "rb")
nf = open(newhedfile, "wb")
for i in range(numimg):
data = of.read(1024)
headerstr = data[0:60 * 4]
if machineonly is not True:
headerstr += intToFourByte(1)
headerstr += intToFourByte(numimg)
else:
headerstr += data[60 * 4:62 * 4]
headerstr += data[62 * 4:68 * 4]
headerstr += intToFourByte(33686018)
headerstr += data[69 * 4:]
nf.write(headerstr)
of.close()
nf.close()
if not os.path.isfile(newhedfile):
apDisplay.printError("failed to imagic header in file %s" % oldhedfile)
oldsize = apFile.fileSize(oldhedfile)
newsize = apFile.fileSize(newhedfile)
if oldsize != newsize:
apDisplay.printError("failed to imagic header in file %s" % oldhedfile)
shutil.move(newhedfile, oldhedfile)
return
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['description'] is None:
apDisplay.printError("substack description was not defined")
if self.params['runname'] is None:
apDisplay.printError("new stack name was not defined")
stackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
if oldstack[-4:] == ".hed":
oldstack = oldstack[:-4]+".img"
stacksize = apFile.fileSize(oldstack)/1024.0/1024.0
if stacksize > 1200:
apDisplay.printError("Stack is too large to read "+str(round(stacksize,1))+" MB")
def readSingleParticleFromStack(filename, partnum=1, boxsize=None, msg=True):
"""
reads a single particle from imagic stack
particle number starts at 1
assumes particles have squares boxes
"""
t0 = time.time()
if partnum < 1:
apDisplay.printError("particle numbering starts at 1")
root=os.path.splitext(filename)[0]
headerfilename=root + ".hed"
datafilename=root + ".img"
if msg is True:
apDisplay.printMsg("reading particle %d from stack %s into memory"%(partnum, os.path.basename(filename)))
### determine boxsize
if boxsize is None:
headerdict = readImagicHeader(headerfilename)
boxsize = headerdict['rows']
if partnum > headerdict['nimg']:
apDisplay.printError("requested particle %d from stack of length %d"%(partnum, headerdict['nimg']))
filesize = apFile.fileSize(datafilename)
### calculate number of bytes per particle
partbytes = boxsize**2*4
if partbytes*partnum > filesize:
apDisplay.printError("requested particle %d from stack of length %d"%(partnum, filesize/partbytes))
### open file
f = open(datafilename, 'rb')
### skip ahead to desired particle
f.seek(partbytes*(partnum-1))
### read particle image
data = f.read(partbytes)
f.close()
shape = (boxsize, boxsize)
partimg = numpy.fromstring(data, dtype=numpy.float32)
try:
partimg = partimg.reshape(boxsize, boxsize)
partimg = numpy.fliplr(partimg)
except:
print partimg
print boxsize, boxsize*boxsize, partimg.shape
apDisplay.printError("could not read particle from stack")
return partimg
def convertStackToXmippData(instack,
outdata,
maskpixrad,
boxsize,
numpart=None):
"""
From http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Img2Data
This program applies a mask to a set of images.
This set is given by a selfile.
After applying the mask the result is storaged as a vector in the following format:
The first line indicates the dimension of the vectors and the number of vectors.
The rest of the lines are the feature vectors.
Each line is a vector and each column is a vectors' component (pixels values inside the mask).
"""
apDisplay.printMsg("Convert stack file to Xmipp data file")
maskfile = "circlemask.spi"
operations.createMask(maskfile, maskpixrad, boxsize)
partlistdocfile = breakupStackIntoSingleFiles(instack, numpart=numpart)
convertcmd = "xmipp_convert_img2data -i %s -mask %s -o %s" % (
partlistdocfile, maskfile, outdata)
proc = subprocess.Popen(
convertcmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
proc.wait()
outfilesize = apFile.fileSize(outdata)
partfilesize = apFile.fileSize(partlistdocfile)
if outfilesize < 2 * partfilesize:
apDisplay.printError(
"Outdata conversion did not work, data file smaller than docfile, %s < %s"
% (apDisplay.bytes(outfilesize), apDisplay.bytes(partfilesize)))
apFile.removeFilePattern("partfiles/*")
return outdata
def createSplitStacks(self):
self.splitStacks = {}
self.splitStacks['left'] = "particles.left.mrc"
self.splitStacks['right'] = "particles.right.mrc"
if not os.path.isfile(self.splitStacks['left']):
pass
elif not os.path.isfile(self.splitStacks['right']):
pass
elif apFile.fileSize(self.splitStacks['left']) < 100:
pass
elif apFile.fileSize(self.splitStacks['right']) < 100:
pass
else:
apDisplay.printMsg(
"split stacks already exist, skipping this step")
return
apDisplay.printColor(
"Step 1A: Splitting input stack %s into two parts" %
(self.params['inputstack']), "purple")
oddfile, evenfile = apImagicFile.splitStackEvenOdd(
self.params['inputstack'], rundir=self.params['rundir'], msg=True)
apDisplay.printColor(
"Step 1B: Converting odd particles from %s into Left MRC stack %s"
% (oddfile, self.splitStacks['left']), "purple")
imagic2mrc.imagic_to_mrc(oddfile, self.splitStacks['left'])
apFile.removeStack(oddfile)
apDisplay.printColor(
"Step 1C: Converting even particles from %s into Right MRC stack %s"
% (evenfile, self.splitStacks['right']), "purple")
imagic2mrc.imagic_to_mrc(evenfile, self.splitStacks['right'])
apFile.removeStack(evenfile)
return
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['description'] is None:
apDisplay.printError("substack description was not defined")
if self.params['runname'] is None:
apDisplay.printError("new stack name was not defined")
stackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
if oldstack[-4:] == ".hed":
oldstack = oldstack[:-4] + ".img"
stacksize = apFile.fileSize(oldstack) / 1024.0 / 1024.0
if stacksize > 1200:
apDisplay.printError("Stack is too large to read " +
str(round(stacksize, 1)) + " MB")
def createMask(maskfile, maskrad, boxsize, dataext=".spi"):
"""
We should use imagefun.filled_circle() instead
"""
apDisplay.printMsg("Creating mask with diameter %.1f and boxsize %d"%(maskrad*2.0,boxsize))
mySpider = spyder.SpiderSession(dataext=dataext, logo=False)
mySpider.toSpiderQuiet("MO",
spyder.fileFilter(maskfile),
"%d,%d" % (boxsize, boxsize),
"C",
str(maskrad),
)
mySpider.close()
if not os.path.isfile(maskfile) or apFile.fileSize(maskfile) < 2:
apDisplay.printError("Failed to create mask file")
return
def createMask(maskfile, maskrad, boxsize, dataext=".spi"):
"""
We should use imagefun.filled_circle() instead
"""
apDisplay.printMsg("Creating mask with diameter %.1f and boxsize %d"%(maskrad*2.0,boxsize))
mySpider = spyder.SpiderSession(dataext=dataext, logo=False)
mySpider.toSpiderQuiet("MO",
spyder.fileFilter(maskfile),
"%d,%d" % (boxsize, boxsize),
"C",
str(maskrad),
)
mySpider.close()
if not os.path.isfile(maskfile) or apFile.fileSize(maskfile) < 2:
apDisplay.printError("Failed to create mask file")
return
def fetchEMDB(self, emdbid, outfile):
# retrieve emdb from web based on emdb id
mapurl = ( "ftp://ftp.ebi.ac.uk/pub/databases/emdb/structures/EMD-%d/map/emd_%d.map.gz"
%(emdbid, emdbid) )
apDisplay.printMsg("retrieving emdb file: "+mapurl)
# uncompress file & save
data = urllib.urlretrieve(mapurl)[0]
g = gzip.open(data,'r').read()
f=open(outfile,'w')
f.write(g)
f.close()
if not os.path.exists(outfile):
apDisplay.printError("Could not retrieve EMDB file")
size = apFile.fileSize(outfile)
apDisplay.printMsg("downloaded file of size "+str(round(size/1024.0,1))+"k")
return outfile
def singleNodeRun(self, iternum):
"""
single node run
"""
### single node run
apDisplay.printMsg("Single node run, iteration %d" % (iternum))
### create individual processor jobs
stackbase = os.path.splitext(os.path.basename(self.refinestackfile))[0]
self.stackfile = "../../%s" % (stackbase)
procjobfiles = self.createMultipleJobs(iternum)
### convert job files to commands
procjobcmds = []
for procjobfile in procjobfiles:
procjobcmd = "sh " + procjobfile
procjobcmds.append(procjobcmd)
### run individual processor jobs
t0 = time.time()
apThread.threadCommands(procjobcmds,
nproc=self.params['nproc'],
pausetime=30.0)
apDisplay.printColor(
"Refinement complete in %s" %
(apDisplay.timeString(time.time() - t0)), "green")
### create combine processor jobs
stackbase = os.path.splitext(os.path.basename(self.reconstackfile))[0]
self.stackfile = "../%s" % (stackbase)
combinejobfile = self.combineMultipleJobs(iternum)
### run combine processor jobs
t0 = time.time()
proc = subprocess.Popen("sh " + combinejobfile, shell=True)
proc.wait()
apDisplay.printColor(
"Volume complete in %s" % (apDisplay.timeString(time.time() - t0)),
"green")
if apFile.fileSize('iter%03d.img' % (iternum)) < 100:
apDisplay.printError("Failed to generate volume for iter %d" %
(iternum))
return
def fetchEMDB(self, emdbid, outfile):
# retrieve emdb from web based on emdb id
mapurl = (
"ftp://ftp.ebi.ac.uk/pub/databases/emdb/structures/EMD-%d/map/emd_%d.map.gz"
% (emdbid, emdbid))
apDisplay.printMsg("retrieving emdb file: " + mapurl)
# uncompress file & save
data = urllib.urlretrieve(mapurl)[0]
g = gzip.open(data, 'r').read()
f = open(outfile, 'w')
f.write(g)
f.close()
if not os.path.exists(outfile):
apDisplay.printError("Could not retrieve EMDB file")
size = apFile.fileSize(outfile)
apDisplay.printMsg("downloaded file of size " +
str(round(size / 1024.0, 1)) + "k")
return outfile
def singleNodeRun(self, iternum):
"""
single node run
"""
### single node run
apDisplay.printMsg("Single node run, iteration %d"%(iternum))
### create individual processor jobs
stackbase = os.path.splitext(os.path.basename(self.refinestackfile))[0]
self.stackfile = "../../%s"%(stackbase)
procjobfiles = self.createMultipleJobs(iternum)
### convert job files to commands
procjobcmds = []
for procjobfile in procjobfiles:
procjobcmd = "sh "+procjobfile
procjobcmds.append(procjobcmd)
### run individual processor jobs
t0 = time.time()
apThread.threadCommands(procjobcmds, nproc=self.params['nproc'], pausetime=30.0)
apDisplay.printColor("Refinement complete in %s"%(apDisplay.timeString(time.time()-t0)), "green")
### create combine processor jobs
stackbase = os.path.splitext(os.path.basename(self.reconstackfile))[0]
self.stackfile = "../%s"%(stackbase)
combinejobfile = self.combineMultipleJobs(iternum)
### run combine processor jobs
t0 = time.time()
proc = subprocess.Popen("sh "+combinejobfile, shell=True)
proc.wait()
apDisplay.printColor("Volume complete in %s"%(apDisplay.timeString(time.time()-t0)), "green")
if apFile.fileSize('iter%03d.img'%(iternum)) < 100:
apDisplay.printError("Failed to generate volume for iter %d"%(iternum))
return
def downloadImagesFromAMI(self):
imglist = []
from urllib import urlretrieve
imgdict = {
110: "06jul12a_00022gr_00037sq_00025hl_00005en.mrc",
111: "06jul12a_00035gr_00063sq_00012hl_00004en.mrc",
112: "06jul12a_00015gr_00028sq_00004hl_00002en.mrc",
113: "06jul12a_00015gr_00028sq_00023hl_00002en.mrc",
114: "06jul12a_00015gr_00028sq_00023hl_00004en.mrc",
115: "06jul12a_00022gr_00013sq_00002hl_00004en.mrc",
116: "06jul12a_00022gr_00013sq_00003hl_00005en.mrc",
}
for key in imgdict.keys():
imgfile = os.path.join(self.params["rundir"], imgdict[key])
url = "http://ami.scripps.edu/redmine/attachments/download/%d/%s" % (key, imgdict[key])
apDisplay.printMsg("Downloading image '%s'" % (imgdict[key]))
urlretrieve(url, imgfile)
if not os.path.isfile(imgfile):
apDisplay.printError("could not download file: %s" % (url))
if apFile.fileSize(imgfile) < 30e6:
apDisplay.printError("error in downloaded file: %s" % (url))
imglist.append(imgfile)
return imglist
def createMontageInMemory(self, apix):
self.cluster_resolution = []
apDisplay.printMsg("Converting files")
### Set binning of images
boxsize = apImagicFile.getBoxsize(self.instack)
bin = 1
while boxsize/bin > 200:
bin+=1
binboxsize = boxsize/bin
### create averages
files = glob.glob(self.timestamp+".[0-9]*")
files.sort(self.sortFile)
montage = []
montagepngs = []
i = 0
for listname in files:
i += 1
apDisplay.printMsg("%d of %d classes"%(i,len(files)))
pngfile = listname+".png"
if not os.path.isfile(listname) or apFile.fileSize(listname) < 1:
### create a ghost particle
sys.stderr.write("skipping "+listname+"\n")
blank = numpy.ones((binboxsize, binboxsize), dtype=numpy.float32)
### add to montage stack
montage.append(blank)
self.cluster_resolution.append(None)
### create png
apImage.arrayToPng(blank, pngfile)
else:
### read particle list
partlist = self.readListFile(listname)
### average particles
partdatalist = apImagicFile.readParticleListFromStack(self.instack, partlist, boxsize, msg=False)
partdataarray = numpy.asarray(partdatalist)
finaldata = partdataarray.mean(0)
if bin > 1:
finaldata = apImage.binImg(finaldata, bin)
### add to montage stack
montage.append(finaldata)
res = apFourier.spectralSNR(partdatalist, apix)
self.cluster_resolution.append(res)
### create png
apImage.arrayToPng(finaldata, pngfile)
### check for png file
if os.path.isfile(pngfile):
montagepngs.append(pngfile)
else:
apDisplay.printError("failed to create montage")
stackname = "kerdenstack"+self.timestamp+".hed"
apImagicFile.writeImagic(montage, stackname)
### create montage
montagecmd = ("montage -geometry +4+4 -tile %dx%d "%(self.params['xdim'], self.params['ydim']))
for monpng in montagepngs:
montagecmd += monpng+" "
montagecmd += "montage.png"
apEMAN.executeEmanCmd(montagecmd, showcmd=True, verbose=False)
time.sleep(1)
apFile.removeFilePattern(self.timestamp+".*.png")
return bin
def readImagic(filename, first=1, last=None, msg=True):
"""
Rudimentary Imagic stack reader
Could be improved with more sophisticated error testing and header parsing
Currently only reads image data as floats
Currently reads header information for only first image in stack
"""
t0 = time.time()
if first < 1:
apDisplay.printError("particle numbering starts at 1")
if last is not None and first > last:
apDisplay.printError(
"requested first particle %d is greater than last particle %d" %
(first, last))
if msg is True:
apDisplay.printMsg("reading stack from disk into memory: " +
os.path.basename(filename))
if last is not None:
apDisplay.printMsg("particles %d through %d" % (first, last))
root = os.path.splitext(filename)[0]
headerfilename = root + ".hed"
datafilename = root + ".img"
### check file size, no more than 2 GB is possible
### it takes double memory on machine to read stack
filesize = apFile.fileSize(datafilename)
if first is None and last is None and filesize > bytelimit:
apDisplay.printError("Stack is too large to read %s" %
(apDisplay.bytes(filesize)))
### read stack header
headerdict = readImagicHeader(headerfilename)
### determine amount of memory needed
partbytes = 4 * headerdict['rows'] * headerdict['lines']
if last is None:
last = headerdict['nimg']
elif last > headerdict['nimg']:
apDisplay.printWarning(
"requested particle %d from stack of length %d" %
(last, headerdict['nimg']))
last = headerdict['nimg']
numpart = last - first + 1
if partbytes * numpart > filesize:
apDisplay.printError("requested particle %d from stack of length %d" %
(last, filesize / partbytes))
if partbytes * numpart > bytelimit:
apDisplay.printError(
"Stack is too large to read %d particles, requesting %s" %
(numpart, apDisplay.bytes(partbytes * numpart)))
### read stack images
images = readImagicData(datafilename, headerdict, first, numpart)
stack = {'header': headerdict, 'images': images}
if msg is True:
apDisplay.printMsg("read %d particles equaling %s in size" %
(numpart, apDisplay.bytes(partbytes * numpart)))
apDisplay.printMsg("finished in " +
apDisplay.timeString(time.time() - t0))
return stack
def mergeStacks(stacklist, mergestack, msg=True):
### initialization
t0 = time.time()
apFile.removeStack(mergestack, warn=msg)
root = os.path.splitext(mergestack)[0]
mergeheader = root + ".hed"
mergedata = root + ".img"
### merge data files
fout = file(mergedata, 'wb')
numpart = 0
totalsize = 0
for stackfile in stacklist:
stackdatafile = os.path.splitext(stackfile)[0] + ".img"
### size checks
npart = apFile.numImagesInStack(stackdatafile)
size = apFile.fileSize(stackdatafile)
if msg is True:
apDisplay.printMsg("%d particles in %s (%s)" %
(npart, stackdatafile, apDisplay.bytes(size)))
totalsize += size
numpart += npart
fin = file(stackdatafile, 'rb')
shutil.copyfileobj(fin, fout, 65536)
fin.close()
fout.close()
if numpart < 1:
apDisplay.printError("found %d particles" % (numpart))
if msg is True:
apDisplay.printMsg("found %d particles" % (numpart))
finalsize = apFile.fileSize(mergedata)
if finalsize != totalsize:
apDisplay.printError(
"size mismatch %s vs. %s" %
(apDisplay.bytes(finalsize), apDisplay.bytes(totalsize)))
if msg is True:
apDisplay.printMsg(
"size match %s vs. %s" %
(apDisplay.bytes(finalsize), apDisplay.bytes(totalsize)))
### merge header files
#apDisplay.printError("not finished")
mergehead = open(mergeheader, 'wb')
partnum = 1
totalsize = 0
for stackfile in stacklist:
headerfilename = os.path.splitext(stackfile)[0] + ".hed"
headfile = open(headerfilename, 'rb')
### size checks
size = apFile.fileSize(headerfilename)
if msg is True:
apDisplay.printMsg("%s (%d kB)" % (headerfilename, size / 1024))
totalsize += size
#apDisplay.printMsg("%d\t%s"%(npart, stackfile))
i = 0
npart = apFile.numImagesInStack(stackfile)
while i < npart:
#print i, npart, partnum
### read old header
data = headfile.read(1024)
### start new string
headerstr = ""
### first image number
headerstr += intToFourByte(partnum)
### number of images, less one
headerstr += intToFourByte(numpart - partnum)
### always 0,1 ???
headerstr += intToFourByte(0)
headerstr += intToFourByte(1)
### creation date: day, month, year, hour, min, sec
headerstr += intToFourByte(time.localtime()[2])
headerstr += intToFourByte(
time.localtime()[1]) #eman always uses month-1?
headerstr += intToFourByte(time.localtime()[0])
headerstr += intToFourByte(time.localtime()[3])
headerstr += intToFourByte(time.localtime()[4])
headerstr += intToFourByte(time.localtime()[5])
### append other header info, 4 character per item
headerstr += data[10 * 4:60 * 4]
### number of z slices
headerstr += intToFourByte(1)
### first image number, EMAN does this
headerstr += intToFourByte(partnum)
### append other header info, 4 character per item
headerstr += data[62 * 4:68 * 4]
headerstr += intToFourByte(33686018)
headerstr += data[69 * 4:]
mergehead.write(headerstr)
partnum += 1
i += 1
mergehead.close()
if msg is True:
apDisplay.printMsg("wrote %d particles to file %s" %
(numpart, mergestack))
finalsize = apFile.fileSize(mergeheader)
if finalsize != totalsize:
apDisplay.printError(
"size mismatch %s vs. %s" %
(apDisplay.bytes(finalsize), apDisplay.bytes(totalsize)))
if msg is True:
apDisplay.printMsg(
"size match %s vs. %s" %
(apDisplay.bytes(finalsize), apDisplay.bytes(totalsize)))
apDisplay.printMsg(
"finished stack merge of %s in %s" %
(mergestack, apDisplay.timeString(time.time() - t0)))
def readParticleListFromStack(filename, partlist, boxsize=None, msg=True):
"""
reads a single particle from imagic stack
particle number starts at 1
assumes particles have squares boxes
"""
t0 = time.time()
### sort list
partlist.sort()
firstpartnum = partlist[0]
lastpartnum = partlist[len(partlist) - 1]
if firstpartnum < 1:
apDisplay.printError("particle numbering starts at 1")
root = os.path.splitext(filename)[0]
headerfilename = root + ".hed"
datafilename = root + ".img"
### determine boxsize
if boxsize is None:
headerdict = readImagicHeader(headerfilename)
boxsize = headerdict['rows']
if lastpartnum > headerdict['nimg']:
apDisplay.printError(
"requested particle %d from stack %s of length %d" %
(lastpartnum, os.path.basename(datafilename),
headerdict['nimg']))
filesize = apFile.fileSize(datafilename)
### calculate number of bytes per particle
partbytes = boxsize**2 * 4
if partbytes * lastpartnum > filesize:
apDisplay.printError(
"requested particle %d from stack %s of length %d" %
(lastpartnum, os.path.basename(datafilename),
filesize / partbytes))
### open file
f = open(datafilename, 'rb')
partdatalist = []
prevpartnum = 0
unames = os.uname()
for partnum in partlist:
if msg is True:
apDisplay.printMsg(
"reading particle %d from stack %s into memory" %
(partnum, os.path.basename(datafilename)))
seekpos = partbytes * (partnum - prevpartnum - 1)
### for 64 bit machines, skip to desired particles
if unames[-1].find('64') >= 0:
f.seek(seekpos)
### for 32-bit machines, seek incrementally
else:
seekpos = int(seekpos) % 2**32
f.seek(0)
if seekpos > sys.maxint:
while seekpos > sys.maxint:
f.seek(sys.maxint, 1)
seekpos -= sys.maxint
f.seek(seekpos, 1)
### read particle image
data = f.read(partbytes)
shape = (boxsize, boxsize)
partimg = numpy.fromstring(data, dtype=numpy.float32)
try:
partimg = partimg.reshape(boxsize, boxsize)
partimg = numpy.fliplr(partimg)
except:
print partimg
print boxsize, boxsize * boxsize, partimg.shape
apDisplay.printError("could not read particle from stack")
partdatalist.append(partimg)
f.close()
return partdatalist
def mergeStacks(stacklist, mergestack):
### initialization
t0 = time.time()
apFile.removeStack(mergestack)
root=os.path.splitext(mergestack)[0]
mergeheader = root+".hed"
mergedata = root+".img"
### merge data files
fout = file(mergedata, 'wb')
numpart = 0
totalsize = 0
for stackfile in stacklist:
stackdatafile = os.path.splitext(stackfile)[0]+ ".img"
### size checks
npart = apFile.numImagesInStack(stackdatafile)
size = apFile.fileSize(stackdatafile)
apDisplay.printMsg("%d particles in %s (%s)"%(npart, stackdatafile, apDisplay.bytes(size)))
totalsize += size
numpart += npart
fin = file(stackdatafile, 'rb')
shutil.copyfileobj(fin, fout, 65536)
fin.close()
fout.close()
if numpart < 1:
apDisplay.printError("found %d particles"%(numpart))
apDisplay.printMsg("found %d particles"%(numpart))
finalsize = apFile.fileSize(mergedata)
if finalsize != totalsize:
apDisplay.printError("size mismatch %s vs. %s"%(apDisplay.bytes(finalsize), apDisplay.bytes(totalsize)))
apDisplay.printMsg("size match %s vs. %s"%(apDisplay.bytes(finalsize), apDisplay.bytes(totalsize)))
### merge header files
#apDisplay.printError("not finished")
mergehead = open(mergeheader, 'wb')
partnum = 1
totalsize = 0
for stackfile in stacklist:
headerfilename = os.path.splitext(stackfile)[0]+ ".hed"
headfile = open(headerfilename, 'rb')
### size checks
size = apFile.fileSize(headerfilename)
apDisplay.printMsg("%s (%d kB)"%(headerfilename, size/1024))
totalsize += size
#apDisplay.printMsg("%d\t%s"%(npart, stackfile))
i = 0
npart = apFile.numImagesInStack(stackfile)
while i < npart:
#print i, npart, partnum
### read old header
data = headfile.read(1024)
### start new string
headerstr = ""
### first image number
headerstr += intToFourByte(partnum)
### number of images, less one
headerstr += intToFourByte(numpart-partnum)
### always 0,1 ???
headerstr += intToFourByte(0)
headerstr += intToFourByte(1)
### creation date: day, month, year, hour, min, sec
headerstr += intToFourByte(time.localtime()[2])
headerstr += intToFourByte(time.localtime()[1]) #eman always uses month-1?
headerstr += intToFourByte(time.localtime()[0])
headerstr += intToFourByte(time.localtime()[3])
headerstr += intToFourByte(time.localtime()[4])
headerstr += intToFourByte(time.localtime()[5])
### append other header info, 4 character per item
headerstr += data[10*4:60*4]
### number of z slices
headerstr += intToFourByte(1)
### first image number, EMAN does this
headerstr += intToFourByte(partnum)
### append other header info, 4 character per item
headerstr += data[62*4:68*4]
headerstr += intToFourByte(33686018)
headerstr += data[69*4:]
mergehead.write(headerstr)
partnum += 1
i += 1
mergehead.close()
apDisplay.printMsg("wrote %d particles"%(numpart))
finalsize = apFile.fileSize(mergeheader)
if finalsize != totalsize:
apDisplay.printError("size mismatch %s vs. %s"%(apDisplay.bytes(finalsize), apDisplay.bytes(totalsize)))
apDisplay.printMsg("size match %s vs. %s"%(apDisplay.bytes(finalsize), apDisplay.bytes(totalsize)))
apDisplay.printMsg("finished stack merge in "+apDisplay.timeString(time.time()-t0))
def readParticleListFromStack(filename, partlist, boxsize=None, msg=True):
"""
reads a single particle from imagic stack
particle number starts at 1
assumes particles have squares boxes
"""
t0 = time.time()
### sort list
partlist.sort()
firstpartnum = partlist[0]
lastpartnum = partlist[len(partlist)-1]
if firstpartnum < 1:
apDisplay.printError("particle numbering starts at 1")
root=os.path.splitext(filename)[0]
headerfilename=root + ".hed"
datafilename=root + ".img"
### determine boxsize
if boxsize is None:
headerdict = readImagicHeader(headerfilename)
boxsize = headerdict['rows']
if lastpartnum > headerdict['nimg']:
apDisplay.printError("requested particle %d from stack %s of length %d"
%(lastpartnum, os.path.basename(datafilename), headerdict['nimg']))
filesize = apFile.fileSize(datafilename)
### calculate number of bytes per particle
partbytes = boxsize**2*4
if partbytes*lastpartnum > filesize:
apDisplay.printError("requested particle %d from stack %s of length %d"
%(lastpartnum, os.path.basename(datafilename), filesize/partbytes))
### open file
f = open(datafilename, 'rb')
partdatalist = []
prevpartnum = 0
unames=os.uname()
for partnum in partlist:
if msg is True:
apDisplay.printMsg("reading particle %d from stack %s into memory"
%(partnum, os.path.basename(datafilename)))
seekpos = partbytes*(partnum-prevpartnum-1)
### for 64 bit machines, skip to desired particles
if unames[-1].find('64') >= 0:
f.seek(seekpos)
### for 32-bit machines, seek incrementally
else:
seekpos = int(seekpos)%2**32
f.seek(0)
if seekpos > sys.maxint:
while seekpos > sys.maxint:
f.seek(sys.maxint,1)
seekpos-=sys.maxint
f.seek(seekpos,1)
### read particle image
data = f.read(partbytes)
shape = (boxsize, boxsize)
partimg = numpy.fromstring(data, dtype=numpy.float32)
try:
partimg = partimg.reshape(boxsize, boxsize)
partimg = numpy.fliplr(partimg)
except:
print partimg
print boxsize, boxsize*boxsize, partimg.shape
apDisplay.printError("could not read particle from stack")
partdatalist.append(partimg)
f.close()
return partdatalist
