def applyEnvelope(self, inimage, outimage, scaleFactor=1, msg=False):
"""
input path to image and envelope, output amplitude-adjusted image
"""
if msg is True:
apDisplay.printColor("now applying envelope function to: "+inimage, "cyan")
if self.envamp is None:
self.prepareEnvelope(scaleFactor)
### read image
im = mrc.read(inimage)
### fourier transform
imfft = self.real_fft2d(im)
### mutliply real envelope function by image fft
newfft = self.envamp * imfft
### inverse transform
newimg = self.inverse_real_fft2d(newfft)
### normalize between 0 and 1
newimg = (newimg-newimg.mean()) / newimg.std()
### save image
mrc.write(newimg, outimage)
### workaround for now
time.sleep(0.1)
return
def makeProjection(filename, xsize=512):
mrcpath = filename
dirpath = os.path.dirname(mrcpath)
apDisplay.printMsg("Reading 3D recon %s" % mrcpath)
array = mrc.read(mrcpath)
shape = array.shape
xsize = min(xsize, shape[2])
# default for full tomogram is XZY
if shape[0] > shape[1]:
renders = {
"a": {"axis": 0, "axisname": "z"},
"b": {"axis": 1, "axisname": "y"},
"c": {"axis": 2, "axisname": "x"},
}
else:
renders = {
"a": {"axis": 1, "axisname": "y"},
"b": {"axis": 0, "axisname": "z"},
"c": {"axis": 2, "axisname": "x"},
}
keys = renders.keys()
keys.sort()
for key in keys:
apDisplay.printMsg("project to axis %s" % renders[key]["axisname"])
pictpath = os.path.join(dirpath, "projection" + key)
axis = renders[key]["axis"]
slice = numpy.sum(array[:, :, :], axis=axis) / (shape[axis])
mrc.write(slice, pictpath + ".mrc")
# adjust and shrink each image
array2jpg(pictpath, slice, size=xsize)
def medianVolume(self):
volpath = os.path.join(self.params['rundir'], "volumes/*a.mrc")
mrcfiles = glob.glob(volpath)
volumes = []
for filename in mrcfiles:
if os.path.isfile(filename):
vol = mrc.read(filename)
print filename, vol.shape
volumes.append(vol)
volarray = numpy.asarray(volumes, dtype=numpy.float32)
try:
medarray = numpy.median(volarray, axis=0)
except:
medarray = numpy.median(volarray)
medfile = os.path.join(self.params['rundir'], "volumes/medianVolume.mrc")
print medfile, medarray.shape
mrc.write(medarray, medfile)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
uploadcmd = ( ("uploadModel.py --projectid=%d --session=%s --file=%s "
+"--apix=%.3f --sym=%s --name=satmedian-recon%d.mrc --res=30 --description='%s %d'")
%(self.params['projectid'], sessiondata['name'], medfile,
apix, self.params['symmname'], self.params['reconid'],
"SAT selected median volume for recon", self.params['reconid'], ) )
apDisplay.printColor(uploadcmd, "purple")
f = open("upload.sh", "w")
f.write(uploadcmd+"\n")
f.close()
def medianVolume(self):
volpath = os.path.join(self.params['rundir'], "volumes/*a.mrc")
mrcfiles = glob.glob(volpath)
volumes = []
for filename in mrcfiles:
if os.path.isfile(filename):
vol = mrc.read(filename)
print filename, vol.shape
volumes.append(vol)
volarray = numpy.asarray(volumes, dtype=numpy.float32)
try:
medarray = numpy.median(volarray, axis=0)
except:
medarray = numpy.median(volarray)
medfile = os.path.join(self.params['rundir'], "volumes/medianVolume.mrc")
print medfile, medarray.shape
mrc.write(medarray, medfile)
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
uploadcmd = ( ("uploadModel.py --projectid=%d --session=%s --file=%s "
+"--apix=%.3f --sym=%s --name=satmedian-recon%d.mrc --res=30 --description='%s %d'")
%(self.params['projectid'], sessiondata['name'], medfile,
apix, self.params['symmname'], self.params['reconid'],
"SAT selected median volume for recon", self.params['reconid'], ) )
apDisplay.printColor(uploadcmd, "purple")
f = open("upload.sh", "w")
f.write(uploadcmd+"\n")
f.close()
def writeVarianceImage(imagicfile, varmrcfile):
imgdict = readImagic(imagicfile)
if imgdict is None:
return
vararray = imgdict['images'].std(0)
mrc.write(vararray, varmrcfile)
return vararray
def get_hole_stats(self, image, coord, radius):
## select the region of interest
rmin = int(coord[0] - radius)
rmax = int(coord[0] + radius)
cmin = int(coord[1] - radius)
cmax = int(coord[1] + radius)
## beware of boundaries
if rmin < 0 or rmax >= image.shape[
0] or cmin < 0 or cmax > image.shape[1]:
return None
subimage = image[rmin:rmax + 1, cmin:cmax + 1]
if self.save_mrc:
mrc.write(subimage, 'hole.mrc')
center = subimage.shape[0] / 2.0, subimage.shape[1] / 2.0
mask = self.circle.get(subimage.shape, center, 0, radius)
if self.save_mrc:
mrc.write(mask, 'holemask.mrc')
im = numpy.ravel(subimage)
mask = numpy.ravel(mask)
roi = numpy.compress(mask, im)
mean = arraystats.mean(roi)
std = arraystats.std(roi)
n = len(roi)
return {'mean': mean, 'std': std, 'n': n}
def getImageFiles(imgtree, rawdir, link, copy):
#This function should replace linkImageFiles in all calls (i.e. in tomoaligner.py and everywhere else)
filenamelist = []
newimgtree = []
for imagedata in imgtree:
#set up names
imgpath = imagedata['session']['image path']
presetname = imagedata['preset']['name']
imgprefix = presetname + imagedata['filename'].split(presetname)[-1]
imgname = imgprefix + '.mrc'
filenamelist.append(imgprefix)
destpath = os.path.join(rawdir, imgname)
newimgtree.append(destpath)
imgfullpath = os.path.join(imgpath, imagedata['filename'] + '.mrc')
if link == "True":
#create symlinks to files
if os.path.islink(destpath):
os.remove(destpath)
if not os.path.isfile(destpath):
os.symlink(imgfullpath, destpath)
elif copy == "True":
shutil.copy(imgfullpath, destpath)
#Y-flip raw images, normalize them, and convert them to float32 because Protomo
image = mrc.read(destpath)
image = numpy.flipud(image)
image = imagenorm.normStdev(image)
image = numpy.float32(image)
mrc.write(image, destpath)
#else: just return values
return filenamelist, newimgtree
def getImageFiles(imgtree, rawdir, link, copy): #This function should replace linkImageFiles in all calls (i.e. in tomoaligner.py and everywhere else) filenamelist = [] newimgtree=[] for imagedata in imgtree: #set up names imgpath=imagedata['session']['image path'] presetname=imagedata['preset']['name'] imgprefix=presetname+imagedata['filename'].split(presetname)[-1] imgname=imgprefix+'.mrc' filenamelist.append(imgprefix) destpath = os.path.join(rawdir,imgname) newimgtree.append(destpath) imgfullpath = os.path.join(imgpath,imagedata['filename']+'.mrc') if link == "True": #create symlinks to files if os.path.islink(destpath): os.remove(destpath) if not os.path.isfile(destpath): os.symlink(imgfullpath,destpath) elif copy == "True": shutil.copy(imgfullpath,destpath) #Y-flip raw images, normalize them, and convert them to float32 because Protomo image=mrc.read(destpath) image=numpy.flipud(image) image=imagenorm.normStdev(image) image=numpy.float32(image) mrc.write(image,destpath) #else: just return values return filenamelist, newimgtree
def register(image1, image2):
trim = 8
image1 = image1[trim:-trim, trim:-trim]
image2 = image2[trim:-trim, trim:-trim]
fft1 = scipy.fftpack.fft2(image1)
fft2 = scipy.fftpack.fft2(image2)
fft1 = scipy.fftpack.fftshift(fft1, axes=[0])
fft2 = scipy.fftpack.fftshift(fft2, axes=[0])
c = int(fft1.shape[0] / 2.0)
fft1 = fft1[:,:c+1]
fft2 = fft2[:,:c+1]
mag1 = numpy.abs(fft1)
mag2 = numpy.abs(fft2)
mrc.write(mag1, 'mag1.mrc')
mrc.write(mag2, 'mag2.mrc')
center = c,0
output_shape = c,c
print 'P1'
p1 = polar_transform(mag1, output_shape, center)
#scipy.misc.imsave('p1.jpg', p1)
mrc.write(p1, 'p1.mrc')
print 'P2'
p2 = polar_transform(mag2, output_shape, center)
#scipy.misc.imsave('p2.jpg', p2)
mrc.write(p2, 'p2.mrc')
pc = correlator.phase_correlate(p1, p2, zero=False)
#pc = correlator.cross_correlate(p1, p2)
mrc.write(pc, 'pc.mrc')
def applyEnvelope(self, inimage, outimage, scaleFactor=1, msg=False):
"""
input path to image and envelope, output amplitude-adjusted image
"""
if msg is True:
apDisplay.printColor("now applying envelope function to: "+inimage, "cyan")
if self.envamp is None:
self.prepareEnvelope(scaleFactor)
### read image
im = mrc.read(inimage)
### fourier transform
imfft = self.real_fft2d(im)
### mutliply real envelope function by image fft
newfft = self.envamp * imfft
### inverse transform
newimg = self.inverse_real_fft2d(newfft)
### normalize between 0 and 1
newimg = (newimg-newimg.mean()) / newimg.std()
### save image
mrc.write(newimg, outimage)
### workaround for now
time.sleep(0.1)
return
def writeVarianceImage(imagicfile, varmrcfile):
imgdict = readImagic(imagicfile)
if imgdict is None:
return
vararray = imgdict['images'].std(0)
mrc.write(vararray, varmrcfile)
return vararray
def processParticles(self, imgdata, partdatas, shiftdata):
self.shortname = apDisplay.short(imgdata['filename'])
### if only selected points along helix,
### fill in points with helical step
if self.params['helicalstep']:
apix = apDatabase.getPixelSize(imgdata)
partdatas = self.fillWithHelicalStep(partdatas, apix)
### run batchboxer
self.boxedpartdatas, self.imgstackfile, self.partmeantree = self.boxParticlesFromImage(imgdata, partdatas, shiftdata)
if self.boxedpartdatas is None:
self.stats['lastpeaks'] = 0
apDisplay.printWarning("no particles were boxed from "+self.shortname+"\n")
self.badprocess = True
return None
self.stats['lastpeaks'] = len(self.boxedpartdatas)
apDisplay.printMsg("do not break function now otherwise it will corrupt stack")
#time.sleep(1.0)
### merge image particles into big stack
totalpart = self.mergeImageStackIntoBigStack(self.imgstackfile, imgdata)
### create a stack average every so often
if self.stats['lastpeaks'] > 0:
totalPartices = self.existingParticleNumber+self.stats['peaksum']+self.stats['lastpeaks']
logpeaks = math.log(totalPartices)
if logpeaks > self.logpeaks:
self.logpeaks = math.ceil(logpeaks)
numpeaks = math.ceil(math.exp(self.logpeaks))
apDisplay.printMsg("writing averaging stack, next average at %d particles"%(numpeaks))
mrc.write(self.summedParticles/float(totalPartices), "average.mrc")
return totalpart
def shiftToCenter(infile, shiftfile, isEMAN=False):
'''
EMAN defines the rotation origin differently from other packages.
Therefore, it needs to be recenterred according to the package
after using EMAN proc3d rotation functions.
'''
# center of rotation for eman is not at length/2.
if isEMAN:
formatoffset = getEmanCenter()
prefix = ''
else:
formatoffset = (0, 0, 0)
prefix = 'non-'
apDisplay.printMsg('Shifting map center for %sEMAN usage' % (prefix, ))
# Find center of mass of the density map
a = mrc.read(infile)
t = a.mean() + 2 * a.std()
numpy.putmask(a, a >= t, t)
numpy.putmask(a, a < t, 0)
center = ndimage.center_of_mass(a)
offset = (center[0] + formatoffset[0] - a.shape[0] / 2,
center[1] + formatoffset[1] - a.shape[1] / 2,
center[2] + formatoffset[2] - a.shape[2] / 2)
offset = (-offset[0], -offset[1], -offset[2])
apDisplay.printMsg('Shifting map center by (x,y,z)=(%.2f,%.2f,%.2f)' %
(offset[2], offset[1], offset[0]))
# shift the map
a = mrc.read(infile)
a = ndimage.interpolation.shift(a, offset)
mrc.write(a, shiftfile)
h = mrc.readHeaderFromFile(infile)
mrc.update_file_header(shiftfile, h)
def createSingleImage(inputimages,
globaloutput,
outfilename,
outformat,
outtext,
outxml=False):
n = len(inputimages)
tiles = []
for i, input in enumerate(inputimages):
print 'inserting %d of %d' % (i + 1, n)
tile = globaloutput.insertImage(input, outtext=outtext)
tiles.append(tile)
'''
if targetinputs:
for input,target in targetinputs:
globaloutput.insertImage(input, target)
print 'inserted %s targets' % (len(globaloutput.targets),)
for target in globaloutput.targets:
print 'T', target
markTarget(globaloutput.image, target)
'''
if outtext is not None:
f = open(outtext, 'w')
if outxml:
xmldoc = getMosaicXMLData(tiles)
xmldoc.writexml(f, " ", "", "\n", "UTF-8")
else:
lines = [str(tile) + "\n" for tile in tiles]
f.writelines(lines)
f.close()
if outfilename is not None:
mrc.write(globaloutput.image, outfilename)
def combine_polished_stacks(self):
oldmovieid = None
nmic = 0
for part in self.stackparts:
self.dd.setImageData(part['particle']['image'])
movieid = part['particle']['image'].dbid
alignpairdata = self.dd.getAlignImagePairData(
None, query_source=not self.dd.getIsAligned())
if alignpairdata is False:
apDisplay.printWarning(
'Image not used for nor a result of alignment.')
if movieid != oldmovieid:
ddstack = orig_dd_file = alignpairdata['source'][
'filename'] + "_lmbfgs.mrc"
particlestack = os.path.join(self.params['rundir'],
"Particles", ddstack)
a = mrc.read(particlestack)
apDisplay.printMsg("appending stack %s" % ddstack)
if oldmovieid is None:
mrc.write(a, "polished.mrc")
else:
mrc.append(a, "polished.mrc")
oldmovieid = movieid
def create_template(self):
fromimage = 'original'
if self.__results[fromimage] is None:
raise RuntimeError('need image %s before creating template' %
(fromimage, ))
self.configure_template()
# read template file
filename = self.template_config['template filename']
tempim = self.read_hole_template(filename)
# create template of proper size
shape = self.__results[fromimage].shape
center = (0, 0)
filediameter = self.template_config['file diameter']
diameter = self.template_config['template diameter']
scale = float(diameter) / filediameter
im2 = scipy.ndimage.zoom(tempim, scale)
origshape = im2.shape
edgevalue = im2[0, 0]
template = edgevalue * numpy.ones(shape, im2.dtype)
offset = ((shape[0] - origshape[0]) / 2, (shape[1] - origshape[1]) / 2)
template[offset[0]:offset[0] + origshape[0],
offset[1]:offset[1] + origshape[1]] = im2
shift = (shape[0] / 2, shape[1] / 2)
template = scipy.ndimage.shift(template, shift, mode='wrap')
template = template.astype(numpy.float32)
self.__update_result('template', template)
if self.save_mrc:
mrc.write(template, 'template.mrc')
def averageStack(self, stackfile):
avgfile = 'avg.mrc'
a = mrc.read(stackfile)
a = np.sum(a, axis=0)
a = (a - a.min()) / (a.max() - a.min())
mrc.write(a, avgfile)
return avgfile
def processAndSaveFFT(self, imgdata, fftpath):
if os.path.isfile(fftpath):
print "FFT file found"
if fftpath in self.freqdict.keys():
print "Freq found"
return False
print "Freq not found"
print "creating FFT file: ", fftpath
### downsize and filter leginon image
if self.params['uncorrected']:
imgarray = imagefilter.correctImage(imgdata, params)
else:
imgarray = imgdata['image']
### calculate power spectra
apix = apDatabase.getPixelSize(imgdata)
fftarray, freq = ctfpower.power(imgarray, apix, mask_radius=0.5, fieldsize=self.params['fieldsize'])
#fftarray = imagefun.power(fftarray, mask_radius=1)
fftarray = ndimage.median_filter(fftarray, 2)
## preform a rotational average and remove peaks
rotfftarray = ctftools.rotationalAverage2D(fftarray)
stdev = rotfftarray.std()
rotplus = rotfftarray + stdev*4
fftarray = numpy.where(fftarray > rotplus, rotfftarray, fftarray)
### save to jpeg
self.freqdict[fftpath] = freq
mrc.write(fftarray, fftpath)
self.saveFreqFile()
return True
def averageStack(self, stackfile):
avgfile = "avg.mrc"
a = mrc.read(stackfile)
a = np.sum(a, axis=0)
a = (a - a.min()) / (a.max() - a.min())
mrc.write(a, avgfile)
return avgfile
def correlate_template(self):
fromimage = 'edges'
if None in (self.__results[fromimage], self.__results['template']):
raise RuntimeError('need image %s and template before correlation' % (fromimage,))
edges = self.__results[fromimage]
template = self.__results['template']
cortype = self.correlation_config['cortype']
corfilt = self.correlation_config['corfilt']
if cortype == 'cross':
cc = correlator.cross_correlate(edges, template)
elif cortype == 'phase':
cc = correlator.phase_correlate(edges, template, zero=False)
else:
raise RuntimeError('bad correlation type: %s' % (cortype,))
cc = numpy.absolute(cc)
if corfilt is not None:
kernel = convolver.gaussian_kernel(*corfilt)
self.edgefinder.setKernel(kernel)
cc = self.edgefinder.convolve(image=cc)
#cc = imagefun.zscore(smooth)
#cc = imagefun.zscore(cc)
self.__update_result('correlation', cc)
if self.save_mrc:
mrc.write(cc, 'correlation.mrc')
def writeTemp(imdata, newarray):
path = imdata['session']['image path']
filename = imdata['filename'] + '.mrc'
tmppath = os.path.split(path)[:-1] + ('tmp', )
tmppath = os.path.join(*tmppath)
fullname = os.path.join(tmppath, filename)
mrc.write(newarray, fullname)
def create_template(self, ring_list=None,tilt_axis=None,tilt_angle=None):
'''
This creates the template image that will be correlated
with the edge image. This will fail if there is no
existing edge image, which is necessary to determine the
size of the template.
'''
fromimage = 'edges'
if self.__results[fromimage] is None:
raise RuntimeError('need image %s before creating template' % (fromimage,))
self.configure_template(ring_list,tilt_axis,tilt_angle)
shape = self.__results[fromimage].shape
center = (0,0)
ring_list = self.template_config['ring_list']
template = numpy.zeros(shape, numpy.int8)
tilt_axis = self.template_config['tilt_axis']
tltangle = self.template_config['tilt_angle']
tltaxis=(tilt_axis*numpy.pi/180)
for ring in ring_list:
temp = self.oval.get(shape, center, ring[0], ring[1],tltangle,tltaxis)
template = template | temp
template = template.astype(numpy.float32)
#template = imagefun.zscore(template)
self.__update_result('template', template)
if self.save_mrc:
mrc.write(template, 'template.mrc')
def shiftMRCStartToZero(filename):
h = mrc.readHeaderFromFile(filename)
if h['nxstart'] != 0 or h['nystart'] != 0:
apDisplay.printMsg("Shifting image header start to zero on %s" %
(os.path.basename(filename)))
a = mrc.read(filename)
mrc.write(a, filename)
def writeTemp(imdata, newarray):
path = imdata['session']['image path']
filename = imdata['filename'] + '.mrc'
tmppath = os.path.split(path)[:-1] + ('tmp',)
tmppath = os.path.join(*tmppath)
fullname = os.path.join(tmppath, filename)
mrc.write(newarray, fullname)
def shiftToCenter(infile,shiftfile,isEMAN=False):
'''
EMAN defines the rotation origin differently from other packages.
Therefore, it needs to be recenterred according to the package
after using EMAN proc3d rotation functions.
'''
# center of rotation for eman is not at length/2.
if isEMAN:
formatoffset = getEmanCenter()
prefix = ''
else:
formatoffset = (0,0,0)
prefix = 'non-'
apDisplay.printMsg('Shifting map center for %sEMAN usage' % (prefix,))
# Find center of mass of the density map
a = mrc.read(infile)
t = a.mean()+2*a.std()
numpy.putmask(a,a>=t,t)
numpy.putmask(a,a<t,0)
center = ndimage.center_of_mass(a)
offset = (center[0]+formatoffset[0]-a.shape[0]/2,center[1]+formatoffset[1]-a.shape[1]/2,center[2]+formatoffset[2]-a.shape[2]/2)
offset = (-offset[0],-offset[1],-offset[2])
apDisplay.printMsg('Shifting map center by (x,y,z)=(%.2f,%.2f,%.2f)' % (offset[2],offset[1],offset[0]))
# shift the map
a = mrc.read(infile)
a = ndimage.interpolation.shift(a,offset)
mrc.write(a,shiftfile)
h = mrc.readHeaderFromFile(infile)
mrc.update_file_header(shiftfile,h)
def create_template(self):
fromimage = 'original'
if self.__results[fromimage] is None:
raise RuntimeError('need image %s before creating template' % (fromimage,))
self.configure_template()
# read template file
filename = self.template_config['template filename']
tempim = self.read_hole_template(filename)
# create template of proper size
shape = self.__results[fromimage].shape
center = (0,0)
filediameter = self.template_config['file diameter']
diameter = self.template_config['template diameter']
scale = float(diameter) / filediameter
im2 = scipy.ndimage.zoom(tempim, scale)
origshape = im2.shape
edgevalue = im2[0,0]
template = edgevalue * numpy.ones(shape, im2.dtype)
offset = ( (shape[0]-origshape[0])/2, (shape[1]-origshape[1])/2 )
template[offset[0]:offset[0]+origshape[0], offset[1]:offset[1]+origshape[1]] = im2
shift = (shape[0]/2, shape[1]/2)
template = scipy.ndimage.shift(template, shift, mode='wrap')
template = template.astype(numpy.float32)
self.__update_result('template', template)
if self.save_mrc:
mrc.write(template, 'template.mrc')
def createSingleImage(inputimages, globaloutput, outfilename, outformat, outtext, outxml=False): n = len(inputimages) tiles = [] for i,input in enumerate(inputimages): print 'inserting %d of %d' % (i+1,n) tile = globaloutput.insertImage(input, outtext=outtext) tiles.append(tile) ''' if targetinputs: for input,target in targetinputs: globaloutput.insertImage(input, target) print 'inserted %s targets' % (len(globaloutput.targets),) for target in globaloutput.targets: print 'T', target markTarget(globaloutput.image, target) ''' if outtext is not None: f = open(outtext, 'w') if outxml: xmldoc = getMosaicXMLData(tiles) xmldoc.writexml(f, " ", "", "\n", "UTF-8") else: lines = [ str(tile)+"\n" for tile in tiles ] f.writelines(lines) f.close() if outfilename is not None: mrc.write(globaloutput.image, outfilename)
def correlate_template(self):
'''
Correlate template that is already created and configured.
'''
fromimage = 'original'
if self.__results[fromimage] is None or self.__results['template'] is None:
raise RuntimeError('need image %s and template before correlation' % (fromimage,))
edges = self.__results[fromimage]
edges = self.maskBlack(edges)
template = self.__results['template']
cortype = self.correlation_config['cortype']
corfilt = self.correlation_config['corfilt']
if cortype == 'cross':
cc = correlator.cross_correlate(edges, template)
elif cortype == 'phase':
cc = correlator.phase_correlate(edges, template, zero=False)
else:
raise RuntimeError('bad correlation type: %s' % (cortype,))
if corfilt is not None:
kernel = convolver.gaussian_kernel(*corfilt)
self.convolver.setKernel(kernel)
cc = self.convolver.convolve(image=cc)
self.__update_result('correlation', cc)
if self.save_mrc:
mrc.write(cc, 'correlation.mrc')
def filterAndChimera(density, res=30, apix=None, box=None, chimtype='snapshot',
contour=None, zoom=1.0, sym='c1', color=None, silhouette=True, mass=None):
"""
filter volume and then create a few snapshots for viewing on the web
"""
if isValidVolume(density) is False:
apDisplay.printError("Volume file %s is not valid"%(density))
if box is None:
boxdims = apFile.getBoxSize(density)
box = boxdims[0]
### if eotest failed, filter to 30
if not res or str(res) == 'nan':
res = 30
### low pass filter the volume to 60% of reported res
tmpf = os.path.abspath(density+'.tmp.mrc')
density = os.path.abspath(density)
filtres = 0.6*res
shrinkby = 1
if box is not None and box > 250:
shrinkby = int(math.ceil(box/160.0))
if box % (2*shrinkby) == 0:
### box is divisible by shrink by
lpcmd = ('proc3d %s %s apix=%.3f tlp=%.2f shrink=%d origin=0,0,0 norm=0,1'
% (density, tmpf, apix, filtres, shrinkby))
else:
### box not divisible by shrink by, need a clip
clip = math.floor(box/shrinkby/2.0)*2*shrinkby
lpcmd = ('proc3d %s %s apix=%.3f tlp=%.2f shrink=%d origin=0,0,0 norm=0,1 clip=%d,%d,%d'
% (density, tmpf, apix, filtres, shrinkby, clip, clip, clip))
else:
lpcmd = ('proc3d %s %s apix=%.3f tlp=%.2f origin=0,0,0 norm=0,1'
% (density, tmpf, apix, filtres))
apDisplay.printMsg("Low pass filtering model for images")
proc = subprocess.Popen(lpcmd, shell=True)
proc.wait()
### flatten solvent
vol = mrc.read(tmpf)
numpy.where(vol < 0, 0.0, vol)
mrc.write(vol, tmpf)
del vol
### contour volume to mass
if mass is not None:
setVolumeMass(tmpf, apix*shrinkby, mass)
contour = 1.0
### set pixelsize and origin
recmd = "proc3d %s %s apix=%.3f origin=0,0,0"%(tmpf, tmpf, apix)
proc = subprocess.Popen(recmd, shell=True)
proc.wait()
### render images
renderSlice(density, box=box, tmpfile=tmpf, sym=sym)
if chimtype != 'snapshot':
renderAnimation(tmpf, contour, zoom, sym, color, silhouette, name=density)
elif chimtype != 'animate':
renderSnapshots(tmpf, contour, zoom, sym, color, silhouette, name=density)
apFile.removeFile(tmpf)
def emanMrcToStack(partlist):
apFile.removeStack("emanmrc.hed", warn=False)
for part in partlist:
mrc.write(part, "temp.mrc")
emancmd = "proc2d temp.mrc emanmrc.hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
apFile.removeFile("temp.mrc")
return
def emanMrcToStack(partlist):
apFile.removeStack("emanmrc.hed", warn=False)
for part in partlist:
mrc.write(part, "temp.mrc")
emancmd = "proc2d temp.mrc emanmrc.hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
apFile.removeFile("temp.mrc")
return
def find_edges(self):
'''
find edges on the original image
'''
if self.__results['original'] is None:
raise RuntimeError('no original image to find edges on')
sourceim = self.__results['original']
filt = self.edges_config['filter']
sigma = self.edges_config['sigma']
ab = self.edges_config['abs']
lpsig = self.edges_config['lpsig']
edgethresh = self.edges_config['thresh']
edgesflag = self.edges_config['edges']
kernel = convolver.gaussian_kernel(lpsig)
n = len(kernel)
self.edgefinder.setKernel(kernel)
smooth = self.edgefinder.convolve(image=sourceim)
if not edgesflag:
edges = smooth
elif filt == 'laplacian3':
kernel = convolver.laplacian_kernel3
self.edgefinder.setKernel(kernel)
edges = self.edgefinder.convolve(image=smooth)
elif filt == 'laplacian5':
kernel = convolver.laplacian_kernel5
self.edgefinder.setKernel(kernel)
edges = self.edgefinder.convolve(image=smooth)
elif filt == 'laplacian-gaussian':
kernel = convolver.laplacian_of_gaussian_kernel(n,sigma)
self.edgefinder.setKernel(kernel)
edges = self.edgefinder.convolve(image=smooth)
elif filt == 'sobel':
self.edgefinder.setImage(smooth)
kernel1 = convolver.sobel_row_kernel
kernel2 = convolver.sobel_col_kernel
edger = self.edgefinder.convolve(kernel=kernel1)
edgec = self.edgefinder.convolve(kernel=kernel2)
edges = numpy.hypot(edger,edgec)
## zero the image edge effects
edges[:n] = 0
edges[:,:n] = 0
edges[:,-n:] = 0
edges[-n:] = 0
else:
raise RuntimeError('no such filter type: %s' % (filt,))
if ab and edgesflag:
edges = numpy.absolute(edges)
if edgethresh and edgesflag:
edges = imagefun.threshold(edges, edgethresh)
self.__update_result('edges', edges)
if self.save_mrc:
mrc.write(edges, 'edges.mrc')
def projectFullZ(processdir,
runname,
seriesname,
bin=1,
rotx=True,
flipyz=False):
"""
# Command for projecting full tomogram to z-axis
#
# Tomography reconstruction y and z are usually flipped
# full tomogram mrc file axes [x,z,y]
# clip average command need [x,y,z]
clip flipyz 09feb18c_002_full.rec temp.mrc
clip avg -2d -iz 0-199 temp.mrc projection.mrc
"""
inputparams = {
'recon': os.path.join(processdir, seriesname + "_full.rec"),
'temp': os.path.join(processdir, "temp.rec"),
'project': os.path.join(processdir, seriesname + "_zproject.mrc"),
}
fullshape = apFile.getMrcFileShape(inputparams['recon'])
commands = []
if rotx or flipyz:
op = ''
if rotx:
op += ' rotx'
if flipyz:
op += ' flipyz'
lookup = {'y': 0, 'z': 1}
inputparams['3d'] = inputparams['temp']
commands.append("$clip" + op + " %s %s" % (
inputparams['recon'],
inputparams['temp'],
))
else:
lookup = {'y': 1, 'z': 0}
inputparams['3d'] = inputparams['recon']
# limit slices for projection to 200 to save time.
zcenter = int(fullshape[lookup['z']] / 2)
zstart = max(0, zcenter - 100)
zend = min(fullshape[lookup['z']] - 1, zcenter + 99)
commands.append(
"$clip avg -2d -iz %d-%d %s %s" % (
zstart,
zend,
inputparams['3d'],
inputparams['project'],
), )
writeCommandAndRun(
processdir, 'projectZ', commands,
[inputparams['temp'], inputparams['project'], 'projectZ.log'])
if bin > 1:
# unbin the projection
a = mrc.read(inputparams['project'])
b = apImage.scaleImage(a, bin)
mrc.write(b, inputparams['project'])
return inputparams['project']
def arrayToMrc(numer, filename, msg=True):
"""
takes a numpy and writes a Mrc
"""
#numer = numpy.asarray(numer, dtype=numpy.float32)
if msg is True:
apDisplay.printMsg("writing MRC: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
mrc.write(numer, filename)
return
def mergeResults(self):
allout = "all_out.mrc"
for proc in range(self.params["nproc"]):
infilepath = "proc%03d/out.mrc" % (proc)
a = mrc.read(infilepath)
if proc == 0:
mrc.write(a, allout)
else:
mrc.append(a, allout)
return allout
def arrayToMrc(numer, filename, msg=True):
"""
takes a numpy and writes a Mrc
"""
#numer = numpy.asarray(numer, dtype=numpy.float32)
if msg is True:
apDisplay.printMsg("writing MRC: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
mrc.write(numer, filename)
return
def mergeResults(self):
allout = 'all_out.mrc'
for proc in range(self.params['nproc']):
infilepath = 'proc%03d/out.mrc' % (proc)
a = mrc.read(infilepath)
if proc == 0:
mrc.write(a, allout)
else:
mrc.append(a, allout)
return allout
def mergeResults(self):
allout = 'all_out.mrc'
for proc in range(self.params['nproc']):
infilepath = 'proc%03d/out.mrc' % (proc)
a = mrc.read(infilepath)
if proc == 0:
mrc.write(a,allout)
else:
mrc.append(a,allout)
return allout
def processVolume(self, spivolfile, iternum=0):
### set values
apix = apStack.getStackPixelSizeFromStackId(
self.params['tiltstackid']) * self.params['tiltbin']
boxsize = self.getBoxSize()
rawspifile = os.path.join(
self.params['rundir'],
"rawvolume%s-%03d.spi" % (self.timestamp, iternum))
mrcvolfile = os.path.join(
self.params['rundir'],
"volume%s-%03d.mrc" % (self.timestamp, iternum))
lowpass = self.params['lowpassvol']
### copy original to raw file
shutil.copy(spivolfile, rawspifile)
### convert to mrc
emancmd = ("proc3d " + spivolfile + " " + mrcvolfile +
" norm=0,1 apix=" + str(apix))
apEMAN.executeEmanCmd(emancmd, verbose=False)
### median filter
rawvol = mrc.read(mrcvolfile)
medvol = ndimage.median_filter(rawvol, size=self.params['median'])
mrc.write(medvol, mrcvolfile)
### low pass filter
emancmd = ("proc3d " + mrcvolfile + " " + mrcvolfile +
" center norm=0,1 apix=" + str(apix) + " lp=" +
str(lowpass))
apEMAN.executeEmanCmd(emancmd, verbose=False)
### set origin
emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " origin=0,0,0 "
apEMAN.executeEmanCmd(emancmd, verbose=False)
### mask volume
emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " mask=" + str(
self.params['radius'])
apEMAN.executeEmanCmd(emancmd, verbose=False)
### convert to spider
apFile.removeFile(spivolfile)
emancmd = "proc3d " + mrcvolfile + " " + spivolfile + " spidersingle"
apEMAN.executeEmanCmd(emancmd, verbose=False)
### image with chimera
if self.params['skipchimera'] is False:
if self.params['mass'] is not None:
apDisplay.printMsg("Using scale by mass method")
apChimera.setVolumeMass(mrcvolfile,
apix=apix,
mass=self.params['mass'])
apChimera.renderSnapshots(mrcvolfile, self.params['contour'],
self.params['zoom'], 'c1')
return mrcvolfile
def createTiles(inputs, tiledict, tilesize, row1=None, row2=None, col1=None, col2=None):
blank = numpy.zeros((tilesize,tilesize), numpy.float32)
if None in (row1,row2,col1,col2):
tileindices = tiledict.keys()
else:
tileindices = []
for rowi in range(row1, row2+1):
for coli in range(col1, col2+1):
tileindices.append((rowi,coli))
f = open('outputinfo', 'w')
f.close()
tilestotal = len(tileindices)
t0 = time.time()
for i,tileindex in enumerate(tileindices):
tileargs = tiledict[tileindex]
print 'Creating tile:', tileindex
output = MontageImage(*tileargs['args'], **tileargs['kwargs'])
for input in inputs:
output.insertImage(input)
print ' Inserted %d images' % (output.inserted,)
if output.inserted:
outim = output.image
else:
outim = blank
f = open('outputinfo', 'a')
r,c = tileindex
x,y = output.scope['stage position']['x'], output.scope['stage position']['y']
f.write('%d\t%d\t%e\t%e\n' % (r,c,x,y))
f.close()
mrc.write(outim, '%d_%d.mrc' % tileindex)
'''
jpeg.save(output.image, clip[0], clip[1], '%d_%d.jpg' % tileindex, 90)
'''
tilesdone = i+1
tilesleft = tilestotal - tilesdone
elapsed = time.time() - t0
tilespersec = tilesdone / elapsed
secleft = tilesleft * tilespersec
hrleft = secleft / 3600.0
hrleft = int(hrleft)
secleft = secleft - hrleft * 3600.0
minleft = secleft / 60.0
minleft = int(minleft)
secleft = secleft - minleft * 60.0
secleft = int(secleft)
print ' Done %d of %d, Avg: %.2f tiles/sec, Estimated time left: %02d:%02d:%02d' % (tilesdone,tilestotal,tilespersec,hrleft,minleft,secleft)
def onEdgeFinding(self,evt):
if not self.panel.selectiontool.isTargeting('Auto Create Contours'):
return
point = self.panel.view2image((evt.m_x,evt.m_y))
ndarray = mrc.read(os.path.join(self.appionloop.params['rundir'], self.appionloop.imgtree[self.index]['filename']+'.dwn.mrc'))
mrc.write(ndarray, os.path.join(self.appionloop.params['rundir'], 'beforefilter'+'.dwn.mrc'))
negative = False
if self.filters:
ndarray = ndimage.gaussian_filter(ndarray,1)
ndarray = ndimage.gaussian_gradient_magnitude(ndarray,2)
markers = []
for i in range(3):
for j in range(3):
if i!=0 or j!=0:
markers.append((point[0]-1+i,point[1]-1+j))
markers = (1,2,3,4,5,6,7,8)
#ndarray = ndimage.watershed_ift(ndarray,markers)
ndarray = ndimage.laplace(ndarray)
ndarray = ndimage.gaussian_filter(ndarray,1)
#ndarray = apImage.preProcessImage(ndarray,params=self.appionloop.params)
negative = True
mrc.write(ndarray, os.path.join(self.appionloop.params['rundir'], 'afterfilter'+'.dwn.mrc'))
delta = .1
targets = []
radius = 20
size = 50
rangeSize = 50
maker = PixelCurveMaker()
maker._init_(size,rangeSize);
for theta in range(size):
theta +=0
theta*=math.pi*2/rangeSize
for rad in range(size):
try:
if negative:
maker.addData(theta,rad,127-ndarray[int(point[1]+rad*math.sin(theta))][int(point[0]+rad*math.cos(theta))])
else:
maker.addData(theta,rad,ndarray[int(point[1]+rad*math.sin(theta))][int(point[0]+rad*math.cos(theta))])
except IndexError:
maker.addData(theta,rad,0)
maker.makeCalculations()
s = self.filterSelectorChoices[self.filterSelector.GetSelection()]
dilate = 2
if s == 'Latex Bead':
dilate = 0
for theta in range(size):
theta += 0
theta*=math.pi*2/rangeSize
targets.append((point[0]+(dilate+maker.getData(theta))*math.cos(theta),point[1]+(dilate+maker.getData(theta))*math.sin(theta)))
self.addPolyParticle(targets)
#this section draws all of the contours that the algorithm considers - useful for debugging
'''
def writeMrcStack(path, stackname, mrc_files, binning=1):
apDisplay.printMsg("Writing MRC stack file... ")
stackname = os.path.join(path, stackname)
im = mrc.read(mrc_files[0])
image = imagefun.bin(im, binning)
mrc.write(image,stackname)
del mrc_files[0]
for mrcfile in mrc_files:
im = mrc.read(mrcfile)
image = imagefun.bin(im, binning)
mrc.append(image, stackname)
def readUploadInfo(self, info=None):
if info is None:
# example
info = ['test.mrc', '2e-10', '1', '1', '50000', '-2e-6', '120000']
apDisplay.printMsg('reading image info')
try:
uploadedInfo = {}
uploadedInfo['original filepath'] = os.path.abspath(info[0])
uploadedInfo['unbinned pixelsize'] = float(info[1])
if uploadedInfo['unbinned pixelsize'] > 1e-6:
apDisplay.printError(
"pixel size is bigger than a micron, that is ridiculous")
uploadedInfo['binning'] = {'x': int(info[2]), 'y': int(info[3])}
uploadedInfo['magnification'] = int(info[4])
uploadedInfo['defocus'] = float(info[5])
uploadedInfo['high tension'] = int(info[6])
if len(info) > 7:
uploadedInfo['stage a'] = float(info[7]) * math.pi / 180.0
# add other items in the dictionary and set to instrument in the function
# setInfoToInstrument if needed
except:
apDisplay.printError("Bad batch file parameters")
if not os.path.isfile(uploadedInfo['original filepath']):
apDisplay.printWarning("Original File %s does not exist" %
uploadedInfo['original filepath'])
apDisplay.printWarning("Skip Uploading")
return None
uploadedInfo['filename'] = self.setNewFilename(
uploadedInfo['original filepath'])
newimgfilepath = os.path.join(self.params['rundir'],
uploadedInfo['filename'] + ".tmp.mrc")
### convert to mrc in new session directory if not mrc:
if self.params['filetype'] != "mrc":
if not os.path.isfile(newimgfilepath):
emancmd = "proc2d %s %s edgenorm flip mrc" % (
uploadedInfo['original filepath'], newimgfilepath)
apEMAN.executeEmanCmd(emancmd)
if not os.path.exists(newimgfilepath):
apDisplay.printError(
"image conversion to mrc did not execute properly")
uploadedInfo['original filepath'] = newimgfilepath
tmpimage = mrc.read(uploadedInfo['original filepath'])
# invert image density
if self.params['invert'] is True:
tmpimage *= -1.0
mrc.write(tmpimage, uploadedInfo['original filepath'])
shape = tmpimage.shape
uploadedInfo['dimension'] = {'x': shape[1], 'y': shape[0]}
uploadedInfo['session'] = self.session
uploadedInfo['pixel size'] = uploadedInfo[
'unbinned pixelsize'] * uploadedInfo['binning']['x']
return uploadedInfo
def processVolume(self, spivolfile, cnum, iternum=0):
### set values
apix = apStack.getStackPixelSizeFromStackId(
self.params['tiltstackid']) * self.params['bin']
boxsize = self.getBoxSize()
volfilename = os.path.splitext(spivolfile)[0]
rawspifile = volfilename + "-raw.spi"
mrcvolfile = volfilename + ".mrc"
lowpass = self.params['lowpassvol']
### copy original to raw file
shutil.copy(spivolfile, rawspifile)
### convert to mrc
emancmd = ("proc3d " + spivolfile + " " + mrcvolfile +
" norm=0,1 apix=" + str(apix))
apEMAN.executeEmanCmd(emancmd, verbose=False)
### median filter
rawvol = mrc.read(mrcvolfile)
medvol = ndimage.median_filter(rawvol, size=self.params['median'])
mrc.write(medvol, mrcvolfile)
### low pass filter
emancmd = ("proc3d " + mrcvolfile + " " + mrcvolfile +
" center norm=0,1 apix=" + str(apix) + " lp=" +
str(lowpass))
apEMAN.executeEmanCmd(emancmd, verbose=False)
### set origin
emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " origin=0,0,0 "
apEMAN.executeEmanCmd(emancmd, verbose=False)
### mask volume
emancmd = "proc3d " + mrcvolfile + " " + mrcvolfile + " mask=" + str(
self.params['radius'])
apEMAN.executeEmanCmd(emancmd, verbose=False)
### convert to spider
apFile.removeFile(spivolfile)
emancmd = "proc3d " + mrcvolfile + " " + spivolfile + " spidersingle"
apEMAN.executeEmanCmd(emancmd, verbose=False)
### image with chimera
if self.params['skipchimera'] is False:
apChimera.renderSnapshots(mrcvolfile, self.params['contour'],
self.params['zoom'], 'c1')
animationthread = threading.Thread(
target=apChimera.renderAnimation,
args=(mrcvolfile, self.params['contour'], self.params['zoom'],
'c1'))
animationthread.setDaemon(1)
animationthread.start()
return mrcvolfile
def writeMrcStack(path, stackname, mrc_files, binning=1):
apDisplay.printMsg("Writing MRC stack file... ")
stackname = os.path.join(path, stackname)
im = mrc.read(mrc_files[0])
image = imagefun.bin(im, binning)
mrc.write(image, stackname)
del mrc_files[0]
for mrcfile in mrc_files:
im = mrc.read(mrcfile)
image = imagefun.bin(im, binning)
mrc.append(image, stackname)
def makeStack(self, tiltseries, mrc_files):
stackname = self.getFilename(tiltseries) + '.st'
stackname = os.path.join(self.settings['path'], stackname)
im = mrc.read(mrc_files[0])
image = imagefun.bin(im, int(self.settings['binning']))
mrc.write(image, stackname)
#shutil.copy(mrc_files[0], stackname)
del mrc_files[0]
for mrcfile in mrc_files:
im = mrc.read(mrcfile)
image = imagefun.bin(im, int(self.settings['binning']))
mrc.append(image, stackname)
def start(self):
mrcfile = self.params['file']
### bin the volume
if self.params['bin'] is not None and self.params['bin'] > 1:
apDisplay.printMsg("Binning volume")
newmrcfile = os.path.join(os.getcwd(), "binned.mrc")
voldata = mrc.read(mrcfile)
voldata = imagefun.bin3(voldata, self.params['bin'])
mrc.write(voldata, newmrcfile)
del voldata
self.params['apix'] *= self.params['bin']
if os.path.isfile(newmrcfile):
mrcfile = newmrcfile
### scale by mass
if self.params['mass'] is not None:
apDisplay.printMsg("Using scale by mass method")
newmrcfile = os.path.join(os.getcwd(), "setmass.mrc")
shutil.copy(self.params['file'], newmrcfile)
apChimera.setVolumeMass(newmrcfile, apix=self.params['apix'], mass=self.params['mass'])
self.params['contour'] = 1.0
if os.path.isfile(newmrcfile):
mrcfile = newmrcfile
### print stats
box = apVolume.getModelDimensions(mrcfile)
apDisplay.printColor("Box: %d Apix: %.2f File: %s"%
(box, self.params['apix'], os.path.basename(mrcfile)), "green")
### animation
if self.params['type'] != "snapshot":
apDisplay.printMsg("Creating animation")
apChimera.renderAnimation(mrcfile, contour=self.params['contour'],
zoom=self.params['zoom'], sym=self.params['sym'],
color=self.params['color'], xvfb=self.params['xvfb'],
name=self.params['file'], silhouette=self.params['silhouette'])
### snapshot
if self.params['type'] != "animate":
apDisplay.printMsg("Creating snapshots")
apChimera.renderSnapshots(mrcfile, contour=self.params['contour'],
zoom=self.params['zoom'], sym=self.params['sym'],
color=self.params['color'], xvfb=self.params['xvfb'],
pdb=self.params['pdb'], name=self.params['file'],
silhouette=self.params['silhouette'])
### clean up
if self.params['mass'] is not None or self.params['bin'] is not None:
images = glob.glob(mrcfile+"*")
for img in images:
newimg = re.sub(mrcfile, self.params['file'], img)
shutil.move(img, newimg)
apFile.removeFile(mrcfile)
def threshold_correlation(self, threshold=None):
'''
Threshold the correlation image.
'''
if self.__results['correlation'] is None:
raise RuntimeError('need correlation image to threshold')
self.configure_threshold(threshold)
cc = self.__results['correlation']
t = imagefun.threshold(cc, self.threshold)
self.__update_result('threshold', t)
if self.save_mrc:
mrc.write(t, 'threshold.mrc')
def makeMovie(self,framepaths_wild,moviepath):
files = os.listdir(self.params['rundir'])
moviepath += '.mrc'
for filename in files:
bits = framepaths_wild.split('*')
if bits[0] in filename:
print filename
array = mrc.read(os.path.join(self.params['rundir'],filename))
if os.path.isfile(moviepath):
mrc.append(array,moviepath)
else:
mrc.write(array,moviepath)
def makeStack(self, tiltseries, mrc_files):
stackname = self.getFilename(tiltseries) + '.st'
stackname = os.path.join(self.settings['path'], stackname)
im = mrc.read(mrc_files[0])
image = imagefun.bin(im, int(self.settings['binning']))
mrc.write(image,stackname)
#shutil.copy(mrc_files[0], stackname)
del mrc_files[0]
for mrcfile in mrc_files:
im = mrc.read(mrcfile)
image = imagefun.bin(im, int(self.settings['binning']))
mrc.append(image, stackname)
def makeMovie(self,framepaths_wild,moviepath):
files = os.listdir(self.params['rundir'])
moviepath += '.mrc'
for filename in files:
bits = framepaths_wild.split('*')
if bits[0] in filename:
print filename
array = mrc.read(os.path.join(self.params['rundir'],filename))
if os.path.isfile(moviepath):
mrc.append(array,moviepath)
else:
mrc.write(array,moviepath)
def mark_holes(self):
if self.__results['holes'] is None or self.__results['original'] is None:
raise RuntimeError('need original image and holes before marking holes')
image = self.__results['original']
im = image.copy()
value = arraystats.min(im)
for hole in self.__results['holes']:
coord = hole.stats['center']
imagefun.mark_image(im, coord, value)
self.__update_result('markedholes', im)
if self.save_mrc:
mrc.write(im, 'markedholes.mrc')
