def read(cls,
cleftId,
bound1Id,
bound2Id,
file,
clean=True,
byteOrder=None,
dataType=None,
arrayOrder=None,
shape=None):
"""
Reads segmented image (label filed) from a file and sets required ids.
Each of the id args (cleftId, bound1Id, bound2Id) can be a single int
or a list (array) in which cases a (boundary or cleft) is formed form
all specified ids.
If file is in em or mrc format (file extension em or mrc) only the file
argument is needed. If the file is in the raw data format (file
extension raw) all arguments are required.
Arguments:
- bound1Id, bound2Id, cleftId: ids of boundaries 1, 2 and cleft
- file: file name
- clean: if true, only the segments corresponding to ids are kept
- byteOrder: '<' (little-endian), '>' (big-endian)
- dataType: any of the numpy types, e.g.: 'int8', 'int16', 'int32',
'float32', 'float64'
- arrayOrder: 'C' (z-axis fastest), or 'F' (x-axis fastest)
- shape: (x_dim, y_dim, z_dim)
Returns:
- instance of Segment
"""
# read file
from pyto.io.image_io import ImageIO as ImageIO
fi = ImageIO()
fi.read(file=file,
byteOrder=byteOrder,
dataType=dataType,
arrayOrder=arrayOrder,
shape=shape)
# make new instance
cleft = cls(data=fi.data,
cleftId=cleftId,
bound1Id=bound1Id,
bound2Id=bound2Id,
copy=False,
clean=clean)
return cleft
def sortPaths(self, paths, mode='num', sequence=None):
"""
Sorts list of file path and returns it.
If arg mode is 'num', paths are sorted by series number, which are
expected to be between the rightmost '_' and '.' in the file names.
If arg mode is 'tilt_angle', paths are sorted by tilt angle.
If arg mode is 'sequence', paths are sorted by arg sequence that has
to have the same length as paths.
Arguments:
- paths: list of file paths (with or without directory)
- mode: sorting mode, 'num', 'tilt_angle', or 'sequence'
- sequence: sequence of values corresponding to paths
Returns sorted list of paths.
"""
if mode == 'tilt_angle':
# get tilt angles from file headers
unsorted = []
for file_ in paths:
image = ImageIO(file=file_)
image.readHeader()
unsorted.append(image.tiltAngle)
elif mode == 'num':
# get file numbers
unsorted = [int(self.splitBase(base=file_)[1]) for file_ in paths]
elif mode == 'sequence':
unsorted = sequence
else:
raise ValueError('Mode ', mode, " not understood. Allowed values ",
"are 'num', 'tilt_angle' and 'sequence'.")
# sort file numbers and paths
unsorted = numpy.array(unsorted)
sort_ind = unsorted.argsort()
sorted_paths = [paths[ind] for ind in sort_ind]
return sorted_paths
def setUp(self):
"""
Sets absolute path to this file directory and saves it as self.dir
"""
# set absolute path to current dir
working_dir = os.getcwd()
file_dir, name = os.path.split(__file__)
self.dir = os.path.join(working_dir, file_dir)
# make raw file
self.raw_shape = (4, 3, 2)
self.raw_dtype = 'int16'
self.raw_data = numpy.arange(24, dtype=self.raw_dtype).reshape(
self.raw_shape)
raw = ImageIO()
self.raw_file_name = 'data.raw'
raw.write(file=self.raw_file_name, data=self.raw_data)
def modify(cls, old, new, fun, fun_kwargs={}, memmap=True):
"""
Reads an image (arg old), modifies old.data using function
passed as arg fun and writes an image containing the modified
data as a new image (arg new).
The function passes (arg fun) has to have signature
fun(Image, **fun_kwargs)
and to return image data (ndarray).
Meant for mrc files. The new image will have exactly the same
header as the old image, except for the shape, length and
min/max/mean values, which are set according to the new image
data.
Also works if old is an mrc and new is a raw file.
Arguments:
- old: old mrc image file name
- new: new (subtomogram) mrc image file name
- fun: function that takes old.data as an argument
- memmap: if True, read memory map instead of the whole image
Returns an instance of this class that holds the new image. This
instance contains attribute image_io (pyto.io.ImageIO) that
was used to write the new file.
"""
# read header
from pyto.io.image_io import ImageIO as ImageIO
image_io = ImageIO()
image_io.readHeader(file=old)
# read image
image = cls.read(file=old, memmap=memmap)
# modify data
data = fun(image, **fun_kwargs)
# write new (modified data)
image_io.setData(data=data)
image_io.write(file=new, pixel=image_io.pixel)
#image_io.setFileFormat(file_=new)
#if image_io.fileFormat == 'mrc':
# image_io.write(file=new, pixel=image_io.pixel)
#elif image_io.fileFormat == 'raw':
# image_io.write(file=new, pixel=image_io.pixel)
#
image.image_io = image_io
return image
def fix(self, mode=None, microscope=None, out=None, pad=0, start=1):
"""
Fixes wrong data in headers.
Mode determines which values are fixed. Currently defined modes are:
- 'polara_fei-tomo': images obtained on Polara (at MPI of
Biochemistry) using FEI tomography package and saved in EM
format.
- 'krios_fei-tomo': images obtained on Krios (at MPI of
Biochemistry) using FEI tomography package and saved in EM
format.
- 'cm300': images from cm300 in EM format
- None: doesn't do anyhing
If mode is 'polara_fei-tomo', then arg microscope has to be specified.
The allowed values are specified in microscope_db.py. Currently (r564)
these are: 'polara-1_01-07', 'polara-1_01-09' and 'polara-2_01-09'.
Works for individual images only (not for stacks), because stacks
are typically recorded by SerialEM and do not need fixing.
Arguments:
- out: directory where the fixed images are written. The fixed and
the original images have the same base names.
- mode: fix mode
"""
# parse arguments
#if path is None: path = self.path
#if mode is None: mode = self.mode
# check for out, pad and start also?
# make out directory if it doesn't exists
if (out is not None) and (not os.path.isdir(out)):
os.makedirs(out)
# loop over images
images_iter = self.images(out=out, pad=pad, start=start)
for (in_path, out_path) in images_iter:
# read image file
image = ImageIO(file=in_path)
image.read()
# fix
image.fix(mode=mode, microscope=microscope)
# write fixed files
image.write(file=out_path)
def read_pixel_size(cls, image_path):
"""
Reads pixel size from an image file.
Raises ValueError if pixel size can not be read from the image
Argument:
- image_path: image path
Returns: pixel size in A
"""
image_io = ImageIO()
if image_path.endswith('.st'):
image_io.readHeader(file=image_path, fileFormat='mrc')
else:
image_io.readHeader(file=image_path)
if image_io.pixel is not None:
if isinstance(image_io.pixel, (list, tuple)):
pixel_a = 10 * image_io.pixel[0]
else:
pixel_a = 10 * image_io.pixel
else:
raise ValueError("Pixel size could not be found from image " +
image_path +
". Please specify pixel_a as an argument.")
# in case of 0 pix size
if pixel_a == 0:
raise ValueError("Pixel size could not be found from image " +
image_path +
". Please specify pixel_a as an argument.")
return pixel_a
def load_tomo(fname, mmap=False):
"""
Loads a tomogram in MRC, EM or VTI format and converts it into a numpy
format.
Args:
fname (str): full path to the tomogram, has to end with '.mrc', '.em' or
'.vti'
mmap (boolean, optional): if True (default False) a numpy.memmap object
is loaded instead of numpy.ndarray, which means that data are not
loaded completely to memory, this is useful only for very large
tomograms. Only valid with formats MRC and EM. VERY IMPORTANT: This
subclass of ndarray has some unpleasant interaction with some
operations, because it does not quite fit properly as a subclass of
numpy.ndarray
Returns:
numpy.ndarray or numpy.memmap object
"""
# Input parsing
stem, ext = os.path.splitext(fname)
if mmap and (not (ext == '.mrc' or (ext == '.em'))):
raise pexceptions.PySegInputError(
expr='load_tomo',
msg=('mmap option is only valid for MRC or EM formats, current ' +
ext))
elif ext == '.mrc':
image = ImageIO()
image.readMRC(fname, memmap=mmap)
im_data = image.data
elif ext == '.em':
image = ImageIO()
image.readEM(fname, memmap=mmap)
im_data = image.data
elif ext == '.vti':
reader = vtk.vtkXMLImageDataReader()
reader.SetFileName(fname)
reader.Update()
im_data = vti_to_numpy(reader.GetOutput())
else:
raise pexceptions.PySegInputError(
expr='load_tomo', msg='{} is non valid format.'.format(ext))
# For avoiding 2D arrays
if len(im_data.shape) == 2:
im_data = np.reshape(im_data, (im_data.shape[0], im_data.shape[1], 1))
return im_data
def testPixelSize(self):
"""
Tests pixel size in read and write
"""
# arrays
#ar_int8_2 = numpy.arange(24, dtype='int8').reshape((4,3,2))
ar_int16_2 = numpy.arange(24, dtype='int16').reshape((4, 3, 2))
#
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16_2,
pixel=2.1)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_almost_equal(file_in.pixel, 2.1)
def cut(cls, old, new, inset, memmap=True):
"""
Reads an image (arg old), cuts a subtomo defined by arg inset
and writes the new image (arg new).
Meant for mrc files. The new image will have exactly the same
header as the old image, except for the shape, length and
min/max/mean values, which are set according to the new image
data.
Arguments:
- old: old mrc image file name
- new: new (subtomogram) mrc image file name
- inset: defines the subtomogram
- memmap: if True, read memory map instead of the whole image
Returns an instance of this class that holds the new image. This
instance contains attribute image_io (pyto.io.ImageIO) that
was used to write the new file.
"""
# read header
from pyto.io.image_io import ImageIO as ImageIO
image_io = ImageIO()
image_io.readHeader(file=old)
# read image and cut inset
image = cls.read(file=old, memmap=memmap)
image.useInset(inset=inset)
# write new
image_io.setData(data=image.data)
image_io.write(file=new, pixel=image_io.pixel)
#
image.image_io = image_io
return image
def find(cls,
image_dir,
image_prefix,
ctf_dir,
params,
pixel_a=None,
flatten='auto',
tool='ctffind',
executable=None,
param_file='ctf_params.txt',
fast=False,
max_images=None,
plot_ctf=True,
plot_ps=True,
b_plot=True,
exp_f_plot=False,
show_legend=True,
plot_phases=True,
plot_defoci=True,
plot_resolution=True,
print_results=True,
print_validation=False):
"""
Determines and shows CTF fits for multiple images.
All files located in (arg) image_dir whose namess start with (arg)
image_prefix and that have extension mrc, em or st are selected
for the ctf determination.
If a selected file is 3D (image stack), and arg flatten is True or
'auto', all z-slices are summed up (saved in ctf_dir) and the ctf
is detemined on the resulting (flattened. Alternatively, if arg
flatten is False, z-slices are extracted, saved in ctf_dir and
analyzed separately.
All resulting files, as well as the extraced or flattened images
(in case of 3D files) are saved or moved to directory ctf_dir.
CTF is determined using external tools. Current options are:
- CTFFIND
- gCTF
These tools have to be installed externally.
Parameters for the ctf tools are specified as a dictionary (arg params).
Parameters used for both ctffind and gctf are:
- 'pixel_a', 'voltage', 'cs', 'amp', 'box', 'min_res', 'max_res',
'min_def', 'max_def', 'def_step', 'astig', 'phase',
'min_phase', 'max_phase', 'phase_step'
Voltage ('voltage') should always be specified. The pixel size
(pixel_a) has to be specified in case it can not be read from
the image header. All other parameters are optional, if they are
not specified the ctffind / gctg default values are used.
The default values should be fine for single particle images.
Parameter recommendations for phase plate images are given in
the ctffind / gctf documentation.
In case of ctffind, arg params can also be a list containing the
parameter values in the same order as specified above, starting
with voltage.
Important for ctffind: Because the required arguments differ between
versions 4.0 and 4.1, as well as depend on values specified, it is
not guaranteed that the dictionary form of arg params will work.
In case of problems, specify params as a list.
In addition, all other gctf arguments can also be specified
(without '--'). It is suggested to use:
'do_EPA':'', 'do_validation':''
Parameter units are the same as in the ctf deterimantion tools.
Intended for use in an environment such as Jupyter notebook.
Arguments:
- image_dir: directory where images reside
- image prefix: beginning of image file(s)
- ctf_dir: directory where the ctf determination results and
extracted images are saved
- pixel_a: pixel size in A
- params: ctf determination parameters
- flatten: indicated whether 3D images should be flatten (True or
'auto') or not (False).
- tool: name of the ctf detmination tool
- executable: ctf tool executable
- param_file: name of the temporary parameter file
- fast: flag indicating whether ctffind --fast option is used
- print_results: flag indicating if phase and defoci found
are printed for each analyzed image
- plot_ctf: flag indicating whether ctf is plotted for each
analyzed image
- show_legend: flag indicating whether a legend is shown on ctf graphs
- plot_phases, plot_defoci: flags indicating whether a graph
containing phases and defoci of all images respectivelly are plotted
- max_images: max number if image analyzed, for testing
Returns an instance of this class. The following attributes are all
lists where elements correspond to individual images:
- image_path_orig: image path of the input file
- image_path: image path of the image that is actually used
to deterime ctf. It differs from image_path_orig if the original
(input) image is a stack that is flattened or used to extract slices
- image_inds: index of a slice extracted for a stack
- ctf_path: path of the ctf fit image
- defocus_1, defocus_2, defocus: defoci along the two axes and the
mean defocus in um
- angle: defocus (astigmatism) angle
- phase: phase shift in multiples of pi
- resolution: resolution in nm
- ccc: correlation coefficient
- pixel_a: pixel size in A
- b_factor: b-factor (gctf only)
"""
# initialize
index = 0
new = cls()
print_head = True
if plot_ctf and fast:
print("Warning: CTF will not be plotted because fast execution" +
" was chosen")
# check which ctf tool to use
if tool == 'ctffind':
if executable is None:
executable = 'ctffind'
elif tool == 'gctf':
if executable is None:
executable = 'gctf'
else:
raise ValueError("CTF determination tool " + str(tool) +
" was not understood.")
new.tool = tool
# cftfind on all images
file_list = np.sort(os.listdir(image_dir))
for image_name in file_list:
# skip files that are not images
if not image_name.startswith(image_prefix): continue
if not (image_name.endswith('.mrc') or image_name.endswith('.st')
or image_name.endswith('.em')):
continue
if image_name.endswith('ctf.mrc'): continue
# set input image path
image_path = os.path.join(image_dir, image_name)
# figure out if to flatten or not (just once, assume all files
# are the same)
im_io = ImageIO(file=image_path)
if image_name.endswith('.st'):
im_io.readHeader(fileFormat='mrc')
else:
im_io.readHeader()
z_dim = im_io.shape[2]
n_digits = int(np.ceil(np.log10(z_dim)))
if isinstance(flatten, bool):
pass
elif isinstance(flatten, basestring) and (flatten == 'auto'):
if z_dim > 1:
flatten = True
else:
flatten = False
else:
raise ValueError("Argument flatten: " + str(flatten) +
" was not understood.")
# load stack and prepare image name, if need to extract images
if (z_dim > 1) and not flatten:
image_dir, image_name = os.path.split(image_path)
image_base, image_extension = image_name.rsplit('.', 1)
image_name_new_tmplt = (image_base + '_%0' + str(n_digits) +
'd.mrc')
if image_name.endswith('.st'):
stack = Image.read(image_path,
memmap=True,
fileFormat='mrc')
else:
stack = Image.read(image_path, memmap=True)
else:
image_path_to_read = image_path
# find ctf of the current image or stack
for image_in_stack_ind in range(z_dim):
# extract and save images if needed
if (z_dim > 1) and not flatten:
if not os.path.exists(ctf_dir): os.makedirs(ctf_dir)
image_path_to_read = os.path.join(
ctf_dir, (image_name_new_tmplt % image_in_stack_ind))
one_image = Image()
one_image.data = stack.data[:, :, image_in_stack_ind]
one_image.write(file=image_path_to_read,
pixel=stack.pixelsize)
# save image path retlated
new.image_path_orig.append(image_path)
new.image_inds.append(image_in_stack_ind)
new.image_path.append(image_path_to_read)
# find ctf
if tool == 'ctffind':
# ctffind
res_one = cls.ctffind(image_path=image_path_to_read,
flatten=flatten,
ctf_dir=ctf_dir,
executable=executable,
pixel_a=pixel_a,
params=params,
param_file=param_file,
fast=fast,
print_head=print_head,
print_results=print_results,
plot_ctf=plot_ctf,
show_legend=show_legend)
elif tool == 'gctf':
# gctf
res_one = cls.gctf(image_path=image_path_to_read,
params=params,
pixel_a=pixel_a,
flatten=flatten,
ctf_dir=ctf_dir,
executable=executable,
plot_ctf=plot_ctf,
plot_ps=plot_ps,
b_plot=b_plot,
exp_f_plot=exp_f_plot,
show_legend=show_legend,
print_results=print_results,
print_head=print_head,
print_validation=print_validation)
# save gctf specific data
try:
new.b_factor.append(res_one['b_factor'])
except AttributeError:
new.b_factor = [res_one['b_factor']]
for name, value in list(res_one.items()):
if name.startswith(cls.validation_prefix):
try:
previous_val = getattr(new, name)
previous_val.append(value)
setattr(new, name, previous_val)
except AttributeError:
setattr(new, name, [value])
else:
raise ValueError("Sorry tool: " + tool + " was not found.")
# save data common for ctffind and gctf
new.phases.append(res_one["phase"])
new.defoci.append(res_one["defocus"])
new.defoci_1.append(res_one['defocus_1'])
new.defoci_2.append(res_one['defocus_2'])
new.resolution.append(res_one['resolution'])
new.pixel_a.append(res_one['pixel_a'])
new.angle.append(res_one['angle'])
new.ctf_path.append(res_one['ctf_path'])
# keep track of n images processed so far
print_head = False
index = index + 1
if (max_images is not None) and (index > max_images): break
if flatten: break
# plot phases
if plot_phases:
plt.figure()
plt.bar(list(range(index)), new.phases)
plt.plot([0, index], [0.5, 0.5], 'r--')
plt.ylabel('Phase shift [$\pi$]')
plt.xlabel('Images')
plt.title("Phase shift summary")
# plot defocus
if plot_defoci:
plt.figure()
plt.bar(list(range(index)), new.defoci)
plt.ylabel('Defocus [$\mu m$]')
plt.xlabel('Images')
plt.title("Defocus summary")
# plot resolution
if plot_resolution:
plt.figure()
plt.bar(list(range(index)), new.resolution)
plt.ylabel('Resolution [nm]')
plt.xlabel('Images')
plt.title("Resolution summary")
return new
def gen_surface(tomo, lbl=1, mask=True, purge_ratio=1, field=False,
mode_2d=False, verbose=False):
"""
Generates a VTK PolyData surface from a segmented tomogram.
Args:
tomo (numpy.ndarray or str): the input segmentation as numpy ndarray or
the file name in MRC, EM or VTI format
lbl (int, optional): label for the foreground, default 1
mask (boolean, optional): if True (default), the input segmentation is
used as mask for the surface
purge_ratio (int, optional): if greater than 1 (default), then 1 every
purge_ratio points of the segmentation are randomly deleted
field (boolean, optional): if True (default False), additionally returns
the polarity distance scalar field
mode_2d (boolean, optional): needed for polarity distance calculation
(if field is True), if True (default False), ...
verbose (boolean, optional): if True (default False), prints out
messages for checking the progress
Returns:
- output surface (vtk.vtkPolyData)
- polarity distance scalar field (np.ndarray), if field is True
"""
# Check input format
if isinstance(tomo, str):
fname, fext = os.path.splitext(tomo)
# if fext == '.fits':
# tomo = pyfits.getdata(tomo)
if fext == '.mrc':
hold = ImageIO()
hold.readMRC(file=tomo)
tomo = hold.data
elif fext == '.em':
hold = ImageIO()
hold.readEM(file=tomo)
tomo = hold.data
elif fext == '.vti':
reader = vtk.vtkXMLImageDataReader()
reader.SetFileName(tomo)
reader.Update()
tomo = vti_to_numpy(reader.GetOutput())
else:
error_msg = 'Format %s not readable.' % fext
raise pexceptions.PySegInputError(expr='gen_surface', msg=error_msg)
elif not isinstance(tomo, np.ndarray):
error_msg = 'Input must be either a file name or a ndarray.'
raise pexceptions.PySegInputError(expr='gen_surface', msg=error_msg)
# Load file with the cloud of points
nx, ny, nz = tomo.shape
cloud = vtk.vtkPolyData()
points = vtk.vtkPoints()
cloud.SetPoints(points)
if purge_ratio <= 1:
for x in range(nx):
for y in range(ny):
for z in range(nz):
if tomo[x, y, z] == lbl:
points.InsertNextPoint(x, y, z)
else:
count = 0
mx_value = purge_ratio - 1
purge = np.random.randint(0, purge_ratio+1, nx*ny*nz)
for x in range(nx):
for y in range(ny):
for z in range(nz):
if purge[count] == mx_value:
if tomo[x, y, z] == lbl:
points.InsertNextPoint(x, y, z)
count += 1
if verbose:
print 'Cloud of points loaded...'
# Creating the isosurface
surf = vtk.vtkSurfaceReconstructionFilter()
# surf.SetSampleSpacing(2)
surf.SetSampleSpacing(purge_ratio)
# surf.SetNeighborhoodSize(10)
surf.SetInputData(cloud)
contf = vtk.vtkContourFilter()
contf.SetInputConnection(surf.GetOutputPort())
# if thick is None:
contf.SetValue(0, 0)
# else:
# contf.SetValue(0, thick)
# Sometimes the contouring algorithm can create a volume whose gradient
# vector and ordering of polygon (using the right hand rule) are
# inconsistent. vtkReverseSense cures this problem.
reverse = vtk.vtkReverseSense()
reverse.SetInputConnection(contf.GetOutputPort())
reverse.ReverseCellsOn()
reverse.ReverseNormalsOn()
reverse.Update()
rsurf = reverse.GetOutput()
if verbose:
print 'Isosurfaces generated...'
# Translate and scale to the proper positions
cloud.ComputeBounds()
rsurf.ComputeBounds()
xmin, xmax, ymin, ymax, zmin, zmax = cloud.GetBounds()
rxmin, rxmax, rymin, rymax, rzmin, rzmax = rsurf.GetBounds()
scale_x = (xmax-xmin) / (rxmax-rxmin)
scale_y = (ymax-ymin) / (rymax-rymin)
denom = rzmax - rzmin
num = zmax - xmin
if (denom == 0) or (num == 0):
scale_z = 1
else:
scale_z = (zmax-zmin) / (rzmax-rzmin)
transp = vtk.vtkTransform()
transp.Translate(xmin, ymin, zmin)
transp.Scale(scale_x, scale_y, scale_z)
transp.Translate(-rxmin, -rymin, -rzmin)
tpd = vtk.vtkTransformPolyDataFilter()
tpd.SetInputData(rsurf)
tpd.SetTransform(transp)
tpd.Update()
tsurf = tpd.GetOutput()
if verbose:
print 'Rescaled and translated...'
# Masking according to distance to the original segmentation
if mask:
tomod = scipy.ndimage.morphology.distance_transform_edt(
np.invert(tomo == lbl))
for i in range(tsurf.GetNumberOfCells()):
# Check if all points which made up the polygon are in the mask
points_cell = tsurf.GetCell(i).GetPoints()
count = 0
for j in range(0, points_cell.GetNumberOfPoints()):
x, y, z = points_cell.GetPoint(j)
if (tomod[int(round(x)), int(round(y)), int(round(z))]
> MAX_DIST_SURF):
count += 1
if count > 0:
tsurf.DeleteCell(i)
# Release free memory
tsurf.RemoveDeletedCells()
if verbose:
print 'Mask applied...'
# Field distance
if field:
# Get normal attributes
norm_flt = vtk.vtkPolyDataNormals()
norm_flt.SetInputData(tsurf)
norm_flt.ComputeCellNormalsOn()
norm_flt.AutoOrientNormalsOn()
norm_flt.ConsistencyOn()
norm_flt.Update()
tsurf = norm_flt.GetOutput()
# for i in range(tsurf.GetPointData().GetNumberOfArrays()):
# array = tsurf.GetPointData().GetArray(i)
# if array.GetNumberOfComponents() == 3:
# break
array = tsurf.GetCellData().GetNormals()
# Build membrane mask
tomoh = np.ones(shape=tomo.shape, dtype=np.bool)
tomon = np.ones(shape=(tomo.shape[0], tomo.shape[1], tomo.shape[2], 3),
dtype=TypesConverter().vtk_to_numpy(array))
# for i in range(tsurf.GetNumberOfCells()):
# points_cell = tsurf.GetCell(i).GetPoints()
# for j in range(0, points_cell.GetNumberOfPoints()):
# x, y, z = points_cell.GetPoint(j)
# # print x, y, z, array.GetTuple(j)
# x, y, z = int(round(x)), int(round(y)), int(round(z))
# tomo[x, y, z] = False
# tomon[x, y, z, :] = array.GetTuple(j)
for i in range(tsurf.GetNumberOfCells()):
points_cell = tsurf.GetCell(i).GetPoints()
for j in range(0, points_cell.GetNumberOfPoints()):
x, y, z = points_cell.GetPoint(j)
# print x, y, z, array.GetTuple(j)
x, y, z = int(round(x)), int(round(y)), int(round(z))
if tomo[x, y, z] == lbl:
tomoh[x, y, z] = False
tomon[x, y, z, :] = array.GetTuple(i)
# Distance transform
tomod, ids = scipy.ndimage.morphology.distance_transform_edt(
tomoh, return_indices=True)
# Compute polarity
if mode_2d:
for x in range(nx):
for y in range(ny):
for z in range(nz):
i_x, i_y, i_z = (ids[0, x, y, z], ids[1, x, y, z],
ids[2, x, y, z])
norm = tomon[i_x, i_y, i_z]
norm[2] = 0
pnorm = (i_x, i_y, 0)
p = (x, y, 0)
dprod = dot_norm(np.asarray(p, dtype=np.float),
np.asarray(pnorm, dtype=np.float),
np.asarray(norm, dtype=np.float))
tomod[x, y, z] = tomod[x, y, z] * np.sign(dprod)
else:
for x in range(nx):
for y in range(ny):
for z in range(nz):
i_x, i_y, i_z = (ids[0, x, y, z], ids[1, x, y, z],
ids[2, x, y, z])
hold_norm = tomon[i_x, i_y, i_z]
norm = hold_norm
# norm[0] = (-1) * hold_norm[1]
# norm[1] = hold_norm[0]
# norm[2] = hold_norm[2]
pnorm = (i_x, i_y, i_z)
p = (x, y, z)
dprod = dot_norm(np.asarray(pnorm, dtype=np.float),
np.asarray(p, dtype=np.float),
np.asarray(norm, dtype=np.float))
tomod[x, y, z] = tomod[x, y, z] * np.sign(dprod)
if verbose:
print 'Distance field generated...'
return tsurf, tomod
if verbose:
print 'Finished!'
return tsurf
def save_numpy(array, fname):
"""
Saves a numpy array to a file in MRC, EM or VTI format.
Args:
array (numpy.ndarray): input array
fname (str): full path to the tomogram, has to end with '.mrc', '.em' or
'.vti'
Returns:
None
"""
_, ext = os.path.splitext(fname)
# Parse input array for fulfilling format requirements
if (ext == '.mrc') or (ext == '.em'):
if ((array.dtype != 'ubyte') and (array.dtype != 'int16') and
(array.dtype != 'float32')):
array = array.astype('float32')
# if (len(array.shape) == 3) and (array.shape[2] == 1):
# array = np.reshape(array, (array.shape[0], array.shape[1]))
if ext == '.vti':
pname, fnameh = os.path.split(fname)
save_vti(numpy_to_vti(array), fnameh, pname)
# elif ext == '.fits':
# warnings.resetwarnings()
# warnings.filterwarnings('ignore', category=UserWarning, append=True)
# pyfits.writeto(fname, array, clobber=True, output_verify='silentfix')
# warnings.resetwarnings()
# warnings.filterwarnings('always', category=UserWarning, append=True)
elif ext == '.mrc':
img = ImageIO()
# img.setData(np.transpose(array, (1, 0, 2)))
img.setData(array)
img.writeMRC(fname)
elif ext == '.em':
img = ImageIO()
# img.setData(np.transpose(array, (1, 0, 2)))
img.setData(array)
img.writeEM(fname)
else:
error_msg = 'Format not valid %s.' % ext
raise pexceptions.PySegInputError(expr='save_numpy', msg=error_msg)
def sort(self,
seq=None,
out=None,
pad=0,
start=1,
fix_mode=None,
microscope=None,
limit=None,
limit_mode='std',
size=5,
byte_order=None,
test=False):
"""
Sorts series, fixes image headers, corrects the data and writes the
sorted and corrected images.
A series is sorted by tilt angle that is read from each image, or by the
elements of seq (not necessarily angles) if given.
Sorted images names have the form:
directory/name + number + extension.
where out is: directory/name_.extension and numbers start with start
argument and are padded with pad (argument) number of zeros (see
self.images).
Argument fix_mode detemines how the headers are fixed:
- None: no fixing
- 'polara_fei-tomo': for series obtained on polara with the FEI tomo
software
- 'krios_fei-tomo': for series obtained on krios with the FEI tomo
software
- 'cm300': for series from CM300
(see pyto.io.image_io).
If fix_mode is 'polara_fei-tomo', then arg microscope has to be
specified. The allowed values are specified in microscope_db.py.
Currently (r564) these are: 'polara-1_01-07', 'polara-1_01-09' and
'polara-2_01-09'.
If fix_mode is None, microscope does nor need to be specified.
Series recorded by SerialEM typically do not need fixing.
Works for individual images only (not for stacks).
Arguments:
- seq: if specified this sequence used for sorting projection,
otherwise tilt angles are used
- out: sorted series path
- pad: number of digits of a projection number in the sorted
series file name
- start: start number for sorted series file names
- fix_mode: determined if and how headers are fixed
- microscope: microscope type, see pyto.io.microscope_db
- limit_mode: 'abs' or 'std'
- limit: absolute greyscale limit if limit_mode is 'abs', or the
number of stds above and below the mean
- size: size of the subarray used to find the replacement value
for a voxel that's out of the limits
- byte_order: '<' or '>', default is the byte order of the machine
- test: if True all steps are done except writing corrected images
"""
# shortcuts
path = self.path
match_mode = self.mode
# make tilt angles - file names dictionary
# ToDo: use self.sortPath() instead
in_paths = []
images_iter = self.images(mode=match_mode)
if seq is None:
seq = []
# get tilt angles from file headers
for in_path in images_iter:
image = ImageIO(file=in_path)
image.readHeader()
seq.append(image.tiltAngle)
in_paths.append(in_path)
image.file_.close()
else:
# use seq to sort
for in_path in images_iter:
in_paths.append(in_path)
# sort (note: dictionary angle:in_path fails for multiple files with
# same angles)
seq_arr = numpy.array(seq)
in_paths_arr = numpy.array(in_paths)
sort_ind = seq_arr.argsort()
sorted_seq = seq_arr[sort_ind]
sorted_in_paths = in_paths_arr[sort_ind]
# parse out and make out directory if it doesn't exists
if out is not None:
out_dir, out_base_pat = os.path.split(out)
if out_dir == '':
out_dir = '.'
if (not test) and (not os.path.isdir(out_dir)):
os.makedirs(out_dir)
else:
out_path = None
# initialize file counter
if start is None:
ind = None
else:
ind = start - 1
# loop over sorted in files
for (item, in_path) in zip(sorted_seq, sorted_in_paths):
if ind is not None: ind += 1
# parse in file path
in_dir, in_base = os.path.split(in_path)
if out is not None:
# make out file path
out_base = self.convertBase(in_base=in_base,
out_base=out_base_pat,
pad=pad,
index=ind)
out_path = os.path.join(out_dir, out_base)
# read files
im_io = ImageIO(file=in_path)
im_io.read()
# log
logging.info("%5.1f: %s -> %s %6.1f %6.1f" \
% (item, in_path, out_path, \
im_io.data.mean(), im_io.data.std()))
# fix
if fix_mode is not None:
im_io.fix(mode=fix_mode, microscope=microscope)
# limit
if limit is not None:
image = Image(im_io.data)
image.limit(limit=limit, mode=limit_mode, size=size)
# write fixed file
if not test:
if byte_order is None:
byte_order = im_io.machineByteOrder
im_io.write(file=out_path,
byteOrder=byte_order,
data=image.data)
else:
logging.info(" " + str(item) + ": " + in_path + " -> "\
+ str(out_path))
def write(
self, file, byteOrder=None, dataType=None, fileFormat=None,
arrayOrder=None, shape=None, length=None, pixel=1, casting=u'unsafe',
header=False, existing=False):
"""
Writes image to a file in em, mrc or raw format.
If fileFormat is not given it is determined from the file extension.
Data (image) has to be specified by arg data or previously set
self.data attribute.
Data type and shape are determined by args dataType and shape,
or by the data type and shape of the data, in this order..
If data type (determined as described above) is not one of the
data types used for the specified file format (ubyte, int16, float32,
complex64 for mrc and uint8, uint16, int32, float32, float64,
complex64 for em), then the value of arg dataType has to be one of the
appropriate data types. Otherwise an exception is raised.
If data type (determined as described above) is different from the
type of actual data, the data is converted to the data type. Note that
if these two types are incompatible according to arg casting, an
exception is raised.
The data is converted to the (prevously determined) shape and array
order and then written. That means that the shape and the array order
may be changed in the original array (argument data or self.data).
However, array order is not written the header.
Additional header parameters are determined for mrc format. Nxstart,
nystart and nzstart are set to 0, while mx, my and mz to the
corresponding data size (grid size). xlen, ylen and zlen are taken from
arg length if given, or obtained by multiplying data size with pixel
size (in nm).
If arg header is True and if the file type corresponding to arg file
is the same as self.fileFormat, a header consisting of self.header
(list) and self.extendedHeaderString (str) are written as file
header. The extended header should be used only for mrc format.
Arguments:
- file: file name
- fileFormat: 'em', 'mrc', or 'raw'
- byteOrder: '<' (little-endian), '>' (big-endian)
- dataType: any of the numpy types, e.g.: 'int8', 'int16', 'int32',
'float32', 'float64'
- arrayOrder: 'C' (z-axis fastest), or 'F' (x-axis fastest)
- shape: (x_dim, y_dim, z_dim)
- length: (list aor ndarray) length in each dimension in nm (used
only for mrc format)
- pixel: pixel size in nm (used only for mrc format if length is
None)
- casting: Controls what kind of data casting may occur: 'no',
'equiv', 'safe', 'same_kind', 'unsafe'. Identical to numpy.astype()
method.
- header: flag indicating if self.header is written as file header
- existing: flag indicating whether the already existing file_io
attribute is used for writting
Returns file instance.
"""
# write file
from pyto.io.image_io import ImageIO as ImageIO
if existing and (self.file_io is not None):
fi = self.file_io
fi = ImageIO()
# get file format of the file to be written
fi.setFileFormat(file_=file, fileFormat=fileFormat)
new_file_format = fi.fileFormat
# set header to pass as argument, if new and old file types are the same
header_arg = None
extended = None
try:
if header and (new_file_format == self.fileFormat):
header_arg = self.header
extended = self.extendedHeaderString
except AttributeError:
pass
# write
fi.write(
file=file, data=self.data, fileFormat=fileFormat,
byteOrder=byteOrder, dataType=dataType, arrayOrder=arrayOrder,
shape=shape, length=length, pixel=pixel, casting=casting,
header=header_arg, extended=extended)
return fi.file_
def read(cls, file, fileFormat=None, byteOrder=None, dataType=None,
arrayOrder=None, shape=None, header=False, memmap=False):
"""
Reads image from a file and saves the data in numpy.array format.
Images can be in em, mrc and raw format. Unless arg fileFormat is
specified, file format is determined from the extension:
- 'em' and 'EM' for em format
- 'mrc', 'rec' and 'mrcs' for mrc format
- 'raw', 'dat' and 'RAW' for raw format
If file is in em or mrc format (file extension em or mrc) only the file
argument is needed. If the file is in the raw data format (file
extension raw) all arguments (except fileFormat) are required.
If arg header is True, file header (em, mrc or raw) is saved as
attribute header. In any case attribute fileFormat is set.
For mrc and em files, array order is determined from arg arrayOrder,
from self.arrayOrder, or it is set to the default ("F") in
this order. Data is read according the determined array order.
That is, array order is not read from the file header.
If arg memmap is True, instead into a nparray, the data is read to
a memory map. That means that the complete data is not read into
the memory, but the required parts are read on demand. This is useful
when working with large images, but might not always work properly
because the memory map is not quite properly a subclass of
numpy.ndarray (from Numpy doc).
Data from mrc files is always read as 3D because mrc header always
contains lengths for all three dimensions (the length of the last
dimeension is 1 for 2D images). In such cases one can obtain the
2D ndarray using:
self.data.reshape(self.data.shape[0:2])
Sets attributes:
- data: (ndarray) image data
- pixelsize: (float or list of floats) pixel size in nm, 1 if pixel
size not known
- length: (list or ndarray) length in each dimension in nm
- fileFormat: file format ('em', 'mrc', or 'raw')
- memmap: from the argument
- header: header
Arguments:
- file: file name
- fileFormat: 'em', 'mrc', or 'raw'
- byteOrder: '<' (little-endian), '>' (big-endian)
- dataType: any of the numpy types, e.g.: 'int8', 'int16', 'int32',
'float32', 'float64'
- arrayOrder: 'C' (z-axis fastest), or 'F' (x-axis fastest)
- shape: (x_dim, y_dim, z_dim)
- header: flag indicating if file header is read and saved
- memmap: Flag indicating if the data is read to a memory map,
instead of reading it into a ndarray
Returns:
- instance of this class
"""
# read file
from pyto.io.image_io import ImageIO as ImageIO
fi = ImageIO()
fi.read(
file=file, fileFormat=fileFormat, byteOrder=byteOrder,
dataType=dataType, arrayOrder=arrayOrder, shape=shape,
memmap=memmap)
# instantiate object and set fullInset
object = cls(data=fi.data)
# is this needed (ugly)
object.saveFull()
# save file header
object.header = None
if header:
object.header = fi.header
try:
object.extendedHeaderString = fi.extendedHeaderString
except AttributeError:
object.extendedHeaderString = None
object.fileFormat = fi.fileFormat
# save pixel size
try:
object.pixelsize = fi.pixelsize
except (AttributeError, ValueError):
object.pixelsize = 1
# save length
try:
object.length = fi.length
except (AttributeError, ValueError):
object.length = numpy.asarray(fi.data.shape) \
* numpy.asarray(object.pixelsize)
# save memmap
object.memmap = fi.memmap
# save file io object
# removed because if object.data is changed, object.file_io.data will
# still hold a copy of data
#object.file_io = fi
# save file type
#try:
# object.fileType = fi.fileType
#except (AttributeError, ValueError):
# object.fileType = None
return object
def getDose(self, conversion=1, mode='pixel_size', projection_dose=False):
"""
Calculates dose for the whole series (in e/A^2) and the average counts
per pixel..
Logs average, std, min and max counts for each image to INFO.
Mode determines how the pixels are converted to A^2:
- 'pixel_size': use pixelsize from the image header (default)
- 'cm300_fix': use only for cm300 series with headers that were not
corrected
Arguments:
- conversion: number of electrons per pixel (if known microscope use
pyto.io.microscope_db.conversion[microscope]).
- mode: 'pixel_size' should be used, other moders were used in
special cases
- projection_dose: flag indicating if dose [e/A^2] for each
projection is returned
Returns total_dose, mean_count:
- total dose in e/A^2
- mean count in counts/pixel
- projection_dose (if projection_dose=True) 2D ndarray containing
tilt angle and projection dose for all projections
(shape n_projections x 2)
"""
# set images to z positions if projections are in a stack, or
# to projection file names if individual projection files
if self.stack:
image_stack = Image.read(file=self.path, memmap=True)
images = numpy.array(list(range(image_stack.data.shape[2])))
images = images[numpy.argsort(self.tiltAngles)]
else:
images = list(self.images())
images = self.sortPaths(paths=images, mode='tilt_angle')
tot_dose = 0
tot_count = 0
num = 0.
proj_dose = []
for fi in images:
# read single projection file
if self.stack:
image = Image(data=image_stack.data[:, :, fi])
if isinstance(image_stack.pixelsize, (numpy.ndarray, list)):
image.pixelsize = image_stack.pixelsize[0]
else:
image.pixelsize = image_stack.pixelsize
image.tiltAngle = self.tiltAngles[fi]
else:
image = ImageIO(file=fi)
image.read()
# calculate
mean = image.data.mean()
tot_count += mean
num += 1
if mode == 'pixel_size':
loc_dose = mean / (conversion * image.pixelsize**2)
elif (mode == 'cm300_fix') or (image.pixelsize == 2.048):
ccd_pixelsize = 30000
pixelsize = ccd_pixelsize / image.magnification
loc_dose = mean / (conversion * pixelsize**2)
else:
raise ValueError("Sorry, dose calculation mode: " + mode +
" is not recognized.")
tot_dose += loc_dose
# save and print stats for an individual image
proj_dose.append([image.tiltAngle, loc_dose / 100.])
logging.info('%s, (%5.1f): mean=%6.1f, std=%6.1f, min=%d, max=%d' %
(fi, image.tiltAngle, mean, image.data.std(),
image.data.min(), image.data.max()))
# mean counts
mean_count = tot_count / num
# total dose per A^2
tot_dose = tot_dose / 100
# print total values
logging.info("Mean count per pixel: %6.1f" % mean_count)
logging.info("Total electron dose per A^2: %6.1f" % tot_dose)
if projection_dose:
return tot_dose, mean_count, numpy.asarray(proj_dose)
else:
return tot_dose, mean_count
def save_numpy(array, fname):
"""
Saves a numpy array to a file in MRC, EM or VTI format.
Args:
array (numpy.ndarray): input array
fname (str): full path to the tomogram, has to end with '.mrc', '.em' or
'.vti'
Returns:
None
"""
_, ext = os.path.splitext(fname)
# Parse input array for fulfilling format requirements
if (ext == '.mrc') or (ext == '.em'):
if ((array.dtype != 'ubyte') and (array.dtype != 'int16')
and (array.dtype != 'float32')):
array = array.astype('float32')
# if (len(array.shape) == 3) and (array.shape[2] == 1):
# array = np.reshape(array, (array.shape[0], array.shape[1]))
if ext == '.vti':
pname, fnameh = os.path.split(fname)
save_vti(numpy_to_vti(array), fnameh, pname)
elif ext == '.mrc':
img = ImageIO()
# img.setData(np.transpose(array, (1, 0, 2)))
img.setData(array)
img.writeMRC(fname)
elif ext == '.em':
img = ImageIO()
# img.setData(np.transpose(array, (1, 0, 2)))
img.setData(array)
img.writeEM(fname)
else:
raise pexceptions.PySegInputError(
expr='save_numpy', msg='Format not valid {}.'.format(ext))
def testRead(self):
"""
Tests reading EM and MRC files
"""
# EM tomo
em = ImageIO()
em.read(file=os.path.join(self.dir, "bin-2.em"))
expected = numpy.array([[-0.0242, -0.0250, 0.0883],
[0.0640, 0.0071, -0.1300],
[-0.0421, -0.0392, -0.0312]])
np_test.assert_almost_equal(em.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0573, 0.0569, 0.0386],
[0.1309, 0.1211, -0.0881],
[-0.0110, -0.0240, 0.0347]])
np_test.assert_almost_equal(em.data[150:153, 20:23, 10],
expected,
decimal=4)
np_test.assert_equal(em.byteOrder, '<')
np_test.assert_equal(em.arrayOrder, 'F')
np_test.assert_equal(em.dataType, 'float32')
np_test.assert_equal(em.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(em.memmap, False)
# EM tomo with memory map
em.read(file=os.path.join(self.dir, "bin-2.em"), memmap=True)
expected = numpy.array([[-0.0242, -0.0250, 0.0883],
[0.0640, 0.0071, -0.1300],
[-0.0421, -0.0392, -0.0312]])
np_test.assert_almost_equal(em.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0573, 0.0569, 0.0386],
[0.1309, 0.1211, -0.0881],
[-0.0110, -0.0240, 0.0347]])
np_test.assert_almost_equal(em.data[150:153, 20:23, 10],
expected,
decimal=4)
np_test.assert_equal(em.byteOrder, '<')
np_test.assert_equal(em.arrayOrder, 'F')
np_test.assert_equal(em.dataType, 'float32')
np_test.assert_equal(em.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(em.memmap, True)
# EM, big-endian
em = ImageIO()
em.read(file=os.path.join(self.dir, "mac-file.em"))
np_test.assert_equal(em.byteOrder, '>')
# EM, little-endian
em = ImageIO()
em.read(file=os.path.join(self.dir, "pc-file.em"))
np_test.assert_equal(em.byteOrder, '<')
em.read(file=os.path.join(self.dir, "pc-file.em"), memmap=True)
np_test.assert_equal(em.byteOrder, '<')
# MRC tomo
mrc = ImageIO()
mrc.read(file=os.path.join(self.dir, "bin-2.mrc"))
expected = numpy.array([[-0.0242, -0.0250, 0.0883],
[0.0640, 0.0071, -0.1300],
[-0.0421, -0.0392, -0.0312]])
np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0573, 0.0569, 0.0386],
[0.1309, 0.1211, -0.0881],
[-0.0110, -0.0240, 0.0347]])
np_test.assert_almost_equal(mrc.data[150:153, 20:23, 10],
expected,
decimal=4)
np_test.assert_equal(mrc.byteOrder, '<')
np_test.assert_equal(mrc.arrayOrder, 'F')
np_test.assert_equal(mrc.dataType, 'float32')
np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(mrc.memmap, False)
# MRC tomo with memmap
mrc = ImageIO()
mrc.read(file=os.path.join(self.dir, "bin-2.mrc"), memmap=True)
expected = numpy.array([[-0.0242, -0.0250, 0.0883],
[0.0640, 0.0071, -0.1300],
[-0.0421, -0.0392, -0.0312]])
np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0573, 0.0569, 0.0386],
[0.1309, 0.1211, -0.0881],
[-0.0110, -0.0240, 0.0347]])
np_test.assert_almost_equal(mrc.data[150:153, 20:23, 10],
expected,
decimal=4)
np_test.assert_equal(mrc.byteOrder, '<')
np_test.assert_equal(mrc.arrayOrder, 'F')
np_test.assert_equal(mrc.dataType, 'float32')
np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(mrc.memmap, True)
# MRC tomo with extended header
mrc = ImageIO()
mrc.read(file=os.path.join(self.dir, "bin-2_ext.mrc"), memmap=False)
expected = numpy.array([[-0.0242, -0.0250, 0.0883],
[0.0640, 0.0071, -0.1300],
[-0.0421, -0.0392, -0.0312]])
np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0573, 0.0569, 0.0386],
[0.1309, 0.1211, -0.0881],
[-0.0110, -0.0240, 0.0347]])
np_test.assert_almost_equal(mrc.data[150:153, 20:23, 10],
expected,
decimal=4)
np_test.assert_equal(mrc.byteOrder, '<')
np_test.assert_equal(mrc.arrayOrder, 'F')
np_test.assert_equal(mrc.dataType, 'float32')
np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(mrc.memmap, False)
np_test.assert_equal(mrc.extendedHeaderLength, 5120)
# MRC tomo with extended header and with memmap
mrc = ImageIO()
mrc.read(file=os.path.join(self.dir, "bin-2_ext.mrc"), memmap=True)
expected = numpy.array([[-0.0242, -0.0250, 0.0883],
[0.0640, 0.0071, -0.1300],
[-0.0421, -0.0392, -0.0312]])
np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0573, 0.0569, 0.0386],
[0.1309, 0.1211, -0.0881],
[-0.0110, -0.0240, 0.0347]])
np_test.assert_almost_equal(mrc.data[150:153, 20:23, 10],
expected,
decimal=4)
np_test.assert_equal(mrc.byteOrder, '<')
np_test.assert_equal(mrc.arrayOrder, 'F')
np_test.assert_equal(mrc.dataType, 'float32')
np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(mrc.memmap, True)
np_test.assert_equal(mrc.extendedHeaderLength, 5120)
# another MRC tomo (generated by and)
mrc = ImageIO()
mrc.read(file=os.path.join(self.dir, "and-tomo.mrc"))
expected = numpy.array([[-0.0329, -0.0006, -0.0698],
[-0.0101, -0.1196, -0.1295],
[0.0844, -0.0400, -0.0716]])
np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0019, -0.0085, 0.0036],
[0.0781, 0.0279, -0.0365],
[0.0210, -0.0193, -0.0355]])
np_test.assert_almost_equal(mrc.data[150:153, 20:23, 60],
expected,
decimal=4)
np_test.assert_equal(mrc.dataType, 'float32')
np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(mrc.memmap, False)
# another MRC tomo (generated by and) with memmap
mrc = ImageIO()
mrc.read(file=os.path.join(self.dir, "and-tomo.mrc"), memmap=True)
expected = numpy.array([[-0.0329, -0.0006, -0.0698],
[-0.0101, -0.1196, -0.1295],
[0.0844, -0.0400, -0.0716]])
np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0019, -0.0085, 0.0036],
[0.0781, 0.0279, -0.0365],
[0.0210, -0.0193, -0.0355]])
np_test.assert_almost_equal(mrc.data[150:153, 20:23, 60],
expected,
decimal=4)
np_test.assert_equal(mrc.dataType, 'float32')
np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(mrc.memmap, True)
# mrc with the opposite byte order
mrc2 = ImageIO()
mrc2.read(file=os.path.join(self.dir, "swapped_byte_order.mrc"))
expected = numpy.array([[0.000, 0.000], [-0.341, -6.702],
[0.782, -11.780], [0.327, -14.298],
[-0.691, -17.411], [-0.337, -18.076],
[-0.669, -19.157], [-0.799, -20.400],
[-0.793, -21.286], [-1.008, -21.386]])
np_test.assert_almost_equal(mrc2.data[:, :, 0], expected, decimal=3)
np_test.assert_equal(mrc2.memmap, False)
raised = False
try:
mrc2.read(file=os.path.join(self.dir, "swapped_byte_order.mrc"),
memmap=True)
except ValueError:
raised = True
np_test.assert_equal(raised, True)
np_test.assert_equal(mrc2.memmap, True)
# new style header mrc
mrc_new = ImageIO()
mrc_new.read(file=os.path.join(self.dir, 'new-head_int16.mrc'))
np_test.assert_equal(mrc_new.dataType, 'int16')
np_test.assert_equal(mrc_new.data.dtype, numpy.dtype('int16'))
np_test.assert_equal(mrc_new.byteOrder, '<')
np_test.assert_equal(mrc_new.arrayOrder, 'F')
np_test.assert_equal(mrc_new.shape, (40, 30, 20))
np_test.assert_equal(mrc_new.pixel, [0.4, 0.4, 0.4])
np_test.assert_equal(mrc_new.pixelsize, 0.4)
np_test.assert_equal(mrc_new.data[14, 8, 10], -14)
np_test.assert_equal(mrc_new.data[15, 23, 12], 10)
np_test.assert_equal(mrc_new.data[23, 29, 16], 2)
np_test.assert_equal(mrc_new.memmap, False)
# new style header mrc
mrc_new = ImageIO()
mrc_new.read(file=os.path.join(self.dir, 'new-head_int16.mrc'),
memmap=True)
np_test.assert_equal(mrc_new.dataType, 'int16')
np_test.assert_equal(mrc_new.data.dtype, numpy.dtype('int16'))
np_test.assert_equal(mrc_new.byteOrder, '<')
np_test.assert_equal(mrc_new.arrayOrder, 'F')
np_test.assert_equal(mrc_new.shape, (40, 30, 20))
np_test.assert_equal(mrc_new.pixel, [0.4, 0.4, 0.4])
np_test.assert_equal(mrc_new.pixelsize, 0.4)
np_test.assert_equal(mrc_new.data[14, 8, 10], -14)
np_test.assert_equal(mrc_new.data[15, 23, 12], 10)
np_test.assert_equal(mrc_new.data[23, 29, 16], 2)
np_test.assert_equal(mrc_new.memmap, True)
np_test.assert_equal(mrc_new.n_labels, 9)
np_test.assert_equal(len(mrc_new.labels), 9)
desired = (
b"COMBINEFFT: Combined FFT from two tomograms " +
b"07-Oct-13 17:15:24")
np_test.assert_equal(len(mrc_new.labels[3]), 80)
np_test.assert_equal(mrc_new.labels[3][:len(desired)], desired)
desired = (
b"NEWSTACK: Images copied 10-Oct-13 18:00:03"
)
np_test.assert_equal(len(mrc_new.labels[6]), 80)
np_test.assert_equal(mrc_new.labels[6][:len(desired)], desired)
# test raw file
raw = ImageIO()
raw.read(file=self.raw_file_name,
dataType=self.raw_dtype,
shape=self.raw_shape)
np_test.assert_equal(raw.data, self.raw_data)
np_test.assert_equal(raw.memmap, False)
# test raw file with memmap
raw = ImageIO()
raw.read(file=self.raw_file_name,
dataType=self.raw_dtype,
shape=self.raw_shape,
memmap=True)
np_test.assert_equal(raw.data, self.raw_data)
np_test.assert_equal(raw.memmap, True)
def testWrite(self):
"""
Tests write (and implicitly read), for em, mrc and raw format.
"""
# arrays
ar_uint8 = numpy.array([54, 200, 5, 7, 45, 123],
dtype='uint8').reshape((3, 1, 2))
ar_int8 = numpy.array([54, 2, -5, 7, 45, 123], dtype='uint8').reshape(
(3, 1, 2))
ar_uint16 = numpy.array([1034, 546, 248, 40000, 2345, 365, 4876, 563],
dtype='uint16').reshape((2, 2, 2))
ar_int16 = numpy.array([1034, 546, -248, 156, 2345, 365, -4876, 563],
dtype='int16').reshape((2, 2, 2))
ar_int32 = numpy.array(
[1034, 56546, -223448, 156, 2345, 2**31 - 10, -884876, 563],
dtype='int32').reshape((2, 2, 2))
ar_uint32 = numpy.array(
[1034, 56546, 223448, 156, 2345, 365, 884876, 2**32 - 10],
dtype='uint32').reshape((2, 2, 2))
ar_int8_2 = numpy.arange(24, dtype='int8').reshape((4, 3, 2))
ar_int16_2 = numpy.arange(24, dtype='int16').reshape((4, 3, 2))
ar2_int16 = numpy.array([1034, 546, -248, 156, 2345, 365, -4876, 563],
dtype='int16').reshape((2, 4))
ar_int16_f = numpy.array([1034, 546, -248, 156, 2345, 365, -4876, 563],
dtype='int16',
order='F').reshape((2, 2, 2))
ar_int16_c = numpy.array([1034, 546, -248, 156, 2345, 365, -4876, 563],
dtype='int16',
order='C').reshape((2, 2, 2))
# em uint8
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'), data=ar_uint8)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'uint8')
np_test.assert_equal(file_in.data, ar_uint8)
# em uint16
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'), data=ar_uint16)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'uint16')
np_test.assert_equal(file_in.data, ar_uint16)
# em int16 converted to int32, safe casting
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'),
data=ar_int16,
dataType='int32',
casting='safe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'int32')
np_test.assert_equal(file_in.data, ar_int16)
# em int16, safe casting
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.em'),
'data': ar_int16,
'casting': 'safe'
})
# em int16 converted to uint16, unsafe casting
file_out = ImageIO()
print("int16 to uint16")
file_out.write(file=os.path.join(self.dir, '_test.em'),
data=ar_int16,
dataType='uint16',
casting='unsafe')
print("int16 to uint16 end")
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'uint16')
np_test.assert_equal(file_in.data.dtype, numpy.dtype('uint16'))
np_test.assert_equal(file_in.data[0, 1, 0] == ar_int16[0, 1, 0], False)
# em int16 to uint16, safe casting
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.em'),
'data': ar_int16,
'dataType': 'uint16',
'casting': 'safe'
})
# em uint16 to int16, unsafe casting
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.em'),
'data': ar_uint16,
'dataType': 'int16',
'casting': 'unsafe'
})
# em uint32 to int32, safe casting
print("uint32 to int32 safe")
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.em'),
'data': ar_uint32,
'dataType': 'int32',
'casting': 'safe'
})
# em uint32 converted to int32, unsafe casting
print("uint32 to int32")
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'),
data=ar_uint32,
dataType='int32',
casting='unsafe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'int32')
#np_test.assert_equal(file_in.data, ar_uint32) should fail
np_test.assert_equal(file_in.data[0, 0, 0] == ar_uint32[0, 0, 0], True)
np_test.assert_equal(file_in.data[1, 1, 1] == ar_uint32[1, 1, 1],
False)
# em uint32 to float32, safe casting
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.em'),
'data': ar_uint32,
'dataType': 'float32',
'casting': 'safe'
})
# em uint32 to float32, unsafe casting
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'),
data=ar_uint32,
dataType='float32',
casting='unsafe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'float32')
#np_test.assert_almost_equal(file_in.data, ar_uint32) should fail
np_test.assert_equal(file_in.data[0, 0, 0] == ar_uint32[0, 0, 0], True)
np_test.assert_equal(file_in.data[1, 1, 1] == ar_uint32[1, 1, 1],
False)
# em int32 to float32, unsafe casting
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'),
data=ar_int32,
dataType='float32',
casting='unsafe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'float32')
#np_test.assert_almost_equal(file_in.data, ar_int32) should fail
np_test.assert_equal(file_in.data[0, 0, 0] == ar_int32[0, 0, 0], True)
np_test.assert_equal(file_in.data[1, 0, 1] == ar_int32[1, 0, 1], False)
# em int32 to float64, safe casting
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'),
data=ar_int32,
dataType='float64',
casting='safe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'float64')
np_test.assert_almost_equal(file_in.data, ar_int32)
# mrc data type and shape from args
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int8_2,
shape=(2, 3, 4),
dataType='int16')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.shape, (2, 3, 4))
# mrc data type and shape from previously given data
file_out = ImageIO()
file_out.setData(ar_int16_2)
file_out.write(file=os.path.join(self.dir, '_test.mrc'))
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.shape, (4, 3, 2))
# mrc data type and shape from attributes
file_out = ImageIO()
file_out.data = ar_int8_2
file_out.shape = (2, 3, 4)
file_out.dataType = 'int16'
file_out.write(file=os.path.join(self.dir, '_test.mrc'))
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.shape, (2, 3, 4))
# mrc data type and shape from data
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16_2)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.shape, (4, 3, 2))
# mrc uint8, same as ubyte
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_uint8)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'ubyte')
np_test.assert_almost_equal(file_in.data, ar_uint8)
# mrc uint16
file_out = ImageIO()
np_test.assert_raises((KeyError, TypeError), file_out.write, **{
'file': os.path.join(self.dir, '_test.mrc'),
'data': ar_uint16
})
# mrc uint16 to int16, safe casting
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.mrc'),
'data': ar_uint16,
'dataType': 'ubyte',
'casting': 'safe'
})
# mrc uint16 to int16, unsafe casting
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_uint16,
dataType='int16',
casting='unsafe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
#np_test.assert_almost_equal(file_in.data, ar_uint16) should fail
np_test.assert_equal(file_in.data[0, 0, 0] == ar_uint16[0, 0, 0], True)
np_test.assert_equal(file_in.data[0, 1, 1] == ar_uint16[0, 1, 1],
False)
# mrc int16
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16,
pixel=2.3)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.data, ar_int16)
np_test.assert_equal(file_in.pixel, [2.3, 2.3, 2.3])
np_test.assert_equal(file_in.pixelsize, 2.3)
# mrc int16 2D
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar2_int16,
pixel=3.4)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.data[:, :, 0], ar2_int16)
np_test.assert_equal(file_in.pixelsize, 3.4)
# mrc int8 to int16
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int8,
dataType='int16',
casting='safe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.data, ar_int8)
# mrc int32
file_out = ImageIO()
np_test.assert_raises((KeyError, TypeError), file_out.write, **{
'file': os.path.join(self.dir, '_test.mrc'),
'data': ar_int32
})
# mrc int32 to int16
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.mrc'),
'data': ar_int32,
'dataType': 'int16',
'casting': 'safe'
})
# mrc int32 to float32
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.mrc'),
'data': ar_int32,
'dataType': 'float32',
'casting': 'safe'
})
# mrc int32 to complex64
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.mrc'),
'data': ar_int32,
'dataType': 'complex64',
'casting': 'safe'
})
# raw int16
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.raw'), data=ar_int16)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.raw'),
dataType='int16',
shape=(2, 2, 2))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.data, ar_int16)
# raw int8 to int16
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.raw'),
data=ar_int8,
dataType='int16')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.raw'),
dataType='int16',
shape=(3, 1, 2))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.data, ar_int8)
# raw int16 to int8
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.raw'),
'data': ar_int16,
'dataType': 'int8',
'casting': 'safe'
})
# explain error messages printed before
print("It's fine if few error messages were printed just before " +
"this line, because they have been caught.")
# shape param
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16,
dataType='int16')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'),
dataType='int16')
np_test.assert_equal(file_in.data.shape, (2, 2, 2))
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16,
dataType='int16',
shape=(1, 4, 2))
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'),
dataType='int16')
np_test.assert_equal(file_in.data.shape, (1, 4, 2))
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16,
dataType='int16',
shape=(4, 2))
file_in.readHeader(file=os.path.join(self.dir, '_test.mrc'))
file_in.read(file=os.path.join(self.dir, '_test.mrc'),
dataType='int16')
np_test.assert_equal(file_in.data.shape, (4, 2, 1))
file_in.read(file=os.path.join(self.dir, '_test.mrc'),
dataType='int16',
shape=(2, 2, 2))
np_test.assert_equal(file_in.data.shape, (2, 2, 2))
# array order C, read write default (F)
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16_c)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.data, ar_int16_c)
# array order C, read write C
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16_c,
arrayOrder='C')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'), arrayOrder='C')
np_test.assert_equal(file_in.data, ar_int16_c)
# array order F, read write default (F)
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16_f)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.data, ar_int16_f)
# array order F, read write F
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16_f,
arrayOrder='F')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'), arrayOrder='F')
np_test.assert_equal(file_in.data, ar_int16_f)