def read(cls,
cleftId,
bound1Id,
bound2Id,
file,
clean=True,
byteOrder=None,
dataType=None,
arrayOrder=None,
shape=None):
"""
Reads segmented image (label filed) from a file and sets required ids.
Each of the id args (cleftId, bound1Id, bound2Id) can be a single int
or a list (array) in which cases a (boundary or cleft) is formed form
all specified ids.
If file is in em or mrc format (file extension em or mrc) only the file
argument is needed. If the file is in the raw data format (file
extension raw) all arguments are required.
Arguments:
- bound1Id, bound2Id, cleftId: ids of boundaries 1, 2 and cleft
- file: file name
- clean: if true, only the segments corresponding to ids are kept
- byteOrder: '<' (little-endian), '>' (big-endian)
- dataType: any of the numpy types, e.g.: 'int8', 'int16', 'int32',
'float32', 'float64'
- arrayOrder: 'C' (z-axis fastest), or 'F' (x-axis fastest)
- shape: (x_dim, y_dim, z_dim)
Returns:
- instance of Segment
"""
# read file
from pyto.io.image_io import ImageIO as ImageIO
fi = ImageIO()
fi.read(file=file,
byteOrder=byteOrder,
dataType=dataType,
arrayOrder=arrayOrder,
shape=shape)
# make new instance
cleft = cls(data=fi.data,
cleftId=cleftId,
bound1Id=bound1Id,
bound2Id=bound2Id,
copy=False,
clean=clean)
return cleft
def fix(self, mode=None, microscope=None, out=None, pad=0, start=1):
"""
Fixes wrong data in headers.
Mode determines which values are fixed. Currently defined modes are:
- 'polara_fei-tomo': images obtained on Polara (at MPI of
Biochemistry) using FEI tomography package and saved in EM
format.
- 'krios_fei-tomo': images obtained on Krios (at MPI of
Biochemistry) using FEI tomography package and saved in EM
format.
- 'cm300': images from cm300 in EM format
- None: doesn't do anyhing
If mode is 'polara_fei-tomo', then arg microscope has to be specified.
The allowed values are specified in microscope_db.py. Currently (r564)
these are: 'polara-1_01-07', 'polara-1_01-09' and 'polara-2_01-09'.
Works for individual images only (not for stacks), because stacks
are typically recorded by SerialEM and do not need fixing.
Arguments:
- out: directory where the fixed images are written. The fixed and
the original images have the same base names.
- mode: fix mode
"""
# parse arguments
#if path is None: path = self.path
#if mode is None: mode = self.mode
# check for out, pad and start also?
# make out directory if it doesn't exists
if (out is not None) and (not os.path.isdir(out)):
os.makedirs(out)
# loop over images
images_iter = self.images(out=out, pad=pad, start=start)
for (in_path, out_path) in images_iter:
# read image file
image = ImageIO(file=in_path)
image.read()
# fix
image.fix(mode=mode, microscope=microscope)
# write fixed files
image.write(file=out_path)
def testPixelSize(self):
"""
Tests pixel size in read and write
"""
# arrays
#ar_int8_2 = numpy.arange(24, dtype='int8').reshape((4,3,2))
ar_int16_2 = numpy.arange(24, dtype='int16').reshape((4, 3, 2))
#
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16_2,
pixel=2.1)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_almost_equal(file_in.pixel, 2.1)
def read(cls, file, fileFormat=None, byteOrder=None, dataType=None,
arrayOrder=None, shape=None, header=False, memmap=False):
"""
Reads image from a file and saves the data in numpy.array format.
Images can be in em, mrc and raw format. Unless arg fileFormat is
specified, file format is determined from the extension:
- 'em' and 'EM' for em format
- 'mrc', 'rec' and 'mrcs' for mrc format
- 'raw', 'dat' and 'RAW' for raw format
If file is in em or mrc format (file extension em or mrc) only the file
argument is needed. If the file is in the raw data format (file
extension raw) all arguments (except fileFormat) are required.
If arg header is True, file header (em, mrc or raw) is saved as
attribute header. In any case attribute fileFormat is set.
For mrc and em files, array order is determined from arg arrayOrder,
from self.arrayOrder, or it is set to the default ("F") in
this order. Data is read according the determined array order.
That is, array order is not read from the file header.
If arg memmap is True, instead into a nparray, the data is read to
a memory map. That means that the complete data is not read into
the memory, but the required parts are read on demand. This is useful
when working with large images, but might not always work properly
because the memory map is not quite properly a subclass of
numpy.ndarray (from Numpy doc).
Data from mrc files is always read as 3D because mrc header always
contains lengths for all three dimensions (the length of the last
dimeension is 1 for 2D images). In such cases one can obtain the
2D ndarray using:
self.data.reshape(self.data.shape[0:2])
Sets attributes:
- data: (ndarray) image data
- pixelsize: (float or list of floats) pixel size in nm, 1 if pixel
size not known
- length: (list or ndarray) length in each dimension in nm
- fileFormat: file format ('em', 'mrc', or 'raw')
- memmap: from the argument
- header: header
Arguments:
- file: file name
- fileFormat: 'em', 'mrc', or 'raw'
- byteOrder: '<' (little-endian), '>' (big-endian)
- dataType: any of the numpy types, e.g.: 'int8', 'int16', 'int32',
'float32', 'float64'
- arrayOrder: 'C' (z-axis fastest), or 'F' (x-axis fastest)
- shape: (x_dim, y_dim, z_dim)
- header: flag indicating if file header is read and saved
- memmap: Flag indicating if the data is read to a memory map,
instead of reading it into a ndarray
Returns:
- instance of this class
"""
# read file
from pyto.io.image_io import ImageIO as ImageIO
fi = ImageIO()
fi.read(
file=file, fileFormat=fileFormat, byteOrder=byteOrder,
dataType=dataType, arrayOrder=arrayOrder, shape=shape,
memmap=memmap)
# instantiate object and set fullInset
object = cls(data=fi.data)
# is this needed (ugly)
object.saveFull()
# save file header
object.header = None
if header:
object.header = fi.header
try:
object.extendedHeaderString = fi.extendedHeaderString
except AttributeError:
object.extendedHeaderString = None
object.fileFormat = fi.fileFormat
# save pixel size
try:
object.pixelsize = fi.pixelsize
except (AttributeError, ValueError):
object.pixelsize = 1
# save length
try:
object.length = fi.length
except (AttributeError, ValueError):
object.length = numpy.asarray(fi.data.shape) \
* numpy.asarray(object.pixelsize)
# save memmap
object.memmap = fi.memmap
# save file io object
# removed because if object.data is changed, object.file_io.data will
# still hold a copy of data
#object.file_io = fi
# save file type
#try:
# object.fileType = fi.fileType
#except (AttributeError, ValueError):
# object.fileType = None
return object
def testRead(self):
"""
Tests reading EM and MRC files
"""
# EM tomo
em = ImageIO()
em.read(file=os.path.join(self.dir, "bin-2.em"))
expected = numpy.array([[-0.0242, -0.0250, 0.0883],
[0.0640, 0.0071, -0.1300],
[-0.0421, -0.0392, -0.0312]])
np_test.assert_almost_equal(em.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0573, 0.0569, 0.0386],
[0.1309, 0.1211, -0.0881],
[-0.0110, -0.0240, 0.0347]])
np_test.assert_almost_equal(em.data[150:153, 20:23, 10],
expected,
decimal=4)
np_test.assert_equal(em.byteOrder, '<')
np_test.assert_equal(em.arrayOrder, 'F')
np_test.assert_equal(em.dataType, 'float32')
np_test.assert_equal(em.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(em.memmap, False)
# EM tomo with memory map
em.read(file=os.path.join(self.dir, "bin-2.em"), memmap=True)
expected = numpy.array([[-0.0242, -0.0250, 0.0883],
[0.0640, 0.0071, -0.1300],
[-0.0421, -0.0392, -0.0312]])
np_test.assert_almost_equal(em.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0573, 0.0569, 0.0386],
[0.1309, 0.1211, -0.0881],
[-0.0110, -0.0240, 0.0347]])
np_test.assert_almost_equal(em.data[150:153, 20:23, 10],
expected,
decimal=4)
np_test.assert_equal(em.byteOrder, '<')
np_test.assert_equal(em.arrayOrder, 'F')
np_test.assert_equal(em.dataType, 'float32')
np_test.assert_equal(em.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(em.memmap, True)
# EM, big-endian
em = ImageIO()
em.read(file=os.path.join(self.dir, "mac-file.em"))
np_test.assert_equal(em.byteOrder, '>')
# EM, little-endian
em = ImageIO()
em.read(file=os.path.join(self.dir, "pc-file.em"))
np_test.assert_equal(em.byteOrder, '<')
em.read(file=os.path.join(self.dir, "pc-file.em"), memmap=True)
np_test.assert_equal(em.byteOrder, '<')
# MRC tomo
mrc = ImageIO()
mrc.read(file=os.path.join(self.dir, "bin-2.mrc"))
expected = numpy.array([[-0.0242, -0.0250, 0.0883],
[0.0640, 0.0071, -0.1300],
[-0.0421, -0.0392, -0.0312]])
np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0573, 0.0569, 0.0386],
[0.1309, 0.1211, -0.0881],
[-0.0110, -0.0240, 0.0347]])
np_test.assert_almost_equal(mrc.data[150:153, 20:23, 10],
expected,
decimal=4)
np_test.assert_equal(mrc.byteOrder, '<')
np_test.assert_equal(mrc.arrayOrder, 'F')
np_test.assert_equal(mrc.dataType, 'float32')
np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(mrc.memmap, False)
# MRC tomo with memmap
mrc = ImageIO()
mrc.read(file=os.path.join(self.dir, "bin-2.mrc"), memmap=True)
expected = numpy.array([[-0.0242, -0.0250, 0.0883],
[0.0640, 0.0071, -0.1300],
[-0.0421, -0.0392, -0.0312]])
np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0573, 0.0569, 0.0386],
[0.1309, 0.1211, -0.0881],
[-0.0110, -0.0240, 0.0347]])
np_test.assert_almost_equal(mrc.data[150:153, 20:23, 10],
expected,
decimal=4)
np_test.assert_equal(mrc.byteOrder, '<')
np_test.assert_equal(mrc.arrayOrder, 'F')
np_test.assert_equal(mrc.dataType, 'float32')
np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(mrc.memmap, True)
# MRC tomo with extended header
mrc = ImageIO()
mrc.read(file=os.path.join(self.dir, "bin-2_ext.mrc"), memmap=False)
expected = numpy.array([[-0.0242, -0.0250, 0.0883],
[0.0640, 0.0071, -0.1300],
[-0.0421, -0.0392, -0.0312]])
np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0573, 0.0569, 0.0386],
[0.1309, 0.1211, -0.0881],
[-0.0110, -0.0240, 0.0347]])
np_test.assert_almost_equal(mrc.data[150:153, 20:23, 10],
expected,
decimal=4)
np_test.assert_equal(mrc.byteOrder, '<')
np_test.assert_equal(mrc.arrayOrder, 'F')
np_test.assert_equal(mrc.dataType, 'float32')
np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(mrc.memmap, False)
np_test.assert_equal(mrc.extendedHeaderLength, 5120)
# MRC tomo with extended header and with memmap
mrc = ImageIO()
mrc.read(file=os.path.join(self.dir, "bin-2_ext.mrc"), memmap=True)
expected = numpy.array([[-0.0242, -0.0250, 0.0883],
[0.0640, 0.0071, -0.1300],
[-0.0421, -0.0392, -0.0312]])
np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0573, 0.0569, 0.0386],
[0.1309, 0.1211, -0.0881],
[-0.0110, -0.0240, 0.0347]])
np_test.assert_almost_equal(mrc.data[150:153, 20:23, 10],
expected,
decimal=4)
np_test.assert_equal(mrc.byteOrder, '<')
np_test.assert_equal(mrc.arrayOrder, 'F')
np_test.assert_equal(mrc.dataType, 'float32')
np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(mrc.memmap, True)
np_test.assert_equal(mrc.extendedHeaderLength, 5120)
# another MRC tomo (generated by and)
mrc = ImageIO()
mrc.read(file=os.path.join(self.dir, "and-tomo.mrc"))
expected = numpy.array([[-0.0329, -0.0006, -0.0698],
[-0.0101, -0.1196, -0.1295],
[0.0844, -0.0400, -0.0716]])
np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0019, -0.0085, 0.0036],
[0.0781, 0.0279, -0.0365],
[0.0210, -0.0193, -0.0355]])
np_test.assert_almost_equal(mrc.data[150:153, 20:23, 60],
expected,
decimal=4)
np_test.assert_equal(mrc.dataType, 'float32')
np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(mrc.memmap, False)
# another MRC tomo (generated by and) with memmap
mrc = ImageIO()
mrc.read(file=os.path.join(self.dir, "and-tomo.mrc"), memmap=True)
expected = numpy.array([[-0.0329, -0.0006, -0.0698],
[-0.0101, -0.1196, -0.1295],
[0.0844, -0.0400, -0.0716]])
np_test.assert_almost_equal(mrc.data[50:53, 120:123, 40],
expected,
decimal=4)
expected = numpy.array([[-0.0019, -0.0085, 0.0036],
[0.0781, 0.0279, -0.0365],
[0.0210, -0.0193, -0.0355]])
np_test.assert_almost_equal(mrc.data[150:153, 20:23, 60],
expected,
decimal=4)
np_test.assert_equal(mrc.dataType, 'float32')
np_test.assert_equal(mrc.data.dtype, numpy.dtype('float32'))
np_test.assert_equal(mrc.memmap, True)
# mrc with the opposite byte order
mrc2 = ImageIO()
mrc2.read(file=os.path.join(self.dir, "swapped_byte_order.mrc"))
expected = numpy.array([[0.000, 0.000], [-0.341, -6.702],
[0.782, -11.780], [0.327, -14.298],
[-0.691, -17.411], [-0.337, -18.076],
[-0.669, -19.157], [-0.799, -20.400],
[-0.793, -21.286], [-1.008, -21.386]])
np_test.assert_almost_equal(mrc2.data[:, :, 0], expected, decimal=3)
np_test.assert_equal(mrc2.memmap, False)
raised = False
try:
mrc2.read(file=os.path.join(self.dir, "swapped_byte_order.mrc"),
memmap=True)
except ValueError:
raised = True
np_test.assert_equal(raised, True)
np_test.assert_equal(mrc2.memmap, True)
# new style header mrc
mrc_new = ImageIO()
mrc_new.read(file=os.path.join(self.dir, 'new-head_int16.mrc'))
np_test.assert_equal(mrc_new.dataType, 'int16')
np_test.assert_equal(mrc_new.data.dtype, numpy.dtype('int16'))
np_test.assert_equal(mrc_new.byteOrder, '<')
np_test.assert_equal(mrc_new.arrayOrder, 'F')
np_test.assert_equal(mrc_new.shape, (40, 30, 20))
np_test.assert_equal(mrc_new.pixel, [0.4, 0.4, 0.4])
np_test.assert_equal(mrc_new.pixelsize, 0.4)
np_test.assert_equal(mrc_new.data[14, 8, 10], -14)
np_test.assert_equal(mrc_new.data[15, 23, 12], 10)
np_test.assert_equal(mrc_new.data[23, 29, 16], 2)
np_test.assert_equal(mrc_new.memmap, False)
# new style header mrc
mrc_new = ImageIO()
mrc_new.read(file=os.path.join(self.dir, 'new-head_int16.mrc'),
memmap=True)
np_test.assert_equal(mrc_new.dataType, 'int16')
np_test.assert_equal(mrc_new.data.dtype, numpy.dtype('int16'))
np_test.assert_equal(mrc_new.byteOrder, '<')
np_test.assert_equal(mrc_new.arrayOrder, 'F')
np_test.assert_equal(mrc_new.shape, (40, 30, 20))
np_test.assert_equal(mrc_new.pixel, [0.4, 0.4, 0.4])
np_test.assert_equal(mrc_new.pixelsize, 0.4)
np_test.assert_equal(mrc_new.data[14, 8, 10], -14)
np_test.assert_equal(mrc_new.data[15, 23, 12], 10)
np_test.assert_equal(mrc_new.data[23, 29, 16], 2)
np_test.assert_equal(mrc_new.memmap, True)
np_test.assert_equal(mrc_new.n_labels, 9)
np_test.assert_equal(len(mrc_new.labels), 9)
desired = (
b"COMBINEFFT: Combined FFT from two tomograms " +
b"07-Oct-13 17:15:24")
np_test.assert_equal(len(mrc_new.labels[3]), 80)
np_test.assert_equal(mrc_new.labels[3][:len(desired)], desired)
desired = (
b"NEWSTACK: Images copied 10-Oct-13 18:00:03"
)
np_test.assert_equal(len(mrc_new.labels[6]), 80)
np_test.assert_equal(mrc_new.labels[6][:len(desired)], desired)
# test raw file
raw = ImageIO()
raw.read(file=self.raw_file_name,
dataType=self.raw_dtype,
shape=self.raw_shape)
np_test.assert_equal(raw.data, self.raw_data)
np_test.assert_equal(raw.memmap, False)
# test raw file with memmap
raw = ImageIO()
raw.read(file=self.raw_file_name,
dataType=self.raw_dtype,
shape=self.raw_shape,
memmap=True)
np_test.assert_equal(raw.data, self.raw_data)
np_test.assert_equal(raw.memmap, True)
def testWrite(self):
"""
Tests write (and implicitly read), for em, mrc and raw format.
"""
# arrays
ar_uint8 = numpy.array([54, 200, 5, 7, 45, 123],
dtype='uint8').reshape((3, 1, 2))
ar_int8 = numpy.array([54, 2, -5, 7, 45, 123], dtype='uint8').reshape(
(3, 1, 2))
ar_uint16 = numpy.array([1034, 546, 248, 40000, 2345, 365, 4876, 563],
dtype='uint16').reshape((2, 2, 2))
ar_int16 = numpy.array([1034, 546, -248, 156, 2345, 365, -4876, 563],
dtype='int16').reshape((2, 2, 2))
ar_int32 = numpy.array(
[1034, 56546, -223448, 156, 2345, 2**31 - 10, -884876, 563],
dtype='int32').reshape((2, 2, 2))
ar_uint32 = numpy.array(
[1034, 56546, 223448, 156, 2345, 365, 884876, 2**32 - 10],
dtype='uint32').reshape((2, 2, 2))
ar_int8_2 = numpy.arange(24, dtype='int8').reshape((4, 3, 2))
ar_int16_2 = numpy.arange(24, dtype='int16').reshape((4, 3, 2))
ar2_int16 = numpy.array([1034, 546, -248, 156, 2345, 365, -4876, 563],
dtype='int16').reshape((2, 4))
ar_int16_f = numpy.array([1034, 546, -248, 156, 2345, 365, -4876, 563],
dtype='int16',
order='F').reshape((2, 2, 2))
ar_int16_c = numpy.array([1034, 546, -248, 156, 2345, 365, -4876, 563],
dtype='int16',
order='C').reshape((2, 2, 2))
# em uint8
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'), data=ar_uint8)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'uint8')
np_test.assert_equal(file_in.data, ar_uint8)
# em uint16
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'), data=ar_uint16)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'uint16')
np_test.assert_equal(file_in.data, ar_uint16)
# em int16 converted to int32, safe casting
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'),
data=ar_int16,
dataType='int32',
casting='safe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'int32')
np_test.assert_equal(file_in.data, ar_int16)
# em int16, safe casting
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.em'),
'data': ar_int16,
'casting': 'safe'
})
# em int16 converted to uint16, unsafe casting
file_out = ImageIO()
print("int16 to uint16")
file_out.write(file=os.path.join(self.dir, '_test.em'),
data=ar_int16,
dataType='uint16',
casting='unsafe')
print("int16 to uint16 end")
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'uint16')
np_test.assert_equal(file_in.data.dtype, numpy.dtype('uint16'))
np_test.assert_equal(file_in.data[0, 1, 0] == ar_int16[0, 1, 0], False)
# em int16 to uint16, safe casting
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.em'),
'data': ar_int16,
'dataType': 'uint16',
'casting': 'safe'
})
# em uint16 to int16, unsafe casting
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.em'),
'data': ar_uint16,
'dataType': 'int16',
'casting': 'unsafe'
})
# em uint32 to int32, safe casting
print("uint32 to int32 safe")
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.em'),
'data': ar_uint32,
'dataType': 'int32',
'casting': 'safe'
})
# em uint32 converted to int32, unsafe casting
print("uint32 to int32")
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'),
data=ar_uint32,
dataType='int32',
casting='unsafe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'int32')
#np_test.assert_equal(file_in.data, ar_uint32) should fail
np_test.assert_equal(file_in.data[0, 0, 0] == ar_uint32[0, 0, 0], True)
np_test.assert_equal(file_in.data[1, 1, 1] == ar_uint32[1, 1, 1],
False)
# em uint32 to float32, safe casting
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.em'),
'data': ar_uint32,
'dataType': 'float32',
'casting': 'safe'
})
# em uint32 to float32, unsafe casting
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'),
data=ar_uint32,
dataType='float32',
casting='unsafe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'float32')
#np_test.assert_almost_equal(file_in.data, ar_uint32) should fail
np_test.assert_equal(file_in.data[0, 0, 0] == ar_uint32[0, 0, 0], True)
np_test.assert_equal(file_in.data[1, 1, 1] == ar_uint32[1, 1, 1],
False)
# em int32 to float32, unsafe casting
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'),
data=ar_int32,
dataType='float32',
casting='unsafe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'float32')
#np_test.assert_almost_equal(file_in.data, ar_int32) should fail
np_test.assert_equal(file_in.data[0, 0, 0] == ar_int32[0, 0, 0], True)
np_test.assert_equal(file_in.data[1, 0, 1] == ar_int32[1, 0, 1], False)
# em int32 to float64, safe casting
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.em'),
data=ar_int32,
dataType='float64',
casting='safe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.em'))
np_test.assert_equal(file_in.dataType, 'float64')
np_test.assert_almost_equal(file_in.data, ar_int32)
# mrc data type and shape from args
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int8_2,
shape=(2, 3, 4),
dataType='int16')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.shape, (2, 3, 4))
# mrc data type and shape from previously given data
file_out = ImageIO()
file_out.setData(ar_int16_2)
file_out.write(file=os.path.join(self.dir, '_test.mrc'))
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.shape, (4, 3, 2))
# mrc data type and shape from attributes
file_out = ImageIO()
file_out.data = ar_int8_2
file_out.shape = (2, 3, 4)
file_out.dataType = 'int16'
file_out.write(file=os.path.join(self.dir, '_test.mrc'))
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.shape, (2, 3, 4))
# mrc data type and shape from data
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16_2)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.shape, (4, 3, 2))
# mrc uint8, same as ubyte
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'), data=ar_uint8)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'ubyte')
np_test.assert_almost_equal(file_in.data, ar_uint8)
# mrc uint16
file_out = ImageIO()
np_test.assert_raises((KeyError, TypeError), file_out.write, **{
'file': os.path.join(self.dir, '_test.mrc'),
'data': ar_uint16
})
# mrc uint16 to int16, safe casting
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.mrc'),
'data': ar_uint16,
'dataType': 'ubyte',
'casting': 'safe'
})
# mrc uint16 to int16, unsafe casting
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_uint16,
dataType='int16',
casting='unsafe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
#np_test.assert_almost_equal(file_in.data, ar_uint16) should fail
np_test.assert_equal(file_in.data[0, 0, 0] == ar_uint16[0, 0, 0], True)
np_test.assert_equal(file_in.data[0, 1, 1] == ar_uint16[0, 1, 1],
False)
# mrc int16
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16,
pixel=2.3)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.data, ar_int16)
np_test.assert_equal(file_in.pixel, [2.3, 2.3, 2.3])
np_test.assert_equal(file_in.pixelsize, 2.3)
# mrc int16 2D
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar2_int16,
pixel=3.4)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.data[:, :, 0], ar2_int16)
np_test.assert_equal(file_in.pixelsize, 3.4)
# mrc int8 to int16
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int8,
dataType='int16',
casting='safe')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.data, ar_int8)
# mrc int32
file_out = ImageIO()
np_test.assert_raises((KeyError, TypeError), file_out.write, **{
'file': os.path.join(self.dir, '_test.mrc'),
'data': ar_int32
})
# mrc int32 to int16
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.mrc'),
'data': ar_int32,
'dataType': 'int16',
'casting': 'safe'
})
# mrc int32 to float32
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.mrc'),
'data': ar_int32,
'dataType': 'float32',
'casting': 'safe'
})
# mrc int32 to complex64
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.mrc'),
'data': ar_int32,
'dataType': 'complex64',
'casting': 'safe'
})
# raw int16
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.raw'), data=ar_int16)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.raw'),
dataType='int16',
shape=(2, 2, 2))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.data, ar_int16)
# raw int8 to int16
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.raw'),
data=ar_int8,
dataType='int16')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.raw'),
dataType='int16',
shape=(3, 1, 2))
np_test.assert_equal(file_in.dataType, 'int16')
np_test.assert_equal(file_in.data, ar_int8)
# raw int16 to int8
file_out = ImageIO()
np_test.assert_raises(
TypeError, file_out.write, **{
'file': os.path.join(self.dir, '_test.raw'),
'data': ar_int16,
'dataType': 'int8',
'casting': 'safe'
})
# explain error messages printed before
print("It's fine if few error messages were printed just before " +
"this line, because they have been caught.")
# shape param
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16,
dataType='int16')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'),
dataType='int16')
np_test.assert_equal(file_in.data.shape, (2, 2, 2))
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16,
dataType='int16',
shape=(1, 4, 2))
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'),
dataType='int16')
np_test.assert_equal(file_in.data.shape, (1, 4, 2))
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16,
dataType='int16',
shape=(4, 2))
file_in.readHeader(file=os.path.join(self.dir, '_test.mrc'))
file_in.read(file=os.path.join(self.dir, '_test.mrc'),
dataType='int16')
np_test.assert_equal(file_in.data.shape, (4, 2, 1))
file_in.read(file=os.path.join(self.dir, '_test.mrc'),
dataType='int16',
shape=(2, 2, 2))
np_test.assert_equal(file_in.data.shape, (2, 2, 2))
# array order C, read write default (F)
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16_c)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.data, ar_int16_c)
# array order C, read write C
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16_c,
arrayOrder='C')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'), arrayOrder='C')
np_test.assert_equal(file_in.data, ar_int16_c)
# array order F, read write default (F)
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16_f)
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'))
np_test.assert_equal(file_in.data, ar_int16_f)
# array order F, read write F
file_out = ImageIO()
file_out.write(file=os.path.join(self.dir, '_test.mrc'),
data=ar_int16_f,
arrayOrder='F')
file_in = ImageIO()
file_in.read(file=os.path.join(self.dir, '_test.mrc'), arrayOrder='F')
np_test.assert_equal(file_in.data, ar_int16_f)
def sort(self,
seq=None,
out=None,
pad=0,
start=1,
fix_mode=None,
microscope=None,
limit=None,
limit_mode='std',
size=5,
byte_order=None,
test=False):
"""
Sorts series, fixes image headers, corrects the data and writes the
sorted and corrected images.
A series is sorted by tilt angle that is read from each image, or by the
elements of seq (not necessarily angles) if given.
Sorted images names have the form:
directory/name + number + extension.
where out is: directory/name_.extension and numbers start with start
argument and are padded with pad (argument) number of zeros (see
self.images).
Argument fix_mode detemines how the headers are fixed:
- None: no fixing
- 'polara_fei-tomo': for series obtained on polara with the FEI tomo
software
- 'krios_fei-tomo': for series obtained on krios with the FEI tomo
software
- 'cm300': for series from CM300
(see pyto.io.image_io).
If fix_mode is 'polara_fei-tomo', then arg microscope has to be
specified. The allowed values are specified in microscope_db.py.
Currently (r564) these are: 'polara-1_01-07', 'polara-1_01-09' and
'polara-2_01-09'.
If fix_mode is None, microscope does nor need to be specified.
Series recorded by SerialEM typically do not need fixing.
Works for individual images only (not for stacks).
Arguments:
- seq: if specified this sequence used for sorting projection,
otherwise tilt angles are used
- out: sorted series path
- pad: number of digits of a projection number in the sorted
series file name
- start: start number for sorted series file names
- fix_mode: determined if and how headers are fixed
- microscope: microscope type, see pyto.io.microscope_db
- limit_mode: 'abs' or 'std'
- limit: absolute greyscale limit if limit_mode is 'abs', or the
number of stds above and below the mean
- size: size of the subarray used to find the replacement value
for a voxel that's out of the limits
- byte_order: '<' or '>', default is the byte order of the machine
- test: if True all steps are done except writing corrected images
"""
# shortcuts
path = self.path
match_mode = self.mode
# make tilt angles - file names dictionary
# ToDo: use self.sortPath() instead
in_paths = []
images_iter = self.images(mode=match_mode)
if seq is None:
seq = []
# get tilt angles from file headers
for in_path in images_iter:
image = ImageIO(file=in_path)
image.readHeader()
seq.append(image.tiltAngle)
in_paths.append(in_path)
image.file_.close()
else:
# use seq to sort
for in_path in images_iter:
in_paths.append(in_path)
# sort (note: dictionary angle:in_path fails for multiple files with
# same angles)
seq_arr = numpy.array(seq)
in_paths_arr = numpy.array(in_paths)
sort_ind = seq_arr.argsort()
sorted_seq = seq_arr[sort_ind]
sorted_in_paths = in_paths_arr[sort_ind]
# parse out and make out directory if it doesn't exists
if out is not None:
out_dir, out_base_pat = os.path.split(out)
if out_dir == '':
out_dir = '.'
if (not test) and (not os.path.isdir(out_dir)):
os.makedirs(out_dir)
else:
out_path = None
# initialize file counter
if start is None:
ind = None
else:
ind = start - 1
# loop over sorted in files
for (item, in_path) in zip(sorted_seq, sorted_in_paths):
if ind is not None: ind += 1
# parse in file path
in_dir, in_base = os.path.split(in_path)
if out is not None:
# make out file path
out_base = self.convertBase(in_base=in_base,
out_base=out_base_pat,
pad=pad,
index=ind)
out_path = os.path.join(out_dir, out_base)
# read files
im_io = ImageIO(file=in_path)
im_io.read()
# log
logging.info("%5.1f: %s -> %s %6.1f %6.1f" \
% (item, in_path, out_path, \
im_io.data.mean(), im_io.data.std()))
# fix
if fix_mode is not None:
im_io.fix(mode=fix_mode, microscope=microscope)
# limit
if limit is not None:
image = Image(im_io.data)
image.limit(limit=limit, mode=limit_mode, size=size)
# write fixed file
if not test:
if byte_order is None:
byte_order = im_io.machineByteOrder
im_io.write(file=out_path,
byteOrder=byte_order,
data=image.data)
else:
logging.info(" " + str(item) + ": " + in_path + " -> "\
+ str(out_path))