def test_load_save_cycle(self, save_path_nordif):
s = load(PATTERN_FILE)
scan_time_string = s.original_metadata["nordif_header"][80][10:18]
scan_time = time.strptime(scan_time_string, "%H:%M:%S")
scan_time = datetime.timedelta(
hours=scan_time.tm_hour,
minutes=scan_time.tm_min,
seconds=scan_time.tm_sec,
).total_seconds()
assert (s.metadata.Acquisition_instrument.SEM.Detector.EBSD.scan_time
== scan_time)
assert s.metadata.General.title == "Pattern"
s.save(save_path_nordif, overwrite=True)
with pytest.warns(UserWarning): # No background pattern in directory
s_reload = load(save_path_nordif, setting_file=SETTING_FILE)
assert np.allclose(s.data, s_reload.data)
# Add static background and change filename to make metadata equal
s_reload.metadata.Acquisition_instrument.SEM.Detector.EBSD.static_background = imread(
BG_FILE)
s_reload.metadata.General.original_filename = (
s.metadata.General.original_filename)
s_reload.metadata.General.title = s.metadata.General.title
np.testing.assert_equal(s_reload.metadata.as_dictionary(),
s.metadata.as_dictionary())
# Delete reference to close np.memmap file
del s_reload
def test_save_load_1d_nav(self, save_path_hdf5):
"""Save-load cycle of signals with one navigation dimension."""
desired_shape = (3, 60, 60)
desired_nav_extent = (0, 3)
s = nickel_ebsd_small()
# One column of patterns
s_y_only = s.inav[0]
s_y_only.save(save_path_hdf5)
s_y_only2 = load(save_path_hdf5)
assert s_y_only2.data.shape == desired_shape
assert s_y_only2.axes_manager.navigation_axes[0].name == "y"
assert s_y_only2.axes_manager.navigation_extent == desired_nav_extent
# One row of patterns
s_x_only = s.inav[:, 0]
s_x_only.save(save_path_hdf5, overwrite=True)
s_x_only2 = load(save_path_hdf5)
assert s_x_only2.data.shape == desired_shape
assert s_x_only2.axes_manager.navigation_axes[0].name == "x"
assert s_x_only2.axes_manager.navigation_extent == desired_nav_extent
# Maintain axis name
s_y_only2.axes_manager["y"].name = "x"
s_y_only2.save(save_path_hdf5, overwrite=True)
s_x_only3 = load(save_path_hdf5)
assert s_x_only3.data.shape == desired_shape
assert s_x_only3.axes_manager.navigation_axes[0].name == "x"
assert s_x_only3.axes_manager.navigation_extent == desired_nav_extent
def test_load_multiple(self, scan_group_names):
if scan_group_names == KIKUCHIPY_FILE_GROUP_NAMES + ["Scan 3"]:
with pytest.warns(UserWarning,
match="Scan 'Scan 3' is not among "):
s1, s2 = load(KIKUCHIPY_FILE,
scan_group_names=scan_group_names)
elif scan_group_names == ["Scan 3"]:
with pytest.raises(OSError,
match="Scan 'Scan 3' is not among the"):
_ = load(KIKUCHIPY_FILE, scan_group_names=scan_group_names)
return 0
elif scan_group_names == KIKUCHIPY_FILE_GROUP_NAMES:
s1, s2 = load(KIKUCHIPY_FILE,
scan_group_names=KIKUCHIPY_FILE_GROUP_NAMES)
else: # scan_group_names == "Scan 2"
s2 = load(KIKUCHIPY_FILE, scan_group_names=scan_group_names)
assert s2.metadata.General.title == "patterns Scan 2"
s1 = load(KIKUCHIPY_FILE)
assert np.allclose(s1.data, s2.data)
with pytest.raises(
AssertionError,
match="\nItems are not equal:\nkey='title'\nkey='General'\n\n "
"ACTUAL: 'patterns My awes0m4 ...'\n DESIRED: 'patterns Scan 2'",
):
np.testing.assert_equal(s1.metadata.as_dictionary(),
s2.metadata.as_dictionary())
s2.metadata.General.title = s1.metadata.General.title
np.testing.assert_equal(s1.metadata.as_dictionary(),
s2.metadata.as_dictionary())
def test_load_inplace(self, lazy):
if lazy:
with pytest.raises(ValueError):
_ = load(PATTERN_FILE, lazy=lazy, mmap_mode="r+")
else:
s = load(PATTERN_FILE, lazy=lazy, mmap_mode="r+")
assert s.axes_manager.as_dictionary() == AXES_MANAGER
def test_load_save_cycle(self, save_path_hdf5, remove_phases):
s = load(KIKUCHIPY_FILE)
# Check that metadata is read correctly
assert s.metadata.Acquisition_instrument.SEM.Detector.EBSD.xpc == -5.64
assert s.metadata.General.title == "patterns My awes0m4 ..."
if remove_phases:
del s.metadata.Sample.Phases
s.save(save_path_hdf5, overwrite=True)
s_reload = load(save_path_hdf5)
np.testing.assert_equal(s.data, s_reload.data)
# Change data set name and package version to make metadata equal, and
# redo deleting of phases
s_reload.metadata.General.title = s.metadata.General.title
ebsd_node = metadata_nodes("ebsd")
s_reload.metadata.set_item(
ebsd_node + ".version", s.metadata.get_item(ebsd_node + ".version")
)
if remove_phases:
s.metadata.Sample.set_item(
"Phases", s_reload.metadata.Sample.Phases
)
np.testing.assert_equal(
s_reload.metadata.as_dictionary(), s.metadata.as_dictionary()
)
def test_load_save_lazy(self, save_path_hdf5):
s = load(KIKUCHIPY_FILE, lazy=True)
assert isinstance(s.data, da.Array)
s.save(save_path_hdf5)
s_reload = load(save_path_hdf5, lazy=True)
assert s.data.shape == s_reload.data.shape
with pytest.raises(OSError, match="Cannot write to an already open"):
s_reload.save(save_path_hdf5, add_scan=True, scan_number=2)
def test_load_save_lazy(self, save_path_nordif):
s = load(PATTERN_FILE, lazy=True)
assert isinstance(s.data, da.Array)
s.save(save_path_nordif, overwrite=True)
with pytest.warns(UserWarning): # No background pattern in directory
s_reload = load(
save_path_nordif, lazy=True, setting_file=SETTING_FILE
)
assert s.data.shape == s_reload.data.shape
def test_load_with_padding(self, save_path_hdf5, lazy):
s = load(KIKUCHIPY_FILE)
s.save(save_path_hdf5)
new_n_columns = 4
with File(save_path_hdf5, mode="r+") as f:
f["Scan 1/EBSD/Header/n_columns"][()] = new_n_columns
with pytest.warns(UserWarning, match="Will attempt to load by zero"):
s_reload = load(save_path_hdf5, lazy=lazy)
AXES_MANAGER["axis-1"]["size"] = new_n_columns
assert_dictionary(s_reload.axes_manager.as_dictionary(), AXES_MANAGER)
def test_save_to_existing_file(self, save_path_hdf5):
s = load(KIKUCHIPY_FILE)
s.save(save_path_hdf5)
with pytest.warns(UserWarning, match="Your terminal does not"):
s.save(save_path_hdf5, scan_number=2)
with pytest.raises(ValueError, match="overwrite parameter can"):
s.save(save_path_hdf5,
scan_number=2,
overwrite="False",
add_scan=False)
s.save(save_path_hdf5, scan_number=2, overwrite=False, add_scan=False)
with pytest.raises(OSError, match="Scan 'Scan 2' is not among the"):
_ = load(save_path_hdf5, scan_group_names="Scan 2")
def test_load_bruker(self):
with File(BRUKER_FILE, mode="r+") as f:
grid = f["Scan 0/EBSD/Header/Grid Type"]
grid[()] = "hexagonal".encode()
with pytest.raises(IOError, match="Only square grids are"):
_ = load(BRUKER_FILE)
with File(BRUKER_FILE, mode="r+") as f:
grid = f["Scan 0/EBSD/Header/Grid Type"]
grid[()] = "isometric".encode()
s = load(BRUKER_FILE)
assert s.data.shape == (3, 3, 60, 60)
assert_dictionary(s.axes_manager.as_dictionary(), AXES_MANAGER)
def test_load_edax(self):
with File(EDAX_FILE, mode="r+") as f:
grid = f["Scan 1/EBSD/Header/Grid Type"]
grid[()] = "HexGrid".encode()
with pytest.raises(IOError, match="Only square grids are"):
_ = load(EDAX_FILE)
with File(EDAX_FILE, mode="r+") as f:
grid = f["Scan 1/EBSD/Header/Grid Type"]
grid[()] = "SqrGrid".encode()
s = load(EDAX_FILE)
assert s.data.shape == (3, 3, 60, 60)
assert_dictionary(s.axes_manager.as_dictionary(), AXES_MANAGER)
def test_load(self, filename):
if filename == "im_not_here.h5":
with pytest.raises(IOError, match="No filename matches"):
_ = load(filename)
else:
s = load(KIKUCHIPY_FILE)
with tempfile.TemporaryDirectory() as tmp:
file_path = os.path.join(tmp, "supported.h5")
s.save(file_path)
new_file_path = os.path.join(tmp, filename)
os.rename(file_path, new_file_path)
with pytest.raises(IOError, match="Could not read"):
_ = load(new_file_path)
gc.collect()
def test_read_patterns(self, save_path_hdf5):
s = EBSD((255 * np.random.rand(10, 3, 5, 5)).astype(np.uint8))
s.save(save_path_hdf5)
with File(save_path_hdf5, mode="r+") as f:
del f["Scan 1/EBSD/Data/patterns"]
with pytest.raises(KeyError, match="Could not find patterns"):
_ = load(save_path_hdf5)
def test_read_cutoff(self, save_path_nordif):
scan_size = (10, 3)
scan_size_reloaded = (10, 20)
pattern_size = (5, 5)
data_shape = scan_size + pattern_size
s = EBSD((255 * np.random.rand(*data_shape)).astype(np.uint8))
s.save(save_path_nordif, overwrite=True)
# Reload data but with a scan_size bigger than available file bytes,
# so that the data has to be padded
with pytest.warns(UserWarning): # No background or setting files
s_reload = load(
save_path_nordif,
scan_size=scan_size_reloaded[::-1],
pattern_size=pattern_size,
)
# To check if the data padding works as expected, the original data is
# padded and compared to the reloaded data
cut_data = s.data.flatten()
pw = [
(
0,
(scan_size_reloaded[1] - scan_size[1])
* scan_size[0]
* np.prod(pattern_size),
)
]
cut_data = np.pad(cut_data, pw, mode="constant")
cut_data = cut_data.reshape(scan_size_reloaded + pattern_size)
assert np.allclose(cut_data, s_reload.data)
def test_file_reader(self):
s = load(EMSOFT_FILE)
axes_manager = setup_axes_manager(["energy", "height", "width"])
assert s.data.shape == (11, 13, 13)
assert s.axes_manager.as_dictionary() == axes_manager
assert_dictionary(s.metadata.as_dictionary(), METADATA)
def test_save_load_0d_nav(self, save_path_hdf5):
"""Save-load cycle of a signal with no navigation dimension."""
s = nickel_ebsd_small()
s0 = s.inav[0, 0]
s0.save(save_path_hdf5)
s1 = load(save_path_hdf5)
assert s1.data.shape == (60, 60)
assert s1.axes_manager.navigation_axes == ()
def test_file_reader(self):
"""Test correct data shape, axes properties and metadata."""
s = load(EMSOFT_FILE)
assert isinstance(s.xmap, CrystalMap)
assert s.data.shape == (10, 10, 10)
assert s.axes_manager["dx"].scale == 70
assert s.axes_manager["dx"].units == "um"
assert s.axes_manager["x"].units == "px"
def test_write_data_single(self, save_path_nordif):
pattern_size = (5, 5)
s = EBSD((255 * np.random.rand(*pattern_size)).astype(np.uint8))
s.save(save_path_nordif, overwrite=True)
with pytest.warns(UserWarning): # No background or setting files
s_reload = load(
save_path_nordif, scan_size=1, pattern_size=pattern_size
)
assert np.allclose(s.data, s_reload.data)
def test_load_readonly(self):
s = load(KIKUCHIPY_FILE, lazy=True)
k = next(
filter(
lambda x: isinstance(x, str) and x.startswith("array-original"
),
s.data.dask.keys(),
))
mm = s.data.dask[k]
assert isinstance(mm, Dataset)
def test_read_lazily_no_chunks(self):
# First, make sure the data image dataset is not actually chunked
f = File(KIKUCHIPY_FILE_NO_CHUNKS, mode="r")
data_dset = f["Scan 1/EBSD/Data/patterns"]
assert data_dset.chunks is None
f.close()
# Then, make sure it can be read correctly
s = load(KIKUCHIPY_FILE_NO_CHUNKS, lazy=True)
assert s.data.chunks == ((60, ), (60, ))
def test_save_extensions(self, extension):
s = load(KIKUCHIPY_FILE)
with tempfile.TemporaryDirectory() as tmp:
file_path = os.path.join(tmp, "supported" + extension)
if extension == "":
s.save(file_path)
assert os.path.isfile(file_path + ".h5") is True
else: # extension == '.h4'
with pytest.raises(ValueError, match="'h4' does not"):
s.save(file_path)
gc.collect()
def test_save_fresh(self, save_path_nordif):
scan_size = (10, 3)
pattern_size = (5, 5)
data_shape = scan_size + pattern_size
s = EBSD((255 * np.random.rand(*data_shape)).astype(np.uint8))
s.save(save_path_nordif, overwrite=True)
with pytest.warns(UserWarning): # No background or setting files
s_reload = load(save_path_nordif,
scan_size=scan_size[::-1],
pattern_size=pattern_size)
assert np.allclose(s.data, s_reload.data)
def test_load(self, setting_file):
s = load(PATTERN_FILE, setting_file=SETTING_FILE)
assert s.data.shape == (3, 3, 60, 60)
assert s.axes_manager.as_dictionary() == AXES_MANAGER
static_bg = imread(BG_FILE)
assert np.allclose(
s.metadata.Acquisition_instrument.SEM.Detector.EBSD.static_background,
static_bg,
)
def test_projection_lambert(self):
s = load(
EMSOFT_FILE,
projection="lambert",
hemisphere="both",
)
axes_manager = setup_axes_manager()
assert s.data.shape == (2, 11, 13, 13)
assert s.axes_manager.as_dictionary() == axes_manager
def test_load_readonly(self):
s = load(PATTERN_FILE, lazy=True)
k = next(
filter(
lambda x: isinstance(x, str) and x.startswith("array-original"
),
s.data.dask.keys(),
))
mm = s.data.dask[k]
assert isinstance(mm, np.memmap)
assert not mm.flags["WRITEABLE"]
def test_scan_size(self, scan_size):
"""Scan size parameter works as expected."""
s = load(EMSOFT_FILE, scan_size=scan_size)
sy, sx = (10, 10)
if isinstance(scan_size, int):
desired_shape = (scan_size, sy, sx)
else:
desired_shape = scan_size + (sy, sx)
assert s.data.shape == desired_shape
def test_load_parameters(self, nav_shape, sig_shape):
if nav_shape is None and sig_shape is None:
with pytest.raises(ValueError):
_ = load(
PATTERN_FILE,
setting_file="Setting.txt",
scan_size=nav_shape,
pattern_size=sig_shape,
)
else:
if sum(nav_shape + sig_shape) > 126:
# Check if zero padding user warning is raised if sum of data
# shape is bigger than file size
with pytest.warns(UserWarning):
s = load(PATTERN_FILE,
scan_size=nav_shape,
pattern_size=sig_shape)
else:
s = load(PATTERN_FILE,
scan_size=nav_shape,
pattern_size=sig_shape)
assert s.data.shape == nav_shape[::-1] + sig_shape
def test_load_manufacturer(self, save_path_hdf5):
s = EBSD((255 * np.random.rand(10, 3, 5, 5)).astype(np.uint8))
s.save(save_path_hdf5)
# Change manufacturer
with File(save_path_hdf5, mode="r+") as f:
manufacturer = f["manufacturer"]
manufacturer[()] = "Nope".encode()
with pytest.raises(
OSError,
match="Manufacturer Nope not among recognised manufacturers"):
_ = load(save_path_hdf5)
def test_save_multiple(self, save_path_hdf5, scan_number):
s1, s2 = load(KIKUCHIPY_FILE,
scan_group_names=KIKUCHIPY_FILE_GROUP_NAMES)
s1.save(save_path_hdf5)
error = "Invalid scan number"
with pytest.raises(OSError, match=error), pytest.warns(UserWarning):
s2.save(save_path_hdf5, add_scan=True)
if scan_number == 1:
with pytest.raises(OSError,
match=error), pytest.warns(UserWarning):
s2.save(save_path_hdf5, add_scan=True, scan_number=scan_number)
else:
s2.save(save_path_hdf5, add_scan=True, scan_number=scan_number)
def test_load_lazy(self, projection):
"""The Lambert projection's southern hemisphere is stored
chunked.
"""
s = load(EMSOFT_FILE,
projection=projection,
hemisphere="south",
lazy=True)
assert isinstance(s, LazyEBSDMasterPattern)
s.compute()
assert isinstance(s, EBSDMasterPattern)