def _parse_sb_size(s, reference, sb_position, sb_size, parallel):
# Default value is 1/2 distance between sideband and central band
if sb_size is None:
if reference is None:
sb_size = s.estimate_sideband_size(sb_position, parallel=parallel)
else:
sb_size = reference.estimate_sideband_size(sb_position,
parallel=parallel)
else:
if not isinstance(sb_size, BaseSignal):
if isinstance(sb_size, (np.ndarray, daArray)) and sb_size.size > 1:
# transpose if np.array of multiple instances
sb_size = BaseSignal(sb_size).T
else:
sb_size = BaseSignal(sb_size)
if isinstance(sb_size.data, daArray):
sb_size = sb_size.as_lazy()
if sb_size.axes_manager.navigation_size != s.axes_manager.navigation_size:
if sb_size.axes_manager.navigation_size:
raise ValueError('Sideband size dimensions do not match '
'neither reference nor hologram dimensions.')
# sb_position navdim=0, therefore map function should not iterate:
else:
sb_size_temp = np.float64(sb_size.data)
else:
sb_size_temp = sb_size.deepcopy()
return sb_size, sb_size_temp
def _spikes_diagnosis(self, signal_mask=None, navigation_mask=None):
"""Plots a histogram to help in choosing the threshold for
spikes removal.
Parameters
----------
signal_mask: boolean array
Restricts the operation to the signal locations not marked
as True (masked)
navigation_mask: boolean array
Restricts the operation to the navigation locations not
marked as True (masked).
See also
--------
spikes_removal_tool
"""
self._check_signal_dimension_equals_one()
dc = self.data
if signal_mask is not None:
dc = dc[..., ~signal_mask]
if navigation_mask is not None:
dc = dc[~navigation_mask, :]
der = np.abs(np.diff(dc, 1, -1))
n = ((~navigation_mask).sum()
if navigation_mask else self.axes_manager.navigation_size)
# arbitrary cutoff for number of spectra necessary before histogram
# data is compressed by finding maxima of each spectrum
tmp = BaseSignal(der) if n < 2000 else BaseSignal(np.ravel(
der.max(-1)))
# get histogram signal using smart binning and plot
tmph = tmp.get_histogram()
tmph.plot()
# Customize plot appearance
plt.gca().set_title('')
plt.gca().fill_between(tmph.axes_manager[0].axis,
tmph.data,
facecolor='#fddbc7',
interpolate=True,
color='none')
ax = tmph._plot.signal_plot.ax
axl = tmph._plot.signal_plot.ax_lines[0]
axl.set_line_properties(color='#b2182b')
plt.xlabel('Derivative magnitude')
plt.ylabel('Log(Counts)')
ax.set_yscale('log')
ax.set_ylim(10**-1, plt.ylim()[1])
ax.set_xlim(plt.xlim()[0], 1.1 * plt.xlim()[1])
plt.draw()
def __setattr__(self, key, value):
if key in ['_double_lines', '_lazy_attributes']:
super().__setattr__(key, value)
return
if key == 'binned':
warnings.warn(
'Use of the `binned` attribute in metadata is '
'going to be deprecated in v2.0. Set the '
'`axis.is_binned` attribute instead. ',
VisibleDeprecationWarning)
if key.startswith('_sig_'):
key = key[5:]
from hyperspy.signal import BaseSignal
value = BaseSignal(**value)
slugified_key = str(slugify(key, valid_variable_name=True))
if isinstance(value, dict):
if slugified_key in self.__dict__.keys():
self.__dict__[slugified_key]['_dtb_value_'].add_dictionary(
value, double_lines=self._double_lines)
return
else:
value = DictionaryTreeBrowser(value,
double_lines=self._double_lines,
lazy=False)
super().__setattr__(slugified_key, {'key': key, '_dtb_value_': value})
def test_signal_to_dictionary(self, tree):
s = BaseSignal([1.0, 2, 3])
s.axes_manager[0].name = "x"
s.axes_manager[0].units = "ly"
tree.set_item("Some name", s)
d = tree.as_dictionary()
np.testing.assert_array_equal(d["_sig_Some name"]["data"], s.data)
d["_sig_Some name"]["data"] = 0
assert {
"Node1": {
"leaf11": 11,
"Node11": {
"leaf111": 111
},
},
"Node2": {
"leaf21": 21,
"Node21": {
"leaf211": 211
},
},
"_sig_Some name": {
"attributes": {
"_lazy": False,
"ragged": False
},
"axes": [{
"_type": "UniformDataAxis",
"name": "x",
"navigate": False,
"is_binned": False,
"offset": 0.0,
"scale": 1.0,
"size": 3,
"units": "ly",
}],
"data":
0,
"learning_results": {},
"metadata": {
"General": {
"title": ""
},
"Signal": {
"signal_type": ""
},
"_HyperSpy": {
"Folding": {
"original_axes_manager": None,
"original_shape": None,
"unfolded": False,
"signal_unfolded": False,
}
},
},
"original_metadata": {},
"tmp_parameters": {},
},
} == d
def test_axes_configuration_binning(tmp_path, file):
fname = tmp_path / file
s = BaseSignal(np.zeros((2, 2, 2)))
s.axes_manager.signal_axes[-1].is_binned = True
s.save(fname)
s = load(fname)
assert s.axes_manager.signal_axes[-1].is_binned
def test_compression_opts(self, tmp_path):
self.filename = tmp_path / 'testfile.zspy'
from numcodecs import Blosc
comp = Blosc(cname='zstd', clevel=1, shuffle=Blosc.SHUFFLE)
BaseSignal([1, 2, 3]).save(self.filename, compressor=comp)
f = zarr.open(self.filename.__str__(), mode='r+')
d = f['Experiments/__unnamed__/data']
assert (d.compressor == comp)\
def test_signal_to_dictionary(self):
tree = self.tree
s = BaseSignal([1., 2, 3])
s.axes_manager[0].name = 'x'
s.axes_manager[0].units = 'ly'
tree.set_item('Some name', s)
d = tree.as_dictionary()
np.testing.assert_array_equal(d['_sig_Some name']['data'], s.data)
d['_sig_Some name']['data'] = 0
assert ({
"Node1": {
"leaf11": 11,
"Node11": {
"leaf111": 111
},
},
"Node2": {
"leaf21": 21,
"Node21": {
"leaf211": 211
},
},
"_sig_Some name": {
'attributes': {
'_lazy': False
},
'axes': [{
'name': 'x',
'navigate': False,
'offset': 0.0,
'scale': 1.0,
'size': 3,
'units': 'ly'
}],
'data':
0,
'learning_results': {},
'metadata': {
'General': {
'title': ''
},
'Signal': {
'binned': False,
'signal_type': ''
},
'_HyperSpy': {
'Folding': {
'original_axes_manager': None,
'original_shape': None,
'unfolded': False,
'signal_unfolded': False
}
}
},
'original_metadata': {},
'tmp_parameters': {}
}
} == d)
def test_passing_compression_opts_saving(tmp_path):
filename = tmp_path / 'testfile.hdf5'
BaseSignal([1, 2, 3]).save(filename, compression_opts=8)
f = h5py.File(filename, mode='r+')
d = f['Experiments/__unnamed__/data']
assert d.compression_opts == 8
assert d.compression == 'gzip'
f.close()
def test_general_type_not_working(self):
s = self.s
s.metadata.set_item('test', (BaseSignal([1]), 0.1, 'test_string'))
s.save('tmp.hdf5', overwrite=True)
l = load('tmp.hdf5')
nt.assert_is_instance(l.metadata.test, tuple)
nt.assert_is_instance(l.metadata.test[0], Signal1D)
nt.assert_is_instance(l.metadata.test[1], float)
nt.assert_is_instance(l.metadata.test[2], str)
def test_hdf5_extension(tmpfilepath):
try:
hspy_extension = preferences.General.hspy_extension
preferences.General.hspy_extension = False
s = BaseSignal([0])
s.save(tmpfilepath)
assert os.path.exists(tmpfilepath + ".hdf5")
finally:
preferences.General.hspy_extension = hspy_extension
def setup_method(self, method):
s = BaseSignal(np.random.random((3, 2, 5)))
s.axes_manager.set_signal_dimension(1)
s.axes_manager[0].name = "x"
s.axes_manager[1].name = "y"
s.axes_manager[2].name = "E"
s.axes_manager[2].scale = 0.5
s.metadata.General.title = 'test'
self.signal = s
def test_general_type_not_working(self, tmpfilepath):
s = self.s
s.metadata.set_item('test', (BaseSignal([1]), 0.1, 'test_string'))
s.save(tmpfilepath)
l = load(tmpfilepath + ".hspy")
assert isinstance(l.metadata.test, tuple)
assert isinstance(l.metadata.test[0], Signal1D)
assert isinstance(l.metadata.test[1], float)
assert isinstance(l.metadata.test[2], str)
def test_lazy_metadata_arrays(tmpfilepath):
s = BaseSignal([1, 2, 3])
s.metadata.array = np.arange(10.)
s.save(tmpfilepath)
l = load(tmpfilepath + ".hspy", lazy=True)
# Can't deepcopy open hdf5 file handles
with pytest.raises(TypeError):
l.deepcopy()
del l
def test_general_type_not_working(self, tmp_path, file):
s = self.s
s.metadata.set_item('test', (BaseSignal([1]), 0.1, 'test_string'))
fname = tmp_path / file
s.save(fname)
l = load(fname)
assert isinstance(l.metadata.test, tuple)
assert isinstance(l.metadata.test[0], Signal1D)
assert isinstance(l.metadata.test[1], float)
assert isinstance(l.metadata.test[2], str)
def test_save_ragged_array(tmpfilepath):
a = np.array([0, 1])
b = np.array([0, 1, 2])
s = BaseSignal(np.array([a, b])).T
filename = os.path.join(tmpfilepath, "test_save_ragged_array.hspy")
s.save(filename)
s1 = load(filename)
for i in range(len(s.data)):
np.testing.assert_allclose(s.data[i], s1.data[i])
assert s.__class__ == s1.__class__
def test_save_ragged_array(tmp_path):
a = np.array([0, 1])
b = np.array([0, 1, 2])
s = BaseSignal(np.array([a, b], dtype=object)).T
fname = tmp_path / 'test_save_ragged_array.hspy'
s.save(fname)
s1 = load(fname)
for i in range(len(s.data)):
np.testing.assert_allclose(s.data[i], s1.data[i])
assert s.__class__ == s1.__class__
def test_axes_configuration(tmp_path, file):
fname = tmp_path / file
s = BaseSignal(np.zeros((2, 2, 2, 2, 2)))
s.axes_manager.signal_axes[0].navigate = True
s.axes_manager.signal_axes[0].navigate = True
s.save(fname, overwrite=True)
s = load(fname)
assert s.axes_manager.navigation_axes[0].index_in_array == 4
assert s.axes_manager.navigation_axes[1].index_in_array == 3
assert s.axes_manager.signal_dimension == 3
def test_add_signal_in_dictionary(self, tree):
s = BaseSignal([1.0, 2, 3])
s.axes_manager[0].name = "x"
s.axes_manager[0].units = "ly"
tree.add_dictionary({"_sig_signal name": s._to_dictionary()})
assert isinstance(tree.signal_name, BaseSignal)
np.testing.assert_array_equal(tree.signal_name.data, s.data)
assert tree.signal_name.metadata.as_dictionary(
) == s.metadata.as_dictionary()
assert (tree.signal_name.axes_manager._get_axes_dicts() ==
s.axes_manager._get_axes_dicts())
def test_add_signal_in_dictionary(self):
tree = self.tree
s = BaseSignal([1., 2, 3])
s.axes_manager[0].name = 'x'
s.axes_manager[0].units = 'ly'
tree.add_dictionary({"_sig_signal name": s._to_dictionary()})
assert isinstance(tree.signal_name, BaseSignal)
np.testing.assert_array_equal(tree.signal_name.data, s.data)
assert (tree.signal_name.metadata.as_dictionary() ==
s.metadata.as_dictionary())
assert (tree.signal_name.axes_manager._get_axes_dicts() ==
s.axes_manager._get_axes_dicts())
def test_save_ragged_dim2(tmp_path, file):
x = np.empty(5, dtype=object)
for i in range(1, 6):
x[i - 1] = np.array([list(range(i)), list(range(i))])
s = BaseSignal(x, ragged=True)
filename = tmp_path / file
s.save(filename)
s2 = load(filename)
for i, j in zip(s.data,s2.data):
np.testing.assert_array_equal(i,j)
def setup_method(self, method):
s = BaseSignal(np.empty((5, 5, 5)))
s.save('tmp.hdf5', overwrite=True)
self.shape = (10000, 10000, 100)
del s
f = h5py.File('tmp.hdf5', mode='r+')
s = f['Experiments/__unnamed__']
del s['data']
s.create_dataset('data',
shape=self.shape,
dtype='float64',
chunks=True)
f.close()
def test_two_peaks(self):
if self.s._lazy:
pytest.skip(
"Lazy Signals don't work properly with 0 dimension data")
s = self.s.deepcopy()
shifts = BaseSignal([1.0])
s.shift1D(shifts)
self.s = self.s.isig[10:] + s
width, left, right = self.s.estimate_peak_width(window=None,
return_interval=True)
assert np.isnan(width.data).all()
assert np.isnan(left.data).all()
assert np.isnan(right.data).all()
def test_lazy_loading(tmp_path):
s = BaseSignal(np.empty((5, 5, 5)))
fname = tmp_path / 'tmp.hdf5'
s.save(fname, overwrite=True)
shape = (10000, 10000, 100)
del s
f = h5py.File(fname, mode='r+')
s = f['Experiments/__unnamed__']
del s['data']
s.create_dataset('data', shape=shape, dtype='float64', chunks=True)
f.close()
s = load(fname, lazy=True)
assert shape == s.data.shape
assert isinstance(s.data, da.Array)
assert s._lazy
s.close_file()
def create_model(self, signal_dict, model_letter):
_logger.debug('Creating model in worker {}'.format(self.identity))
sig = BaseSignal(**signal_dict)
sig._assign_subclass()
self.model = sig.models[model_letter].restore()
self.model.signal.data = self.model.signal.data.copy()
for component in self.model:
component.active_is_multidimensional = False
component.active = True
for par in component.parameters:
par.map = par.map.copy()
var = self.model.signal.metadata.Signal.Noise_properties.variance
if isinstance(var, BaseSignal):
var.data = var.data.copy()
if self.model.low_loss is not None:
self.model.low_loss.data = self.model.low_loss.data.copy()
def as_signal(self, field='values'):
"""Get a parameter map as a signal object.
Please note that this method only works when the navigation
dimension is greater than 0.
Parameters
----------
field : {'values', 'std', 'is_set'}
Raises
------
NavigationDimensionError : if the navigation dimension is 0
"""
from hyperspy.signal import BaseSignal
s = BaseSignal(data=self.map[field],
axes=self._axes_manager._get_navigation_axes_dicts())
if self.component is not None and \
self.component.active_is_multidimensional:
s.data[np.logical_not(self.component._active_array)] = np.nan
s.metadata.General.title = ("%s parameter" % self.name
if self.component is None
else "%s parameter of %s component" %
(self.name, self.component.name))
for axis in s.axes_manager._axes:
axis.navigate = False
if self._number_of_elements > 1:
s.axes_manager._append_axis(
size=self._number_of_elements,
name=self.name,
navigate=True)
s._assign_subclass()
if field == "values":
# Add the variance if available
std = self.as_signal(field="std")
if not np.isnan(std.data).all():
std.data = std.data ** 2
std.metadata.General.title = "Variance"
s.metadata.set_item(
"Signal.Noise_properties.variance", std)
return s
def __setattr__(self, key, value):
if key.startswith('_sig_'):
key = key[5:]
from hyperspy.signal import BaseSignal
value = BaseSignal(**value)
slugified_key = str(slugify(key, valid_variable_name=True))
if isinstance(value, dict):
if self.has_item(slugified_key):
self.get_item(slugified_key).add_dictionary(
value, double_lines=self._double_lines)
return
else:
value = DictionaryTreeBrowser(value,
double_lines=self._double_lines)
super(DictionaryTreeBrowser, self).__setattr__(slugified_key, {
'key': key,
'_dtb_value_': value
})
def reconstruct_object(flags, value):
""" Reconstructs the value (if necessary) after having saved it in a
dictionary
"""
if not isinstance(flags, list):
flags = parse_flag_string(flags)
if 'sig' in flags:
if isinstance(value, dict):
from hyperspy.signal import BaseSignal
value = BaseSignal(**value)
value._assign_subclass()
return value
if 'fn' in flags:
ifdill, thing = value
if ifdill is None:
return thing
if ifdill in [True, 'True', b'True']:
return dill.loads(thing)
# should not be reached
raise ValueError("The object format is not recognized")
return value
def estimate_parameters(self, signal, x1, x2, only_current=False,
out=False):
"""Estimate the parameters for the power law component by the two area
method.
Parameters
----------
signal : Signal1D instance
x1 : float
Defines the left limit of the spectral range to use for the
estimation.
x2 : float
Defines the right limit of the spectral range to use for the
estimation.
only_current : bool
If False, estimates the parameters for the full dataset.
out : bool
If True, returns the result arrays directly without storing in the
parameter maps/values. The returned order is (A, r).
Returns
-------
{bool, tuple of values}
"""
super(PowerLaw, self)._estimate_parameters(signal)
axis = signal.axes_manager.signal_axes[0]
i1, i2 = axis.value_range_to_indices(x1, x2)
if not (i2 + i1) % 2 == 0:
i2 -= 1
if i2 == i1:
i2 += 2
i3 = (i2 + i1) // 2
x1 = axis.index2value(i1)
x2 = axis.index2value(i2)
x3 = axis.index2value(i3)
if only_current is True:
s = signal.get_current_signal()
else:
s = signal
if s._lazy:
import dask.array as da
log = da.log
I1 = s.isig[i1:i3].integrate1D(2j).data
I2 = s.isig[i3:i2].integrate1D(2j).data
else:
from hyperspy.signal import BaseSignal
shape = s.data.shape[:-1]
I1_s = BaseSignal(np.empty(shape, dtype='float'))
I2_s = BaseSignal(np.empty(shape, dtype='float'))
# Use the `out` parameters to avoid doing the deepcopy
s.isig[i1:i3].integrate1D(2j, out=I1_s)
s.isig[i3:i2].integrate1D(2j, out=I2_s)
I1 = I1_s.data
I2 = I2_s.data
log = np.log
with np.errstate(divide='raise'):
try:
r = 2 * log(I1 / I2) / log(x2 / x1)
k = 1 - r
A = k * I2 / (x2 ** k - x3 ** k)
if s._lazy:
r = r.map_blocks(np.nan_to_num)
A = A.map_blocks(np.nan_to_num)
else:
r = np.nan_to_num(r)
A = np.nan_to_num(A)
except (RuntimeWarning, FloatingPointError):
_logger.warning('Power law paramaters estimation failed '
'because of a "divide by zero" error.')
return False
if only_current is True:
self.r.value = r
self.A.value = A
return True
if out:
return A, r
else:
if self.A.map is None:
self._create_arrays()
self.A.map['values'][:] = A
self.A.map['is_set'][:] = True
self.r.map['values'][:] = r
self.r.map['is_set'][:] = True
self.fetch_stored_values()
return True
def test_crop_right(self):
s = self.s
shifts = BaseSignal([-0.1])
s.shift1D(shifts, crop=True)
np.testing.assert_allclose(s.axes_manager[0].axis,
np.arange(0., 1.8, 0.2))
def stack(signal_list,
axis=None,
new_axis_name='stack_element',
lazy=None,
**kwargs):
"""Concatenate the signals in the list over a given axis or a new axis.
The title is set to that of the first signal in the list.
Parameters
----------
signal_list : list of BaseSignal instances
axis : {None, int, str}
If None, the signals are stacked over a new axis. The data must
have the same dimensions. Otherwise the
signals are stacked over the axis given by its integer index or
its name. The data must have the same shape, except in the dimension
corresponding to `axis`.
new_axis_name : string
The name of the new axis when `axis` is None.
If an axis with this name already
exists it automatically append '-i', where `i` are integers,
until it finds a name that is not yet in use.
lazy: {bool, None}
Returns a LazySignal if True. If None, only returns lazy rezult if at
least one is lazy.
Returns
-------
signal : BaseSignal instance (or subclass, determined by the objects in
signal list)
Examples
--------
>>> data = np.arange(20)
>>> s = hs.stack([hs.signals.Signal1D(data[:10]),
... hs.signals.Signal1D(data[10:])])
>>> s
<Signal1D, title: Stack of , dimensions: (2, 10)>
>>> s.data
array([[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
[10, 11, 12, 13, 14, 15, 16, 17, 18, 19]])
"""
from hyperspy.signals import BaseSignal
import dask.array as da
from numbers import Number
# TODO: remove next time
deprecated = ['mmap', 'mmap_dir']
warn_str = "'{}' argument is deprecated, please use 'lazy' instead"
for k in deprecated:
if k in kwargs:
lazy = True
warnings.warn(warn_str.format(k), VisibleDeprecationWarning)
axis_input = copy.deepcopy(axis)
signal_list = list(signal_list)
# Get the real signal with the most axes to get metadata/class/etc
# first = sorted(filter(lambda _s: isinstance(_s, BaseSignal), signal_list),
# key=lambda _s: _s.data.ndim)[-1]
first = next(filter(lambda _s: isinstance(_s, BaseSignal), signal_list))
# Cast numbers as signals. Will broadcast later
for i, _s in enumerate(signal_list):
if isinstance(_s, BaseSignal):
pass
elif isinstance(_s, Number):
sig = BaseSignal(_s)
signal_list[i] = sig
else:
raise ValueError("{} type cannot be stacked.".format(type(_s)))
if lazy is None:
lazy = any(_s._lazy for _s in signal_list)
if not isinstance(lazy, bool):
raise ValueError("'lazy' argument has to be None, True or False")
# Cast all as lazy if required
for i, _s in enumerate(signal_list):
if not _s._lazy:
signal_list[i] = _s.as_lazy()
if len(signal_list) > 1:
newlist = broadcast_signals(*signal_list, ignore_axis=axis_input)
if axis is not None:
step_sizes = [s.axes_manager[axis].size for s in newlist]
axis = newlist[0].axes_manager[axis]
datalist = [s.data for s in newlist]
newdata = da.stack(datalist, axis=0) if axis is None else \
da.concatenate(datalist, axis=axis.index_in_array)
if axis_input is None:
signal = first.__class__(newdata)
signal._lazy = True
signal._assign_subclass()
signal.axes_manager._axes[1:] = copy.deepcopy(
newlist[0].axes_manager._axes)
axis_name = new_axis_name
axis_names = [
axis_.name for axis_ in signal.axes_manager._axes[1:]
]
j = 1
while axis_name in axis_names:
axis_name = new_axis_name + "_%i" % j
j += 1
eaxis = signal.axes_manager._axes[0]
eaxis.name = axis_name
eaxis.navigate = True # This triggers _update_parameters
signal.metadata = copy.deepcopy(first.metadata)
# Get the title from 1st object
signal.metadata.General.title = ("Stack of " +
first.metadata.General.title)
signal.original_metadata = DictionaryTreeBrowser({})
else:
signal = newlist[0]._deepcopy_with_new_data(newdata)
signal._lazy = True
signal._assign_subclass()
signal.get_dimensions_from_data()
signal.original_metadata.add_node('stack_elements')
for i, obj in enumerate(signal_list):
signal.original_metadata.stack_elements.add_node('element%i' % i)
node = signal.original_metadata.stack_elements['element%i' % i]
node.original_metadata = \
obj.original_metadata.as_dictionary()
node.metadata = \
obj.metadata.as_dictionary()
if axis_input is None:
axis_input = signal.axes_manager[-1 + 1j].index_in_axes_manager
step_sizes = 1
signal.metadata._HyperSpy.set_item('Stacking_history.axis', axis_input)
signal.metadata._HyperSpy.set_item('Stacking_history.step_sizes',
step_sizes)
if np.all([
s.metadata.has_item('Signal.Noise_properties.variance')
for s in signal_list
]):
variance = stack([
s.metadata.Signal.Noise_properties.variance
for s in signal_list
], axis)
signal.metadata.set_item('Signal.Noise_properties.variance',
variance)
else:
signal = signal_list[0]
# Leave as lazy or compute
if lazy:
signal = signal.as_lazy()
else:
signal.compute(False)
return signal
