def _parse_sb_size(s, reference, sb_position, sb_size, parallel): # Default value is 1/2 distance between sideband and central band if sb_size is None: if reference is None: sb_size = s.estimate_sideband_size(sb_position, parallel=parallel) else: sb_size = reference.estimate_sideband_size(sb_position, parallel=parallel) else: if not isinstance(sb_size, BaseSignal): if isinstance(sb_size, (np.ndarray, daArray)) and sb_size.size > 1: # transpose if np.array of multiple instances sb_size = BaseSignal(sb_size).T else: sb_size = BaseSignal(sb_size) if isinstance(sb_size.data, daArray): sb_size = sb_size.as_lazy() if sb_size.axes_manager.navigation_size != s.axes_manager.navigation_size: if sb_size.axes_manager.navigation_size: raise ValueError('Sideband size dimensions do not match ' 'neither reference nor hologram dimensions.') # sb_position navdim=0, therefore map function should not iterate: else: sb_size_temp = np.float64(sb_size.data) else: sb_size_temp = sb_size.deepcopy() return sb_size, sb_size_temp
def _spikes_diagnosis(self, signal_mask=None, navigation_mask=None): """Plots a histogram to help in choosing the threshold for spikes removal. Parameters ---------- signal_mask: boolean array Restricts the operation to the signal locations not marked as True (masked) navigation_mask: boolean array Restricts the operation to the navigation locations not marked as True (masked). See also -------- spikes_removal_tool """ self._check_signal_dimension_equals_one() dc = self.data if signal_mask is not None: dc = dc[..., ~signal_mask] if navigation_mask is not None: dc = dc[~navigation_mask, :] der = np.abs(np.diff(dc, 1, -1)) n = ((~navigation_mask).sum() if navigation_mask else self.axes_manager.navigation_size) # arbitrary cutoff for number of spectra necessary before histogram # data is compressed by finding maxima of each spectrum tmp = BaseSignal(der) if n < 2000 else BaseSignal(np.ravel( der.max(-1))) # get histogram signal using smart binning and plot tmph = tmp.get_histogram() tmph.plot() # Customize plot appearance plt.gca().set_title('') plt.gca().fill_between(tmph.axes_manager[0].axis, tmph.data, facecolor='#fddbc7', interpolate=True, color='none') ax = tmph._plot.signal_plot.ax axl = tmph._plot.signal_plot.ax_lines[0] axl.set_line_properties(color='#b2182b') plt.xlabel('Derivative magnitude') plt.ylabel('Log(Counts)') ax.set_yscale('log') ax.set_ylim(10**-1, plt.ylim()[1]) ax.set_xlim(plt.xlim()[0], 1.1 * plt.xlim()[1]) plt.draw()
def __setattr__(self, key, value): if key in ['_double_lines', '_lazy_attributes']: super().__setattr__(key, value) return if key == 'binned': warnings.warn( 'Use of the `binned` attribute in metadata is ' 'going to be deprecated in v2.0. Set the ' '`axis.is_binned` attribute instead. ', VisibleDeprecationWarning) if key.startswith('_sig_'): key = key[5:] from hyperspy.signal import BaseSignal value = BaseSignal(**value) slugified_key = str(slugify(key, valid_variable_name=True)) if isinstance(value, dict): if slugified_key in self.__dict__.keys(): self.__dict__[slugified_key]['_dtb_value_'].add_dictionary( value, double_lines=self._double_lines) return else: value = DictionaryTreeBrowser(value, double_lines=self._double_lines, lazy=False) super().__setattr__(slugified_key, {'key': key, '_dtb_value_': value})
def test_signal_to_dictionary(self, tree): s = BaseSignal([1.0, 2, 3]) s.axes_manager[0].name = "x" s.axes_manager[0].units = "ly" tree.set_item("Some name", s) d = tree.as_dictionary() np.testing.assert_array_equal(d["_sig_Some name"]["data"], s.data) d["_sig_Some name"]["data"] = 0 assert { "Node1": { "leaf11": 11, "Node11": { "leaf111": 111 }, }, "Node2": { "leaf21": 21, "Node21": { "leaf211": 211 }, }, "_sig_Some name": { "attributes": { "_lazy": False, "ragged": False }, "axes": [{ "_type": "UniformDataAxis", "name": "x", "navigate": False, "is_binned": False, "offset": 0.0, "scale": 1.0, "size": 3, "units": "ly", }], "data": 0, "learning_results": {}, "metadata": { "General": { "title": "" }, "Signal": { "signal_type": "" }, "_HyperSpy": { "Folding": { "original_axes_manager": None, "original_shape": None, "unfolded": False, "signal_unfolded": False, } }, }, "original_metadata": {}, "tmp_parameters": {}, }, } == d
def test_axes_configuration_binning(tmp_path, file): fname = tmp_path / file s = BaseSignal(np.zeros((2, 2, 2))) s.axes_manager.signal_axes[-1].is_binned = True s.save(fname) s = load(fname) assert s.axes_manager.signal_axes[-1].is_binned
def test_compression_opts(self, tmp_path): self.filename = tmp_path / 'testfile.zspy' from numcodecs import Blosc comp = Blosc(cname='zstd', clevel=1, shuffle=Blosc.SHUFFLE) BaseSignal([1, 2, 3]).save(self.filename, compressor=comp) f = zarr.open(self.filename.__str__(), mode='r+') d = f['Experiments/__unnamed__/data'] assert (d.compressor == comp)\
def test_signal_to_dictionary(self): tree = self.tree s = BaseSignal([1., 2, 3]) s.axes_manager[0].name = 'x' s.axes_manager[0].units = 'ly' tree.set_item('Some name', s) d = tree.as_dictionary() np.testing.assert_array_equal(d['_sig_Some name']['data'], s.data) d['_sig_Some name']['data'] = 0 assert ({ "Node1": { "leaf11": 11, "Node11": { "leaf111": 111 }, }, "Node2": { "leaf21": 21, "Node21": { "leaf211": 211 }, }, "_sig_Some name": { 'attributes': { '_lazy': False }, 'axes': [{ 'name': 'x', 'navigate': False, 'offset': 0.0, 'scale': 1.0, 'size': 3, 'units': 'ly' }], 'data': 0, 'learning_results': {}, 'metadata': { 'General': { 'title': '' }, 'Signal': { 'binned': False, 'signal_type': '' }, '_HyperSpy': { 'Folding': { 'original_axes_manager': None, 'original_shape': None, 'unfolded': False, 'signal_unfolded': False } } }, 'original_metadata': {}, 'tmp_parameters': {} } } == d)
def test_passing_compression_opts_saving(tmp_path): filename = tmp_path / 'testfile.hdf5' BaseSignal([1, 2, 3]).save(filename, compression_opts=8) f = h5py.File(filename, mode='r+') d = f['Experiments/__unnamed__/data'] assert d.compression_opts == 8 assert d.compression == 'gzip' f.close()
def test_general_type_not_working(self): s = self.s s.metadata.set_item('test', (BaseSignal([1]), 0.1, 'test_string')) s.save('tmp.hdf5', overwrite=True) l = load('tmp.hdf5') nt.assert_is_instance(l.metadata.test, tuple) nt.assert_is_instance(l.metadata.test[0], Signal1D) nt.assert_is_instance(l.metadata.test[1], float) nt.assert_is_instance(l.metadata.test[2], str)
def test_hdf5_extension(tmpfilepath): try: hspy_extension = preferences.General.hspy_extension preferences.General.hspy_extension = False s = BaseSignal([0]) s.save(tmpfilepath) assert os.path.exists(tmpfilepath + ".hdf5") finally: preferences.General.hspy_extension = hspy_extension
def setup_method(self, method): s = BaseSignal(np.random.random((3, 2, 5))) s.axes_manager.set_signal_dimension(1) s.axes_manager[0].name = "x" s.axes_manager[1].name = "y" s.axes_manager[2].name = "E" s.axes_manager[2].scale = 0.5 s.metadata.General.title = 'test' self.signal = s
def test_general_type_not_working(self, tmpfilepath): s = self.s s.metadata.set_item('test', (BaseSignal([1]), 0.1, 'test_string')) s.save(tmpfilepath) l = load(tmpfilepath + ".hspy") assert isinstance(l.metadata.test, tuple) assert isinstance(l.metadata.test[0], Signal1D) assert isinstance(l.metadata.test[1], float) assert isinstance(l.metadata.test[2], str)
def test_lazy_metadata_arrays(tmpfilepath): s = BaseSignal([1, 2, 3]) s.metadata.array = np.arange(10.) s.save(tmpfilepath) l = load(tmpfilepath + ".hspy", lazy=True) # Can't deepcopy open hdf5 file handles with pytest.raises(TypeError): l.deepcopy() del l
def test_general_type_not_working(self, tmp_path, file): s = self.s s.metadata.set_item('test', (BaseSignal([1]), 0.1, 'test_string')) fname = tmp_path / file s.save(fname) l = load(fname) assert isinstance(l.metadata.test, tuple) assert isinstance(l.metadata.test[0], Signal1D) assert isinstance(l.metadata.test[1], float) assert isinstance(l.metadata.test[2], str)
def test_save_ragged_array(tmpfilepath): a = np.array([0, 1]) b = np.array([0, 1, 2]) s = BaseSignal(np.array([a, b])).T filename = os.path.join(tmpfilepath, "test_save_ragged_array.hspy") s.save(filename) s1 = load(filename) for i in range(len(s.data)): np.testing.assert_allclose(s.data[i], s1.data[i]) assert s.__class__ == s1.__class__
def test_save_ragged_array(tmp_path): a = np.array([0, 1]) b = np.array([0, 1, 2]) s = BaseSignal(np.array([a, b], dtype=object)).T fname = tmp_path / 'test_save_ragged_array.hspy' s.save(fname) s1 = load(fname) for i in range(len(s.data)): np.testing.assert_allclose(s.data[i], s1.data[i]) assert s.__class__ == s1.__class__
def test_axes_configuration(tmp_path, file): fname = tmp_path / file s = BaseSignal(np.zeros((2, 2, 2, 2, 2))) s.axes_manager.signal_axes[0].navigate = True s.axes_manager.signal_axes[0].navigate = True s.save(fname, overwrite=True) s = load(fname) assert s.axes_manager.navigation_axes[0].index_in_array == 4 assert s.axes_manager.navigation_axes[1].index_in_array == 3 assert s.axes_manager.signal_dimension == 3
def test_add_signal_in_dictionary(self, tree): s = BaseSignal([1.0, 2, 3]) s.axes_manager[0].name = "x" s.axes_manager[0].units = "ly" tree.add_dictionary({"_sig_signal name": s._to_dictionary()}) assert isinstance(tree.signal_name, BaseSignal) np.testing.assert_array_equal(tree.signal_name.data, s.data) assert tree.signal_name.metadata.as_dictionary( ) == s.metadata.as_dictionary() assert (tree.signal_name.axes_manager._get_axes_dicts() == s.axes_manager._get_axes_dicts())
def test_add_signal_in_dictionary(self): tree = self.tree s = BaseSignal([1., 2, 3]) s.axes_manager[0].name = 'x' s.axes_manager[0].units = 'ly' tree.add_dictionary({"_sig_signal name": s._to_dictionary()}) assert isinstance(tree.signal_name, BaseSignal) np.testing.assert_array_equal(tree.signal_name.data, s.data) assert (tree.signal_name.metadata.as_dictionary() == s.metadata.as_dictionary()) assert (tree.signal_name.axes_manager._get_axes_dicts() == s.axes_manager._get_axes_dicts())
def test_save_ragged_dim2(tmp_path, file): x = np.empty(5, dtype=object) for i in range(1, 6): x[i - 1] = np.array([list(range(i)), list(range(i))]) s = BaseSignal(x, ragged=True) filename = tmp_path / file s.save(filename) s2 = load(filename) for i, j in zip(s.data,s2.data): np.testing.assert_array_equal(i,j)
def setup_method(self, method): s = BaseSignal(np.empty((5, 5, 5))) s.save('tmp.hdf5', overwrite=True) self.shape = (10000, 10000, 100) del s f = h5py.File('tmp.hdf5', mode='r+') s = f['Experiments/__unnamed__'] del s['data'] s.create_dataset('data', shape=self.shape, dtype='float64', chunks=True) f.close()
def test_two_peaks(self): if self.s._lazy: pytest.skip( "Lazy Signals don't work properly with 0 dimension data") s = self.s.deepcopy() shifts = BaseSignal([1.0]) s.shift1D(shifts) self.s = self.s.isig[10:] + s width, left, right = self.s.estimate_peak_width(window=None, return_interval=True) assert np.isnan(width.data).all() assert np.isnan(left.data).all() assert np.isnan(right.data).all()
def test_lazy_loading(tmp_path): s = BaseSignal(np.empty((5, 5, 5))) fname = tmp_path / 'tmp.hdf5' s.save(fname, overwrite=True) shape = (10000, 10000, 100) del s f = h5py.File(fname, mode='r+') s = f['Experiments/__unnamed__'] del s['data'] s.create_dataset('data', shape=shape, dtype='float64', chunks=True) f.close() s = load(fname, lazy=True) assert shape == s.data.shape assert isinstance(s.data, da.Array) assert s._lazy s.close_file()
def create_model(self, signal_dict, model_letter): _logger.debug('Creating model in worker {}'.format(self.identity)) sig = BaseSignal(**signal_dict) sig._assign_subclass() self.model = sig.models[model_letter].restore() self.model.signal.data = self.model.signal.data.copy() for component in self.model: component.active_is_multidimensional = False component.active = True for par in component.parameters: par.map = par.map.copy() var = self.model.signal.metadata.Signal.Noise_properties.variance if isinstance(var, BaseSignal): var.data = var.data.copy() if self.model.low_loss is not None: self.model.low_loss.data = self.model.low_loss.data.copy()
def as_signal(self, field='values'): """Get a parameter map as a signal object. Please note that this method only works when the navigation dimension is greater than 0. Parameters ---------- field : {'values', 'std', 'is_set'} Raises ------ NavigationDimensionError : if the navigation dimension is 0 """ from hyperspy.signal import BaseSignal s = BaseSignal(data=self.map[field], axes=self._axes_manager._get_navigation_axes_dicts()) if self.component is not None and \ self.component.active_is_multidimensional: s.data[np.logical_not(self.component._active_array)] = np.nan s.metadata.General.title = ("%s parameter" % self.name if self.component is None else "%s parameter of %s component" % (self.name, self.component.name)) for axis in s.axes_manager._axes: axis.navigate = False if self._number_of_elements > 1: s.axes_manager._append_axis( size=self._number_of_elements, name=self.name, navigate=True) s._assign_subclass() if field == "values": # Add the variance if available std = self.as_signal(field="std") if not np.isnan(std.data).all(): std.data = std.data ** 2 std.metadata.General.title = "Variance" s.metadata.set_item( "Signal.Noise_properties.variance", std) return s
def __setattr__(self, key, value): if key.startswith('_sig_'): key = key[5:] from hyperspy.signal import BaseSignal value = BaseSignal(**value) slugified_key = str(slugify(key, valid_variable_name=True)) if isinstance(value, dict): if self.has_item(slugified_key): self.get_item(slugified_key).add_dictionary( value, double_lines=self._double_lines) return else: value = DictionaryTreeBrowser(value, double_lines=self._double_lines) super(DictionaryTreeBrowser, self).__setattr__(slugified_key, { 'key': key, '_dtb_value_': value })
def reconstruct_object(flags, value): """ Reconstructs the value (if necessary) after having saved it in a dictionary """ if not isinstance(flags, list): flags = parse_flag_string(flags) if 'sig' in flags: if isinstance(value, dict): from hyperspy.signal import BaseSignal value = BaseSignal(**value) value._assign_subclass() return value if 'fn' in flags: ifdill, thing = value if ifdill is None: return thing if ifdill in [True, 'True', b'True']: return dill.loads(thing) # should not be reached raise ValueError("The object format is not recognized") return value
def estimate_parameters(self, signal, x1, x2, only_current=False, out=False): """Estimate the parameters for the power law component by the two area method. Parameters ---------- signal : Signal1D instance x1 : float Defines the left limit of the spectral range to use for the estimation. x2 : float Defines the right limit of the spectral range to use for the estimation. only_current : bool If False, estimates the parameters for the full dataset. out : bool If True, returns the result arrays directly without storing in the parameter maps/values. The returned order is (A, r). Returns ------- {bool, tuple of values} """ super(PowerLaw, self)._estimate_parameters(signal) axis = signal.axes_manager.signal_axes[0] i1, i2 = axis.value_range_to_indices(x1, x2) if not (i2 + i1) % 2 == 0: i2 -= 1 if i2 == i1: i2 += 2 i3 = (i2 + i1) // 2 x1 = axis.index2value(i1) x2 = axis.index2value(i2) x3 = axis.index2value(i3) if only_current is True: s = signal.get_current_signal() else: s = signal if s._lazy: import dask.array as da log = da.log I1 = s.isig[i1:i3].integrate1D(2j).data I2 = s.isig[i3:i2].integrate1D(2j).data else: from hyperspy.signal import BaseSignal shape = s.data.shape[:-1] I1_s = BaseSignal(np.empty(shape, dtype='float')) I2_s = BaseSignal(np.empty(shape, dtype='float')) # Use the `out` parameters to avoid doing the deepcopy s.isig[i1:i3].integrate1D(2j, out=I1_s) s.isig[i3:i2].integrate1D(2j, out=I2_s) I1 = I1_s.data I2 = I2_s.data log = np.log with np.errstate(divide='raise'): try: r = 2 * log(I1 / I2) / log(x2 / x1) k = 1 - r A = k * I2 / (x2 ** k - x3 ** k) if s._lazy: r = r.map_blocks(np.nan_to_num) A = A.map_blocks(np.nan_to_num) else: r = np.nan_to_num(r) A = np.nan_to_num(A) except (RuntimeWarning, FloatingPointError): _logger.warning('Power law paramaters estimation failed ' 'because of a "divide by zero" error.') return False if only_current is True: self.r.value = r self.A.value = A return True if out: return A, r else: if self.A.map is None: self._create_arrays() self.A.map['values'][:] = A self.A.map['is_set'][:] = True self.r.map['values'][:] = r self.r.map['is_set'][:] = True self.fetch_stored_values() return True
def test_crop_right(self): s = self.s shifts = BaseSignal([-0.1]) s.shift1D(shifts, crop=True) np.testing.assert_allclose(s.axes_manager[0].axis, np.arange(0., 1.8, 0.2))
def stack(signal_list, axis=None, new_axis_name='stack_element', lazy=None, **kwargs): """Concatenate the signals in the list over a given axis or a new axis. The title is set to that of the first signal in the list. Parameters ---------- signal_list : list of BaseSignal instances axis : {None, int, str} If None, the signals are stacked over a new axis. The data must have the same dimensions. Otherwise the signals are stacked over the axis given by its integer index or its name. The data must have the same shape, except in the dimension corresponding to `axis`. new_axis_name : string The name of the new axis when `axis` is None. If an axis with this name already exists it automatically append '-i', where `i` are integers, until it finds a name that is not yet in use. lazy: {bool, None} Returns a LazySignal if True. If None, only returns lazy rezult if at least one is lazy. Returns ------- signal : BaseSignal instance (or subclass, determined by the objects in signal list) Examples -------- >>> data = np.arange(20) >>> s = hs.stack([hs.signals.Signal1D(data[:10]), ... hs.signals.Signal1D(data[10:])]) >>> s <Signal1D, title: Stack of , dimensions: (2, 10)> >>> s.data array([[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]]) """ from hyperspy.signals import BaseSignal import dask.array as da from numbers import Number # TODO: remove next time deprecated = ['mmap', 'mmap_dir'] warn_str = "'{}' argument is deprecated, please use 'lazy' instead" for k in deprecated: if k in kwargs: lazy = True warnings.warn(warn_str.format(k), VisibleDeprecationWarning) axis_input = copy.deepcopy(axis) signal_list = list(signal_list) # Get the real signal with the most axes to get metadata/class/etc # first = sorted(filter(lambda _s: isinstance(_s, BaseSignal), signal_list), # key=lambda _s: _s.data.ndim)[-1] first = next(filter(lambda _s: isinstance(_s, BaseSignal), signal_list)) # Cast numbers as signals. Will broadcast later for i, _s in enumerate(signal_list): if isinstance(_s, BaseSignal): pass elif isinstance(_s, Number): sig = BaseSignal(_s) signal_list[i] = sig else: raise ValueError("{} type cannot be stacked.".format(type(_s))) if lazy is None: lazy = any(_s._lazy for _s in signal_list) if not isinstance(lazy, bool): raise ValueError("'lazy' argument has to be None, True or False") # Cast all as lazy if required for i, _s in enumerate(signal_list): if not _s._lazy: signal_list[i] = _s.as_lazy() if len(signal_list) > 1: newlist = broadcast_signals(*signal_list, ignore_axis=axis_input) if axis is not None: step_sizes = [s.axes_manager[axis].size for s in newlist] axis = newlist[0].axes_manager[axis] datalist = [s.data for s in newlist] newdata = da.stack(datalist, axis=0) if axis is None else \ da.concatenate(datalist, axis=axis.index_in_array) if axis_input is None: signal = first.__class__(newdata) signal._lazy = True signal._assign_subclass() signal.axes_manager._axes[1:] = copy.deepcopy( newlist[0].axes_manager._axes) axis_name = new_axis_name axis_names = [ axis_.name for axis_ in signal.axes_manager._axes[1:] ] j = 1 while axis_name in axis_names: axis_name = new_axis_name + "_%i" % j j += 1 eaxis = signal.axes_manager._axes[0] eaxis.name = axis_name eaxis.navigate = True # This triggers _update_parameters signal.metadata = copy.deepcopy(first.metadata) # Get the title from 1st object signal.metadata.General.title = ("Stack of " + first.metadata.General.title) signal.original_metadata = DictionaryTreeBrowser({}) else: signal = newlist[0]._deepcopy_with_new_data(newdata) signal._lazy = True signal._assign_subclass() signal.get_dimensions_from_data() signal.original_metadata.add_node('stack_elements') for i, obj in enumerate(signal_list): signal.original_metadata.stack_elements.add_node('element%i' % i) node = signal.original_metadata.stack_elements['element%i' % i] node.original_metadata = \ obj.original_metadata.as_dictionary() node.metadata = \ obj.metadata.as_dictionary() if axis_input is None: axis_input = signal.axes_manager[-1 + 1j].index_in_axes_manager step_sizes = 1 signal.metadata._HyperSpy.set_item('Stacking_history.axis', axis_input) signal.metadata._HyperSpy.set_item('Stacking_history.step_sizes', step_sizes) if np.all([ s.metadata.has_item('Signal.Noise_properties.variance') for s in signal_list ]): variance = stack([ s.metadata.Signal.Noise_properties.variance for s in signal_list ], axis) signal.metadata.set_item('Signal.Noise_properties.variance', variance) else: signal = signal_list[0] # Leave as lazy or compute if lazy: signal = signal.as_lazy() else: signal.compute(False) return signal