def _spikes_diagnosis(self, signal_mask=None, navigation_mask=None): """Plots a histogram to help in choosing the threshold for spikes removal. Parameters ---------- signal_mask: boolean array Restricts the operation to the signal locations not marked as True (masked) navigation_mask: boolean array Restricts the operation to the navigation locations not marked as True (masked). See also -------- spikes_removal_tool """ self._check_signal_dimension_equals_one() dc = self.data if signal_mask is not None: dc = dc[..., ~signal_mask] if navigation_mask is not None: dc = dc[~navigation_mask, :] der = np.abs(np.diff(dc, 1, -1)) n = ((~navigation_mask).sum() if navigation_mask else self.axes_manager.navigation_size) # arbitrary cutoff for number of spectra necessary before histogram # data is compressed by finding maxima of each spectrum tmp = Signal(der) if n < 2000 else Signal(np.ravel(der.max(-1))) # get histogram signal using smart binning and plot tmph = tmp.get_histogram() tmph.plot() # Customize plot appearance plt.gca().set_title('') plt.gca().fill_between(tmph.axes_manager[0].axis, tmph.data, facecolor='#fddbc7', interpolate=True, color='none') ax = tmph._plot.signal_plot.ax axl = tmph._plot.signal_plot.ax_lines[0] axl.set_line_properties(color='#b2182b') plt.xlabel('Derivative magnitude') plt.ylabel('Log(Counts)') ax.set_yscale('log') ax.set_ylim(10**-1, plt.ylim()[1]) ax.set_xlim(plt.xlim()[0], 1.1 * plt.xlim()[1]) plt.draw()
def as_signal(self, field='values'): """Get a parameter map as a signal object. Please note that this method only works when the navigation dimension is greater than 0. Parameters ---------- field : {'values', 'std', 'is_set'} Raises ------ NavigationDimensionError : if the navigation dimension is 0 """ from hyperspy.signal import Signal if self._axes_manager.navigation_dimension == 0: raise NavigationDimensionError(0, '>0') s = Signal(data=self.map[field], axes=self._axes_manager._get_navigation_axes_dicts()) s.metadata.General.title = self.name for axis in s.axes_manager._axes: axis.navigate = False if self._number_of_elements > 1: s.axes_manager.append_axis(size=self._number_of_elements, name=self.name, navigate=True) return s
def reconstruct_object(flags, value): """ Reconstructs the value (if necessary) after having saved it in a dictionary """ if not isinstance(flags, list): flags = parse_flag_string(flags) if 'sig' in flags: if isinstance(value, dict): from hyperspy.signal import Signal value = Signal(**value) value._assign_subclass() return value if 'fn' in flags: ifdill, thing = value if ifdill is None: return thing if ifdill in [False, 'False']: return types.FunctionType(marshal.loads(thing), globals()) if ifdill in [True, 'True']: if not dill_avail: raise ValueError( "the dictionary was constructed using " "\"dill\" package, which is not available on the system") else: return dill.loads(thing) # should not be reached raise ValueError("The object format is not recognized") return value
def setUp(self): self.signal = Signal(np.arange(2 * 4 * 6).reshape(2, 4, 6)) self.signal.axes_manager[0].name = "x" self.signal.axes_manager[1].name = "y" self.signal.axes_manager[2].name = "E" self.signal.axes_manager[0].scale = 0.5 self.data = self.signal.data.copy()
def as_signal(self, field='values'): """Get a parameter map as a signal object. Please note that this method only works when the navigation dimension is greater than 0. Parameters ---------- field : {'values', 'std', 'is_set'} Raises ------ NavigationDimensionError : if the navigation dimension is 0 """ from hyperspy.signal import Signal if self._axes_manager.navigation_dimension == 0: raise NavigationDimensionError(0, '>0') s = Signal(data=self.map[field], axes=self._axes_manager._get_navigation_axes_dicts()) if self.component.active_is_multidimensional: s.data[np.logical_not(self.component._active_array)] = np.nan s.metadata.General.title = ("%s parameter" % self.name if self.component is None else "%s parameter of %s component" % (self.name, self.component.name)) for axis in s.axes_manager._axes: axis.navigate = False if self._number_of_elements > 1: s.axes_manager.append_axis(size=self._number_of_elements, name=self.name, navigate=True) return s
def test_signal_iterator(): s = Signal(np.arange(3).reshape((3, 1))) nt.assert_equal(s.next().data[0], 0) # If the following fails it can be because the iteration index was not # restarted for i, signal in enumerate(s): nt.assert_equal(i, signal.data[0])
def setUp(self): s = Signal(np.ones((3, 2, 5))) s.axes_manager[0].name = "x" s.axes_manager[1].name = "y" s.axes_manager[2].name = "E" s.axes_manager[2].scale = 0.5 s.metadata.General.title = 'test' self.signal = s
def test_signal_to_dictionary(self): tree = self.tree s = Signal([1., 2, 3]) s.axes_manager[0].name = 'x' s.axes_manager[0].units = 'ly' tree.set_item('Some name', s) d = tree.as_dictionary() nose.tools.assert_true(np.all(d['_sig_Some name']['data'] == s.data)) d['_sig_Some name']['data'] = 0 nose.tools.assert_equal( { "Node1": { "leaf11": 11, "Node11": { "leaf111": 111 }, }, "Node2": { "leaf21": 21, "Node21": { "leaf211": 211 }, }, "_sig_Some name": { 'axes': [{ 'name': 'x', 'navigate': False, 'offset': 0.0, 'scale': 1.0, 'size': 3, 'units': 'ly' }], 'data': 0, 'learning_results': {}, 'metadata': { 'General': { 'title': '' }, 'Signal': { 'binned': False, 'record_by': '', 'signal_origin': '', 'signal_type': '' }, '_HyperSpy': { 'Folding': { 'original_axes_manager': None, 'original_shape': None, 'unfolded': False, 'signal_unfolded': False } } }, 'original_metadata': {}, 'tmp_parameters': {} } }, d)
def setUp(self): self.signal = Signal(np.arange(5*10).reshape(5,10)) self.signal.axes_manager[0].name = "x" self.signal.axes_manager[1].name = "E" self.signal.axes_manager[0].scale = 0.5 self.signal.mapped_parameters.set_item('splitting.axis', 0) self.signal.mapped_parameters.set_item( 'splitting.step_sizes',[2,2]) self.data = self.signal.data.copy()
def test_general_type_not_working(self): s = self.s s.metadata.set_item('test', (Signal([1]), 0.1, 'test_string')) s.save('tmp.hdf5', overwrite=True) l = load('tmp.hdf5') nt.assert_is_instance(l.metadata.test, tuple) nt.assert_is_instance(l.metadata.test[0], Signal) nt.assert_is_instance(l.metadata.test[1], float) nt.assert_is_instance(l.metadata.test[2], unicode)
def setUp(self): s = Signal(np.empty((5, 5, 5))) s.save('tmp.hdf5', overwrite=True) self.shape = (10000, 10000, 100) del s f = h5py.File('tmp.hdf5', model='r+') s = f['Experiments/__unnamed__'] del s['data'] s.create_dataset('data', shape=self.shape, dtype='float64', chunks=True) f.close()
def test_add_signal_in_dictionary(self): tree = self.tree s = Signal([1., 2, 3]) s.axes_manager[0].name = 'x' s.axes_manager[0].units = 'ly' tree.add_dictionary({"_sig_signal name": s._to_dictionary()}) nose.tools.assert_is_instance(tree.signal_name, Signal) nose.tools.assert_true(np.all(tree.signal_name.data == s.data)) nose.tools.assert_equal(tree.signal_name.metadata.as_dictionary(), s.metadata.as_dictionary()) nose.tools.assert_equal( tree.signal_name.axes_manager._get_axes_dicts(), s.axes_manager._get_axes_dicts())
def __setattr__(self, key, value): if key.startswith('_sig_'): key = key[5:] from hyperspy.signal import Signal value = Signal(**value) slugified_key = str(slugify(key, valid_variable_name=True)) if isinstance(value, dict): if self.has_item(slugified_key): self.get_item(slugified_key).add_dictionary(value) return else: value = DictionaryTreeBrowser(value) super(DictionaryTreeBrowser, self).__setattr__( slugified_key, {'key': key, '_dtb_value_': value})
def setUp(self): self.s1 = Signal(np.arange(6))
def setUp(self): self.s1 = Signal(np.array((1, -1, 4, -3)))
def setUp(self): self.gaussian = Gaussian() self.gaussian._axes_manager = Signal(np.zeros((3, 3, 1))).axes_manager
def setUp(self): self.s1 = Signal(np.arange(10).reshape(2, 5)) self.s1.axes_manager.set_signal_dimension(2) self.s2 = Signal(np.arange(6))
def setUp(self): self.s = Signal(np.random.random((2,3,4)))
def setUp(self): self.s = Signal(np.arange(2))
def setUp(self): self.signal = Signal(np.arange(24).reshape((2, 3, 4))) self.data = self.signal.data.copy() self.signal.axes_manager._axes[0].navigate = False self.signal.axes_manager._axes[1].navigate = True self.signal.axes_manager._axes[2].navigate = False
def setUp(self): self.s = Signal(np.empty((2, 3, 4, 5))) self.s.axes_manager.set_signal_dimension(2)
def setUp(self): self.signal = Signal(np.arange(10)) self.signal.axes_manager[0].scale = 0.5 self.signal.axes_manager[0].offset = 0.25 self.data = self.signal.data.copy()
def setUp(self): self.signal = Signal(np.arange(10)) self.data = self.signal.data.copy()
def setUp(self): self.signal = Signal(np.arange(2**4).reshape((2, 2, 2, 2))) self.data = self.signal.data.copy()
def multi_kernel(ind, m_dic, values, optional_components, _args, result_q, test_dict): import hyperspy.api as hs from hyperspy.signal import Signal from multiprocessing import current_process from itertools import combinations, product # from collections import Iterable import numpy as np import copy from hyperspy.utils.model_selection import AICc import dill def generate_values_iterator(compnames, vals, turned_on_component_inds): turned_on_names = [compnames[i] for i in turned_on_component_inds] tmp = [] name_list = [] # TODO: put changing _position parameter of each component at the # beginning for _comp_n, _comp in vals.items(): if _comp_n in turned_on_names: for par_n, par in _comp.items(): if not isinstance(par, list): par = [ par, ] tmp.append(par) name_list.append((_comp_n, par_n)) return name_list, product(*tmp) def send_good_results(model, previous_switching, cur_p, result_q, ind): result = copy.deepcopy(model.as_dictionary()) for num, a_i_m in enumerate(previous_switching): result['components'][num]['active_is_multidimensional'] = a_i_m result['current'] = cur_p._identity result_q.put((ind, result, True)) test = dill.loads(test_dict) cur_p = current_process() previous_switching = [] comb = [] AICc_fraction = 0.99 comp_dict = m_dic['models']['z']['_dict']['components'] for num, comp in enumerate(comp_dict): previous_switching.append(comp['active_is_multidimensional']) comp['active_is_multidimensional'] = False for comp in optional_components: comp_dict[comp]['active'] = False for num in range(len(optional_components) + 1): for comp in combinations(optional_components, num): comb.append(comp) best_AICc, best_dof = np.inf, np.inf best_comb, best_values, best_names = None, None, None component_names = [c['name'] for c in comp_dict] sig = Signal(**m_dic) sig._assign_subclass() model = sig.models.z.restore() for combination in comb: # iterate all component combinations for component in combination: model[component].active = True on_comps = [i for i, _c in enumerate(model) if _c.active] name_list, iterator = generate_values_iterator(component_names, values, on_comps) ifgood = False for it in iterator: # iterate all parameter value combinations for (comp_n, par_n), val in zip(name_list, it): try: getattr(model[comp_n], par_n).value = val except: pass model.fit(**_args) # only perform iterations until we find a solution that we think is # good enough ifgood = test.test(model, (0, )) if ifgood: break if ifgood: # get model with best chisq here, and test model validation if len(comb) == 1: send_good_results(model, previous_switching, cur_p, result_q, ind) new_AICc = AICc(model) if new_AICc < AICc_fraction * best_AICc or \ (np.abs(new_AICc - best_AICc) <= np.abs(AICc_fraction * best_AICc) and len(model.p0) < best_dof): best_values = [ getattr(model[comp_n], par_n).value for comp_n, par_n in name_list ] best_names = name_list best_comb = combination best_AICc = new_AICc best_dof = len(model.p0) for component in combination: model[component].active = False # take the best overall combination of components and parameters: if best_comb is None: result_q.put((ind, None, False)) else: for component in best_comb: model[component].active = True for (comp_n, par_n), val in zip(best_names, best_values): try: getattr(model[comp_n], par_n).value = val except: pass model.fit(**_args) send_good_results(model, previous_switching, cur_p, result_q, ind)
def setUp(self): self.s = Signal(np.empty((2, 3, 4, 5))) self.s.axes_manager.set_signal_dimension(2) self.s.estimate_poissonian_noise_variance()
def setUp(self): self.s1 = Signal(np.arange(20).reshape(2, 2, 5)) self.s2 = Signal(np.arange(4).reshape(2, 2)) self.s2.axes_manager.set_signal_dimension(2) self.s3 = Signal(np.arange(4).reshape(1, 2, 2)) self.s3.axes_manager.set_signal_dimension(2)
def setUp(self): self.signal = Signal(np.arange(5 * 10).reshape(5, 10)) self.signal.axes_manager[0].name = "x" self.signal.axes_manager[1].name = "E" self.signal.axes_manager[0].scale = 0.5 self.data = self.signal.data.copy()
def setUp(self): self.s1 = Signal(np.arange(20).reshape(2, 2, 5)) self.s2 = Signal(np.arange(5)) self.s3 = Signal(np.arange(5).reshape(1, 5))
def setUp(self): self.s = Signal([0.1])