def _spikes_diagnosis(self, signal_mask=None, navigation_mask=None):
"""Plots a histogram to help in choosing the threshold for
spikes removal.
Parameters
----------
signal_mask: boolean array
Restricts the operation to the signal locations not marked
as True (masked)
navigation_mask: boolean array
Restricts the operation to the navigation locations not
marked as True (masked).
See also
--------
spikes_removal_tool
"""
self._check_signal_dimension_equals_one()
dc = self.data
if signal_mask is not None:
dc = dc[..., ~signal_mask]
if navigation_mask is not None:
dc = dc[~navigation_mask, :]
der = np.abs(np.diff(dc, 1, -1))
n = ((~navigation_mask).sum()
if navigation_mask else self.axes_manager.navigation_size)
# arbitrary cutoff for number of spectra necessary before histogram
# data is compressed by finding maxima of each spectrum
tmp = Signal(der) if n < 2000 else Signal(np.ravel(der.max(-1)))
# get histogram signal using smart binning and plot
tmph = tmp.get_histogram()
tmph.plot()
# Customize plot appearance
plt.gca().set_title('')
plt.gca().fill_between(tmph.axes_manager[0].axis,
tmph.data,
facecolor='#fddbc7',
interpolate=True,
color='none')
ax = tmph._plot.signal_plot.ax
axl = tmph._plot.signal_plot.ax_lines[0]
axl.set_line_properties(color='#b2182b')
plt.xlabel('Derivative magnitude')
plt.ylabel('Log(Counts)')
ax.set_yscale('log')
ax.set_ylim(10**-1, plt.ylim()[1])
ax.set_xlim(plt.xlim()[0], 1.1 * plt.xlim()[1])
plt.draw()
def as_signal(self, field='values'):
"""Get a parameter map as a signal object.
Please note that this method only works when the navigation
dimension is greater than 0.
Parameters
----------
field : {'values', 'std', 'is_set'}
Raises
------
NavigationDimensionError : if the navigation dimension is 0
"""
from hyperspy.signal import Signal
if self._axes_manager.navigation_dimension == 0:
raise NavigationDimensionError(0, '>0')
s = Signal(data=self.map[field],
axes=self._axes_manager._get_navigation_axes_dicts())
s.metadata.General.title = self.name
for axis in s.axes_manager._axes:
axis.navigate = False
if self._number_of_elements > 1:
s.axes_manager.append_axis(size=self._number_of_elements,
name=self.name,
navigate=True)
return s
def reconstruct_object(flags, value):
""" Reconstructs the value (if necessary) after having saved it in a
dictionary
"""
if not isinstance(flags, list):
flags = parse_flag_string(flags)
if 'sig' in flags:
if isinstance(value, dict):
from hyperspy.signal import Signal
value = Signal(**value)
value._assign_subclass()
return value
if 'fn' in flags:
ifdill, thing = value
if ifdill is None:
return thing
if ifdill in [False, 'False']:
return types.FunctionType(marshal.loads(thing), globals())
if ifdill in [True, 'True']:
if not dill_avail:
raise ValueError(
"the dictionary was constructed using "
"\"dill\" package, which is not available on the system")
else:
return dill.loads(thing)
# should not be reached
raise ValueError("The object format is not recognized")
return value
def setUp(self):
self.signal = Signal(np.arange(2 * 4 * 6).reshape(2, 4, 6))
self.signal.axes_manager[0].name = "x"
self.signal.axes_manager[1].name = "y"
self.signal.axes_manager[2].name = "E"
self.signal.axes_manager[0].scale = 0.5
self.data = self.signal.data.copy()
def as_signal(self, field='values'):
"""Get a parameter map as a signal object.
Please note that this method only works when the navigation
dimension is greater than 0.
Parameters
----------
field : {'values', 'std', 'is_set'}
Raises
------
NavigationDimensionError : if the navigation dimension is 0
"""
from hyperspy.signal import Signal
if self._axes_manager.navigation_dimension == 0:
raise NavigationDimensionError(0, '>0')
s = Signal(data=self.map[field],
axes=self._axes_manager._get_navigation_axes_dicts())
if self.component.active_is_multidimensional:
s.data[np.logical_not(self.component._active_array)] = np.nan
s.metadata.General.title = ("%s parameter" %
self.name if self.component is None else
"%s parameter of %s component" %
(self.name, self.component.name))
for axis in s.axes_manager._axes:
axis.navigate = False
if self._number_of_elements > 1:
s.axes_manager.append_axis(size=self._number_of_elements,
name=self.name,
navigate=True)
return s
def test_signal_iterator():
s = Signal(np.arange(3).reshape((3, 1)))
nt.assert_equal(s.next().data[0], 0)
# If the following fails it can be because the iteration index was not
# restarted
for i, signal in enumerate(s):
nt.assert_equal(i, signal.data[0])
def setUp(self):
s = Signal(np.ones((3, 2, 5)))
s.axes_manager[0].name = "x"
s.axes_manager[1].name = "y"
s.axes_manager[2].name = "E"
s.axes_manager[2].scale = 0.5
s.metadata.General.title = 'test'
self.signal = s
def test_signal_to_dictionary(self):
tree = self.tree
s = Signal([1., 2, 3])
s.axes_manager[0].name = 'x'
s.axes_manager[0].units = 'ly'
tree.set_item('Some name', s)
d = tree.as_dictionary()
nose.tools.assert_true(np.all(d['_sig_Some name']['data'] == s.data))
d['_sig_Some name']['data'] = 0
nose.tools.assert_equal(
{
"Node1": {
"leaf11": 11,
"Node11": {
"leaf111": 111
},
},
"Node2": {
"leaf21": 21,
"Node21": {
"leaf211": 211
},
},
"_sig_Some name": {
'axes': [{
'name': 'x',
'navigate': False,
'offset': 0.0,
'scale': 1.0,
'size': 3,
'units': 'ly'
}],
'data':
0,
'learning_results': {},
'metadata': {
'General': {
'title': ''
},
'Signal': {
'binned': False,
'record_by': '',
'signal_origin': '',
'signal_type': ''
},
'_HyperSpy': {
'Folding': {
'original_axes_manager': None,
'original_shape': None,
'unfolded': False,
'signal_unfolded': False
}
}
},
'original_metadata': {},
'tmp_parameters': {}
}
}, d)
def setUp(self):
self.signal = Signal(np.arange(5*10).reshape(5,10))
self.signal.axes_manager[0].name = "x"
self.signal.axes_manager[1].name = "E"
self.signal.axes_manager[0].scale = 0.5
self.signal.mapped_parameters.set_item('splitting.axis', 0)
self.signal.mapped_parameters.set_item(
'splitting.step_sizes',[2,2])
self.data = self.signal.data.copy()
def test_general_type_not_working(self):
s = self.s
s.metadata.set_item('test', (Signal([1]), 0.1, 'test_string'))
s.save('tmp.hdf5', overwrite=True)
l = load('tmp.hdf5')
nt.assert_is_instance(l.metadata.test, tuple)
nt.assert_is_instance(l.metadata.test[0], Signal)
nt.assert_is_instance(l.metadata.test[1], float)
nt.assert_is_instance(l.metadata.test[2], unicode)
def setUp(self):
s = Signal(np.empty((5, 5, 5)))
s.save('tmp.hdf5', overwrite=True)
self.shape = (10000, 10000, 100)
del s
f = h5py.File('tmp.hdf5', model='r+')
s = f['Experiments/__unnamed__']
del s['data']
s.create_dataset('data',
shape=self.shape,
dtype='float64',
chunks=True)
f.close()
def test_add_signal_in_dictionary(self):
tree = self.tree
s = Signal([1., 2, 3])
s.axes_manager[0].name = 'x'
s.axes_manager[0].units = 'ly'
tree.add_dictionary({"_sig_signal name": s._to_dictionary()})
nose.tools.assert_is_instance(tree.signal_name, Signal)
nose.tools.assert_true(np.all(tree.signal_name.data == s.data))
nose.tools.assert_equal(tree.signal_name.metadata.as_dictionary(),
s.metadata.as_dictionary())
nose.tools.assert_equal(
tree.signal_name.axes_manager._get_axes_dicts(),
s.axes_manager._get_axes_dicts())
def __setattr__(self, key, value):
if key.startswith('_sig_'):
key = key[5:]
from hyperspy.signal import Signal
value = Signal(**value)
slugified_key = str(slugify(key, valid_variable_name=True))
if isinstance(value, dict):
if self.has_item(slugified_key):
self.get_item(slugified_key).add_dictionary(value)
return
else:
value = DictionaryTreeBrowser(value)
super(DictionaryTreeBrowser, self).__setattr__(
slugified_key,
{'key': key, '_dtb_value_': value})
def setUp(self):
self.s1 = Signal(np.arange(6))
def setUp(self):
self.s1 = Signal(np.array((1, -1, 4, -3)))
def setUp(self):
self.gaussian = Gaussian()
self.gaussian._axes_manager = Signal(np.zeros((3, 3, 1))).axes_manager
def setUp(self):
self.s1 = Signal(np.arange(10).reshape(2, 5))
self.s1.axes_manager.set_signal_dimension(2)
self.s2 = Signal(np.arange(6))
def setUp(self):
self.s = Signal(np.random.random((2,3,4)))
def setUp(self):
self.s = Signal(np.arange(2))
def setUp(self):
self.signal = Signal(np.arange(24).reshape((2, 3, 4)))
self.data = self.signal.data.copy()
self.signal.axes_manager._axes[0].navigate = False
self.signal.axes_manager._axes[1].navigate = True
self.signal.axes_manager._axes[2].navigate = False
def setUp(self):
self.s = Signal(np.empty((2, 3, 4, 5)))
self.s.axes_manager.set_signal_dimension(2)
def setUp(self):
self.signal = Signal(np.arange(10))
self.signal.axes_manager[0].scale = 0.5
self.signal.axes_manager[0].offset = 0.25
self.data = self.signal.data.copy()
def setUp(self):
self.signal = Signal(np.arange(10))
self.data = self.signal.data.copy()
def setUp(self):
self.signal = Signal(np.arange(2**4).reshape((2, 2, 2, 2)))
self.data = self.signal.data.copy()
def multi_kernel(ind, m_dic, values, optional_components, _args, result_q,
test_dict):
import hyperspy.api as hs
from hyperspy.signal import Signal
from multiprocessing import current_process
from itertools import combinations, product
# from collections import Iterable
import numpy as np
import copy
from hyperspy.utils.model_selection import AICc
import dill
def generate_values_iterator(compnames, vals, turned_on_component_inds):
turned_on_names = [compnames[i] for i in turned_on_component_inds]
tmp = []
name_list = []
# TODO: put changing _position parameter of each component at the
# beginning
for _comp_n, _comp in vals.items():
if _comp_n in turned_on_names:
for par_n, par in _comp.items():
if not isinstance(par, list):
par = [
par,
]
tmp.append(par)
name_list.append((_comp_n, par_n))
return name_list, product(*tmp)
def send_good_results(model, previous_switching, cur_p, result_q, ind):
result = copy.deepcopy(model.as_dictionary())
for num, a_i_m in enumerate(previous_switching):
result['components'][num]['active_is_multidimensional'] = a_i_m
result['current'] = cur_p._identity
result_q.put((ind, result, True))
test = dill.loads(test_dict)
cur_p = current_process()
previous_switching = []
comb = []
AICc_fraction = 0.99
comp_dict = m_dic['models']['z']['_dict']['components']
for num, comp in enumerate(comp_dict):
previous_switching.append(comp['active_is_multidimensional'])
comp['active_is_multidimensional'] = False
for comp in optional_components:
comp_dict[comp]['active'] = False
for num in range(len(optional_components) + 1):
for comp in combinations(optional_components, num):
comb.append(comp)
best_AICc, best_dof = np.inf, np.inf
best_comb, best_values, best_names = None, None, None
component_names = [c['name'] for c in comp_dict]
sig = Signal(**m_dic)
sig._assign_subclass()
model = sig.models.z.restore()
for combination in comb:
# iterate all component combinations
for component in combination:
model[component].active = True
on_comps = [i for i, _c in enumerate(model) if _c.active]
name_list, iterator = generate_values_iterator(component_names, values,
on_comps)
ifgood = False
for it in iterator:
# iterate all parameter value combinations
for (comp_n, par_n), val in zip(name_list, it):
try:
getattr(model[comp_n], par_n).value = val
except:
pass
model.fit(**_args)
# only perform iterations until we find a solution that we think is
# good enough
ifgood = test.test(model, (0, ))
if ifgood:
break
if ifgood:
# get model with best chisq here, and test model validation
if len(comb) == 1:
send_good_results(model, previous_switching, cur_p, result_q,
ind)
new_AICc = AICc(model)
if new_AICc < AICc_fraction * best_AICc or \
(np.abs(new_AICc - best_AICc) <= np.abs(AICc_fraction * best_AICc)
and len(model.p0) < best_dof):
best_values = [
getattr(model[comp_n], par_n).value
for comp_n, par_n in name_list
]
best_names = name_list
best_comb = combination
best_AICc = new_AICc
best_dof = len(model.p0)
for component in combination:
model[component].active = False
# take the best overall combination of components and parameters:
if best_comb is None:
result_q.put((ind, None, False))
else:
for component in best_comb:
model[component].active = True
for (comp_n, par_n), val in zip(best_names, best_values):
try:
getattr(model[comp_n], par_n).value = val
except:
pass
model.fit(**_args)
send_good_results(model, previous_switching, cur_p, result_q, ind)
def setUp(self):
self.s = Signal(np.empty((2, 3, 4, 5)))
self.s.axes_manager.set_signal_dimension(2)
self.s.estimate_poissonian_noise_variance()
def setUp(self):
self.s1 = Signal(np.arange(20).reshape(2, 2, 5))
self.s2 = Signal(np.arange(4).reshape(2, 2))
self.s2.axes_manager.set_signal_dimension(2)
self.s3 = Signal(np.arange(4).reshape(1, 2, 2))
self.s3.axes_manager.set_signal_dimension(2)
def setUp(self):
self.signal = Signal(np.arange(5 * 10).reshape(5, 10))
self.signal.axes_manager[0].name = "x"
self.signal.axes_manager[1].name = "E"
self.signal.axes_manager[0].scale = 0.5
self.data = self.signal.data.copy()
def setUp(self):
self.s1 = Signal(np.arange(20).reshape(2, 2, 5))
self.s2 = Signal(np.arange(5))
self.s3 = Signal(np.arange(5).reshape(1, 5))
def setUp(self):
self.s = Signal([0.1])
