def __init__(self, model, workers=None, setup=True, random_state=None, **kwargs):
# constants:
if workers is None:
workers = max(1, cpu_count() - 1)
self.model = model
self.metadata = DictionaryTreeBrowser()
self._scale = 1.0
# -1 -> done pixel, use
# -2 -> done, ignore when diffusion
# 0 -> bad fit/no info
# >0 -> select when turn comes
self.metadata.add_node('marker')
self.metadata.add_node('goodness_test')
marker = np.empty(self.model.axes_manager.navigation_shape[::-1])
marker.fill(self._scale)
self.metadata.marker = marker
self.strategies = StrategyList(self)
self.strategies.append(ReducedChiSquaredStrategy())
self.strategies.append(HistogramStrategy())
self._active_strategy_ind = 0
self.update_every = max(10, workers * 2) # some sensible number....
from hyperspy.samfire_utils.fit_tests import red_chisq_test
self.metadata.goodness_test = red_chisq_test(tolerance=1.0)
self.metadata._gt_dump = None
from hyperspy.samfire_utils.samfire_kernel import single_kernel
self.single_kernel = single_kernel
self._workers = workers
if len(kwargs) or setup:
self._setup(**kwargs)
self.refresh_database()
self.random_state = check_random_state(random_state)
def get_low_loss_eels_line_scan_signal(add_noise=True, random_state=None):
"""Get an artificial low loss electron energy loss line scan spectrum.
The zero loss peak is offset by 4.1 eV.
Parameters
----------
%s
%s
Example
-------
>>> s = hs.datasets.artificial_data.get_low_loss_eels_signal()
>>> s.plot()
See also
--------
artificial_low_loss_line_scan_signal : :py:class:`~hyperspy._signals.eels.EELSSpectrum`
"""
from hyperspy.signals import EELSSpectrum
from hyperspy import components1d
random_state = check_random_state(random_state)
x = np.arange(-100, 400, 0.5)
zero_loss = components1d.Gaussian(A=100, centre=4.1, sigma=1)
plasmon = components1d.Gaussian(A=100, centre=60, sigma=20)
data_signal = zero_loss.function(x)
data_signal += plasmon.function(x)
data = np.zeros((12, len(x)))
for i in range(12):
data[i] += data_signal
if add_noise:
data[i] += random_state.uniform(size=len(x)) * 0.7
s = EELSSpectrum(data)
s.axes_manager.signal_axes[0].offset = x[0]
s.axes_manager.signal_axes[0].scale = x[1] - x[0]
s.metadata.General.title = 'Artifical low loss EEL spectrum'
s.axes_manager.signal_axes[0].name = 'Electron energy loss'
s.axes_manager.signal_axes[0].units = 'eV'
s.axes_manager.navigation_axes[0].name = 'Probe position'
s.axes_manager.navigation_axes[0].units = 'nm'
s.set_microscope_parameters(beam_energy=200,
convergence_angle=26,
collection_angle=20)
return s
def get_low_loss_eels_signal(add_noise=True, random_state=None):
"""Get an artificial low loss electron energy loss spectrum.
The zero loss peak is offset by 4.1 eV.
Parameters
----------
%s
%s
Example
-------
>>> s = hs.datasets.artificial_data.get_low_loss_eels_signal()
>>> s.plot()
See also
--------
get_core_loss_eels_signal, get_core_loss_eels_model,
get_low_loss_eels_line_scan_signal, get_core_loss_eels_line_scan_signal
"""
random_state = check_random_state(random_state)
x = np.arange(-100, 400, 0.5)
zero_loss = components1d.Gaussian(A=100, centre=4.1, sigma=1)
plasmon = components1d.Gaussian(A=100, centre=60, sigma=20)
data = zero_loss.function(x)
data += plasmon.function(x)
if add_noise:
data += random_state.uniform(size=len(x)) * 0.7
s = signals.EELSSpectrum(data)
s.axes_manager[0].offset = x[0]
s.axes_manager[0].scale = x[1] - x[0]
s.metadata.General.title = 'Artifical low loss EEL spectrum'
s.axes_manager[0].name = 'Electron energy loss'
s.axes_manager[0].units = 'eV'
s.set_microscope_parameters(beam_energy=200,
convergence_angle=26,
collection_angle=20)
return s
def __init__(
self,
rank,
store_error=False,
lambda1=1.0,
kappa=1.0,
method="PGD",
subspace_learning_rate=1.0,
subspace_momentum=0.5,
random_state=None,
):
"""Creates Online Robust NMF instance that can learn a representation.
Parameters
----------
rank : int
The rank of the representation (number of components/factors)
store_error : bool, default False
If True, stores the sparse error matrix.
lambda1 : float
Nuclear norm regularization parameter.
kappa : float
Step-size for projection solver.
method : {'PGD', 'RobustPGD', 'MomentumSGD'}, default 'PGD'
* 'PGD' - Proximal gradient descent
* 'RobustPGD' - Robust proximal gradient descent
* 'MomentumSGD' - Stochastic gradient descent with momentum
subspace_learning_rate : float
Learning rate for the 'MomentumSGD' method. Should be a
float > 0.0
subspace_momentum : float
Momentum parameter for 'MomentumSGD' method, should be
a float between 0 and 1.
random_state : None or int or RandomState instance, default None
Used to initialize the subspace on the first iteration.
"""
self.n_features = None
self.iterating = False
self.t = 0
if store_error:
self.E = []
else:
self.E = None
self.rank = rank
self.robust = False
self.subspace_tracking = False
self.lambda1 = lambda1
self.kappa = kappa
self.subspace_learning_rate = subspace_learning_rate
self.subspace_momentum = subspace_momentum
self.random_state = check_random_state(random_state)
# Check options are valid
if method not in ("PGD", "RobustPGD", "MomentumSGD"):
raise ValueError("'method' not recognised")
if method == "RobustPGD":
self.robust = True
if method == "MomentumSGD":
self.subspace_tracking = True
if subspace_momentum < 0.0 or subspace_momentum > 1:
raise ValueError("'subspace_momentum' must be a float between 0 and 1")
def get_core_loss_eels_signal(add_powerlaw=False,
add_noise=True,
random_state=None):
"""Get an artificial core loss electron energy loss spectrum.
Similar to a Mn-L32 edge from a perovskite oxide.
Some random noise is also added to the spectrum, to simulate
experimental noise.
Parameters
----------
%s
%s
%s
Example
-------
>>> import hs.datasets.artifical_data as ad
>>> s = ad.get_core_loss_eels_signal()
>>> s.plot()
With the powerlaw background
>>> s = ad.get_core_loss_eels_signal(add_powerlaw=True)
>>> s.plot()
To make the noise the same for multiple spectra, which can
be useful for testing fitting routines
>>> s1 = ad.get_core_loss_eels_signal(random_state=10)
>>> s2 = ad.get_core_loss_eels_signal(random_state=10)
>>> (s1.data == s2.data).all()
True
See also
--------
get_core_loss_eels_line_scan_signal, get_low_loss_eels_line_scan_signal,
get_core_loss_eels_model
"""
from hyperspy.signals import EELSSpectrum
from hyperspy import components1d
random_state = check_random_state(random_state)
x = np.arange(400, 800, 1)
arctan = components1d.EELSArctan(A=1, k=0.2, x0=688)
mn_l3_g = components1d.Gaussian(A=100, centre=695, sigma=4)
mn_l2_g = components1d.Gaussian(A=20, centre=720, sigma=4)
data = arctan.function(x)
data += mn_l3_g.function(x)
data += mn_l2_g.function(x)
if add_noise:
data += random_state.uniform(size=len(x)) * 0.7
if add_powerlaw:
powerlaw = components1d.PowerLaw(A=10e8, r=3, origin=0)
data += powerlaw.function(x)
s = EELSSpectrum(data)
s.axes_manager[0].offset = x[0]
s.metadata.General.title = 'Artifical core loss EEL spectrum'
s.axes_manager[0].name = 'Electron energy loss'
s.axes_manager[0].units = 'eV'
s.set_microscope_parameters(beam_energy=200,
convergence_angle=26,
collection_angle=20)
return s
def get_core_loss_eels_line_scan_signal(add_powerlaw=False,
add_noise=True,
random_state=None):
"""Get an artificial core loss electron energy loss line scan spectrum.
Similar to a Mn-L32 and Fe-L32 edge from a perovskite oxide.
Parameters
----------
%s
%s
%s
Example
-------
>>> s = hs.datasets.artificial_data.get_core_loss_eels_line_scan_signal()
>>> s.plot()
See also
--------
get_low_loss_eels_line_scan_signal, get_core_loss_eels_model
"""
from hyperspy.signals import EELSSpectrum
from hyperspy import components1d
random_state = check_random_state(random_state)
x = np.arange(400, 800, 1)
arctan_mn = components1d.EELSArctan(A=1, k=0.2, x0=688)
arctan_fe = components1d.EELSArctan(A=1, k=0.2, x0=612)
mn_l3_g = components1d.Gaussian(A=100, centre=695, sigma=4)
mn_l2_g = components1d.Gaussian(A=20, centre=720, sigma=4)
fe_l3_g = components1d.Gaussian(A=100, centre=605, sigma=4)
fe_l2_g = components1d.Gaussian(A=10, centre=630, sigma=3)
mn_intensity = [1, 1, 1, 1, 1, 1, 0.8, 0.5, 0.2, 0, 0, 0]
fe_intensity = [0, 0, 0, 0, 0, 0, 0.2, 0.5, 0.8, 1, 1, 1]
data = np.zeros((len(mn_intensity), len(x)))
for i in range(len(mn_intensity)):
data[i] += arctan_mn.function(x) * mn_intensity[i]
data[i] += mn_l3_g.function(x) * mn_intensity[i]
data[i] += mn_l2_g.function(x) * mn_intensity[i]
data[i] += arctan_fe.function(x) * fe_intensity[i]
data[i] += fe_l3_g.function(x) * fe_intensity[i]
data[i] += fe_l2_g.function(x) * fe_intensity[i]
if add_noise:
data[i] += random_state.uniform(size=len(x)) * 0.7
if add_powerlaw:
powerlaw = components1d.PowerLaw(A=10e8, r=3, origin=0)
data += powerlaw.function_nd(np.stack([x] * len(mn_intensity)))
if add_powerlaw:
powerlaw = components1d.PowerLaw(A=10e8, r=3, origin=0)
data += powerlaw.function(x)
s = EELSSpectrum(data)
s.axes_manager.signal_axes[0].offset = x[0]
s.metadata.General.title = 'Artifical core loss EEL spectrum'
s.axes_manager.signal_axes[0].name = 'Electron energy loss'
s.axes_manager.signal_axes[0].units = 'eV'
s.axes_manager.navigation_axes[0].name = 'Probe position'
s.axes_manager.navigation_axes[0].units = 'nm'
s.set_microscope_parameters(beam_energy=200,
convergence_angle=26,
collection_angle=20)
return s
def test_random_state_error():
with pytest.raises(ValueError, match="RandomState"):
math_tools.check_random_state("string")
def test_random_state_lazy(seed):
assert isinstance(math_tools.check_random_state(seed, lazy=True), da.random.RandomState)
def test_random_state(seed):
assert isinstance(math_tools.check_random_state(seed), np.random.RandomState)
def rpca_godec(X,
rank,
lambda1=None,
power=0,
tol=1e-3,
maxiter=1000,
random_state=None,
**kwargs):
"""Perform Robust PCA with missing or corrupted data, using the GoDec algorithm.
Decomposes a matrix Y = X + E, where X is low-rank and E
is a sparse error matrix. This algorithm is based on the
Matlab code from [Zhou2011]_. See code here:
https://sites.google.com/site/godecomposition/matrix/artifact-1
Read more in the :ref:`User Guide <mva.rpca>`.
Parameters
----------
X : numpy array, shape (n_features, n_samples)
The matrix of observations.
rank : int
The model dimensionality.
lambda1 : None or float
Regularization parameter.
If None, set to 1 / sqrt(n_features)
power : int, default 0
The number of power iterations used in the initialization
tol : float, default 1e-3
Convergence tolerance
maxiter : int, default 1000
Maximum number of iterations
random_state : None or int or RandomState instance, default None
Used to initialize the subspace on the first iteration.
Returns
-------
Xhat : numpy array, shape (n_features, n_samples)
The low-rank matrix
Ehat : numpy array, shape (n_features, n_samples)
The sparse error matrix
U, S, V : numpy arrays
The results of an SVD on Xhat
References
----------
.. [Zhou2011] Tianyi Zhou and Dacheng Tao, "GoDec: Randomized Low-rank &
Sparse Matrix Decomposition in Noisy Case", ICML-11, (2011), pp. 33-40.
"""
# Get shape
m, n = X.shape
# Operate on transposed matrix for speed
transpose = False
if m < n:
transpose = True
X = X.T
# Get shape
m, n = X.shape
# Check options if None
if lambda1 is None:
_logger.info(
"Threshold 'lambda1' is set to default: 1 / sqrt(n_features)")
lambda1 = 1.0 / np.sqrt(m)
# Initialize L and E
L = X
E = np.zeros(L.shape)
random_state = check_random_state(random_state)
for itr in range(int(maxiter)):
# Initialization with bilateral random projections
Y2 = random_state.normal(size=(n, rank))
for _ in range(power + 1):
Y2 = L.T @ (L @ Y2)
Q, _ = scipy.linalg.qr(Y2, mode="economic")
# Estimate the new low-rank and sparse matrices
Lnew = (L @ Q) @ Q.T
A = L - Lnew + E
L = Lnew
E = _soft_thresh(A, lambda1)
A -= E
L += A
# Check convergence
eps = np.linalg.norm(A)
if eps < tol:
_logger.info(f"Converged to {eps} in {itr} iterations")
break
# Transpose back
if transpose:
L = L.T
E = E.T
# Rescale
Xhat = L
Ehat = E
# Do final SVD
U, S, Vh = svd_solve(Xhat, output_dimension=rank, **kwargs)
V = Vh.T
# Chop small singular values which
# likely arise from numerical noise
# in the SVD.
S[rank:] = 0.0
return Xhat, Ehat, U, S, V
def __init__(
self,
rank,
store_error=False,
lambda1=0.1,
lambda2=1.0,
method="BCD",
init="qr",
training_samples=10,
subspace_learning_rate=1.0,
subspace_momentum=0.5,
random_state=None,
):
"""Creates Online Robust PCA instance that can learn a representation.
Parameters
----------
rank : int
The rank of the representation (number of components/factors)
store_error : bool, default False
If True, stores the sparse error matrix.
lambda1 : float
Nuclear norm regularization parameter.
lambda2 : float
Sparse error regularization parameter.
method : {'CF', 'BCD', 'SGD', 'MomentumSGD'}, default 'BCD'
* 'CF' - Closed-form solver
* 'BCD' - Block-coordinate descent
* 'SGD' - Stochastic gradient descent
* 'MomentumSGD' - Stochastic gradient descent with momentum
init : {'qr', 'rand', np.ndarray}, default 'qr'
* 'qr' - QR-based initialization
* 'rand' - Random initialization
* np.ndarray if the shape [n_features x rank]
training_samples : int
Specifies the number of training samples to use in
the 'qr' initialization.
subspace_learning_rate : float
Learning rate for the 'SGD' and 'MomentumSGD' methods.
Should be a float > 0.0
subspace_momentum : float
Momentum parameter for 'MomentumSGD' method, should be
a float between 0 and 1.
random_state : None or int or RandomState instance, default None
Used to initialize the subspace on the first iteration.
"""
self.n_features = None
self.iterating = False
self.t = 0
if store_error:
self.E = []
else:
self.E = None
self.rank = rank
self.lambda1 = lambda1
self.lambda2 = lambda2
self.method = method
self.init = init
self.training_samples = training_samples
self.subspace_learning_rate = subspace_learning_rate
self.subspace_momentum = subspace_momentum
self.random_state = check_random_state(random_state)
# Check options are valid
if method not in ("CF", "BCD", "SGD", "MomentumSGD"):
raise ValueError("'method' not recognised")
if not isinstance(init, np.ndarray) and init not in ("qr", "rand"):
raise ValueError("'init' not recognised")
if not isinstance(init, np.ndarray):
if init == "qr" and training_samples < rank:
raise ValueError(
"'training_samples' must be >= 'output_dimension'")
if method == "MomentumSGD" and (subspace_momentum > 1.0
or subspace_momentum < 0.0):
raise ValueError(
"'subspace_momentum' must be a float between 0 and 1")
def get_luminescence_signal(navigation_dimension=0,
uniform=False,
add_baseline=False,
add_noise=True,
random_state=None):
"""Get an artificial luminescence signal in wavelength scale (nm, uniform) or
energy scale (eV, non-uniform), simulating luminescence data recorded with a
diffracting spectrometer. Some random noise is also added to the spectrum,
to simulate experimental noise.
Parameters
----------
navigation_dimension: positive int.
The navigation dimension(s) of the signal. 0 = single spectrum,
1 = linescan, 2 = spectral map etc...
uniform: bool.
return uniform (wavelength) or non-uniform (energy) spectrum
add_baseline : bool
If true, adds a constant baseline to the spectrum. Conversion to
energy representation will turn the constant baseline into inverse
powerlaw.
%s
Example
-------
>>> import hyperspy.datasets.artificial_data as ad
>>> s = ad.get_luminescence_signal()
>>> s.plot()
With constant baseline
>>> s = ad.get_luminescence_signal(uniform=True, add_baseline=True)
>>> s.plot()
To make the noise the same for multiple spectra, which can
be useful for testing fitting routines
>>> s1 = ad.get_luminescence_signal(random_state=10)
>>> s2 = ad.get_luminescence_signal(random_state=10)
>>> (s1.data == s2.data).all()
True
2D map
>>> s = ad.get_luminescence_signal(navigation_dimension=2)
>>> s.plot()
See also
--------
get_low_loss_eels_signal,
get_core_loss_eels_signal,
get_low_loss_eels_line_scan_signal,
get_core_loss_eels_line_scan_signal,
get_core_loss_eels_model,
get_atomic_resolution_tem_signal2d,
"""
#Initialisation of random number generator
random_state = check_random_state(random_state)
#Creating a uniform data axis, roughly similar to Horiba iHR320 with a 150 mm-1 grating
nm_axis = UniformDataAxis(
index_in_array=None,
name="Wavelength",
units="nm",
navigate=False,
size=1024,
scale=0.54,
offset=222.495,
is_binned=False,
)
#Artificial luminescence peak
gaussian_peak = components1d.Gaussian(A=5000, centre=375, sigma=25)
if navigation_dimension >= 0:
#Generate empty data (ones)
data = np.ones([10
for i in range(navigation_dimension)] + [nm_axis.size])
#Generate spatial axes
spaxes = [
UniformDataAxis(
index_in_array=None,
name="X{:d}".format(i),
units="um",
navigate=False,
size=10,
scale=2.1,
offset=0,
is_binned=False,
) for i in range(navigation_dimension)
]
#Generate empty signal
sig = signals.Signal1D(data, axes=spaxes + [nm_axis])
sig.metadata.General.title = '{:d}d-map Artificial Luminescence Signal'\
.format(navigation_dimension)
else:
raise ValueError("Value {:d} invalid as navigation dimension.".format(\
navigation_dimension))
#Populating data array, possibly with noise and baseline
sig.data *= gaussian_peak.function(nm_axis.axis)
if add_noise:
sig.data += (random_state.uniform(size=sig.data.shape) - 0.5) * 1.4
if add_baseline:
data += 350.
#if not uniform, transformation into non-uniform axis
if not uniform:
hc = 1239.84198 #nm/eV
#converting to non-uniform axis
sig.axes_manager.signal_axes[0].convert_to_functional_data_axis(\
expression="a/x",
name='Energy',
units='eV',
a=hc,
)
#Reverting the orientation of signal axis to have increasing Energy
sig = sig.isig[::-1]
#Jacobian transformation
Eax = sig.axes_manager.signal_axes[0].axis
sig *= hc / Eax**2
return sig
