def __init__(self, model, workers=None, setup=True, random_state=None, **kwargs): # constants: if workers is None: workers = max(1, cpu_count() - 1) self.model = model self.metadata = DictionaryTreeBrowser() self._scale = 1.0 # -1 -> done pixel, use # -2 -> done, ignore when diffusion # 0 -> bad fit/no info # >0 -> select when turn comes self.metadata.add_node('marker') self.metadata.add_node('goodness_test') marker = np.empty(self.model.axes_manager.navigation_shape[::-1]) marker.fill(self._scale) self.metadata.marker = marker self.strategies = StrategyList(self) self.strategies.append(ReducedChiSquaredStrategy()) self.strategies.append(HistogramStrategy()) self._active_strategy_ind = 0 self.update_every = max(10, workers * 2) # some sensible number.... from hyperspy.samfire_utils.fit_tests import red_chisq_test self.metadata.goodness_test = red_chisq_test(tolerance=1.0) self.metadata._gt_dump = None from hyperspy.samfire_utils.samfire_kernel import single_kernel self.single_kernel = single_kernel self._workers = workers if len(kwargs) or setup: self._setup(**kwargs) self.refresh_database() self.random_state = check_random_state(random_state)
def get_low_loss_eels_line_scan_signal(add_noise=True, random_state=None): """Get an artificial low loss electron energy loss line scan spectrum. The zero loss peak is offset by 4.1 eV. Parameters ---------- %s %s Example ------- >>> s = hs.datasets.artificial_data.get_low_loss_eels_signal() >>> s.plot() See also -------- artificial_low_loss_line_scan_signal : :py:class:`~hyperspy._signals.eels.EELSSpectrum` """ from hyperspy.signals import EELSSpectrum from hyperspy import components1d random_state = check_random_state(random_state) x = np.arange(-100, 400, 0.5) zero_loss = components1d.Gaussian(A=100, centre=4.1, sigma=1) plasmon = components1d.Gaussian(A=100, centre=60, sigma=20) data_signal = zero_loss.function(x) data_signal += plasmon.function(x) data = np.zeros((12, len(x))) for i in range(12): data[i] += data_signal if add_noise: data[i] += random_state.uniform(size=len(x)) * 0.7 s = EELSSpectrum(data) s.axes_manager.signal_axes[0].offset = x[0] s.axes_manager.signal_axes[0].scale = x[1] - x[0] s.metadata.General.title = 'Artifical low loss EEL spectrum' s.axes_manager.signal_axes[0].name = 'Electron energy loss' s.axes_manager.signal_axes[0].units = 'eV' s.axes_manager.navigation_axes[0].name = 'Probe position' s.axes_manager.navigation_axes[0].units = 'nm' s.set_microscope_parameters(beam_energy=200, convergence_angle=26, collection_angle=20) return s
def get_low_loss_eels_signal(add_noise=True, random_state=None): """Get an artificial low loss electron energy loss spectrum. The zero loss peak is offset by 4.1 eV. Parameters ---------- %s %s Example ------- >>> s = hs.datasets.artificial_data.get_low_loss_eels_signal() >>> s.plot() See also -------- get_core_loss_eels_signal, get_core_loss_eels_model, get_low_loss_eels_line_scan_signal, get_core_loss_eels_line_scan_signal """ random_state = check_random_state(random_state) x = np.arange(-100, 400, 0.5) zero_loss = components1d.Gaussian(A=100, centre=4.1, sigma=1) plasmon = components1d.Gaussian(A=100, centre=60, sigma=20) data = zero_loss.function(x) data += plasmon.function(x) if add_noise: data += random_state.uniform(size=len(x)) * 0.7 s = signals.EELSSpectrum(data) s.axes_manager[0].offset = x[0] s.axes_manager[0].scale = x[1] - x[0] s.metadata.General.title = 'Artifical low loss EEL spectrum' s.axes_manager[0].name = 'Electron energy loss' s.axes_manager[0].units = 'eV' s.set_microscope_parameters(beam_energy=200, convergence_angle=26, collection_angle=20) return s
def __init__( self, rank, store_error=False, lambda1=1.0, kappa=1.0, method="PGD", subspace_learning_rate=1.0, subspace_momentum=0.5, random_state=None, ): """Creates Online Robust NMF instance that can learn a representation. Parameters ---------- rank : int The rank of the representation (number of components/factors) store_error : bool, default False If True, stores the sparse error matrix. lambda1 : float Nuclear norm regularization parameter. kappa : float Step-size for projection solver. method : {'PGD', 'RobustPGD', 'MomentumSGD'}, default 'PGD' * 'PGD' - Proximal gradient descent * 'RobustPGD' - Robust proximal gradient descent * 'MomentumSGD' - Stochastic gradient descent with momentum subspace_learning_rate : float Learning rate for the 'MomentumSGD' method. Should be a float > 0.0 subspace_momentum : float Momentum parameter for 'MomentumSGD' method, should be a float between 0 and 1. random_state : None or int or RandomState instance, default None Used to initialize the subspace on the first iteration. """ self.n_features = None self.iterating = False self.t = 0 if store_error: self.E = [] else: self.E = None self.rank = rank self.robust = False self.subspace_tracking = False self.lambda1 = lambda1 self.kappa = kappa self.subspace_learning_rate = subspace_learning_rate self.subspace_momentum = subspace_momentum self.random_state = check_random_state(random_state) # Check options are valid if method not in ("PGD", "RobustPGD", "MomentumSGD"): raise ValueError("'method' not recognised") if method == "RobustPGD": self.robust = True if method == "MomentumSGD": self.subspace_tracking = True if subspace_momentum < 0.0 or subspace_momentum > 1: raise ValueError("'subspace_momentum' must be a float between 0 and 1")
def get_core_loss_eels_signal(add_powerlaw=False, add_noise=True, random_state=None): """Get an artificial core loss electron energy loss spectrum. Similar to a Mn-L32 edge from a perovskite oxide. Some random noise is also added to the spectrum, to simulate experimental noise. Parameters ---------- %s %s %s Example ------- >>> import hs.datasets.artifical_data as ad >>> s = ad.get_core_loss_eels_signal() >>> s.plot() With the powerlaw background >>> s = ad.get_core_loss_eels_signal(add_powerlaw=True) >>> s.plot() To make the noise the same for multiple spectra, which can be useful for testing fitting routines >>> s1 = ad.get_core_loss_eels_signal(random_state=10) >>> s2 = ad.get_core_loss_eels_signal(random_state=10) >>> (s1.data == s2.data).all() True See also -------- get_core_loss_eels_line_scan_signal, get_low_loss_eels_line_scan_signal, get_core_loss_eels_model """ from hyperspy.signals import EELSSpectrum from hyperspy import components1d random_state = check_random_state(random_state) x = np.arange(400, 800, 1) arctan = components1d.EELSArctan(A=1, k=0.2, x0=688) mn_l3_g = components1d.Gaussian(A=100, centre=695, sigma=4) mn_l2_g = components1d.Gaussian(A=20, centre=720, sigma=4) data = arctan.function(x) data += mn_l3_g.function(x) data += mn_l2_g.function(x) if add_noise: data += random_state.uniform(size=len(x)) * 0.7 if add_powerlaw: powerlaw = components1d.PowerLaw(A=10e8, r=3, origin=0) data += powerlaw.function(x) s = EELSSpectrum(data) s.axes_manager[0].offset = x[0] s.metadata.General.title = 'Artifical core loss EEL spectrum' s.axes_manager[0].name = 'Electron energy loss' s.axes_manager[0].units = 'eV' s.set_microscope_parameters(beam_energy=200, convergence_angle=26, collection_angle=20) return s
def get_core_loss_eels_line_scan_signal(add_powerlaw=False, add_noise=True, random_state=None): """Get an artificial core loss electron energy loss line scan spectrum. Similar to a Mn-L32 and Fe-L32 edge from a perovskite oxide. Parameters ---------- %s %s %s Example ------- >>> s = hs.datasets.artificial_data.get_core_loss_eels_line_scan_signal() >>> s.plot() See also -------- get_low_loss_eels_line_scan_signal, get_core_loss_eels_model """ from hyperspy.signals import EELSSpectrum from hyperspy import components1d random_state = check_random_state(random_state) x = np.arange(400, 800, 1) arctan_mn = components1d.EELSArctan(A=1, k=0.2, x0=688) arctan_fe = components1d.EELSArctan(A=1, k=0.2, x0=612) mn_l3_g = components1d.Gaussian(A=100, centre=695, sigma=4) mn_l2_g = components1d.Gaussian(A=20, centre=720, sigma=4) fe_l3_g = components1d.Gaussian(A=100, centre=605, sigma=4) fe_l2_g = components1d.Gaussian(A=10, centre=630, sigma=3) mn_intensity = [1, 1, 1, 1, 1, 1, 0.8, 0.5, 0.2, 0, 0, 0] fe_intensity = [0, 0, 0, 0, 0, 0, 0.2, 0.5, 0.8, 1, 1, 1] data = np.zeros((len(mn_intensity), len(x))) for i in range(len(mn_intensity)): data[i] += arctan_mn.function(x) * mn_intensity[i] data[i] += mn_l3_g.function(x) * mn_intensity[i] data[i] += mn_l2_g.function(x) * mn_intensity[i] data[i] += arctan_fe.function(x) * fe_intensity[i] data[i] += fe_l3_g.function(x) * fe_intensity[i] data[i] += fe_l2_g.function(x) * fe_intensity[i] if add_noise: data[i] += random_state.uniform(size=len(x)) * 0.7 if add_powerlaw: powerlaw = components1d.PowerLaw(A=10e8, r=3, origin=0) data += powerlaw.function_nd(np.stack([x] * len(mn_intensity))) if add_powerlaw: powerlaw = components1d.PowerLaw(A=10e8, r=3, origin=0) data += powerlaw.function(x) s = EELSSpectrum(data) s.axes_manager.signal_axes[0].offset = x[0] s.metadata.General.title = 'Artifical core loss EEL spectrum' s.axes_manager.signal_axes[0].name = 'Electron energy loss' s.axes_manager.signal_axes[0].units = 'eV' s.axes_manager.navigation_axes[0].name = 'Probe position' s.axes_manager.navigation_axes[0].units = 'nm' s.set_microscope_parameters(beam_energy=200, convergence_angle=26, collection_angle=20) return s
def test_random_state_error(): with pytest.raises(ValueError, match="RandomState"): math_tools.check_random_state("string")
def test_random_state_lazy(seed): assert isinstance(math_tools.check_random_state(seed, lazy=True), da.random.RandomState)
def test_random_state(seed): assert isinstance(math_tools.check_random_state(seed), np.random.RandomState)
def rpca_godec(X, rank, lambda1=None, power=0, tol=1e-3, maxiter=1000, random_state=None, **kwargs): """Perform Robust PCA with missing or corrupted data, using the GoDec algorithm. Decomposes a matrix Y = X + E, where X is low-rank and E is a sparse error matrix. This algorithm is based on the Matlab code from [Zhou2011]_. See code here: https://sites.google.com/site/godecomposition/matrix/artifact-1 Read more in the :ref:`User Guide <mva.rpca>`. Parameters ---------- X : numpy array, shape (n_features, n_samples) The matrix of observations. rank : int The model dimensionality. lambda1 : None or float Regularization parameter. If None, set to 1 / sqrt(n_features) power : int, default 0 The number of power iterations used in the initialization tol : float, default 1e-3 Convergence tolerance maxiter : int, default 1000 Maximum number of iterations random_state : None or int or RandomState instance, default None Used to initialize the subspace on the first iteration. Returns ------- Xhat : numpy array, shape (n_features, n_samples) The low-rank matrix Ehat : numpy array, shape (n_features, n_samples) The sparse error matrix U, S, V : numpy arrays The results of an SVD on Xhat References ---------- .. [Zhou2011] Tianyi Zhou and Dacheng Tao, "GoDec: Randomized Low-rank & Sparse Matrix Decomposition in Noisy Case", ICML-11, (2011), pp. 33-40. """ # Get shape m, n = X.shape # Operate on transposed matrix for speed transpose = False if m < n: transpose = True X = X.T # Get shape m, n = X.shape # Check options if None if lambda1 is None: _logger.info( "Threshold 'lambda1' is set to default: 1 / sqrt(n_features)") lambda1 = 1.0 / np.sqrt(m) # Initialize L and E L = X E = np.zeros(L.shape) random_state = check_random_state(random_state) for itr in range(int(maxiter)): # Initialization with bilateral random projections Y2 = random_state.normal(size=(n, rank)) for _ in range(power + 1): Y2 = L.T @ (L @ Y2) Q, _ = scipy.linalg.qr(Y2, mode="economic") # Estimate the new low-rank and sparse matrices Lnew = (L @ Q) @ Q.T A = L - Lnew + E L = Lnew E = _soft_thresh(A, lambda1) A -= E L += A # Check convergence eps = np.linalg.norm(A) if eps < tol: _logger.info(f"Converged to {eps} in {itr} iterations") break # Transpose back if transpose: L = L.T E = E.T # Rescale Xhat = L Ehat = E # Do final SVD U, S, Vh = svd_solve(Xhat, output_dimension=rank, **kwargs) V = Vh.T # Chop small singular values which # likely arise from numerical noise # in the SVD. S[rank:] = 0.0 return Xhat, Ehat, U, S, V
def __init__( self, rank, store_error=False, lambda1=0.1, lambda2=1.0, method="BCD", init="qr", training_samples=10, subspace_learning_rate=1.0, subspace_momentum=0.5, random_state=None, ): """Creates Online Robust PCA instance that can learn a representation. Parameters ---------- rank : int The rank of the representation (number of components/factors) store_error : bool, default False If True, stores the sparse error matrix. lambda1 : float Nuclear norm regularization parameter. lambda2 : float Sparse error regularization parameter. method : {'CF', 'BCD', 'SGD', 'MomentumSGD'}, default 'BCD' * 'CF' - Closed-form solver * 'BCD' - Block-coordinate descent * 'SGD' - Stochastic gradient descent * 'MomentumSGD' - Stochastic gradient descent with momentum init : {'qr', 'rand', np.ndarray}, default 'qr' * 'qr' - QR-based initialization * 'rand' - Random initialization * np.ndarray if the shape [n_features x rank] training_samples : int Specifies the number of training samples to use in the 'qr' initialization. subspace_learning_rate : float Learning rate for the 'SGD' and 'MomentumSGD' methods. Should be a float > 0.0 subspace_momentum : float Momentum parameter for 'MomentumSGD' method, should be a float between 0 and 1. random_state : None or int or RandomState instance, default None Used to initialize the subspace on the first iteration. """ self.n_features = None self.iterating = False self.t = 0 if store_error: self.E = [] else: self.E = None self.rank = rank self.lambda1 = lambda1 self.lambda2 = lambda2 self.method = method self.init = init self.training_samples = training_samples self.subspace_learning_rate = subspace_learning_rate self.subspace_momentum = subspace_momentum self.random_state = check_random_state(random_state) # Check options are valid if method not in ("CF", "BCD", "SGD", "MomentumSGD"): raise ValueError("'method' not recognised") if not isinstance(init, np.ndarray) and init not in ("qr", "rand"): raise ValueError("'init' not recognised") if not isinstance(init, np.ndarray): if init == "qr" and training_samples < rank: raise ValueError( "'training_samples' must be >= 'output_dimension'") if method == "MomentumSGD" and (subspace_momentum > 1.0 or subspace_momentum < 0.0): raise ValueError( "'subspace_momentum' must be a float between 0 and 1")
def get_luminescence_signal(navigation_dimension=0, uniform=False, add_baseline=False, add_noise=True, random_state=None): """Get an artificial luminescence signal in wavelength scale (nm, uniform) or energy scale (eV, non-uniform), simulating luminescence data recorded with a diffracting spectrometer. Some random noise is also added to the spectrum, to simulate experimental noise. Parameters ---------- navigation_dimension: positive int. The navigation dimension(s) of the signal. 0 = single spectrum, 1 = linescan, 2 = spectral map etc... uniform: bool. return uniform (wavelength) or non-uniform (energy) spectrum add_baseline : bool If true, adds a constant baseline to the spectrum. Conversion to energy representation will turn the constant baseline into inverse powerlaw. %s Example ------- >>> import hyperspy.datasets.artificial_data as ad >>> s = ad.get_luminescence_signal() >>> s.plot() With constant baseline >>> s = ad.get_luminescence_signal(uniform=True, add_baseline=True) >>> s.plot() To make the noise the same for multiple spectra, which can be useful for testing fitting routines >>> s1 = ad.get_luminescence_signal(random_state=10) >>> s2 = ad.get_luminescence_signal(random_state=10) >>> (s1.data == s2.data).all() True 2D map >>> s = ad.get_luminescence_signal(navigation_dimension=2) >>> s.plot() See also -------- get_low_loss_eels_signal, get_core_loss_eels_signal, get_low_loss_eels_line_scan_signal, get_core_loss_eels_line_scan_signal, get_core_loss_eels_model, get_atomic_resolution_tem_signal2d, """ #Initialisation of random number generator random_state = check_random_state(random_state) #Creating a uniform data axis, roughly similar to Horiba iHR320 with a 150 mm-1 grating nm_axis = UniformDataAxis( index_in_array=None, name="Wavelength", units="nm", navigate=False, size=1024, scale=0.54, offset=222.495, is_binned=False, ) #Artificial luminescence peak gaussian_peak = components1d.Gaussian(A=5000, centre=375, sigma=25) if navigation_dimension >= 0: #Generate empty data (ones) data = np.ones([10 for i in range(navigation_dimension)] + [nm_axis.size]) #Generate spatial axes spaxes = [ UniformDataAxis( index_in_array=None, name="X{:d}".format(i), units="um", navigate=False, size=10, scale=2.1, offset=0, is_binned=False, ) for i in range(navigation_dimension) ] #Generate empty signal sig = signals.Signal1D(data, axes=spaxes + [nm_axis]) sig.metadata.General.title = '{:d}d-map Artificial Luminescence Signal'\ .format(navigation_dimension) else: raise ValueError("Value {:d} invalid as navigation dimension.".format(\ navigation_dimension)) #Populating data array, possibly with noise and baseline sig.data *= gaussian_peak.function(nm_axis.axis) if add_noise: sig.data += (random_state.uniform(size=sig.data.shape) - 0.5) * 1.4 if add_baseline: data += 350. #if not uniform, transformation into non-uniform axis if not uniform: hc = 1239.84198 #nm/eV #converting to non-uniform axis sig.axes_manager.signal_axes[0].convert_to_functional_data_axis(\ expression="a/x", name='Energy', units='eV', a=hc, ) #Reverting the orientation of signal axis to have increasing Energy sig = sig.isig[::-1] #Jacobian transformation Eax = sig.axes_manager.signal_axes[0].axis sig *= hc / Eax**2 return sig