def test_fitting_with_outlier_removal_niter(): """ Test that FittingWithOutlierRemoval stops prior to reaching niter if the set of masked points has converged and correctly reports the actual number of iterations performed. """ # 2 rows with some noise around a constant level and 1 deviant point: x = np.arange(25) with NumpyRNGContext(_RANDOM_SEED): y = np.random.normal(loc=10., scale=1., size=(2, 25)) y[0, 14] = 100. # Fit 2 models with up to 5 iterations (should only take 2): fitter = FittingWithOutlierRemoval( fitter=LinearLSQFitter(), outlier_func=sigma_clip, niter=5, sigma_lower=3., sigma_upper=3., maxiters=1 ) model, mask = fitter(models.Chebyshev1D(2, n_models=2), x, y) # Confirm that only the deviant point was rejected, in 2 iterations: assert_equal(np.where(mask), [[0], [14]]) assert fitter.fit_info['niter'] == 2 # Refit just the first row without any rejection iterations, to ensure # there are no regressions for that special case: fitter = FittingWithOutlierRemoval( fitter=LinearLSQFitter(), outlier_func=sigma_clip, niter=0, sigma_lower=3., sigma_upper=3., maxiters=1 ) model, mask = fitter(models.Chebyshev1D(2), x, y[0]) # Confirm that there were no iterations or rejected points: assert mask.sum() == 0 assert fitter.fit_info['niter'] == 0
def test_linear_fit_model_set_common_weight(): """Tests fitting multiple models simultaneously.""" init_model = Polynomial1D(degree=2, c0=[1, 1], n_models=2) x = np.arange(10) y_expected = init_model(x, model_set_axis=False) assert y_expected.shape == (2, 10) # Add a bit of random noise with NumpyRNGContext(_RANDOM_SEED): y = y_expected + np.random.normal(0, 0.01, size=y_expected.shape) fitter = LinearLSQFitter() weights = np.ones(10) weights[[0, -1]] = 0 fitted_model = fitter(init_model, x, y, weights=weights) assert_allclose(fitted_model(x, model_set_axis=False), y_expected, rtol=1e-1) # Check that using null weights raises an error # ValueError: On entry to DLASCL parameter number 4 had an illegal value with pytest.raises(ValueError, match='Found NaNs in the coefficient matrix'): with pytest.warns(RuntimeWarning, match=r'invalid value encountered in.*divide'): fitted_model = fitter(init_model, x, y, weights=np.zeros(10))
def test_covariance_std_printing_indexing(self, capsys): """ Test printing methods and indexing. """ # test str representation for Covariance/stds fitter = LinearLSQFitter(calc_uncertainties=True) mod = models.Linear1D() fit_mod = fitter(mod, self.x, mod(self.x)+self.rand) print(fit_mod.cov_matrix) captured = capsys.readouterr() assert "slope | 0.001" in captured.out assert "intercept| -0.006, 0.041" in captured.out print(fit_mod.stds) captured = capsys.readouterr() assert "slope | 0.038" in captured.out assert "intercept| 0.203" in captured.out # test 'pprint' for Covariance/stds print(fit_mod.cov_matrix.pprint(round_val=5, max_lines=1)) captured = capsys.readouterr() assert "slope | 0.00144" in captured.out assert "intercept" not in captured.out print(fit_mod.stds.pprint(max_lines=1, round_val=5)) captured = capsys.readouterr() assert "slope | 0.03799" in captured.out assert "intercept" not in captured.out # test indexing for Covariance class. assert fit_mod.cov_matrix[0, 0] == fit_mod.cov_matrix['slope', 'slope'] # test indexing for stds class. assert fit_mod.stds[1] == fit_mod.stds['intercept']
def test_linear_1d_separate_weights_axis_1(self): model = Polynomial1D(1, model_set_axis=1) fitter = LinearLSQFitter() model = fitter(model, self.x1, self.y1.T, weights=self.w1[..., np.newaxis]) assert_allclose(model.c0, 0.5, atol=1e-12) assert_allclose(model.c1, 2.5, atol=1e-12)
def test_model_set_axis_outputs(): fitter = LinearLSQFitter() model_set = Polynomial2D(1, n_models=2, model_set_axis=2) y2, x2 = np.mgrid[:5, :5] # z = np.moveaxis([x2 + y2, 1 - 0.1 * x2 + 0.2 * y2]), 0, 2) z = np.rollaxis(np.array([x2 + y2, 1 - 0.1 * x2 + 0.2 * y2]), 0, 3) model = fitter(model_set, x2, y2, z) res = model(x2, y2, model_set_axis=False) assert z.shape == res.shape # Test initializing with integer model_set_axis # and evaluating with a different model_set_axis model_set = Polynomial1D(1, c0=[1, 2], c1=[2, 3], n_models=2, model_set_axis=0) y0 = model_set(xx) y1 = model_set(xx.T, model_set_axis=1) assert_allclose(y0[0], y1[:, 0]) assert_allclose(y0[1], y1[:, 1]) model_set = Polynomial1D(1, c0=[[1, 2]], c1=[[2, 3]], n_models=2, model_set_axis=1) y0 = model_set(xx.T) y1 = model_set(xx, model_set_axis=0) assert_allclose(y0[:, 0], y1[0]) assert_allclose(y0[:, 1], y1[1]) with pytest.raises(ValueError): model_set(x)
def test_linear_2d_common_weights(self): model = Polynomial2D(1) fitter = LinearLSQFitter() model = fitter(model, self.x2, self.y2, self.z2, weights=self.w2) assert_allclose(model.c0_0, 1., atol=1e-12) assert_allclose(model.c1_0, -0.1, atol=1e-12) assert_allclose(model.c0_1, 0.2, atol=1e-12)
def trace_fit(layout, side='left', degree=1): '''Linear fit to describe edge of a spectral trace Parameters ---------- layout : [table] a table describing a spectral trace side : [str] can be 'left', 'right' or 'centre' ''' from astropy.modeling.models import Polynomial1D from astropy.modeling.fitting import LinearLSQFitter if side == 'left': xpt = layout['x1'] ypt = layout['y1'] name = 'left edge' elif side == 'centre': xpt = layout['x2'] ypt = layout['y2'] name = 'trace centre' elif side == 'right': xpt = layout['x3'] ypt = layout['y3'] name = 'right edge' else: raise ValueError('Side ' + str(side) + ' not recognized') pinit = Polynomial1D(degree=degree) fitter = LinearLSQFitter() edge_fit = fitter(pinit, ypt, xpt) edge_fit.name = name return edge_fit
def test_linear_1d_separate_weights(self): model = Polynomial1D(1) fitter = LinearLSQFitter() model = fitter(model, self.x1, self.y1, weights=self.w1[np.newaxis, ...]) assert_allclose(model.c0, 0.5, atol=1e-12) assert_allclose(model.c1, 2.5, atol=1e-12)
def test_1d_without_weights_with_sigma_clip(self): model = models.Polynomial1D(0) fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip, niter=3, sigma=3.) fit, mask = fitter(model, self.x1d, self.z1d) assert((~mask).sum() == self.z1d.size - 2) assert(mask[0] and mask[1]) assert_allclose(fit.parameters[0], 0.0, atol=10**(-2)) # with removed outliers mean is 0.0
def test_linear_2d_separate_weights_axis_2(self): model = Polynomial2D(1, model_set_axis=2) fitter = LinearLSQFitter() model = fitter(model, self.x2, self.y2, np.rollaxis(self.z2, 0, 3), weights=self.w2[..., np.newaxis]) assert_allclose(model.c0_0, 1., atol=1e-12) assert_allclose(model.c1_0, -0.1, atol=1e-12) assert_allclose(model.c0_1, 0.2, atol=1e-12)
def test_1d_with_weights_with_sigma_clip(self): """smoke test for #7020 - fails without fitting.py patch because weights does not propagate""" model = models.Polynomial1D(0) fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip, niter=3, sigma=3.) fit, filtered = fitter(model, self.x1d, self.z1d, weights=self.weights1d) assert(fit.parameters[0] > 10**(-2)) # weights pulled it > 0 assert(fit.parameters[0] < 1.0) # outliers didn't pull it out of [-1:1] because they had been removed
def test_2d_without_weights_with_sigma_clip(self): model = models.Polynomial2D(0) fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip, niter=3, sigma=3.) fit, mask = fitter(model, self.x, self.y, self.z) assert((~mask).sum() == self.z.size - 2) assert(mask[0, 0] and mask[0, 1]) assert_allclose(fit.parameters[0], 0.0, atol=10**(-2))
def test_2d_linear_with_weights_with_sigma_clip(self): """same as test above with a linear fitter.""" model = models.Polynomial2D(0) fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip, niter=3, sigma=3.) fit, filtered = fitter(model, self.x, self.y, self.z, weights=self.weights) assert(fit.parameters[0] > 10**(-2)) # weights pulled it > 0 assert(fit.parameters[0] < 1.0) # outliers didn't pull it out of [-1:1] because they had been removed
def test_1d_set_with_common_weights_with_sigma_clip(self): """added for #6819 (1D model set with weights in common)""" model = models.Polynomial1D(0, n_models=2) fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip, niter=3, sigma=3.) z1d = np.array([self.z1d, self.z1d]) fit, filtered = fitter(model, self.x1d, z1d, weights=self.weights1d) assert_allclose(fit.parameters, [0.8, 0.8], atol=1e-14)
def poly_solution(self, order): from astropy.modeling.fitting import LinearLSQFitter from astropy.modeling.models import Polynomial1D poly = Polynomial1D(order) ftr = LinearLSQFitter() poly = ftr(poly, list(self.linepixtowl.keys()), list(self.linepixtowl.values())) return poly
def setup_class(self): self.model = models.Polynomial2D(2) self.y, self.x = np.mgrid[:5, :5] def poly2(x, y): return 1 + 2 * x + 3 * x**2 + 4 * y + 5 * y**2 + 6 * x * y self.z = poly2(self.x, self.y) self.fitter = LinearLSQFitter()
def test_linear_fit_model_set_errors(): init_model = Polynomial1D(degree=2, c0=[1, 1], n_models=2) x = np.arange(10) y = init_model(x, model_set_axis=False) fitter = LinearLSQFitter() with pytest.raises(ValueError): fitter(init_model, x[:5], y) with pytest.raises(ValueError): fitter(init_model, x, y[:, :5])
def test_compound_model_raises_error(self): """Test that if an user tries to use a compound model, raises an error""" with pytest.raises(ValueError) as excinfo: init_model1 = models.Polynomial1D(degree=2, c0=[1, 1], n_models=2) init_model2 = models.Polynomial1D(degree=2, c0=[1, 1], n_models=2) init_model_comp = init_model1 + init_model2 x = np.arange(10) y = init_model_comp(x, model_set_axis=False) fitter = LinearLSQFitter() _ = fitter(init_model_comp, x, y) assert "Model must be simple, not compound" in str(excinfo.value)
def test_non_linear_lsq_fitter_with_weights(self, fitter): """ Tests that issue #11581 has been solved. """ fitter = fitter() np.random.seed(42) norder = 2 fitter2 = LinearLSQFitter() model = models.Polynomial1D(norder) npts = 10000 c = [2.0, -10.0, 7.0] tw = np.random.uniform(0.0, 10.0, npts) tx = np.random.uniform(0.0, 10.0, npts) ty = c[0] + c[1] * tx + c[2] * (tx ** 2) ty += np.random.normal(0.0, 1.5, npts) with pytest.warns(AstropyUserWarning, match=r'Model is linear in parameters'): tf1 = fitter(model, tx, ty, weights=tw) tf2 = fitter2(model, tx, ty, weights=tw) assert_allclose(tf1.parameters, tf2.parameters, atol=10 ** (-16)) assert_allclose(tf1.parameters, c, rtol=10 ** (-2), atol=10 ** (-2)) model = models.Gaussian1D() if isinstance(fitter, TRFLSQFitter) or isinstance(fitter, LMLSQFitter): with pytest.warns(AstropyUserWarning, match=r'The fit may be unsuccessful; *.'): fitter(model, tx, ty, weights=tw) else: fitter(model, tx, ty, weights=tw) model = models.Polynomial2D(norder) nxpts = 100 nypts = 150 npts = nxpts * nypts c = [1.0, 4.0, 7.0, -8.0, -9.0, -3.0] tw = np.random.uniform(0.0, 10.0, npts).reshape(nxpts, nypts) tx = np.random.uniform(0.0, 10.0, npts).reshape(nxpts, nypts) ty = np.random.uniform(0.0, 10.0, npts).reshape(nxpts, nypts) tz = c[0] + c[1] * tx + c[2] * (tx ** 2) + c[3] * ty + c[4] * (ty ** 2) + c[5] * tx * ty tz += np.random.normal(0.0, 1.5, npts).reshape(nxpts, nypts) with pytest.warns(AstropyUserWarning, match=r'Model is linear in parameters'): tf1 = fitter(model, tx, ty, tz, weights=tw) tf2 = fitter2(model, tx, ty, tz, weights=tw) assert_allclose(tf1.parameters, tf2.parameters, atol=10 ** (-16)) assert_allclose(tf1.parameters, c, rtol=10 ** (-2), atol=10 ** (-2))
def measure_SNR(spex_path, data_path, plot_path, star, ranges): """ Measure the SNR of a spectrum, by fitting straight lines to pieces of the spectrum """ # plot the spectrum to define regions without spectral lines fig, ax = plot_spectrum(star, data_path, range=[0.75, 5.6], log=True) # read in all bands and spectra for this star starobs = StarData("%s.dat" % star.lower(), path=data_path, use_corfac=True) # obtain flux values at a few wavelengths and fit a straight line through the data waves, fluxes, uncs = starobs.get_flat_data_arrays(["SpeX_SXD", "SpeX_LXD"]) print(star) for range in ranges: min_indx = np.abs(waves - range[0]).argmin() max_indx = np.abs(waves - range[1]).argmin() func = Linear1D() fit = LinearLSQFitter() fit_result = fit(func, waves[min_indx:max_indx], fluxes[min_indx:max_indx]) residu = fluxes[min_indx:max_indx] - fit_result(waves[min_indx:max_indx]) # calculate the SNR from the data data_sxd = Table.read( spex_path + star + "_sxd.txt", format="ascii", ) data_lxd = Table.read( spex_path + star + "_lxd.txt", format="ascii", ) data = vstack([data_sxd, data_lxd]) data.sort("col1") print("wave_range", range) min_indx2 = np.abs(data["col1"] - range[0]).argmin() max_indx2 = np.abs(data["col1"] - range[1]).argmin() SNR_data = np.nanmedian((data["col2"] / data["col3"])[min_indx2:max_indx2]) print("SNR from data", SNR_data) # calculate the SNR from the noise around the linear fit mean, median, stddev = sigma_clipped_stats(residu) SNR_fit = np.median(fluxes[min_indx:max_indx] / stddev) print("SNR from fit", SNR_fit) # plot the fitted lines on top of the spectrum ax.plot( waves[min_indx:max_indx], fit_result(waves[min_indx:max_indx]), lw=2, alpha=0.8, color="k", ) fig.savefig(plot_path + star + "_SNR_measure.pdf")
def test_linear_fitter_with_weights(): """Regression test for #7035""" Xin, Yin = np.mgrid[0:21, 0:21] fitter = LinearLSQFitter() with NumpyRNGContext(_RANDOM_SEED): zsig = np.random.normal(0, 0.01, size=Xin.shape) p2 = models.Polynomial2D(3) p2.parameters = np.arange(10)/1.2 z = p2(Xin, Yin) pmod = fitter(models.Polynomial2D(3), Xin, Yin, z + zsig, weights=zsig**(-2)) assert_allclose(pmod.parameters, p2.parameters, atol=10 ** (-2))
def test_linear_fit_2d_model_set_common_weight(): init_model = Polynomial2D(degree=2, c1_0=[1, 2], c0_1=[-0.5, 1], n_models=2, fixed={'c1_0': True, 'c0_1': True}) x, y = np.mgrid[0:5, 0:5] zz = np.array([1+x-0.5*y+0.1*x*x, 2*x+y-0.2*y*y]) fitter = LinearLSQFitter() fitted_model = fitter(init_model, x, y, zz, weights=np.ones((5, 5))) assert_allclose(fitted_model(x, y, model_set_axis=False), zz, atol=1e-14)
def test_linear_fit_fixed_parameter(self): """ Tests fitting a polynomial model with a fixed parameter (issue #6135). """ init_model = models.Polynomial1D(degree=2, c1=1) init_model.c1.fixed = True x = np.arange(10) y = 2 + x + 0.5*x*x fitter = LinearLSQFitter() fitted_model = fitter(init_model, x, y) assert_allclose(fitted_model.parameters, [2., 1., 0.5], atol=1e-14)
def test_linear_fitter_with_weights_flat(): """Same as the above #7035 test but with flattened inputs""" Xin, Yin = np.mgrid[0:21, 0:21] Xin, Yin = Xin.flatten(), Yin.flatten() fitter = LinearLSQFitter() with NumpyRNGContext(_RANDOM_SEED): zsig = np.random.normal(0, 0.01, size=Xin.shape) p2 = models.Polynomial2D(3) p2.parameters = np.arange(10)/1.2 z = p2(Xin, Yin) pmod = fitter(models.Polynomial2D(3), Xin, Yin, z + zsig, weights=zsig**(-2)) assert_allclose(pmod.parameters, p2.parameters, atol=10 ** (-2))
def test_fitting_shapes(): """ Test fitting model sets of Linear1D and Planar2D.""" fitter = LinearLSQFitter() model = Linear1D(slope=[1, 2], intercept=[3, 4], n_models=2) y = model(xx) fit_model = fitter(model, x, y) model = Linear1D(slope=[[1, 2]], intercept=[[3, 4]], n_models=2, model_set_axis=1) fit_model = fitter(model, x, y.T) model = Planar2D(slope_x=[1, 2], slope_y=[1, 2], intercept=[3, 4], n_models=2) y = model(xx, xx) fit_model = fitter(model, x, x, y)
def test_linear_fit_fixed_parameter(self): """ Tests fitting a polynomial model with a fixed parameter (issue #6135). """ init_model = models.Polynomial1D(degree=2, c1=1) init_model.c1.fixed = True x = np.arange(10) y = 2 + x + 0.5 * x * x fitter = LinearLSQFitter() with pytest.warns(AstropyUserWarning, match=r'The fit may be poorly conditioned'): fitted_model = fitter(init_model, x, y) assert_allclose(fitted_model.parameters, [2., 1., 0.5], atol=1e-14)
def test_linear_fit_2d_model_set_fixed_parameters(self): """ Tests fitting a 2d polynomial model set with fixed parameters (#6135). """ init_model = models.Polynomial2D(degree=2, c1_0=[1, 2], c0_1=[-0.5, 1], n_models=2, fixed={'c1_0': True, 'c0_1': True}) x, y = np.mgrid[0:5, 0:5] zz = np.array([1+x-0.5*y+0.1*x*x, 2*x+y-0.2*y*y]) fitter = LinearLSQFitter() fitted_model = fitter(init_model, x, y, zz) assert_allclose(fitted_model(x, y, model_set_axis=False), zz, atol=1e-14)
def test_linear_fit_model_set_fixed_parameter(self): """ Tests fitting a polynomial model set with a fixed parameter (#6135). """ init_model = models.Polynomial1D(degree=2, c1=[1, -2], n_models=2) init_model.c1.fixed = True x = np.arange(10) yy = np.array([2 + x + 0.5*x*x, -2*x]) fitter = LinearLSQFitter() fitted_model = fitter(init_model, x, yy) assert_allclose(fitted_model.c0, [2., 0.], atol=1e-14) assert_allclose(fitted_model.c1, [1., -2.], atol=1e-14) assert_allclose(fitted_model.c2, [0.5, 0.], atol=1e-14)
def test_linear_fit_model_set(self): """Tests fitting multiple models simultaneously.""" init_model = models.Polynomial1D(degree=2, c0=[1, 1], n_models=2) x = np.arange(10) y_expected = init_model(x, model_set_axis=False) assert y_expected.shape == (2, 10) # Add a bit of random noise with NumpyRNGContext(_RANDOM_SEED): y = y_expected + np.random.normal(0, 0.01, size=y_expected.shape) fitter = LinearLSQFitter() fitted_model = fitter(init_model, x, y) assert_allclose(fitted_model(x, model_set_axis=False), y_expected, rtol=1e-1)
def trace_fibres(data, slice_positions, slice_width, polynomial_order=2, spectral_axis=0, first_peak_position=None, peak_kwargs={}): if polynomial_order >= len(slice_positions): warn( "Polynomial order ({}) >= number of slices ({}), under-constrained." .format(polynomial_order, len(slice_positions))) try: # Convert CCDData or similar to masked array data = np.ma.array(data.data, mask=data.mask) except AttributeError: data = np.ma.array(data) slice_half_width = slice_width // 2 slice_peaks = [] for i, slice_position in enumerate(slice_positions): start = slice_position - slice_half_width stop = slice_position + slice_half_width + 1 if spectral_axis == 0: data_slice = data[start:stop, :] else: data_slice = data[:, start:stop] projection = data_slice.sum(axis=spectral_axis) if first_peak_position: peak_kwargs.update({'first_peak_position': first_peak_position}) peaks = find_taipan_peaks(projection, **peak_kwargs) # Store first peak from peak finder to use for next slice first_peak_position = peaks.data[0] slice_peaks.append(peaks) slice_peaks = np.array(slice_peaks) n_fibres = slice_peaks.shape[1] traces_init = Polynomial1D(degree=polynomial_order, n_models=n_fibres, c0=slice_peaks[0]) trace_fitter = LinearLSQFitter() traces = trace_fitter(traces_init, slice_positions, slice_peaks.T) return traces