def test_fitting_with_outlier_removal_niter(): """ Test that FittingWithOutlierRemoval stops prior to reaching niter if the set of masked points has converged and correctly reports the actual number of iterations performed. """ # 2 rows with some noise around a constant level and 1 deviant point: x = np.arange(25) with NumpyRNGContext(_RANDOM_SEED): y = np.random.normal(loc=10., scale=1., size=(2, 25)) y[0, 14] = 100. # Fit 2 models with up to 5 iterations (should only take 2): fitter = FittingWithOutlierRemoval( fitter=LinearLSQFitter(), outlier_func=sigma_clip, niter=5, sigma_lower=3., sigma_upper=3., maxiters=1 ) model, mask = fitter(models.Chebyshev1D(2, n_models=2), x, y) # Confirm that only the deviant point was rejected, in 2 iterations: assert_equal(np.where(mask), [[0], [14]]) assert fitter.fit_info['niter'] == 2 # Refit just the first row without any rejection iterations, to ensure # there are no regressions for that special case: fitter = FittingWithOutlierRemoval( fitter=LinearLSQFitter(), outlier_func=sigma_clip, niter=0, sigma_lower=3., sigma_upper=3., maxiters=1 ) model, mask = fitter(models.Chebyshev1D(2), x, y[0]) # Confirm that there were no iterations or rejected points: assert mask.sum() == 0 assert fitter.fit_info['niter'] == 0
def test_with_fitters_and_sigma_clip(self): import scipy.stats as stats np.random.seed(0) c = stats.bernoulli.rvs(0.25, size=self.z.shape) self.z += (np.random.normal(0., 0.2, self.z.shape) + c*np.random.normal(self.z, 2.0, self.z.shape)) guess = self.initial_guess(self.z, np.array([self.y, self.x])) g2_init = models.Gaussian2D(amplitude=guess[0], x_mean=guess[1], y_mean=guess[2], x_stddev=0.75, y_stddev=1.25) # test with Levenberg-Marquardt Least Squares fitter fit = FittingWithOutlierRemoval(LevMarLSQFitter(), sigma_clip, niter=3, sigma=3.) fitted_model, _ = fit(g2_init, self.x, self.y, self.z) assert_allclose(fitted_model.parameters[0:5], self.model_params, atol=1e-1) # test with Sequential Least Squares Programming fitter fit = FittingWithOutlierRemoval(SLSQPLSQFitter(), sigma_clip, niter=3, sigma=3.) fitted_model, _ = fit(g2_init, self.x, self.y, self.z) assert_allclose(fitted_model.parameters[0:5], self.model_params, atol=1e-1) # test with Simplex LSQ fitter fit = FittingWithOutlierRemoval(SimplexLSQFitter(), sigma_clip, niter=3, sigma=3.) fitted_model, _ = fit(g2_init, self.x, self.y, self.z) assert_allclose(fitted_model.parameters[0:5], self.model_params, atol=1e-1)
def test_with_fitters_and_sigma_clip(self): import scipy.stats as stats np.random.seed(0) c = stats.bernoulli.rvs(0.25, size=self.x.shape) self.y += (np.random.normal(0., 0.2, self.x.shape) + c * np.random.normal(3.0, 5.0, self.x.shape)) g_init = models.Gaussian1D(amplitude=1., mean=0, stddev=1.) # test with Levenberg-Marquardt Least Squares fitter fit = FittingWithOutlierRemoval(LevMarLSQFitter(), sigma_clip, niter=3, sigma=3.0) fitted_model, _ = fit(g_init, self.x, self.y) assert_allclose(fitted_model.parameters, self.model_params, rtol=1e-1) # test with Sequential Least Squares Programming fitter fit = FittingWithOutlierRemoval(SLSQPLSQFitter(), sigma_clip, niter=3, sigma=3.0) fitted_model, _ = fit(g_init, self.x, self.y) assert_allclose(fitted_model.parameters, self.model_params, rtol=1e-1) # test with Simplex LSQ fitter fit = FittingWithOutlierRemoval(SimplexLSQFitter(), sigma_clip, niter=3, sigma=3.0) fitted_model, _ = fit(g_init, self.x, self.y) assert_allclose(fitted_model.parameters, self.model_params, atol=1e-1)
def test_2d_without_weights_with_sigma_clip(self): model = models.Polynomial2D(0) fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip, niter=3, sigma=3.) fit, mask = fitter(model, self.x, self.y, self.z) assert((~mask).sum() == self.z.size - 2) assert(mask[0, 0] and mask[0, 1]) assert_allclose(fit.parameters[0], 0.0, atol=10**(-2))
def test_1d_with_weights_with_sigma_clip(self): """smoke test for #7020 - fails without fitting.py patch because weights does not propagate""" model = models.Polynomial1D(0) fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip, niter=3, sigma=3.) fit, filtered = fitter(model, self.x1d, self.z1d, weights=self.weights1d) assert(fit.parameters[0] > 10**(-2)) # weights pulled it > 0 assert(fit.parameters[0] < 1.0) # outliers didn't pull it out of [-1:1] because they had been removed
def test_1d_without_weights_with_sigma_clip(self): model = models.Polynomial1D(0) fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip, niter=3, sigma=3.) fit, mask = fitter(model, self.x1d, self.z1d) assert((~mask).sum() == self.z1d.size - 2) assert(mask[0] and mask[1]) assert_allclose(fit.parameters[0], 0.0, atol=10**(-2)) # with removed outliers mean is 0.0
def test_2d_linear_with_weights_with_sigma_clip(self): """same as test above with a linear fitter.""" model = models.Polynomial2D(0) fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip, niter=3, sigma=3.) fit, filtered = fitter(model, self.x, self.y, self.z, weights=self.weights) assert(fit.parameters[0] > 10**(-2)) # weights pulled it > 0 assert(fit.parameters[0] < 1.0) # outliers didn't pull it out of [-1:1] because they had been removed
def test_1d_set_with_common_weights_with_sigma_clip(self): """added for #6819 (1D model set with weights in common)""" model = models.Polynomial1D(0, n_models=2) fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip, niter=3, sigma=3.) z1d = np.array([self.z1d, self.z1d]) fit, filtered = fitter(model, self.x1d, z1d, weights=self.weights1d) assert_allclose(fit.parameters, [0.8, 0.8], atol=1e-14)
def test_2d_with_weights_with_sigma_clip(self): """smoke test for #7020 - fails without fitting.py patch because weights does not propagate""" model = models.Polynomial2D(0) fitter = FittingWithOutlierRemoval(LevMarLSQFitter(), sigma_clip, niter=3, sigma=3.) with pytest.warns(AstropyUserWarning, match=r'Model is linear in parameters'): fit, filtered = fitter(model, self.x, self.y, self.z, weights=self.weights) assert(fit.parameters[0] > 10**(-2)) # weights pulled it > 0 assert(fit.parameters[0] < 1.0) # outliers didn't pull it out of [-1:1] because they had been removed
def test_with_fitters_and_sigma_clip(self, fitter): import scipy.stats as stats fitter = fitter() np.random.seed(0) c = stats.bernoulli.rvs(0.25, size=self.x.shape) y = self.y + (np.random.normal(0., 0.2, self.x.shape) + c*np.random.normal(3.0, 5.0, self.x.shape)) g_init = models.Gaussian1D(amplitude=1., mean=0, stddev=1.) fit = FittingWithOutlierRemoval(fitter, sigma_clip, niter=3, sigma=3.0) fitted_model, _ = fit(g_init, self.x, y) assert_allclose(fitted_model.parameters, self.model_params, rtol=1e-1)
def test_2d_set_axis_2_fitting_with_outlier_removal(): """Test fitting 2D model set (axis 2) with outlier removal (issue #6819)""" poly_set = models.Polynomial2D(1, n_models=2, model_set_axis=2) fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip, sigma=2.5, niter=3, cenfunc=np.ma.mean, stdfunc=np.ma.std) y, x = np.mgrid[0:5, 0:5] z = np.rollaxis(np.array([x+y, 1-0.1*x+0.2*y]), 0, 3) z[3, 3:5, 0] = 100. # outliers poly_set, filt_z = fitter(poly_set, x, y, z) assert_allclose(poly_set.c0_0, [[[0., 1.]]], atol=1e-14) assert_allclose(poly_set.c1_0, [[[1., -0.1]]], atol=1e-14) assert_allclose(poly_set.c0_1, [[[1., 0.2]]], atol=1e-14)
def test_1d_set_fitting_with_outlier_removal(): """Test model set fitting with outlier removal (issue #6819)""" poly_set = models.Polynomial1D(2, n_models=2) fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip, sigma=2.5, niter=3, cenfunc=np.ma.mean, stdfunc=np.ma.std) x = np.arange(10) y = np.array([2.5*x - 4, 2*x*x + x + 10]) y[1, 5] = -1000 # outlier poly_set, filt_y = fitter(poly_set, x, y) assert_allclose(poly_set.c0, [-4., 10.], atol=1e-14) assert_allclose(poly_set.c1, [2.5, 1.], atol=1e-14) assert_allclose(poly_set.c2, [0., 2.], atol=1e-14)
def correct_outlier_time(time, sigma=3): """Detect time outliers and correct them by interpololation.""" # lazy import of astropy from astropy.modeling.models import Polynomial1D from astropy.modeling.fitting import LinearLSQFitter from astropy.modeling.fitting import FittingWithOutlierRemoval from astropy.stats import sigma_clip # create fitting model model = Polynomial1D(1) fitter = LinearLSQFitter() fitter = FittingWithOutlierRemoval(fitter, sigma_clip, sigma=sigma) # apply model and get mask time = pd.Series(time.astype('int64')) model, mask = fitter(model, np.arange(len(time)), time) # apply mask and interpolation time[mask] = np.nan return pd.to_datetime(time.interpolate(), unit='ns')
def test_with_fitters_and_sigma_clip(self, fitter): import scipy.stats as stats fitter = fitter() np.random.seed(0) c = stats.bernoulli.rvs(0.25, size=self.z.shape) z = self.z + (np.random.normal(0., 0.2, self.z.shape) + c*np.random.normal(self.z, 2.0, self.z.shape)) guess = self.initial_guess(self.z, np.array([self.y, self.x])) g2_init = models.Gaussian2D(amplitude=guess[0], x_mean=guess[1], y_mean=guess[2], x_stddev=0.75, y_stddev=1.25) fit = FittingWithOutlierRemoval(fitter, sigma_clip, niter=3, sigma=3.) fitted_model, _ = fit(g2_init, self.x, self.y, z) assert_allclose(fitted_model.parameters[0:5], self.model_params, atol=1e-1)