def test_fitting_with_outlier_removal_niter():
"""
Test that FittingWithOutlierRemoval stops prior to reaching niter if the
set of masked points has converged and correctly reports the actual number
of iterations performed.
"""
# 2 rows with some noise around a constant level and 1 deviant point:
x = np.arange(25)
with NumpyRNGContext(_RANDOM_SEED):
y = np.random.normal(loc=10., scale=1., size=(2, 25))
y[0, 14] = 100.
# Fit 2 models with up to 5 iterations (should only take 2):
fitter = FittingWithOutlierRemoval(
fitter=LinearLSQFitter(), outlier_func=sigma_clip, niter=5,
sigma_lower=3., sigma_upper=3., maxiters=1
)
model, mask = fitter(models.Chebyshev1D(2, n_models=2), x, y)
# Confirm that only the deviant point was rejected, in 2 iterations:
assert_equal(np.where(mask), [[0], [14]])
assert fitter.fit_info['niter'] == 2
# Refit just the first row without any rejection iterations, to ensure
# there are no regressions for that special case:
fitter = FittingWithOutlierRemoval(
fitter=LinearLSQFitter(), outlier_func=sigma_clip, niter=0,
sigma_lower=3., sigma_upper=3., maxiters=1
)
model, mask = fitter(models.Chebyshev1D(2), x, y[0])
# Confirm that there were no iterations or rejected points:
assert mask.sum() == 0
assert fitter.fit_info['niter'] == 0
def test_linear_fit_model_set_common_weight():
"""Tests fitting multiple models simultaneously."""
init_model = Polynomial1D(degree=2, c0=[1, 1], n_models=2)
x = np.arange(10)
y_expected = init_model(x, model_set_axis=False)
assert y_expected.shape == (2, 10)
# Add a bit of random noise
with NumpyRNGContext(_RANDOM_SEED):
y = y_expected + np.random.normal(0, 0.01, size=y_expected.shape)
fitter = LinearLSQFitter()
weights = np.ones(10)
weights[[0, -1]] = 0
fitted_model = fitter(init_model, x, y, weights=weights)
assert_allclose(fitted_model(x, model_set_axis=False), y_expected,
rtol=1e-1)
# Check that using null weights raises an error
# ValueError: On entry to DLASCL parameter number 4 had an illegal value
with pytest.raises(ValueError,
match='Found NaNs in the coefficient matrix'):
with pytest.warns(RuntimeWarning,
match=r'invalid value encountered in.*divide'):
fitted_model = fitter(init_model, x, y, weights=np.zeros(10))
def test_covariance_std_printing_indexing(self, capsys):
"""
Test printing methods and indexing.
"""
# test str representation for Covariance/stds
fitter = LinearLSQFitter(calc_uncertainties=True)
mod = models.Linear1D()
fit_mod = fitter(mod, self.x, mod(self.x)+self.rand)
print(fit_mod.cov_matrix)
captured = capsys.readouterr()
assert "slope | 0.001" in captured.out
assert "intercept| -0.006, 0.041" in captured.out
print(fit_mod.stds)
captured = capsys.readouterr()
assert "slope | 0.038" in captured.out
assert "intercept| 0.203" in captured.out
# test 'pprint' for Covariance/stds
print(fit_mod.cov_matrix.pprint(round_val=5, max_lines=1))
captured = capsys.readouterr()
assert "slope | 0.00144" in captured.out
assert "intercept" not in captured.out
print(fit_mod.stds.pprint(max_lines=1, round_val=5))
captured = capsys.readouterr()
assert "slope | 0.03799" in captured.out
assert "intercept" not in captured.out
# test indexing for Covariance class.
assert fit_mod.cov_matrix[0, 0] == fit_mod.cov_matrix['slope', 'slope']
# test indexing for stds class.
assert fit_mod.stds[1] == fit_mod.stds['intercept']
def test_linear_1d_separate_weights_axis_1(self):
model = Polynomial1D(1, model_set_axis=1)
fitter = LinearLSQFitter()
model = fitter(model, self.x1, self.y1.T,
weights=self.w1[..., np.newaxis])
assert_allclose(model.c0, 0.5, atol=1e-12)
assert_allclose(model.c1, 2.5, atol=1e-12)
def test_model_set_axis_outputs():
fitter = LinearLSQFitter()
model_set = Polynomial2D(1, n_models=2, model_set_axis=2)
y2, x2 = np.mgrid[:5, :5]
# z = np.moveaxis([x2 + y2, 1 - 0.1 * x2 + 0.2 * y2]), 0, 2)
z = np.rollaxis(np.array([x2 + y2, 1 - 0.1 * x2 + 0.2 * y2]), 0, 3)
model = fitter(model_set, x2, y2, z)
res = model(x2, y2, model_set_axis=False)
assert z.shape == res.shape
# Test initializing with integer model_set_axis
# and evaluating with a different model_set_axis
model_set = Polynomial1D(1,
c0=[1, 2],
c1=[2, 3],
n_models=2,
model_set_axis=0)
y0 = model_set(xx)
y1 = model_set(xx.T, model_set_axis=1)
assert_allclose(y0[0], y1[:, 0])
assert_allclose(y0[1], y1[:, 1])
model_set = Polynomial1D(1,
c0=[[1, 2]],
c1=[[2, 3]],
n_models=2,
model_set_axis=1)
y0 = model_set(xx.T)
y1 = model_set(xx, model_set_axis=0)
assert_allclose(y0[:, 0], y1[0])
assert_allclose(y0[:, 1], y1[1])
with pytest.raises(ValueError):
model_set(x)
def test_linear_2d_common_weights(self):
model = Polynomial2D(1)
fitter = LinearLSQFitter()
model = fitter(model, self.x2, self.y2, self.z2, weights=self.w2)
assert_allclose(model.c0_0, 1., atol=1e-12)
assert_allclose(model.c1_0, -0.1, atol=1e-12)
assert_allclose(model.c0_1, 0.2, atol=1e-12)
def trace_fit(layout, side='left', degree=1):
'''Linear fit to describe edge of a spectral trace
Parameters
----------
layout : [table]
a table describing a spectral trace
side : [str]
can be 'left', 'right' or 'centre'
'''
from astropy.modeling.models import Polynomial1D
from astropy.modeling.fitting import LinearLSQFitter
if side == 'left':
xpt = layout['x1']
ypt = layout['y1']
name = 'left edge'
elif side == 'centre':
xpt = layout['x2']
ypt = layout['y2']
name = 'trace centre'
elif side == 'right':
xpt = layout['x3']
ypt = layout['y3']
name = 'right edge'
else:
raise ValueError('Side ' + str(side) + ' not recognized')
pinit = Polynomial1D(degree=degree)
fitter = LinearLSQFitter()
edge_fit = fitter(pinit, ypt, xpt)
edge_fit.name = name
return edge_fit
def test_linear_1d_separate_weights(self):
model = Polynomial1D(1)
fitter = LinearLSQFitter()
model = fitter(model, self.x1, self.y1,
weights=self.w1[np.newaxis, ...])
assert_allclose(model.c0, 0.5, atol=1e-12)
assert_allclose(model.c1, 2.5, atol=1e-12)
def test_1d_without_weights_with_sigma_clip(self):
model = models.Polynomial1D(0)
fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip,
niter=3, sigma=3.)
fit, mask = fitter(model, self.x1d, self.z1d)
assert((~mask).sum() == self.z1d.size - 2)
assert(mask[0] and mask[1])
assert_allclose(fit.parameters[0], 0.0, atol=10**(-2)) # with removed outliers mean is 0.0
def test_linear_2d_separate_weights_axis_2(self):
model = Polynomial2D(1, model_set_axis=2)
fitter = LinearLSQFitter()
model = fitter(model, self.x2, self.y2, np.rollaxis(self.z2, 0, 3),
weights=self.w2[..., np.newaxis])
assert_allclose(model.c0_0, 1., atol=1e-12)
assert_allclose(model.c1_0, -0.1, atol=1e-12)
assert_allclose(model.c0_1, 0.2, atol=1e-12)
def test_1d_with_weights_with_sigma_clip(self):
"""smoke test for #7020 - fails without fitting.py patch because weights does not propagate"""
model = models.Polynomial1D(0)
fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip,
niter=3, sigma=3.)
fit, filtered = fitter(model, self.x1d, self.z1d, weights=self.weights1d)
assert(fit.parameters[0] > 10**(-2)) # weights pulled it > 0
assert(fit.parameters[0] < 1.0) # outliers didn't pull it out of [-1:1] because they had been removed
def test_2d_without_weights_with_sigma_clip(self):
model = models.Polynomial2D(0)
fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip,
niter=3, sigma=3.)
fit, mask = fitter(model, self.x, self.y, self.z)
assert((~mask).sum() == self.z.size - 2)
assert(mask[0, 0] and mask[0, 1])
assert_allclose(fit.parameters[0], 0.0, atol=10**(-2))
def test_2d_linear_with_weights_with_sigma_clip(self):
"""same as test above with a linear fitter."""
model = models.Polynomial2D(0)
fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip,
niter=3, sigma=3.)
fit, filtered = fitter(model, self.x, self.y, self.z,
weights=self.weights)
assert(fit.parameters[0] > 10**(-2)) # weights pulled it > 0
assert(fit.parameters[0] < 1.0) # outliers didn't pull it out of [-1:1] because they had been removed
def test_1d_set_with_common_weights_with_sigma_clip(self):
"""added for #6819 (1D model set with weights in common)"""
model = models.Polynomial1D(0, n_models=2)
fitter = FittingWithOutlierRemoval(LinearLSQFitter(), sigma_clip,
niter=3, sigma=3.)
z1d = np.array([self.z1d, self.z1d])
fit, filtered = fitter(model, self.x1d, z1d, weights=self.weights1d)
assert_allclose(fit.parameters, [0.8, 0.8], atol=1e-14)
def poly_solution(self, order):
from astropy.modeling.fitting import LinearLSQFitter
from astropy.modeling.models import Polynomial1D
poly = Polynomial1D(order)
ftr = LinearLSQFitter()
poly = ftr(poly, list(self.linepixtowl.keys()),
list(self.linepixtowl.values()))
return poly
def setup_class(self):
self.model = models.Polynomial2D(2)
self.y, self.x = np.mgrid[:5, :5]
def poly2(x, y):
return 1 + 2 * x + 3 * x**2 + 4 * y + 5 * y**2 + 6 * x * y
self.z = poly2(self.x, self.y)
self.fitter = LinearLSQFitter()
def test_linear_fit_model_set_errors():
init_model = Polynomial1D(degree=2, c0=[1, 1], n_models=2)
x = np.arange(10)
y = init_model(x, model_set_axis=False)
fitter = LinearLSQFitter()
with pytest.raises(ValueError):
fitter(init_model, x[:5], y)
with pytest.raises(ValueError):
fitter(init_model, x, y[:, :5])
def test_compound_model_raises_error(self):
"""Test that if an user tries to use a compound model, raises an error"""
with pytest.raises(ValueError) as excinfo:
init_model1 = models.Polynomial1D(degree=2, c0=[1, 1], n_models=2)
init_model2 = models.Polynomial1D(degree=2, c0=[1, 1], n_models=2)
init_model_comp = init_model1 + init_model2
x = np.arange(10)
y = init_model_comp(x, model_set_axis=False)
fitter = LinearLSQFitter()
_ = fitter(init_model_comp, x, y)
assert "Model must be simple, not compound" in str(excinfo.value)
def test_non_linear_lsq_fitter_with_weights(self, fitter):
"""
Tests that issue #11581 has been solved.
"""
fitter = fitter()
np.random.seed(42)
norder = 2
fitter2 = LinearLSQFitter()
model = models.Polynomial1D(norder)
npts = 10000
c = [2.0, -10.0, 7.0]
tw = np.random.uniform(0.0, 10.0, npts)
tx = np.random.uniform(0.0, 10.0, npts)
ty = c[0] + c[1] * tx + c[2] * (tx ** 2)
ty += np.random.normal(0.0, 1.5, npts)
with pytest.warns(AstropyUserWarning, match=r'Model is linear in parameters'):
tf1 = fitter(model, tx, ty, weights=tw)
tf2 = fitter2(model, tx, ty, weights=tw)
assert_allclose(tf1.parameters, tf2.parameters,
atol=10 ** (-16))
assert_allclose(tf1.parameters, c,
rtol=10 ** (-2), atol=10 ** (-2))
model = models.Gaussian1D()
if isinstance(fitter, TRFLSQFitter) or isinstance(fitter, LMLSQFitter):
with pytest.warns(AstropyUserWarning, match=r'The fit may be unsuccessful; *.'):
fitter(model, tx, ty, weights=tw)
else:
fitter(model, tx, ty, weights=tw)
model = models.Polynomial2D(norder)
nxpts = 100
nypts = 150
npts = nxpts * nypts
c = [1.0, 4.0, 7.0, -8.0, -9.0, -3.0]
tw = np.random.uniform(0.0, 10.0, npts).reshape(nxpts, nypts)
tx = np.random.uniform(0.0, 10.0, npts).reshape(nxpts, nypts)
ty = np.random.uniform(0.0, 10.0, npts).reshape(nxpts, nypts)
tz = c[0] + c[1] * tx + c[2] * (tx ** 2) + c[3] * ty + c[4] * (ty ** 2) + c[5] * tx * ty
tz += np.random.normal(0.0, 1.5, npts).reshape(nxpts, nypts)
with pytest.warns(AstropyUserWarning, match=r'Model is linear in parameters'):
tf1 = fitter(model, tx, ty, tz, weights=tw)
tf2 = fitter2(model, tx, ty, tz, weights=tw)
assert_allclose(tf1.parameters, tf2.parameters,
atol=10 ** (-16))
assert_allclose(tf1.parameters, c,
rtol=10 ** (-2), atol=10 ** (-2))
def measure_SNR(spex_path, data_path, plot_path, star, ranges):
"""
Measure the SNR of a spectrum, by fitting straight lines to pieces of the spectrum
"""
# plot the spectrum to define regions without spectral lines
fig, ax = plot_spectrum(star, data_path, range=[0.75, 5.6], log=True)
# read in all bands and spectra for this star
starobs = StarData("%s.dat" % star.lower(), path=data_path, use_corfac=True)
# obtain flux values at a few wavelengths and fit a straight line through the data
waves, fluxes, uncs = starobs.get_flat_data_arrays(["SpeX_SXD", "SpeX_LXD"])
print(star)
for range in ranges:
min_indx = np.abs(waves - range[0]).argmin()
max_indx = np.abs(waves - range[1]).argmin()
func = Linear1D()
fit = LinearLSQFitter()
fit_result = fit(func, waves[min_indx:max_indx], fluxes[min_indx:max_indx])
residu = fluxes[min_indx:max_indx] - fit_result(waves[min_indx:max_indx])
# calculate the SNR from the data
data_sxd = Table.read(
spex_path + star + "_sxd.txt",
format="ascii",
)
data_lxd = Table.read(
spex_path + star + "_lxd.txt",
format="ascii",
)
data = vstack([data_sxd, data_lxd])
data.sort("col1")
print("wave_range", range)
min_indx2 = np.abs(data["col1"] - range[0]).argmin()
max_indx2 = np.abs(data["col1"] - range[1]).argmin()
SNR_data = np.nanmedian((data["col2"] / data["col3"])[min_indx2:max_indx2])
print("SNR from data", SNR_data)
# calculate the SNR from the noise around the linear fit
mean, median, stddev = sigma_clipped_stats(residu)
SNR_fit = np.median(fluxes[min_indx:max_indx] / stddev)
print("SNR from fit", SNR_fit)
# plot the fitted lines on top of the spectrum
ax.plot(
waves[min_indx:max_indx],
fit_result(waves[min_indx:max_indx]),
lw=2,
alpha=0.8,
color="k",
)
fig.savefig(plot_path + star + "_SNR_measure.pdf")
def test_linear_fitter_with_weights():
"""Regression test for #7035"""
Xin, Yin = np.mgrid[0:21, 0:21]
fitter = LinearLSQFitter()
with NumpyRNGContext(_RANDOM_SEED):
zsig = np.random.normal(0, 0.01, size=Xin.shape)
p2 = models.Polynomial2D(3)
p2.parameters = np.arange(10)/1.2
z = p2(Xin, Yin)
pmod = fitter(models.Polynomial2D(3), Xin, Yin, z + zsig, weights=zsig**(-2))
assert_allclose(pmod.parameters, p2.parameters, atol=10 ** (-2))
def test_linear_fit_2d_model_set_common_weight():
init_model = Polynomial2D(degree=2, c1_0=[1, 2], c0_1=[-0.5, 1],
n_models=2,
fixed={'c1_0': True, 'c0_1': True})
x, y = np.mgrid[0:5, 0:5]
zz = np.array([1+x-0.5*y+0.1*x*x, 2*x+y-0.2*y*y])
fitter = LinearLSQFitter()
fitted_model = fitter(init_model, x, y, zz, weights=np.ones((5, 5)))
assert_allclose(fitted_model(x, y, model_set_axis=False), zz,
atol=1e-14)
def test_linear_fit_fixed_parameter(self):
"""
Tests fitting a polynomial model with a fixed parameter (issue #6135).
"""
init_model = models.Polynomial1D(degree=2, c1=1)
init_model.c1.fixed = True
x = np.arange(10)
y = 2 + x + 0.5*x*x
fitter = LinearLSQFitter()
fitted_model = fitter(init_model, x, y)
assert_allclose(fitted_model.parameters, [2., 1., 0.5], atol=1e-14)
def test_linear_fitter_with_weights_flat():
"""Same as the above #7035 test but with flattened inputs"""
Xin, Yin = np.mgrid[0:21, 0:21]
Xin, Yin = Xin.flatten(), Yin.flatten()
fitter = LinearLSQFitter()
with NumpyRNGContext(_RANDOM_SEED):
zsig = np.random.normal(0, 0.01, size=Xin.shape)
p2 = models.Polynomial2D(3)
p2.parameters = np.arange(10)/1.2
z = p2(Xin, Yin)
pmod = fitter(models.Polynomial2D(3), Xin, Yin, z + zsig, weights=zsig**(-2))
assert_allclose(pmod.parameters, p2.parameters, atol=10 ** (-2))
def test_fitting_shapes():
""" Test fitting model sets of Linear1D and Planar2D."""
fitter = LinearLSQFitter()
model = Linear1D(slope=[1, 2], intercept=[3, 4], n_models=2)
y = model(xx)
fit_model = fitter(model, x, y)
model = Linear1D(slope=[[1, 2]], intercept=[[3, 4]], n_models=2, model_set_axis=1)
fit_model = fitter(model, x, y.T)
model = Planar2D(slope_x=[1, 2], slope_y=[1, 2], intercept=[3, 4], n_models=2)
y = model(xx, xx)
fit_model = fitter(model, x, x, y)
def test_linear_fit_fixed_parameter(self):
"""
Tests fitting a polynomial model with a fixed parameter (issue #6135).
"""
init_model = models.Polynomial1D(degree=2, c1=1)
init_model.c1.fixed = True
x = np.arange(10)
y = 2 + x + 0.5 * x * x
fitter = LinearLSQFitter()
with pytest.warns(AstropyUserWarning,
match=r'The fit may be poorly conditioned'):
fitted_model = fitter(init_model, x, y)
assert_allclose(fitted_model.parameters, [2., 1., 0.5], atol=1e-14)
def test_linear_fit_2d_model_set_fixed_parameters(self):
"""
Tests fitting a 2d polynomial model set with fixed parameters (#6135).
"""
init_model = models.Polynomial2D(degree=2, c1_0=[1, 2], c0_1=[-0.5, 1],
n_models=2,
fixed={'c1_0': True, 'c0_1': True})
x, y = np.mgrid[0:5, 0:5]
zz = np.array([1+x-0.5*y+0.1*x*x, 2*x+y-0.2*y*y])
fitter = LinearLSQFitter()
fitted_model = fitter(init_model, x, y, zz)
assert_allclose(fitted_model(x, y, model_set_axis=False), zz,
atol=1e-14)
def test_linear_fit_model_set_fixed_parameter(self):
"""
Tests fitting a polynomial model set with a fixed parameter (#6135).
"""
init_model = models.Polynomial1D(degree=2, c1=[1, -2], n_models=2)
init_model.c1.fixed = True
x = np.arange(10)
yy = np.array([2 + x + 0.5*x*x, -2*x])
fitter = LinearLSQFitter()
fitted_model = fitter(init_model, x, yy)
assert_allclose(fitted_model.c0, [2., 0.], atol=1e-14)
assert_allclose(fitted_model.c1, [1., -2.], atol=1e-14)
assert_allclose(fitted_model.c2, [0.5, 0.], atol=1e-14)
def test_linear_fit_model_set(self):
"""Tests fitting multiple models simultaneously."""
init_model = models.Polynomial1D(degree=2, c0=[1, 1], n_models=2)
x = np.arange(10)
y_expected = init_model(x, model_set_axis=False)
assert y_expected.shape == (2, 10)
# Add a bit of random noise
with NumpyRNGContext(_RANDOM_SEED):
y = y_expected + np.random.normal(0, 0.01, size=y_expected.shape)
fitter = LinearLSQFitter()
fitted_model = fitter(init_model, x, y)
assert_allclose(fitted_model(x, model_set_axis=False), y_expected,
rtol=1e-1)
def trace_fibres(data,
slice_positions,
slice_width,
polynomial_order=2,
spectral_axis=0,
first_peak_position=None,
peak_kwargs={}):
if polynomial_order >= len(slice_positions):
warn(
"Polynomial order ({}) >= number of slices ({}), under-constrained."
.format(polynomial_order, len(slice_positions)))
try:
# Convert CCDData or similar to masked array
data = np.ma.array(data.data, mask=data.mask)
except AttributeError:
data = np.ma.array(data)
slice_half_width = slice_width // 2
slice_peaks = []
for i, slice_position in enumerate(slice_positions):
start = slice_position - slice_half_width
stop = slice_position + slice_half_width + 1
if spectral_axis == 0:
data_slice = data[start:stop, :]
else:
data_slice = data[:, start:stop]
projection = data_slice.sum(axis=spectral_axis)
if first_peak_position:
peak_kwargs.update({'first_peak_position': first_peak_position})
peaks = find_taipan_peaks(projection, **peak_kwargs)
# Store first peak from peak finder to use for next slice
first_peak_position = peaks.data[0]
slice_peaks.append(peaks)
slice_peaks = np.array(slice_peaks)
n_fibres = slice_peaks.shape[1]
traces_init = Polynomial1D(degree=polynomial_order,
n_models=n_fibres,
c0=slice_peaks[0])
trace_fitter = LinearLSQFitter()
traces = trace_fitter(traces_init, slice_positions, slice_peaks.T)
return traces
