def __new__(cls, x, y, order=None, s=None, w=None, bbox=[None]*2, k=3,
ext=0, check_finite=False, outlier_func=sigma_clip,
niter=3, grow=0, debug=False, **outlier_kwargs):
# Decide what sort of spline object we're making
spline_kwargs = {'bbox': bbox, 'k': k, 'ext': ext,
'check_finite': check_finite}
if order is None:
cls_ = UnivariateSpline
spline_args = ()
spline_kwargs['s'] = s
elif s is None:
cls_ = LSQUnivariateSpline
else:
raise ValueError("Both t and s have been specified")
# Both spline classes require sorted x, so do that here. We also
# require unique x values, so we're going to deal with duplicates by
# making duplicated values slightly larger. But we have to do this
# iteratively in case of a basket-case scenario like (1, 1, 1, 1+eps, 2)
# which would become (1, 1+eps, 1+2*eps, 1+eps, 2), which still has
# duplicates and isn't sorted!
# I can't think of any better way to cope with this, other than write
# least-squares spline-fitting code that handles duplicates from scratch
epsf = np.finfo(float).eps
orig_mask = np.zeros(y.shape, dtype=bool)
if isinstance(y, np.ma.masked_array):
if y.mask is not np.ma.nomask:
orig_mask = y.mask.astype(bool)
y = y.data
if w is not None:
orig_mask |= (w == 0)
if debug:
print(y)
print(orig_mask)
iter = 0
full_mask = orig_mask # Will include pixels masked because of "grow"
while iter < niter+1:
last_mask = full_mask
x_to_fit = x.astype(float)
if order is not None:
# Determine actual order to apply based on fraction of unmasked
# pixels, and unmask everything if there are too few good pixels
this_order = int(order * (1 - np.sum(full_mask) / len(full_mask)) + 0.5)
if this_order == 0:
full_mask = np.zeros(x.shape, dtype=bool)
if w is not None and not all(w == 0):
full_mask |= (w == 0)
this_order = int(order * (1 - np.sum(full_mask) / len(full_mask)) + 0.5)
if debug:
print("FULL MASK", full_mask)
xgood = x_to_fit[~full_mask]
while True:
xunique, indices = np.unique(xgood, return_index=True)
if len(indices) == len(xgood):
# All unique x values so continue
break
if order is None:
raise ValueError("Must specify spline order when there are "
"duplicate x values")
for i in range(len(xgood)):
if not (last_mask[i] or i in indices):
xgood[i] *= (1.0 + epsf)
# Space knots equally based on density of unique x values
if order is not None:
knots = [xunique[int(xx+0.5)]
for xx in np.linspace(0, len(xunique)-1, this_order+1)[1:-1]]
spline_args = (knots,)
if debug:
print ("KNOTS", knots)
sort_indices = np.argsort(xgood)
# Create appropriate spline object using current mask
try:
spline = cls_(xgood[sort_indices], y[~full_mask][sort_indices],
*spline_args, w=None if w is None else w[~full_mask][sort_indices],
**spline_kwargs)
except ValueError as e:
if this_order == 0:
avg_y = np.average(y[~full_mask],
weights=None if w is None else w[~full_mask])
spline = lambda xx: avg_y
else:
raise e
spline_y = spline(x)
#masked_residuals = outlier_func(spline_y - masked_y, **outlier_kwargs)
#mask = masked_residuals.mask
# When sigma-clipping, only remove the originally-masked points.
# Note that this differs from the astropy.modeling code because
# the sigma-clipping and spline-fitting are done independently here.
d, mask, v = NDStacker.sigclip(spline_y-y, mask=orig_mask, variance=None,
**outlier_kwargs)
if grow > 0:
maskarray = np.zeros((grow * 2 + 1, len(y)), dtype=bool)
for i in range(-grow, grow + 1):
mx1 = max(i, 0)
mx2 = min(len(y), len(y) + i)
maskarray[grow + i, mx1:mx2] = mask[:mx2 - mx1]
grow_mask = np.logical_or.reduce(maskarray, axis=0)
full_mask = np.logical_or(mask, grow_mask)
else:
full_mask = mask.astype(bool)
# Check if the mask is unchanged
if not np.logical_or.reduce(last_mask ^ full_mask):
break
iter += 1
# Create a standard BSpline object
try:
bspline = BSpline(*spline._eval_args)
except AttributeError:
# Create a spline object that's just a constant
bspline = BSpline(np.r_[(x[0],)*4, (x[-1],)*4],
np.r_[(spline(0),)*4, (0.,)*4], 3)
# Attach the mask and model (may be useful)
bspline.mask = full_mask
bspline.data = spline_y
return bspline
def __new__(cls,
x,
y,
order=None,
s=None,
w=None,
bbox=[None] * 2,
k=3,
ext=0,
check_finite=True,
outlier_func=sigma_clip,
niter=0,
grow=0,
debug=False,
**outlier_kwargs):
if niter is None:
niter = 100 # really should converge by this point
# Decide what sort of spline object we're making
spline_kwargs = {
'bbox': bbox,
'k': k,
'ext': ext,
'check_finite': check_finite
}
if order is None:
cls_ = UnivariateSpline
spline_args = ()
spline_kwargs['s'] = s
elif s is None:
cls_ = LSQUnivariateSpline
else:
raise ValueError("Both t and s have been specified")
# For compatibility with an older version which was using
# NDStacker.sigclip, rename parameters for sigma_clip
if 'lsigma' in outlier_kwargs:
outlier_kwargs['sigma_lower'] = outlier_kwargs.pop('lsigma')
if 'hsigma' in outlier_kwargs:
outlier_kwargs['sigma_upper'] = outlier_kwargs.pop('hsigma')
# Both spline classes require sorted x, so do that here. We also
# require unique x values, so we're going to deal with duplicates by
# making duplicated values slightly larger. But we have to do this
# iteratively in case of a basket-case scenario like (1, 1, 1, 1+eps, 2)
# which would become (1, 1+eps, 1+2*eps, 1+eps, 2), which still has
# duplicates and isn't sorted!
# I can't think of any better way to cope with this, other than write
# least-squares spline-fitting code that handles duplicates from scratch
epsf = np.finfo(float).eps
orig_mask = np.zeros(y.shape, dtype=bool)
if isinstance(y, np.ma.masked_array):
if y.mask is not np.ma.nomask:
orig_mask = y.mask.astype(bool)
y = y.data
if w is not None:
orig_mask |= (w == 0)
if debug:
print('y=', y)
print('orig_mask=', orig_mask.astype(int))
iteration = 0
full_mask = orig_mask # Will include pixels masked because of "grow"
while iteration < niter + 1:
last_mask = full_mask
x_to_fit = x.astype(float)
if order is not None:
# Determine actual order to apply based on fraction of unmasked
# pixels, and unmask everything if there are too few good pixels
this_order = int(order *
(1 - np.sum(full_mask) / full_mask.size) +
0.5)
if this_order == 0 and order > 0:
full_mask = np.zeros(x.shape, dtype=bool)
if w is not None and not all(w == 0):
full_mask |= (w == 0)
this_order = int(order *
(1 - np.sum(full_mask) / full_mask.size) +
0.5)
if debug:
print("FULL MASK", full_mask)
xgood = x_to_fit[~full_mask]
while True:
if debug:
print(f"Iter {iteration}: epsf loop")
xunique, indices = np.unique(xgood, return_index=True)
if indices.size == xgood.size:
# All unique x values so continue
break
if order is None:
raise ValueError(
"Must specify spline order when there are "
"duplicate x values")
for i in range(xgood.size):
if i not in indices:
xgood[i] *= (1.0 + epsf)
# Space knots equally based on density of unique x values
if order is not None:
# Ensure the spline is constrained: num_points - k is max order
this_order = min(this_order, xgood.size - k)
knots = [
xunique[int(xx + 0.5)]
for xx in np.linspace(0, xunique.size - 1, this_order +
1)[1:-1]
]
spline_args = (knots, )
if debug:
print("KNOTS", knots)
sort_indices = np.argsort(xgood)
# Create appropriate spline object using current mask
if order is None or this_order > 0:
spline = cls_(
xgood[sort_indices],
y[~full_mask][sort_indices],
*spline_args,
w=None if w is None else w[~full_mask][sort_indices],
**spline_kwargs)
else:
avg_y = np.average(
y[~full_mask],
weights=None if w is None else w[~full_mask])
spline = lambda xx: avg_y
spline_y = spline(x)
masked_residuals = outlier_func(
np.ma.array(spline_y - y, mask=full_mask), **outlier_kwargs)
mask = masked_residuals.mask
if debug:
print('mask=', mask.astype(int))
if grow > 0:
new_mask = mask ^ full_mask
if new_mask.any():
for i in range(1, grow + 1):
mask[i:] |= new_mask[:-i]
mask[:-i] |= new_mask[i:]
if debug:
print('mask after growth=', mask.astype(int))
full_mask = mask
# Check if the mask is unchanged
if not np.logical_or.reduce(last_mask ^ full_mask):
if debug:
print(f"Iter {iteration}: Breaking")
break
if debug:
print(f"Iter {iteration}: Starting new iteration")
iteration += 1
# Create a standard BSpline object
try:
bspline = BSpline(*spline._eval_args)
except AttributeError:
# Create a spline object that's just a constant
bspline = BSpline(np.r_[(x[0], ) * 4, (x[-1], ) * 4],
np.r_[(spline(0), ) * 4, (0., ) * 4], 3)
# Attach the mask and model (may be useful)
bspline.mask = full_mask
bspline.data = spline_y
return bspline