def _perform(self):
"""
Returns an Argument() with the parameters that depend on this
operation.
"""
self.log.info(f"Running {self.__class__.__name__} action")
box_size = self.cfg['Extract'].getint('background_box_size', 128)
self.log.debug(f" Using box size = {box_size} pixels")
bkg = photutils.Background2D(self.action.args.ccddata,
box_size=box_size,
mask=self.action.args.source_mask,
sigma_clip=stats.SigmaClip())
self.action.args.background = bkg
return self.action.args
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation.
"""
self.log.info(f"Running {self.__class__.__name__} action")
box_size = self.cfg['Extract'].getint('background_box_size', 128)
self.log.debug(f" Using box size = {box_size} pixels")
self.action.args.background = [None] * len(
self.action.args.kd.pixeldata)
for i, pd in enumerate(self.action.args.kd.pixeldata):
bkg = photutils.Background2D(pd,
box_size=box_size,
mask=self.action.args.source_mask[i],
sigma_clip=stats.SigmaClip())
self.action.args.background[i] = bkg
# self.action.args.kd.pixeldata[i].data -= bkg.background
return self.action.args
def weighted_combine(weights,
sci_list,
var_list,
inmask_stack,
sigma_clip=False,
sigma_clip_stack=None,
sigrej=None,
maxiters=5):
r"""
Combine multiple sets of images, all using the same weights and mask.
The multiple sets of images and variances to combine must have the same
shape --- ``(nimgs, nspec, nspat)`` --- and this shape must match the
provided *single* mask set (``inmask_stack``). The provided weights are
broadcast to the necessary shape (see below), where one can provide one
weight per image, one weight per image spatial coordinate (i.e.,
wavelength-dependent weights), or independent weights for each pixel.
Optionally, the image stack can be sigma-clipped by setting
``sigma_clip=True``. If sigma-clipping is requested and no sigma-rejection
thresholds are provided (``sigrej`` is None), the sigma-rejection thresholds
are set *automatically* depending on the number of images to combine. The
default rejection thresholds are 1.1, 1.3, 1.6, 1.9, or 2.0 for,
respectively, 3, 4, 5, 6, or :math:`\geq 7` images. Sigma-clipping cannot
be performed if there are fewer than 3 images. The pixel rejection is based
on a *single* image stack provided by ``sigma_clip_stack``, which does not
necessarily need to be any of the image stacks provided by ``sci_list``.
Pixels rejected by sigma-clipping the ``sigma_clip_stack`` array are applied
to all image stacks in ``sci_list``.
The combined images are collected into the returned image list, where the
order of the list is identical to the input ``sci_list``. The returned mask
and pixel accounting array is identical for all stacked images.
Parameters
----------
weights : `numpy.ndarray`_
Weights to use. Options for the shape of weights are:
- ``(nimgs,)``: a single weight per image in the stack
- ``(nimgs, nspec)``: wavelength dependent weights per image in the
stack
- ``(nimgs, nspec, nspat)``: weights input with the shape of the
image stack
Note that the weights are distinct from the mask, which is dealt with
via the ``inmask_stack`` argument, meaning there should not be any
weights that are set to zero (although in principle this would still
work).
sci_list : :obj:`list`
List of floating-point `numpy.ndarray`_ image groups to stack. Each
image group *must* have the same shape: ``(nimgs, nspec, nspat)``.
var_list : :obj:`list`
List of floating-point `numpy.ndarray`_ images providing the variance
for each image group. The number of image groups and the shape of each
group must match ``sci_list``. These are used to propagate the error in
the combined images.
inmask_stack : `numpy.ndarray`_, boolean, shape (nimgs, nspec, nspat)
Good-pixel mask (True=Good, False=Bad) for the input image stacks. This
single group of good-pixel masks is applied to *all* input image groups.
sigma_clip : :obj:`bool`, optional, default = False
Combine with a mask by sigma clipping the image stack. Stacks can only
be sigma-clipped if there are 3 or more images.
sigma_clip_stack : `numpy.ndarray`_, float, shape (nimgs, nspec, nspat), optional, default = None
The image stack to be used for the sigma clipping. For example, if the
list of images to be combined with the weights is ``[sciimg_stack,
waveimg_stack, tilts_stack]`` and you want to clip based on
``sciimg_stack``, you would set ``sigma_clip_stack=sciimg_stack``.
sigrej : :obj:`int`, :obj:`float`, optional, default = None
Rejection threshold for sigma clipping. If None and ``sigma_clip`` is
True, the rejection threshold is set based on the number of images to
combine; see above. This value is passed directly to
`astropy.stats.SigmaClip`_ as its ``sigma`` parameter.
maxiters : :obj:`int`, optional, default=5
Maximum number of rejection iterations; see `astropy.stats.SigmaClip`_.
Returns
-------
sci_list_out : :obj:`list`
The list of ndarray float combined images with shape ``(nspec, nspat)``.
var_list_out : :obj:`list`
The list of ndarray propagated variance images with shape ``(nspec,
nspat)``.
gpm : boolean `numpy.ndarray`_, shape (nspec, nspat)
Good pixel mask for combined image (True=Good, False=Bad).
nused : integer `numpy.ndarray`_, shape (nspec, nspat)
Number of pixels combined at each location in the stacked images.
"""
shape = img_list_error_check(sci_list, var_list)
nimgs = shape[0]
img_shape = shape[1:]
#nspec = shape[1]
#nspat = shape[2]
if nimgs == 1:
# If only one image is passed in, simply return the input lists of images, but reshaped
# to be (nspec, nspat)
msgs.warn('Cannot combine a single image. Returning input images')
sci_list_out = []
for sci_stack in sci_list:
sci_list_out.append(sci_stack.reshape(img_shape))
var_list_out = []
for var_stack in var_list:
var_list_out.append(var_stack.reshape(img_shape))
gpm = inmask_stack.reshape(img_shape)
nused = gpm.astype(int)
return sci_list_out, var_list_out, gpm, nused
if sigma_clip and nimgs >= 3:
if sigma_clip_stack is None:
msgs.error(
'You must specify sigma_clip_stack; sigma-clipping is based on this array '
'and propagated to the arrays to be stacked.')
if sigrej is None:
# NOTE: If these are changed, make sure to update the doc-string!
if nimgs == 3:
sigrej = 1.1
elif nimgs == 4:
sigrej = 1.3
elif nimgs == 5:
sigrej = 1.6
elif nimgs == 6:
sigrej = 1.9
else:
sigrej = 2.0
# sigma clip if we have enough images
# mask_stack > 0 is a masked value. numpy masked arrays are True for masked (bad) values
data = np.ma.MaskedArray(sigma_clip_stack,
mask=np.logical_not(inmask_stack))
sigclip = stats.SigmaClip(sigma=sigrej,
maxiters=maxiters,
cenfunc='median',
stdfunc=utils.nan_mad_std)
data_clipped, lower, upper = sigclip(data,
axis=0,
masked=True,
return_bounds=True)
mask_stack = np.logical_not(
data_clipped.mask) # mask_stack = True are good values
else:
if sigma_clip and nimgs < 3:
msgs.warn(
'Sigma clipping requested, but you cannot sigma clip with less than 3 '
'images. Proceeding without sigma clipping')
mask_stack = inmask_stack # mask_stack = True are good values
nused = np.sum(mask_stack, axis=0)
weights_stack = broadcast_weights(weights, shape)
weights_mask_stack = weights_stack * mask_stack.astype(float)
weights_sum = np.sum(weights_mask_stack, axis=0)
inv_w_sum = 1. / (weights_sum + (weights_sum == 0.0))
sci_list_out = []
for sci_stack in sci_list:
sci_list_out.append(
np.sum(sci_stack * weights_mask_stack, axis=0) * inv_w_sum)
var_list_out = []
for var_stack in var_list:
var_list_out.append(
np.sum(var_stack * weights_mask_stack**2, axis=0) * inv_w_sum**2)
# Was it masked everywhere?
gpm = np.any(mask_stack, axis=0)
return sci_list_out, var_list_out, gpm, nused
def make_background(data,
sigma=3.,
snr=3.,
npixels=4,
boxsize=(10, 10),
filter_size=(5, 5),
mask_sources=True,
inmask=None):
"""
Use photutils to create a background model from the input data.
data : 2D `~numpy.ndarray`
Data from which to extract background model
sigma : float (default: 2.0)
Number of sigma to use for sigma clipping
snr : float (default: 2.0)
SNR to use when masking sources
npixels : int (default: 7)
Number of connected pixels to use when masking sources
boxsize : tuple or int (default: (7, 7))
Size of box used to create the gridded background map
filter_size : tuple or int (default: (3, 3))
Window size of the 2D median filter to apply to the low-res background map
mask_sources : bool (default: True)
If true, then use `~photutils.make_source_mask` to mask sources before creating background
"""
sigma_clip = stats.SigmaClip(sigma=sigma)
bkg_estimator = photutils.SExtractorBackground()
if inmask is not None:
cov_mask = np.zeros_like(inmask, dtype=bool)
cov_mask[inmask != np.nan] = False
cov_mask[inmask == np.nan] = True
else:
cov_mask = np.zeros_like(data, dtype=bool)
if mask_sources:
src_mask = photutils.make_source_mask(data,
nsigma=snr,
npixels=npixels,
mask=cov_mask)
mask = (cov_mask | src_mask)
bkg = photutils.Background2D(data,
boxsize,
filter_size=filter_size,
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask)
else:
bkg = photutils.Background2D(data,
boxsize,
filter_size=filter_size,
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=cov_mask)
return bkg
def weighted_combine(weights,
sci_list,
var_list,
inmask_stack,
sigma_clip=False,
sigma_clip_stack=None,
sigrej=None,
maxiters=5):
"""
Args:
weights (ndarray):
Weights to use. Options for the shape of weights are:
- (nimgs,) -- a single weight per image in the stack
- (nimgs, nspec) -- wavelength dependent weights per
image in the stack
- (nimgs, nspec, nspat) -- weights input with the shape
of the image stack
Note that the weights are distinct from the mask which is
dealt with via inmask_stack argument so there should not be
any weights that are set to zero (although in principle
this would still work).
sci_list: list
List of float ndarray images (each being an image stack with shape (nimgs, nspec, nspat))
which are to be combined with the weights, inmask_stack, and possibly sigma clipping
var_list: list
List of float ndarray variance images (each being an image stack with shape (nimgs, nspec, nspat))
which are to be combined with proper erorr propagation, i.e.
using the weights**2, inmask_stack, and possibly sigma clipping
inmask_stack: ndarray, boolean, shape (nimgs, nspec, nspat)
Array of input masks for the images. True = Good, False=Bad
sigma_clip: bool, default = False
Combine with a mask by sigma clipping the image stack. Only valid if nimgs > 2
sigma_clip_stack: ndarray, float, shape (nimgs, nspec, nspat), default = None
The image stack to be used for the sigma clipping. For example if
if the list of images to be combined with the weights is [sciimg_stack, waveimg_stack, tilts_stack] you
would be sigma clipping with sciimg_stack, and would set sigma_clip_stack = sciimg_stack
sigrej: int or float, default = None
Rejection threshold for sigma clipping. Code defaults to determining this automatically based
on the numberr of images provided.
maxiters:
Maximum number of iterations for sigma clipping using astropy.stats.SigmaClip
Returns:
tuple: Returns the following:
- sci_list_out: list: The list of ndarray float combined
images with shape (nspec, nspat)
- var_list_out: list: The list of ndarray propagated
variance images with shape (nspec, nspat)
- outmask: bool ndarray, shape (nspec, nspat): Mask for
combined image. True=Good, False=Bad
- nused: int ndarray, shape (nspec, nspat): Image of
integers indicating the number of images that contributed
to each pixel
"""
shape = img_list_error_check(sci_list, var_list)
nimgs = shape[0]
img_shape = shape[1:]
#nspec = shape[1]
#nspat = shape[2]
if nimgs == 1:
# If only one image is passed in, simply return the input lists of images, but reshaped
# to be (nspec, nspat)
msgs.warn('Cannot combine a single image. Returning input images')
sci_list_out = []
for sci_stack in sci_list:
sci_list_out.append(sci_stack.reshape(img_shape))
var_list_out = []
for var_stack in var_list:
var_list_out.append(var_stack.reshape(img_shape))
outmask = inmask_stack.reshape(img_shape)
nused = outmask.astype(int)
return sci_list_out, var_list_out, outmask, nused
if sigma_clip and nimgs >= 3:
if sigma_clip_stack is None:
msgs.error(
'You must specify sigma_clip_stack, i.e. which quantity to use for sigma clipping'
)
if sigrej is None:
if nimgs <= 2:
sigrej = 100.0 # Irrelevant for only 1 or 2 files, we don't sigma clip below
elif nimgs == 3:
sigrej = 1.1
elif nimgs == 4:
sigrej = 1.3
elif nimgs == 5:
sigrej = 1.6
elif nimgs == 6:
sigrej = 1.9
else:
sigrej = 2.0
# sigma clip if we have enough images
# mask_stack > 0 is a masked value. numpy masked arrays are True for masked (bad) values
data = np.ma.MaskedArray(sigma_clip_stack, np.invert(inmask_stack))
sigclip = stats.SigmaClip(sigma=sigrej,
maxiters=maxiters,
cenfunc='median',
stdfunc=utils.nan_mad_std)
data_clipped, lower, upper = sigclip(data,
axis=0,
masked=True,
return_bounds=True)
mask_stack = np.invert(
data_clipped.mask) # mask_stack = True are good values
else:
if sigma_clip and nimgs < 3:
msgs.warn(
'Sigma clipping requested, but you cannot sigma clip with less than 3 images. '
'Proceeding without sigma clipping')
mask_stack = inmask_stack # mask_stack = True are good values
nused = np.sum(mask_stack, axis=0)
weights_stack = broadcast_weights(weights, shape)
weights_mask_stack = weights_stack * mask_stack
weights_sum = np.sum(weights_mask_stack, axis=0)
sci_list_out = []
for sci_stack in sci_list:
sci_list_out.append(
np.sum(sci_stack * weights_mask_stack, axis=0) /
(weights_sum + (weights_sum == 0.0)))
var_list_out = []
for var_stack in var_list:
var_list_out.append(
np.sum(var_stack * weights_mask_stack**2, axis=0) /
(weights_sum + (weights_sum == 0.0))**2)
# Was it masked everywhere?
outmask = np.any(mask_stack, axis=0)
return sci_list_out, var_list_out, outmask, nused
import numpy as np
from astropy import stats
from numpy.random import randn
# Standard sigma clipping of a masked array
randvar = np.ma.MaskedArray(randn(1000, 50), randn(1000, 50) < -5.0)
sigclip = stats.SigmaClip(sigma=2.0, maxiters=5)
filtered_data, lower, upper = sigclip(randvar,
masked=True,
return_bounds=True,
axis=0)
print(lower)
# The same operation but using astropy.stats.mad_std as the stdfunc
sigclip_mad = stats.SigmaClip(sigma=2.0, maxiters=5, stdfunc=stats.mad_std)
filtered_data_mad, lower_mad, upper_mad = sigclip_mad(randvar,
masked=True,
return_bounds=True,
axis=0)
print(lower_mad)
# Note that stats.mad_stad is supposed to be usable with masked arrays
rand_test = np.ma.MaskedArray(randn(10), np.arange(10) > 5)
mad_test = stats.mad_std(rand_test)
print(mad_test)
# The problem is that according to the documentation for astropy.stats.SigmaClip:
"""
stdfunc : {'std'} or callable, optional
The statistic or callable function/object used to compute the
standard deviation about the center value. If set to ``'std'``
then having the optional `bottleneck`_ package installed will