def _perform(self): """ Returns an Argument() with the parameters that depend on this operation. """ self.log.info(f"Running {self.__class__.__name__} action") box_size = self.cfg['Extract'].getint('background_box_size', 128) self.log.debug(f" Using box size = {box_size} pixels") bkg = photutils.Background2D(self.action.args.ccddata, box_size=box_size, mask=self.action.args.source_mask, sigma_clip=stats.SigmaClip()) self.action.args.background = bkg return self.action.args
def _perform(self): """ Returns an Argument() with the parameters that depends on this operation. """ self.log.info(f"Running {self.__class__.__name__} action") box_size = self.cfg['Extract'].getint('background_box_size', 128) self.log.debug(f" Using box size = {box_size} pixels") self.action.args.background = [None] * len( self.action.args.kd.pixeldata) for i, pd in enumerate(self.action.args.kd.pixeldata): bkg = photutils.Background2D(pd, box_size=box_size, mask=self.action.args.source_mask[i], sigma_clip=stats.SigmaClip()) self.action.args.background[i] = bkg # self.action.args.kd.pixeldata[i].data -= bkg.background return self.action.args
def weighted_combine(weights, sci_list, var_list, inmask_stack, sigma_clip=False, sigma_clip_stack=None, sigrej=None, maxiters=5): r""" Combine multiple sets of images, all using the same weights and mask. The multiple sets of images and variances to combine must have the same shape --- ``(nimgs, nspec, nspat)`` --- and this shape must match the provided *single* mask set (``inmask_stack``). The provided weights are broadcast to the necessary shape (see below), where one can provide one weight per image, one weight per image spatial coordinate (i.e., wavelength-dependent weights), or independent weights for each pixel. Optionally, the image stack can be sigma-clipped by setting ``sigma_clip=True``. If sigma-clipping is requested and no sigma-rejection thresholds are provided (``sigrej`` is None), the sigma-rejection thresholds are set *automatically* depending on the number of images to combine. The default rejection thresholds are 1.1, 1.3, 1.6, 1.9, or 2.0 for, respectively, 3, 4, 5, 6, or :math:`\geq 7` images. Sigma-clipping cannot be performed if there are fewer than 3 images. The pixel rejection is based on a *single* image stack provided by ``sigma_clip_stack``, which does not necessarily need to be any of the image stacks provided by ``sci_list``. Pixels rejected by sigma-clipping the ``sigma_clip_stack`` array are applied to all image stacks in ``sci_list``. The combined images are collected into the returned image list, where the order of the list is identical to the input ``sci_list``. The returned mask and pixel accounting array is identical for all stacked images. Parameters ---------- weights : `numpy.ndarray`_ Weights to use. Options for the shape of weights are: - ``(nimgs,)``: a single weight per image in the stack - ``(nimgs, nspec)``: wavelength dependent weights per image in the stack - ``(nimgs, nspec, nspat)``: weights input with the shape of the image stack Note that the weights are distinct from the mask, which is dealt with via the ``inmask_stack`` argument, meaning there should not be any weights that are set to zero (although in principle this would still work). sci_list : :obj:`list` List of floating-point `numpy.ndarray`_ image groups to stack. Each image group *must* have the same shape: ``(nimgs, nspec, nspat)``. var_list : :obj:`list` List of floating-point `numpy.ndarray`_ images providing the variance for each image group. The number of image groups and the shape of each group must match ``sci_list``. These are used to propagate the error in the combined images. inmask_stack : `numpy.ndarray`_, boolean, shape (nimgs, nspec, nspat) Good-pixel mask (True=Good, False=Bad) for the input image stacks. This single group of good-pixel masks is applied to *all* input image groups. sigma_clip : :obj:`bool`, optional, default = False Combine with a mask by sigma clipping the image stack. Stacks can only be sigma-clipped if there are 3 or more images. sigma_clip_stack : `numpy.ndarray`_, float, shape (nimgs, nspec, nspat), optional, default = None The image stack to be used for the sigma clipping. For example, if the list of images to be combined with the weights is ``[sciimg_stack, waveimg_stack, tilts_stack]`` and you want to clip based on ``sciimg_stack``, you would set ``sigma_clip_stack=sciimg_stack``. sigrej : :obj:`int`, :obj:`float`, optional, default = None Rejection threshold for sigma clipping. If None and ``sigma_clip`` is True, the rejection threshold is set based on the number of images to combine; see above. This value is passed directly to `astropy.stats.SigmaClip`_ as its ``sigma`` parameter. maxiters : :obj:`int`, optional, default=5 Maximum number of rejection iterations; see `astropy.stats.SigmaClip`_. Returns ------- sci_list_out : :obj:`list` The list of ndarray float combined images with shape ``(nspec, nspat)``. var_list_out : :obj:`list` The list of ndarray propagated variance images with shape ``(nspec, nspat)``. gpm : boolean `numpy.ndarray`_, shape (nspec, nspat) Good pixel mask for combined image (True=Good, False=Bad). nused : integer `numpy.ndarray`_, shape (nspec, nspat) Number of pixels combined at each location in the stacked images. """ shape = img_list_error_check(sci_list, var_list) nimgs = shape[0] img_shape = shape[1:] #nspec = shape[1] #nspat = shape[2] if nimgs == 1: # If only one image is passed in, simply return the input lists of images, but reshaped # to be (nspec, nspat) msgs.warn('Cannot combine a single image. Returning input images') sci_list_out = [] for sci_stack in sci_list: sci_list_out.append(sci_stack.reshape(img_shape)) var_list_out = [] for var_stack in var_list: var_list_out.append(var_stack.reshape(img_shape)) gpm = inmask_stack.reshape(img_shape) nused = gpm.astype(int) return sci_list_out, var_list_out, gpm, nused if sigma_clip and nimgs >= 3: if sigma_clip_stack is None: msgs.error( 'You must specify sigma_clip_stack; sigma-clipping is based on this array ' 'and propagated to the arrays to be stacked.') if sigrej is None: # NOTE: If these are changed, make sure to update the doc-string! if nimgs == 3: sigrej = 1.1 elif nimgs == 4: sigrej = 1.3 elif nimgs == 5: sigrej = 1.6 elif nimgs == 6: sigrej = 1.9 else: sigrej = 2.0 # sigma clip if we have enough images # mask_stack > 0 is a masked value. numpy masked arrays are True for masked (bad) values data = np.ma.MaskedArray(sigma_clip_stack, mask=np.logical_not(inmask_stack)) sigclip = stats.SigmaClip(sigma=sigrej, maxiters=maxiters, cenfunc='median', stdfunc=utils.nan_mad_std) data_clipped, lower, upper = sigclip(data, axis=0, masked=True, return_bounds=True) mask_stack = np.logical_not( data_clipped.mask) # mask_stack = True are good values else: if sigma_clip and nimgs < 3: msgs.warn( 'Sigma clipping requested, but you cannot sigma clip with less than 3 ' 'images. Proceeding without sigma clipping') mask_stack = inmask_stack # mask_stack = True are good values nused = np.sum(mask_stack, axis=0) weights_stack = broadcast_weights(weights, shape) weights_mask_stack = weights_stack * mask_stack.astype(float) weights_sum = np.sum(weights_mask_stack, axis=0) inv_w_sum = 1. / (weights_sum + (weights_sum == 0.0)) sci_list_out = [] for sci_stack in sci_list: sci_list_out.append( np.sum(sci_stack * weights_mask_stack, axis=0) * inv_w_sum) var_list_out = [] for var_stack in var_list: var_list_out.append( np.sum(var_stack * weights_mask_stack**2, axis=0) * inv_w_sum**2) # Was it masked everywhere? gpm = np.any(mask_stack, axis=0) return sci_list_out, var_list_out, gpm, nused
def make_background(data, sigma=3., snr=3., npixels=4, boxsize=(10, 10), filter_size=(5, 5), mask_sources=True, inmask=None): """ Use photutils to create a background model from the input data. data : 2D `~numpy.ndarray` Data from which to extract background model sigma : float (default: 2.0) Number of sigma to use for sigma clipping snr : float (default: 2.0) SNR to use when masking sources npixels : int (default: 7) Number of connected pixels to use when masking sources boxsize : tuple or int (default: (7, 7)) Size of box used to create the gridded background map filter_size : tuple or int (default: (3, 3)) Window size of the 2D median filter to apply to the low-res background map mask_sources : bool (default: True) If true, then use `~photutils.make_source_mask` to mask sources before creating background """ sigma_clip = stats.SigmaClip(sigma=sigma) bkg_estimator = photutils.SExtractorBackground() if inmask is not None: cov_mask = np.zeros_like(inmask, dtype=bool) cov_mask[inmask != np.nan] = False cov_mask[inmask == np.nan] = True else: cov_mask = np.zeros_like(data, dtype=bool) if mask_sources: src_mask = photutils.make_source_mask(data, nsigma=snr, npixels=npixels, mask=cov_mask) mask = (cov_mask | src_mask) bkg = photutils.Background2D(data, boxsize, filter_size=filter_size, sigma_clip=sigma_clip, bkg_estimator=bkg_estimator, mask=mask) else: bkg = photutils.Background2D(data, boxsize, filter_size=filter_size, sigma_clip=sigma_clip, bkg_estimator=bkg_estimator, mask=cov_mask) return bkg
def weighted_combine(weights, sci_list, var_list, inmask_stack, sigma_clip=False, sigma_clip_stack=None, sigrej=None, maxiters=5): """ Args: weights (ndarray): Weights to use. Options for the shape of weights are: - (nimgs,) -- a single weight per image in the stack - (nimgs, nspec) -- wavelength dependent weights per image in the stack - (nimgs, nspec, nspat) -- weights input with the shape of the image stack Note that the weights are distinct from the mask which is dealt with via inmask_stack argument so there should not be any weights that are set to zero (although in principle this would still work). sci_list: list List of float ndarray images (each being an image stack with shape (nimgs, nspec, nspat)) which are to be combined with the weights, inmask_stack, and possibly sigma clipping var_list: list List of float ndarray variance images (each being an image stack with shape (nimgs, nspec, nspat)) which are to be combined with proper erorr propagation, i.e. using the weights**2, inmask_stack, and possibly sigma clipping inmask_stack: ndarray, boolean, shape (nimgs, nspec, nspat) Array of input masks for the images. True = Good, False=Bad sigma_clip: bool, default = False Combine with a mask by sigma clipping the image stack. Only valid if nimgs > 2 sigma_clip_stack: ndarray, float, shape (nimgs, nspec, nspat), default = None The image stack to be used for the sigma clipping. For example if if the list of images to be combined with the weights is [sciimg_stack, waveimg_stack, tilts_stack] you would be sigma clipping with sciimg_stack, and would set sigma_clip_stack = sciimg_stack sigrej: int or float, default = None Rejection threshold for sigma clipping. Code defaults to determining this automatically based on the numberr of images provided. maxiters: Maximum number of iterations for sigma clipping using astropy.stats.SigmaClip Returns: tuple: Returns the following: - sci_list_out: list: The list of ndarray float combined images with shape (nspec, nspat) - var_list_out: list: The list of ndarray propagated variance images with shape (nspec, nspat) - outmask: bool ndarray, shape (nspec, nspat): Mask for combined image. True=Good, False=Bad - nused: int ndarray, shape (nspec, nspat): Image of integers indicating the number of images that contributed to each pixel """ shape = img_list_error_check(sci_list, var_list) nimgs = shape[0] img_shape = shape[1:] #nspec = shape[1] #nspat = shape[2] if nimgs == 1: # If only one image is passed in, simply return the input lists of images, but reshaped # to be (nspec, nspat) msgs.warn('Cannot combine a single image. Returning input images') sci_list_out = [] for sci_stack in sci_list: sci_list_out.append(sci_stack.reshape(img_shape)) var_list_out = [] for var_stack in var_list: var_list_out.append(var_stack.reshape(img_shape)) outmask = inmask_stack.reshape(img_shape) nused = outmask.astype(int) return sci_list_out, var_list_out, outmask, nused if sigma_clip and nimgs >= 3: if sigma_clip_stack is None: msgs.error( 'You must specify sigma_clip_stack, i.e. which quantity to use for sigma clipping' ) if sigrej is None: if nimgs <= 2: sigrej = 100.0 # Irrelevant for only 1 or 2 files, we don't sigma clip below elif nimgs == 3: sigrej = 1.1 elif nimgs == 4: sigrej = 1.3 elif nimgs == 5: sigrej = 1.6 elif nimgs == 6: sigrej = 1.9 else: sigrej = 2.0 # sigma clip if we have enough images # mask_stack > 0 is a masked value. numpy masked arrays are True for masked (bad) values data = np.ma.MaskedArray(sigma_clip_stack, np.invert(inmask_stack)) sigclip = stats.SigmaClip(sigma=sigrej, maxiters=maxiters, cenfunc='median', stdfunc=utils.nan_mad_std) data_clipped, lower, upper = sigclip(data, axis=0, masked=True, return_bounds=True) mask_stack = np.invert( data_clipped.mask) # mask_stack = True are good values else: if sigma_clip and nimgs < 3: msgs.warn( 'Sigma clipping requested, but you cannot sigma clip with less than 3 images. ' 'Proceeding without sigma clipping') mask_stack = inmask_stack # mask_stack = True are good values nused = np.sum(mask_stack, axis=0) weights_stack = broadcast_weights(weights, shape) weights_mask_stack = weights_stack * mask_stack weights_sum = np.sum(weights_mask_stack, axis=0) sci_list_out = [] for sci_stack in sci_list: sci_list_out.append( np.sum(sci_stack * weights_mask_stack, axis=0) / (weights_sum + (weights_sum == 0.0))) var_list_out = [] for var_stack in var_list: var_list_out.append( np.sum(var_stack * weights_mask_stack**2, axis=0) / (weights_sum + (weights_sum == 0.0))**2) # Was it masked everywhere? outmask = np.any(mask_stack, axis=0) return sci_list_out, var_list_out, outmask, nused
import numpy as np from astropy import stats from numpy.random import randn # Standard sigma clipping of a masked array randvar = np.ma.MaskedArray(randn(1000, 50), randn(1000, 50) < -5.0) sigclip = stats.SigmaClip(sigma=2.0, maxiters=5) filtered_data, lower, upper = sigclip(randvar, masked=True, return_bounds=True, axis=0) print(lower) # The same operation but using astropy.stats.mad_std as the stdfunc sigclip_mad = stats.SigmaClip(sigma=2.0, maxiters=5, stdfunc=stats.mad_std) filtered_data_mad, lower_mad, upper_mad = sigclip_mad(randvar, masked=True, return_bounds=True, axis=0) print(lower_mad) # Note that stats.mad_stad is supposed to be usable with masked arrays rand_test = np.ma.MaskedArray(randn(10), np.arange(10) > 5) mad_test = stats.mad_std(rand_test) print(mad_test) # The problem is that according to the documentation for astropy.stats.SigmaClip: """ stdfunc : {'std'} or callable, optional The statistic or callable function/object used to compute the standard deviation about the center value. If set to ``'std'`` then having the optional `bottleneck`_ package installed will