def fix_pix(img: CCDData) -> CCDData: im = img.data mask = img.mask import scipy.ndimage as ndimage """ taken from https://www.iaa.csic.es/~jmiguel/PANIC/PAPI/html/_modules/reduce/calBPM.html#fixPix (GPLv3) Applies a bad-pixel mask to the input image (im), creating an image with masked values replaced with a bi-linear interpolation from nearby pixels. Probably only good for isolated badpixels. Usage: fixed = fixpix(im, mask, [iraf=]) Inputs: im = the image array mask = an array that is True (or >0) where im contains bad pixels iraf = True use IRAF.fixpix; False use numpy and a loop over all pixels (extremelly low) Outputs: fixed = the corrected image v1.0.0 Michael S. Kelley, UCF, Jan 2008 v1.1.0 Added the option to use IRAF's fixpix. MSK, UMD, 25 Apr 2011 Notes ----- - Non-IRAF algorithm is extremelly slow. """ # create domains around masked pixels dilated = ndimage.binary_dilation(mask) domains, n = ndimage.label(dilated) # loop through each domain, replace bad pixels with the average # from nearest neigboors y, x = np.indices(im.shape, dtype=np.int)[-2:] # x = xarray(im.shape) # y = yarray(im.shape) cleaned = im.copy() for d in (np.arange(n) + 1): # find the current domain i = (domains == d) # extract the sub-image x0, x1 = x[i].min(), x[i].max() + 1 y0, y1 = y[i].min(), y[i].max() + 1 subim = im[y0:y1, x0:x1] submask = mask[y0:y1, x0:x1] # noinspection PyPep8 subgood = (submask == False) # noqa cleaned[i * mask] = subim[subgood].mean() img.data = cleaned return img
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None): masks = [] processed = [] for sci in sci_list: log.info('Loading file: '+sci) log.info('Applying gain correction and flat correction.') with fits.open(sci) as hdr: header = hdr[0].header data = hdr[0].data data[np.isnan(data)] = np.nanmedian(data) raw = CCDData(data,meta=header,unit=u.adu) red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron) log.info('Exposure time of science image is '+str(red.header['TRUITIME']*red.header['COADDONE'])) flat = np.array(ccdproc.flat_correct(red, mflat)) flat[np.isnan(flat)] = np.nanmedian(flat) processed_data = CCDData(flat,unit=u.electron,header=red.header,wcs=red.wcs) log.info('File proccessed.') log.info('Cleaning cosmic rays and creating mask.') mask = make_source_mask(processed_data, nsigma=3, npixels=5) masks.append(mask) clean, com_mask = create_mask.create_mask(sci.replace('.gz',''),processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(raw.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log) processed_data.data = clean log.info('Calculating 2D background.') bkg = Background2D(processed_data, (128, 128), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) log.info('Median background: '+str(np.median(bkg.background))) fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits').replace('.gz',''),bkg.background,overwrite=True) final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['TRUITIME']*red.header['COADDONE']*u.second,propagate_uncertainties=True,handle_meta='first_found') log.info('Background subtracted and image divided by exposure time.') final.write(sci.replace('/raw/','/red/').replace('.gz',''),overwrite=True) processed.append(final) return processed, masks
def kcwi_fits_reader(file): """A reader for KeckData objects. Currently this is a separate function, but should probably be registered as a reader similar to fits_ccddata_reader. Arguments: file -- The filename (or pathlib.Path) of the FITS file to open. """ try: hdul = fits.open(file) except (FileNotFoundError, OSError) as e: print(e) raise e read_imgs = 0 read_tabs = 0 # primary image ccddata = CCDData(hdul['PRIMARY'].data, meta=hdul['PRIMARY'].header, unit='adu') read_imgs += 1 # check for other legal components if 'UNCERT' in hdul: ccddata.uncertainty = hdul['UNCERT'].data read_imgs += 1 if 'FLAGS' in hdul: ccddata.flags = hdul['FLAGS'].data read_imgs += 1 if 'MASK' in hdul: ccddata.mask = hdul['MASK'].data read_imgs += 1 if 'Exposure Events' in hdul: table = hdul['Exposure Events'] read_tabs += 1 else: table = None # prepare for floating point ccddata.data = ccddata.data.astype(np.float64) # Check for CCDCFG keyword if 'CCDCFG' not in ccddata.header: ccdcfg = ccddata.header['CCDSUM'].replace(" ", "") ccdcfg += "%1d" % ccddata.header['CCDMODE'] ccdcfg += "%02d" % ccddata.header['GAINMUL'] ccdcfg += "%02d" % ccddata.header['AMPMNUM'] ccddata.header['CCDCFG'] = ccdcfg if ccddata: if 'BUNIT' in ccddata.header: ccddata.unit = ccddata.header['BUNIT'] if ccddata.uncertainty: ccddata.uncertainty.unit = ccddata.header['BUNIT'] # print("setting image units to " + ccddata.header['BUNIT']) logger.info("<<< read %d imgs and %d tables out of %d hdus in %s" % (read_imgs, read_tabs, len(hdul), file)) return ccddata, table
def test_subtract_dark_fails(): ccd_data = ccd_data_func() # None of these tests check a result so the content of the master # can be anything. ccd_data.header['exptime'] = 30.0 master = ccd_data.copy() # Do we fail if we give one of dark_exposure, data_exposure but not both? with pytest.raises(TypeError): subtract_dark(ccd_data, master, dark_exposure=30 * u.second) with pytest.raises(TypeError): subtract_dark(ccd_data, master, data_exposure=30 * u.second) # Do we fail if we supply dark_exposure and data_exposure and exposure_time with pytest.raises(TypeError): subtract_dark(ccd_data, master, dark_exposure=10 * u.second, data_exposure=10 * u.second, exposure_time='exptime') # Fail if we supply none of the exposure-related arguments? with pytest.raises(TypeError): subtract_dark(ccd_data, master) # Fail if we supply exposure time but not a unit? with pytest.raises(TypeError): subtract_dark(ccd_data, master, exposure_time='exptime') # Fail if ccd_data or master are not CCDData objects? with pytest.raises(TypeError): subtract_dark(ccd_data.data, master, exposure_time='exptime') with pytest.raises(TypeError): subtract_dark(ccd_data, master.data, exposure_time='exptime') # Fail if units do not match... # ...when there is no scaling? master = CCDData(ccd_data) master.unit = u.meter with pytest.raises(u.UnitsError) as e: subtract_dark(ccd_data, master, exposure_time='exptime', exposure_unit=u.second) assert "uncalibrated image" in str(e.value) # Fail when the arrays are not the same size with pytest.raises(ValueError): small_master = CCDData(ccd_data) small_master.data = np.zeros((1, 1)) subtract_dark(ccd_data, small_master)
def trim_ccddata(ccddata: CCDData, left: 'int' = None, right: 'int' = None, bottom: 'int' = None, top: 'int' = None, update_wcs=True): """ :param ccddata: :param left: :param right: :param bottom: :param top: :return: """ shape = ccddata.shape if left is None: left = 0 if right is None: right = shape[1] if bottom is None: bottom = 0 if top is None: top = shape[0] if right < left: raise ValueError('Improper inputs; right is smaller than left.') if top < bottom: raise ValueError('Improper inputs; top is smaller than bottom.') if update_wcs: ccddata.header['CRPIX1'] = ccddata.header['CRPIX1'] - left ccddata.header['CRPIX2'] = ccddata.header['CRPIX2'] - bottom ccddata.data = ccddata.data[bottom:top, left:right] return ccddata
def interpolate(img: CCDData): """ Takes a image with a mask for bad pixels and interpolates over the bad pixels :param img: the image you want to interpolate bad pixels in :return: interpolated image """ # TODO combiner does not care about this and marks it invalid still from astropy.convolution import CustomKernel from astropy.convolution import interpolate_replace_nans # TODO this here doesn't really work all that well -> extended regions cause artifacts at border kernel_array = np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1] ]) / 9 # average of all surrounding pixels # noinspection PyTypeChecker kernel = CustomKernel( kernel_array ) # TODO the original pipeline used fixpix, which says it uses linear interpolation img.data[np.logical_not(img.mask)] = np.NaN img.data = interpolate_replace_nans(img.data, kernel) return img
def vertical_correct(ccd, fitting_sections=None, method='median', sigclip_kw=dict(sigma=2, maxiters=5), dtype='float32', return_pattern=False, update_header=True, verbose=False): ''' Correct vertical strip patterns. Paramters --------- ccd : CCDData, HDU object, HDUList, or ndarray. The CCD to subtract the vertical pattern. fitting_sections : list of two str, optional. The sections to be used for the vertical pattern estimation. This must be identical to the usual FITS section (i.e., that used in SAO ds9 or IRAF, 1-indexing and last-index-inclusive), not in python. **Give it in the order of ``[<upper>, <lower>]`` in FITS y-coordinate.** method : str, optional. One of ``['med', 'avg', 'median', 'average', 'mean']``. sigma, maxiters : float and int, optional A sigma-clipping will be done to remove hot pixels and cosmic rays for estimating the vertical pattern. To turn sigma-clipping off, set ``maxiters=0``. sigclip_kw : dict, optional The keyword arguments for the sigma clipping. dtype : str, dtype, optional The data type to be returned. return_pattern : bool, optional. If `True`, the subtracted pattern will also be returned. Default is `False`. update_header : bool, optional. Whether to update the header if there is any. Return ------ ''' _t = Time.now() data, hdr = _parse_data_header(ccd) if fitting_sections is None: fitting_sections = VERTICALSECTS elif len(fitting_sections) != 2: raise ValueError("fitting_sections must have two elements.") if method in ['median', 'med']: methodstr = 'taking median' fun = np.median idx = 1 elif method in ['average', 'avg', 'mean']: methodstr = 'taking average' fun = np.mean idx = 0 else: raise ValueError( "method not understood; it must be one of [med, avg, median, average, mean]." ) parts = [] # The two horizontal box areas from raw data strips = [] # The estimated patterns (1-D) for sect in fitting_sections: parts.append(data[fitsxy2py(sect)]) try: if sigclip_kw["maxiters"] == 0: clipstr = "no clipping" for part in parts: strips.append(fun(part, axis=0)) except KeyError: pass # The user wants to use default value of maxiters in astropy. clipstr = f"sigma-clipping in astropy (v {astropy.__version__})" if sigclip_kw: clipstr += f", given {sigclip_kw}." else: clipstr += "." for part in parts: clip = sigma_clipped_stats(part, axis=0, **sigclip_kw) strips.append(clip[idx]) ny, nx = data.shape vpattern = np.repeat(strips, ny / 2, axis=0) vsub = data - vpattern.astype(dtype) if update_header and hdr is not None: # add as history add_to_header( hdr, 'h', verbose=verbose, t_ref=_t, s=f"Vertical pattern subtracted using {fitting_sections} by {methodstr} with {clipstr}" ) try: nccd = CCDData(data=vsub, header=hdr) except ValueError: nccd = CCDData(data=vsub, header=hdr, unit='adu') nccd.data = nccd.data.astype(dtype) if return_pattern: return nccd, vpattern return nccd
def bdf_process(ccd, output=None, mbiaspath=None, mdarkpath=None, mflatpath=None, trim_fits_section=None, calc_err=False, unit='adu', gain=None, gain_key="GAIN", gain_unit=u.electron / u.adu, rdnoise=None, rdnoise_key="RDNOISE", rdnoise_unit=u.electron, dark_exposure=None, data_exposure=None, exposure_key="EXPTIME", exposure_unit=u.s, dark_scale=False, normalize_exposure=False, normalize_average=False, flat_min_value=None, flat_norm_value=None, do_crrej=False, crrej_kwargs=None, propagate_crmask=False, verbose_crrej=False, verbose_bdf=True, output_verify='fix', overwrite=True, dtype="float32", uncertainty_dtype="float32"): ''' Do bias, dark and flat process. Parameters ---------- ccd: CCDData The ccd to be processed. output : path-like or None, optional. The path if you want to save the resulting ``ccd`` object. Default is ``None``. mbiaspath, mdarkpath, mflatpath : path-like, optional. The path to master bias, dark, flat FITS files. If ``None``, the corresponding process is not done. trim_fits_section: str, optional Region of ``ccd`` to be trimmed; see ``ccdproc.subtract_overscan`` for details. Default is ``None``. calc_err : bool, optional. Whether to calculate the error map based on Poisson and readnoise error propagation. unit : `~astropy.units.Unit` or str, optional. The units of the data. Default is ``'adu'``. gain, rdnoise : None, float, optional The gain and readnoise value. These are all ignored if ``calc_err=False``. If ``calc_err=True``, it automatically seeks for suitable gain and readnoise value. If ``gain`` or ``readnoise`` is specified, they are interpreted with ``gain_unit`` and ``rdnoise_unit``, respectively. If they are not specified, this function will seek for the header with keywords of ``gain_key`` and ``rdnoise_key``, and interprete the header value in the unit of ``gain_unit`` and ``rdnoise_unit``, respectively. gain_key, rdnoise_key : str, optional See ``gain``, ``rdnoise`` explanation above. These are all ignored if ``calc_err=False``. gain_unit, rdnoise_unit : astropy Unit, optional See ``gain``, ``rdnoise`` explanation above. These are all ignored if ``calc_err=False``. dark_exposure, data_exposure : None, float, astropy Quantity, optional The exposure times of dark and data frame, respectively. They should both be specified or both ``None``. These are all ignored if ``mdarkpath=None``. If both are not specified while ``mdarkpath`` is given, then the code automatically seeks for header's ``exposure_key``. Then interprete the value as the quantity with unit ``exposure_unit``. If ``mdkarpath`` is not ``None``, then these are passed to ``ccdproc.subtract_dark``. exposure_key : str, optional The header keyword for exposure time. Ignored if ``mdarkpath=None``. exposure_unit : astropy Unit, optional. The unit of the exposure time. Ignored if ``mdarkpath=None``. flat_min_value : float or None, optional min_value of `ccdproc.flat_correct`. Minimum value for flat field. The value can either be None and no minimum value is applied to the flat or specified by a float which will replace all values in the flat by the min_value. Default is ``None``. flat_norm_value : float or None, optional norm_value of `ccdproc.flat_correct`. If not ``None``, normalize flat field by this argument rather than the mean of the image. This allows fixing several different flat fields to have the same scale. If this value is negative or 0, a ``ValueError`` is raised. Default is ``None``. crrej_kwargs : dict or None, optional If ``None`` (default), uses some default values defined in ``~.misc.LACOSMIC_KEYS``. It is always discouraged to use default except for quick validity-checking, because even the official L.A. Cosmic codes in different versions (IRAF, IDL, Python, etc) have different default parameters, i.e., there is nothing which can be regarded as default. To see all possible keywords, do ``print(astroscrappy.detect_cosmics.__doc__)`` Also refer to https://nbviewer.jupyter.org/github/ysbach/AO2019/blob/master/Notebooks/07-Cosmic_Ray_Rejection.ipynb propagate_crmask : bool, optional Whether to save (propagate) the mask from CR rejection (``astroscrappy``) to the CCD's mask. Default is ``False``. output_verify : str Output verification option. Must be one of ``"fix"``, ``"silentfix"``, ``"ignore"``, ``"warn"``, or ``"exception"``. May also be any combination of ``"fix"`` or ``"silentfix"`` with ``"+ignore"``, ``+warn``, or ``+exception" (e.g. ``"fix+warn"``). See the astropy documentation below: http://docs.astropy.org/en/stable/io/fits/api/verification.html#verify dtype : str or `numpy.dtype` or None, optional Allows user to set dtype. See `numpy.array` ``dtype`` parameter description. If ``None`` it uses ``np.float64``. Default is ``None``. ''' def _add_and_print(s, header, verbose): header.add_history(s) if verbose: print(s) # Set strings for header history & print (if verbose_bdf) str_bias = "Bias subtracted using {}" str_dark = "Dark subtracted using {}" str_dscale = "Dark scaling {} using {}" str_flat = "Flat corrected using {}" str_trim = "Trim by FITS section {}" str_grd = "From {}, {} = {:.3f} [{}]" # str_grd.format(user/header_key, gain/rdnoise, val, unit) str_e0 = ("Readnoise propagated with Poisson noise (using gain above)" + " of source.") str_ed = "Poisson noise from subtracted dark was propagated." str_ef = "Flat uncertainty was propagated." str_nexp = "Normalized by the exposure time." str_navg = "Normalized by the average value of the frame." str_cr = ("Cosmic-Ray rejected by astroscrappy (v {}), " + "with parameters: {}") # Initial setting proc = CCDData(ccd) hdr_new = proc.header # Add PROCESS key try: _ = hdr_new["PROCESS"] except KeyError: hdr_new["PROCESS"] = ("", "The processed history: see comment.") hdr_new["PROCVER"] = (ccdproc.__version__, "ccdproc version used for processing.") hdr_new.add_comment("PROCESS key can be B (bias), D (dark), F (flat), " + "T (trim), W (WCS astrometry), C(CRrej).") # Set for BIAS if mbiaspath is None: do_bias = False mbias = CCDData(np.zeros_like(ccd), unit=proc.unit) else: do_bias = True mbias = CCDData.read(mbiaspath, unit=unit) hdr_new["PROCESS"] += "B" _add_and_print(str_bias.format(mbiaspath), hdr_new, verbose_bdf) # Set for DARK if mdarkpath is None: do_dark = False mdark = CCDData(np.zeros_like(ccd), unit=proc.unit) else: do_dark = True mdark = CCDData.read(mdarkpath, unit=unit) hdr_new["PROCESS"] += "D" _add_and_print(str_dark.format(mdarkpath), hdr_new, verbose_bdf) if dark_scale: _add_and_print(str_dscale.format(dark_scale, exposure_key), hdr_new, verbose_bdf) # Set for FLAT if mflatpath is None: do_flat = False mflat = CCDData(np.ones_like(ccd), unit=proc.unit) else: do_flat = True mflat = CCDData.read(mflatpath) hdr_new["PROCESS"] += "F" _add_and_print(str_flat.format(mflatpath), hdr_new, verbose_bdf) # Set gain and rdnoise if at least one of calc_err and do_crrej is True. if calc_err or do_crrej: if gain is None: gain_Q = get_from_header(hdr_new, gain_key, unit=gain_unit, verbose=False, default=1.) gain_from = gain_key else: if not isinstance(gain, u.Quantity): gain_Q = gain * gain_unit else: gain_Q = gain gain_from = "User" _add_and_print( str_grd.format(gain_from, "gain", gain_Q.value, gain_Q.unit), hdr_new, verbose_bdf) if rdnoise is None: rdnoise_Q = get_from_header(hdr_new, rdnoise_key, unit=rdnoise_unit, verbose=False, default=1.) rdnoise_from = rdnoise_key else: if not isinstance(rdnoise, u.Quantity): rdnoise_Q = rdnoise * rdnoise_unit else: rdnoise_Q = rdnoise rdnoise_from = "User" _add_and_print( str_grd.format(rdnoise_from, "rdnoise", rdnoise_Q.value, rdnoise_Q.unit), hdr_new, verbose_bdf) # Do TRIM if trim_fits_section is not None: proc = trim_image(proc, trim_fits_section) mbias = trim_image(mbias, trim_fits_section) mdark = trim_image(mdark, trim_fits_section) mflat = trim_image(mflat, trim_fits_section) hdr_new["PROCESS"] += "T" _add_and_print(str_trim.format(trim_fits_section), hdr_new, verbose_bdf) # Do BIAS if do_bias: proc = subtract_bias(proc, mbias) # Do DARK if do_dark: proc = subtract_dark(proc, mdark, dark_exposure=dark_exposure, data_exposure=data_exposure, exposure_time=exposure_key, exposure_unit=exposure_unit, scale=dark_scale) # Make UNCERT extension before doing FLAT # It is better to make_errmap a priori because of mathematical and # computational convenience. See ``if do_flat:`` clause below. if calc_err: err = make_errmap(proc, gain_epadu=gain, subtracted_dark=mdark) proc.uncertainty = StdDevUncertainty(err) _add_and_print(str_e0, hdr_new, verbose_bdf) if do_dark: _add_and_print(str_ed, hdr_new, verbose_bdf) # Do FLAT if do_flat: # Flat error propagation is done automatically by # ``ccdproc.flat_correct``if it has the uncertainty attribute. proc = flat_correct(proc, mflat, min_value=flat_min_value, norm_value=flat_norm_value) if calc_err and mflat.uncertainty is not None: _add_and_print(str_ef, hdr_new, verbose_bdf) # Normalize by the exposure time (e.g., ADU per sec) if normalize_exposure: if data_exposure is None: data_exposure = hdr_new[exposure_key] proc = proc.divide(data_exposure) # uncertainty will also be.. _add_and_print(str_nexp, hdr_new, verbose_bdf) # Normalize by the mean value if normalize_average: avg = np.mean(proc.data) proc = proc.divide(avg) _add_and_print(str_navg, hdr_new, verbose_bdf) # Do CRREJ if do_crrej: import astroscrappy from astroscrappy import detect_cosmics from .misc import LACOSMIC_KEYS if crrej_kwargs is None: crrej_kwargs = LACOSMIC_KEYS warn("You are not specifying CR-rejection parameters and blindly" + " using defaults. It can be dangerous.") if (("B" in hdr_new["PROCESS"]) + ("D" in hdr_new["PROCESS"]) + ("F" in hdr_new["PROCESS"])) < 2: warn("L.A. Cosmic should be run AFTER B/D/F process. " + f"You are running it with {hdr_new['PROCESS']}. " + "See http://www.astro.yale.edu/dokkum/lacosmic/notes.html") # remove the fucxing cosmic rays crmask, cleanarr = detect_cosmics(proc.data, inmask=proc.mask, gain=gain_Q.value, readnoise=rdnoise_Q.value, **crrej_kwargs, verbose=verbose_crrej) # create the new ccd data object # astroscrappy automatically does the gain correction, so return # back to avoid confusion. proc.data = cleanarr / gain_Q.value if propagate_crmask: if proc.mask is None: proc.mask = crmask else: proc.mask = proc.mask + crmask hdr_new["PROCESS"] += "C" _add_and_print(str_cr.format(astroscrappy.__version__, crrej_kwargs), hdr_new, verbose_crrej) proc = CCDData_astype(proc, dtype=dtype, uncertainty_dtype=uncertainty_dtype) proc.header = hdr_new if output is not None: if verbose_bdf: print(f"Writing FITS to {output}... ", end='') proc.write(output, output_verify=output_verify, overwrite=overwrite) if verbose_bdf: print(f"Saved.") return proc