def fix_pix(img: CCDData) -> CCDData:
im = img.data
mask = img.mask
import scipy.ndimage as ndimage
"""
taken from https://www.iaa.csic.es/~jmiguel/PANIC/PAPI/html/_modules/reduce/calBPM.html#fixPix
(GPLv3)
Applies a bad-pixel mask to the input image (im), creating an image with
masked values replaced with a bi-linear interpolation from nearby pixels.
Probably only good for isolated badpixels.
Usage:
fixed = fixpix(im, mask, [iraf=])
Inputs:
im = the image array
mask = an array that is True (or >0) where im contains bad pixels
iraf = True use IRAF.fixpix; False use numpy and a loop over
all pixels (extremelly low)
Outputs:
fixed = the corrected image
v1.0.0 Michael S. Kelley, UCF, Jan 2008
v1.1.0 Added the option to use IRAF's fixpix. MSK, UMD, 25 Apr
2011
Notes
-----
- Non-IRAF algorithm is extremelly slow.
"""
# create domains around masked pixels
dilated = ndimage.binary_dilation(mask)
domains, n = ndimage.label(dilated)
# loop through each domain, replace bad pixels with the average
# from nearest neigboors
y, x = np.indices(im.shape, dtype=np.int)[-2:]
# x = xarray(im.shape)
# y = yarray(im.shape)
cleaned = im.copy()
for d in (np.arange(n) + 1):
# find the current domain
i = (domains == d)
# extract the sub-image
x0, x1 = x[i].min(), x[i].max() + 1
y0, y1 = y[i].min(), y[i].max() + 1
subim = im[y0:y1, x0:x1]
submask = mask[y0:y1, x0:x1]
# noinspection PyPep8
subgood = (submask == False) # noqa
cleaned[i * mask] = subim[subgood].mean()
img.data = cleaned
return img
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None):
masks = []
processed = []
for sci in sci_list:
log.info('Loading file: '+sci)
log.info('Applying gain correction and flat correction.')
with fits.open(sci) as hdr:
header = hdr[0].header
data = hdr[0].data
data[np.isnan(data)] = np.nanmedian(data)
raw = CCDData(data,meta=header,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron)
log.info('Exposure time of science image is '+str(red.header['TRUITIME']*red.header['COADDONE']))
flat = np.array(ccdproc.flat_correct(red, mflat))
flat[np.isnan(flat)] = np.nanmedian(flat)
processed_data = CCDData(flat,unit=u.electron,header=red.header,wcs=red.wcs)
log.info('File proccessed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(processed_data, nsigma=3, npixels=5)
masks.append(mask)
clean, com_mask = create_mask.create_mask(sci.replace('.gz',''),processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(raw.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(processed_data, (128, 128), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
log.info('Median background: '+str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits').replace('.gz',''),bkg.background,overwrite=True)
final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['TRUITIME']*red.header['COADDONE']*u.second,propagate_uncertainties=True,handle_meta='first_found')
log.info('Background subtracted and image divided by exposure time.')
final.write(sci.replace('/raw/','/red/').replace('.gz',''),overwrite=True)
processed.append(final)
return processed, masks
def kcwi_fits_reader(file):
"""A reader for KeckData objects.
Currently this is a separate function, but should probably be
registered as a reader similar to fits_ccddata_reader.
Arguments:
file -- The filename (or pathlib.Path) of the FITS file to open.
"""
try:
hdul = fits.open(file)
except (FileNotFoundError, OSError) as e:
print(e)
raise e
read_imgs = 0
read_tabs = 0
# primary image
ccddata = CCDData(hdul['PRIMARY'].data,
meta=hdul['PRIMARY'].header,
unit='adu')
read_imgs += 1
# check for other legal components
if 'UNCERT' in hdul:
ccddata.uncertainty = hdul['UNCERT'].data
read_imgs += 1
if 'FLAGS' in hdul:
ccddata.flags = hdul['FLAGS'].data
read_imgs += 1
if 'MASK' in hdul:
ccddata.mask = hdul['MASK'].data
read_imgs += 1
if 'Exposure Events' in hdul:
table = hdul['Exposure Events']
read_tabs += 1
else:
table = None
# prepare for floating point
ccddata.data = ccddata.data.astype(np.float64)
# Check for CCDCFG keyword
if 'CCDCFG' not in ccddata.header:
ccdcfg = ccddata.header['CCDSUM'].replace(" ", "")
ccdcfg += "%1d" % ccddata.header['CCDMODE']
ccdcfg += "%02d" % ccddata.header['GAINMUL']
ccdcfg += "%02d" % ccddata.header['AMPMNUM']
ccddata.header['CCDCFG'] = ccdcfg
if ccddata:
if 'BUNIT' in ccddata.header:
ccddata.unit = ccddata.header['BUNIT']
if ccddata.uncertainty:
ccddata.uncertainty.unit = ccddata.header['BUNIT']
# print("setting image units to " + ccddata.header['BUNIT'])
logger.info("<<< read %d imgs and %d tables out of %d hdus in %s" %
(read_imgs, read_tabs, len(hdul), file))
return ccddata, table
def test_subtract_dark_fails():
ccd_data = ccd_data_func()
# None of these tests check a result so the content of the master
# can be anything.
ccd_data.header['exptime'] = 30.0
master = ccd_data.copy()
# Do we fail if we give one of dark_exposure, data_exposure but not both?
with pytest.raises(TypeError):
subtract_dark(ccd_data, master, dark_exposure=30 * u.second)
with pytest.raises(TypeError):
subtract_dark(ccd_data, master, data_exposure=30 * u.second)
# Do we fail if we supply dark_exposure and data_exposure and exposure_time
with pytest.raises(TypeError):
subtract_dark(ccd_data,
master,
dark_exposure=10 * u.second,
data_exposure=10 * u.second,
exposure_time='exptime')
# Fail if we supply none of the exposure-related arguments?
with pytest.raises(TypeError):
subtract_dark(ccd_data, master)
# Fail if we supply exposure time but not a unit?
with pytest.raises(TypeError):
subtract_dark(ccd_data, master, exposure_time='exptime')
# Fail if ccd_data or master are not CCDData objects?
with pytest.raises(TypeError):
subtract_dark(ccd_data.data, master, exposure_time='exptime')
with pytest.raises(TypeError):
subtract_dark(ccd_data, master.data, exposure_time='exptime')
# Fail if units do not match...
# ...when there is no scaling?
master = CCDData(ccd_data)
master.unit = u.meter
with pytest.raises(u.UnitsError) as e:
subtract_dark(ccd_data,
master,
exposure_time='exptime',
exposure_unit=u.second)
assert "uncalibrated image" in str(e.value)
# Fail when the arrays are not the same size
with pytest.raises(ValueError):
small_master = CCDData(ccd_data)
small_master.data = np.zeros((1, 1))
subtract_dark(ccd_data, small_master)
def trim_ccddata(ccddata: CCDData,
left: 'int' = None,
right: 'int' = None,
bottom: 'int' = None,
top: 'int' = None,
update_wcs=True):
"""
:param ccddata:
:param left:
:param right:
:param bottom:
:param top:
:return:
"""
shape = ccddata.shape
if left is None:
left = 0
if right is None:
right = shape[1]
if bottom is None:
bottom = 0
if top is None:
top = shape[0]
if right < left:
raise ValueError('Improper inputs; right is smaller than left.')
if top < bottom:
raise ValueError('Improper inputs; top is smaller than bottom.')
if update_wcs:
ccddata.header['CRPIX1'] = ccddata.header['CRPIX1'] - left
ccddata.header['CRPIX2'] = ccddata.header['CRPIX2'] - bottom
ccddata.data = ccddata.data[bottom:top, left:right]
return ccddata
def interpolate(img: CCDData):
"""
Takes a image with a mask for bad pixels and interpolates over the bad pixels
:param img: the image you want to interpolate bad pixels in
:return: interpolated image
"""
# TODO combiner does not care about this and marks it invalid still
from astropy.convolution import CustomKernel
from astropy.convolution import interpolate_replace_nans
# TODO this here doesn't really work all that well -> extended regions cause artifacts at border
kernel_array = np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1]
]) / 9 # average of all surrounding pixels
# noinspection PyTypeChecker
kernel = CustomKernel(
kernel_array
) # TODO the original pipeline used fixpix, which says it uses linear interpolation
img.data[np.logical_not(img.mask)] = np.NaN
img.data = interpolate_replace_nans(img.data, kernel)
return img
def vertical_correct(ccd,
fitting_sections=None,
method='median',
sigclip_kw=dict(sigma=2, maxiters=5),
dtype='float32',
return_pattern=False,
update_header=True,
verbose=False):
''' Correct vertical strip patterns.
Paramters
---------
ccd : CCDData, HDU object, HDUList, or ndarray.
The CCD to subtract the vertical pattern.
fitting_sections : list of two str, optional.
The sections to be used for the vertical pattern estimation. This must be identical to the
usual FITS section (i.e., that used in SAO ds9 or IRAF, 1-indexing and last-index-inclusive),
not in python. **Give it in the order of ``[<upper>, <lower>]`` in FITS y-coordinate.**
method : str, optional.
One of ``['med', 'avg', 'median', 'average', 'mean']``.
sigma, maxiters : float and int, optional
A sigma-clipping will be done to remove hot pixels and cosmic rays for estimating the vertical
pattern. To turn sigma-clipping off, set ``maxiters=0``.
sigclip_kw : dict, optional
The keyword arguments for the sigma clipping.
dtype : str, dtype, optional
The data type to be returned.
return_pattern : bool, optional.
If `True`, the subtracted pattern will also be returned.
Default is `False`.
update_header : bool, optional.
Whether to update the header if there is any.
Return
------
'''
_t = Time.now()
data, hdr = _parse_data_header(ccd)
if fitting_sections is None:
fitting_sections = VERTICALSECTS
elif len(fitting_sections) != 2:
raise ValueError("fitting_sections must have two elements.")
if method in ['median', 'med']:
methodstr = 'taking median'
fun = np.median
idx = 1
elif method in ['average', 'avg', 'mean']:
methodstr = 'taking average'
fun = np.mean
idx = 0
else:
raise ValueError(
"method not understood; it must be one of [med, avg, median, average, mean]."
)
parts = [] # The two horizontal box areas from raw data
strips = [] # The estimated patterns (1-D)
for sect in fitting_sections:
parts.append(data[fitsxy2py(sect)])
try:
if sigclip_kw["maxiters"] == 0:
clipstr = "no clipping"
for part in parts:
strips.append(fun(part, axis=0))
except KeyError:
pass # The user wants to use default value of maxiters in astropy.
clipstr = f"sigma-clipping in astropy (v {astropy.__version__})"
if sigclip_kw:
clipstr += f", given {sigclip_kw}."
else:
clipstr += "."
for part in parts:
clip = sigma_clipped_stats(part, axis=0, **sigclip_kw)
strips.append(clip[idx])
ny, nx = data.shape
vpattern = np.repeat(strips, ny / 2, axis=0)
vsub = data - vpattern.astype(dtype)
if update_header and hdr is not None:
# add as history
add_to_header(
hdr,
'h',
verbose=verbose,
t_ref=_t,
s=f"Vertical pattern subtracted using {fitting_sections} by {methodstr} with {clipstr}"
)
try:
nccd = CCDData(data=vsub, header=hdr)
except ValueError:
nccd = CCDData(data=vsub, header=hdr, unit='adu')
nccd.data = nccd.data.astype(dtype)
if return_pattern:
return nccd, vpattern
return nccd
def bdf_process(ccd,
output=None,
mbiaspath=None,
mdarkpath=None,
mflatpath=None,
trim_fits_section=None,
calc_err=False,
unit='adu',
gain=None,
gain_key="GAIN",
gain_unit=u.electron / u.adu,
rdnoise=None,
rdnoise_key="RDNOISE",
rdnoise_unit=u.electron,
dark_exposure=None,
data_exposure=None,
exposure_key="EXPTIME",
exposure_unit=u.s,
dark_scale=False,
normalize_exposure=False,
normalize_average=False,
flat_min_value=None,
flat_norm_value=None,
do_crrej=False,
crrej_kwargs=None,
propagate_crmask=False,
verbose_crrej=False,
verbose_bdf=True,
output_verify='fix',
overwrite=True,
dtype="float32",
uncertainty_dtype="float32"):
''' Do bias, dark and flat process.
Parameters
----------
ccd: CCDData
The ccd to be processed.
output : path-like or None, optional.
The path if you want to save the resulting ``ccd`` object.
Default is ``None``.
mbiaspath, mdarkpath, mflatpath : path-like, optional.
The path to master bias, dark, flat FITS files. If ``None``, the
corresponding process is not done.
trim_fits_section: str, optional
Region of ``ccd`` to be trimmed; see
``ccdproc.subtract_overscan`` for details. Default is ``None``.
calc_err : bool, optional.
Whether to calculate the error map based on Poisson and readnoise
error propagation.
unit : `~astropy.units.Unit` or str, optional.
The units of the data.
Default is ``'adu'``.
gain, rdnoise : None, float, optional
The gain and readnoise value. These are all ignored if
``calc_err=False``. If ``calc_err=True``, it automatically seeks
for suitable gain and readnoise value. If ``gain`` or
``readnoise`` is specified, they are interpreted with
``gain_unit`` and ``rdnoise_unit``, respectively. If they are
not specified, this function will seek for the header with
keywords of ``gain_key`` and ``rdnoise_key``, and interprete the
header value in the unit of ``gain_unit`` and ``rdnoise_unit``,
respectively.
gain_key, rdnoise_key : str, optional
See ``gain``, ``rdnoise`` explanation above.
These are all ignored if ``calc_err=False``.
gain_unit, rdnoise_unit : astropy Unit, optional
See ``gain``, ``rdnoise`` explanation above.
These are all ignored if ``calc_err=False``.
dark_exposure, data_exposure : None, float, astropy Quantity, optional
The exposure times of dark and data frame, respectively. They
should both be specified or both ``None``.
These are all ignored if ``mdarkpath=None``.
If both are not specified while ``mdarkpath`` is given, then the
code automatically seeks for header's ``exposure_key``. Then
interprete the value as the quantity with unit
``exposure_unit``.
If ``mdkarpath`` is not ``None``, then these are passed to
``ccdproc.subtract_dark``.
exposure_key : str, optional
The header keyword for exposure time.
Ignored if ``mdarkpath=None``.
exposure_unit : astropy Unit, optional.
The unit of the exposure time.
Ignored if ``mdarkpath=None``.
flat_min_value : float or None, optional
min_value of `ccdproc.flat_correct`.
Minimum value for flat field. The value can either be None and
no minimum value is applied to the flat or specified by a float
which will replace all values in the flat by the min_value.
Default is ``None``.
flat_norm_value : float or None, optional
norm_value of `ccdproc.flat_correct`.
If not ``None``, normalize flat field by this argument rather
than the mean of the image. This allows fixing several different
flat fields to have the same scale. If this value is negative or
0, a ``ValueError``
is raised. Default is ``None``.
crrej_kwargs : dict or None, optional
If ``None`` (default), uses some default values defined in
``~.misc.LACOSMIC_KEYS``. It is always discouraged to use
default except for quick validity-checking, because even the
official L.A. Cosmic codes in different versions (IRAF, IDL,
Python, etc) have different default parameters, i.e., there is
nothing which can be regarded as default.
To see all possible keywords, do
``print(astroscrappy.detect_cosmics.__doc__)``
Also refer to
https://nbviewer.jupyter.org/github/ysbach/AO2019/blob/master/Notebooks/07-Cosmic_Ray_Rejection.ipynb
propagate_crmask : bool, optional
Whether to save (propagate) the mask from CR rejection
(``astroscrappy``) to the CCD's mask.
Default is ``False``.
output_verify : str
Output verification option. Must be one of ``"fix"``,
``"silentfix"``, ``"ignore"``, ``"warn"``, or ``"exception"``.
May also be any combination of ``"fix"`` or ``"silentfix"`` with
``"+ignore"``, ``+warn``, or ``+exception" (e.g.
``"fix+warn"``). See the astropy documentation below:
http://docs.astropy.org/en/stable/io/fits/api/verification.html#verify
dtype : str or `numpy.dtype` or None, optional
Allows user to set dtype. See `numpy.array` ``dtype`` parameter
description. If ``None`` it uses ``np.float64``.
Default is ``None``.
'''
def _add_and_print(s, header, verbose):
header.add_history(s)
if verbose:
print(s)
# Set strings for header history & print (if verbose_bdf)
str_bias = "Bias subtracted using {}"
str_dark = "Dark subtracted using {}"
str_dscale = "Dark scaling {} using {}"
str_flat = "Flat corrected using {}"
str_trim = "Trim by FITS section {}"
str_grd = "From {}, {} = {:.3f} [{}]"
# str_grd.format(user/header_key, gain/rdnoise, val, unit)
str_e0 = ("Readnoise propagated with Poisson noise (using gain above)" +
" of source.")
str_ed = "Poisson noise from subtracted dark was propagated."
str_ef = "Flat uncertainty was propagated."
str_nexp = "Normalized by the exposure time."
str_navg = "Normalized by the average value of the frame."
str_cr = ("Cosmic-Ray rejected by astroscrappy (v {}), " +
"with parameters: {}")
# Initial setting
proc = CCDData(ccd)
hdr_new = proc.header
# Add PROCESS key
try:
_ = hdr_new["PROCESS"]
except KeyError:
hdr_new["PROCESS"] = ("", "The processed history: see comment.")
hdr_new["PROCVER"] = (ccdproc.__version__,
"ccdproc version used for processing.")
hdr_new.add_comment("PROCESS key can be B (bias), D (dark), F (flat), " +
"T (trim), W (WCS astrometry), C(CRrej).")
# Set for BIAS
if mbiaspath is None:
do_bias = False
mbias = CCDData(np.zeros_like(ccd), unit=proc.unit)
else:
do_bias = True
mbias = CCDData.read(mbiaspath, unit=unit)
hdr_new["PROCESS"] += "B"
_add_and_print(str_bias.format(mbiaspath), hdr_new, verbose_bdf)
# Set for DARK
if mdarkpath is None:
do_dark = False
mdark = CCDData(np.zeros_like(ccd), unit=proc.unit)
else:
do_dark = True
mdark = CCDData.read(mdarkpath, unit=unit)
hdr_new["PROCESS"] += "D"
_add_and_print(str_dark.format(mdarkpath), hdr_new, verbose_bdf)
if dark_scale:
_add_and_print(str_dscale.format(dark_scale, exposure_key),
hdr_new, verbose_bdf)
# Set for FLAT
if mflatpath is None:
do_flat = False
mflat = CCDData(np.ones_like(ccd), unit=proc.unit)
else:
do_flat = True
mflat = CCDData.read(mflatpath)
hdr_new["PROCESS"] += "F"
_add_and_print(str_flat.format(mflatpath), hdr_new, verbose_bdf)
# Set gain and rdnoise if at least one of calc_err and do_crrej is True.
if calc_err or do_crrej:
if gain is None:
gain_Q = get_from_header(hdr_new,
gain_key,
unit=gain_unit,
verbose=False,
default=1.)
gain_from = gain_key
else:
if not isinstance(gain, u.Quantity):
gain_Q = gain * gain_unit
else:
gain_Q = gain
gain_from = "User"
_add_and_print(
str_grd.format(gain_from, "gain", gain_Q.value, gain_Q.unit),
hdr_new, verbose_bdf)
if rdnoise is None:
rdnoise_Q = get_from_header(hdr_new,
rdnoise_key,
unit=rdnoise_unit,
verbose=False,
default=1.)
rdnoise_from = rdnoise_key
else:
if not isinstance(rdnoise, u.Quantity):
rdnoise_Q = rdnoise * rdnoise_unit
else:
rdnoise_Q = rdnoise
rdnoise_from = "User"
_add_and_print(
str_grd.format(rdnoise_from, "rdnoise", rdnoise_Q.value,
rdnoise_Q.unit), hdr_new, verbose_bdf)
# Do TRIM
if trim_fits_section is not None:
proc = trim_image(proc, trim_fits_section)
mbias = trim_image(mbias, trim_fits_section)
mdark = trim_image(mdark, trim_fits_section)
mflat = trim_image(mflat, trim_fits_section)
hdr_new["PROCESS"] += "T"
_add_and_print(str_trim.format(trim_fits_section), hdr_new,
verbose_bdf)
# Do BIAS
if do_bias:
proc = subtract_bias(proc, mbias)
# Do DARK
if do_dark:
proc = subtract_dark(proc,
mdark,
dark_exposure=dark_exposure,
data_exposure=data_exposure,
exposure_time=exposure_key,
exposure_unit=exposure_unit,
scale=dark_scale)
# Make UNCERT extension before doing FLAT
# It is better to make_errmap a priori because of mathematical and
# computational convenience. See ``if do_flat:`` clause below.
if calc_err:
err = make_errmap(proc, gain_epadu=gain, subtracted_dark=mdark)
proc.uncertainty = StdDevUncertainty(err)
_add_and_print(str_e0, hdr_new, verbose_bdf)
if do_dark:
_add_and_print(str_ed, hdr_new, verbose_bdf)
# Do FLAT
if do_flat:
# Flat error propagation is done automatically by
# ``ccdproc.flat_correct``if it has the uncertainty attribute.
proc = flat_correct(proc,
mflat,
min_value=flat_min_value,
norm_value=flat_norm_value)
if calc_err and mflat.uncertainty is not None:
_add_and_print(str_ef, hdr_new, verbose_bdf)
# Normalize by the exposure time (e.g., ADU per sec)
if normalize_exposure:
if data_exposure is None:
data_exposure = hdr_new[exposure_key]
proc = proc.divide(data_exposure) # uncertainty will also be..
_add_and_print(str_nexp, hdr_new, verbose_bdf)
# Normalize by the mean value
if normalize_average:
avg = np.mean(proc.data)
proc = proc.divide(avg)
_add_and_print(str_navg, hdr_new, verbose_bdf)
# Do CRREJ
if do_crrej:
import astroscrappy
from astroscrappy import detect_cosmics
from .misc import LACOSMIC_KEYS
if crrej_kwargs is None:
crrej_kwargs = LACOSMIC_KEYS
warn("You are not specifying CR-rejection parameters and blindly" +
" using defaults. It can be dangerous.")
if (("B" in hdr_new["PROCESS"]) + ("D" in hdr_new["PROCESS"]) +
("F" in hdr_new["PROCESS"])) < 2:
warn("L.A. Cosmic should be run AFTER B/D/F process. " +
f"You are running it with {hdr_new['PROCESS']}. " +
"See http://www.astro.yale.edu/dokkum/lacosmic/notes.html")
# remove the fucxing cosmic rays
crmask, cleanarr = detect_cosmics(proc.data,
inmask=proc.mask,
gain=gain_Q.value,
readnoise=rdnoise_Q.value,
**crrej_kwargs,
verbose=verbose_crrej)
# create the new ccd data object
# astroscrappy automatically does the gain correction, so return
# back to avoid confusion.
proc.data = cleanarr / gain_Q.value
if propagate_crmask:
if proc.mask is None:
proc.mask = crmask
else:
proc.mask = proc.mask + crmask
hdr_new["PROCESS"] += "C"
_add_and_print(str_cr.format(astroscrappy.__version__, crrej_kwargs),
hdr_new, verbose_crrej)
proc = CCDData_astype(proc,
dtype=dtype,
uncertainty_dtype=uncertainty_dtype)
proc.header = hdr_new
if output is not None:
if verbose_bdf:
print(f"Writing FITS to {output}... ", end='')
proc.write(output, output_verify=output_verify, overwrite=overwrite)
if verbose_bdf:
print(f"Saved.")
return proc