def test_reject_cosmics(self):
"""
Test that the generic cosmics interface updates the mask
"""
image = Image(self.pix, self.ivar, camera="r0")
reject_cosmic_rays(image)
cosmic = (image.pix > 0)
self.assertTrue(np.all(image.mask[cosmic] & ccdmask.COSMIC))
def main(args) :
if args.outfile is not None :
outfile=args.outfile
else :
outfile=args.infile
log = get_logger()
log.info("starting finding cosmics in %s"%args.infile)
img=image.read_image(args.infile)
if args.ignore_cosmic_ccdmask :
log.warning("ignore cosmic ccdmask for test")
log.debug("ccdmask.COSMIC = %d"%ccdmask.COSMIC)
cosmic_ray_prexisting_mask = img.mask & ccdmask.COSMIC
img._mask &= ~ccdmask.COSMIC #- turn off cosmic mask
reject_cosmic_rays(img)
log.info("writing data and new mask in %s"%outfile)
image.write_image(outfile, img, meta=img.meta)
log.info("done")
def main(args):
if args.outfile is not None:
outfile = args.outfile
else:
outfile = args.infile
log = get_logger()
log.info("starting finding cosmics in %s" % args.infile)
img = image.read_image(args.infile)
if args.ignore_cosmic_ccdmask:
log.warning("ignore cosmic ccdmask for test")
log.debug("ccdmask.COSMIC = %d" % ccdmask.COSMIC)
cosmic_ray_prexisting_mask = img.mask & ccdmask.COSMIC
img._mask &= ~ccdmask.COSMIC #- turn off cosmic mask
reject_cosmic_rays(img)
log.info("writing data and new mask in %s" % outfile)
image.write_image(outfile, img, meta=img.meta)
log.info("done")
def preproc(rawimage,
header,
primary_header,
bias=True,
dark=True,
pixflat=True,
mask=True,
bkgsub=False,
nocosmic=False,
cosmics_nsig=6,
cosmics_cfudge=3.,
cosmics_c2fudge=0.5,
ccd_calibration_filename=None,
nocrosstalk=False,
nogain=False,
overscan_per_row=False,
use_overscan_row=False,
use_savgol=None,
nodarktrail=False,
remove_scattered_light=False,
psf_filename=None,
bias_img=None):
'''
preprocess image using metadata in header
image = ((rawimage-bias-overscan)*gain)/pixflat
Args:
rawimage : 2D numpy array directly from raw data file
header : dict-like metadata, e.g. from FITS header, with keywords
CAMERA, BIASSECx, DATASECx, CCDSECx
where x = A, B, C, D for each of the 4 amplifiers
(also supports old naming convention 1, 2, 3, 4).
primary_header: dict-like metadata fit keywords EXPTIME, DOSVER
DATE-OBS is also required if bias, pixflat, or mask=True
Optional bias, pixflat, and mask can each be:
False: don't apply that step
True: use default calibration data for that night
ndarray: use that array
filename (str or unicode): read HDU 0 and use that
Optional overscan features:
overscan_per_row : bool, Subtract the overscan_col values
row by row from the data.
use_overscan_row : bool, Subtract off the overscan_row
from the data (default: False). Requires ORSEC in
the Header
use_savgol : bool, Specify whether to use Savitsky-Golay filter for
the overscan. (default: False). Requires use_overscan_row=True
to have any effect.
Optional background subtraction with median filtering if bkgsub=True
Optional disabling of cosmic ray rejection if nocosmic=True
Optional disabling of dark trail correction if nodarktrail=True
Optional bias image (testing only) may be provided by bias_img=
Optional tuning of cosmic ray rejection parameters:
cosmics_nsig: number of sigma above background required
cosmics_cfudge: number of sigma inconsistent with PSF required
cosmics_c2fudge: fudge factor applied to PSF
Optional fit and subtraction of scattered light
Returns Image object with member variables:
pix : 2D preprocessed image in units of electrons per pixel
ivar : 2D inverse variance of image
mask : 2D mask of image (0=good)
readnoise : 2D per-pixel readnoise of image
meta : metadata dictionary
TODO: define what keywords are included
preprocessing includes the following steps:
- bias image subtraction
- overscan subtraction (from BIASSEC* keyword defined regions)
- readnoise estimation (from BIASSEC* keyword defined regions)
- gain correction (from GAIN* keywords)
- pixel flat correction
- cosmic ray masking
- propagation of input known bad pixel mask
- inverse variance estimation
Notes:
The bias image is subtracted before any other calculation to remove any
non-uniformities in the overscan regions prior to calculating overscan
levels and readnoise.
The readnoise is an image not just one number per amp, because the pixflat
image also affects the interpreted readnoise.
The inverse variance is estimated from the readnoise and the image itself,
and thus is biased.
'''
log = get_logger()
header = header.copy()
cfinder = None
if ccd_calibration_filename is not False:
cfinder = CalibFinder([header, primary_header],
yaml_file=ccd_calibration_filename)
#- TODO: Check for required keywords first
#- Subtract bias image
camera = header['CAMERA'].lower()
#- convert rawimage to float64 : this is the output format of read_image
rawimage = rawimage.astype(np.float64)
# Savgol
if cfinder and cfinder.haskey("USE_ORSEC"):
use_overscan_row = cfinder.value("USE_ORSEC")
if cfinder and cfinder.haskey("SAVGOL"):
use_savgol = cfinder.value("SAVGOL")
# Set bias image, as desired
if bias_img is None:
bias = get_calibration_image(cfinder, "BIAS", bias)
else:
bias = bias_img
if bias is not False: #- it's an array
if bias.shape == rawimage.shape:
log.info("subtracting bias")
rawimage = rawimage - bias
else:
raise ValueError('shape mismatch bias {} != rawimage {}'.format(
bias.shape, rawimage.shape))
#- Check if this file uses amp names 1,2,3,4 (old) or A,B,C,D (new)
amp_ids = get_amp_ids(header)
#- Double check that we have the necessary keywords
missing_keywords = list()
for prefix in ['CCDSEC', 'BIASSEC']:
for amp in amp_ids:
key = prefix + amp
if not key in header:
log.error('No {} keyword in header'.format(key))
missing_keywords.append(key)
if len(missing_keywords) > 0:
raise KeyError("Missing keywords {}".format(
' '.join(missing_keywords)))
#- Output arrays
ny = 0
nx = 0
for amp in amp_ids:
yy, xx = parse_sec_keyword(header['CCDSEC%s' % amp])
ny = max(ny, yy.stop)
nx = max(nx, xx.stop)
image = np.zeros((ny, nx))
readnoise = np.zeros_like(image)
#- Load mask
mask = get_calibration_image(cfinder, "MASK", mask)
if mask is False:
mask = np.zeros(image.shape, dtype=np.int32)
else:
if mask.shape != image.shape:
raise ValueError('shape mismatch mask {} != image {}'.format(
mask.shape, image.shape))
#- Load dark
dark = get_calibration_image(cfinder, "DARK", dark)
if dark is not False:
if dark.shape != image.shape:
log.error('shape mismatch dark {} != image {}'.format(
dark.shape, image.shape))
raise ValueError('shape mismatch dark {} != image {}'.format(
dark.shape, image.shape))
if cfinder and cfinder.haskey("EXPTIMEKEY"):
exptime_key = cfinder.value("EXPTIMEKEY")
log.info("Using exposure time keyword %s for dark normalization" %
exptime_key)
else:
exptime_key = "EXPTIME"
exptime = primary_header[exptime_key]
log.info("Multiplying dark by exptime %f" % (exptime))
dark *= exptime
for amp in amp_ids:
# Grab the sections
ov_col = parse_sec_keyword(header['BIASSEC' + amp])
if 'ORSEC' + amp in header.keys():
ov_row = parse_sec_keyword(header['ORSEC' + amp])
elif use_overscan_row:
log.error('No ORSEC{} keyword; not using overscan_row'.format(amp))
use_overscan_row = False
if nogain:
gain = 1.
else:
#- Initial teststand data may be missing GAIN* keywords; don't crash
if 'GAIN' + amp in header:
gain = header['GAIN' + amp] #- gain = electrons / ADU
else:
if cfinder and cfinder.haskey('GAIN' + amp):
gain = float(cfinder.value('GAIN' + amp))
log.info('Using GAIN{}={} from calibration data'.format(
amp, gain))
else:
gain = 1.0
log.warning(
'Missing keyword GAIN{} in header and nothing in calib data; using {}'
.format(amp, gain))
#- Add saturation level
if 'SATURLEV' + amp in header:
saturlev = header['SATURLEV' + amp] # in electrons
else:
if cfinder and cfinder.haskey('SATURLEV' + amp):
saturlev = float(cfinder.value('SATURLEV' + amp))
log.info('Using SATURLEV{}={} from calibration data'.format(
amp, saturlev))
else:
saturlev = 200000
log.warning(
'Missing keyword SATURLEV{} in header and nothing in calib data; using 200000'
.format(amp, saturlev))
# Generate the overscan images
raw_overscan_col = rawimage[ov_col].copy()
if use_overscan_row:
raw_overscan_row = rawimage[ov_row].copy()
overscan_row = np.zeros_like(raw_overscan_row)
# Remove overscan_col from overscan_row
raw_overscan_squared = rawimage[ov_row[0], ov_col[1]].copy()
for row in range(raw_overscan_row.shape[0]):
o, r = _overscan(raw_overscan_squared[row])
overscan_row[row] = raw_overscan_row[row] - o
# Now remove the overscan_col
nrows = raw_overscan_col.shape[0]
log.info("nrows in overscan=%d" % nrows)
overscan_col = np.zeros(nrows)
rdnoise = np.zeros(nrows)
if (cfinder and cfinder.haskey('OVERSCAN' + amp)
and cfinder.value("OVERSCAN" + amp).upper()
== "PER_ROW") or overscan_per_row:
log.info(
"Subtracting overscan per row for amplifier %s of camera %s" %
(amp, camera))
for j in range(nrows):
if np.isnan(np.sum(overscan_col[j])):
log.warning(
"NaN values in row %d of overscan of amplifier %s of camera %s"
% (j, amp, camera))
continue
o, r = _overscan(raw_overscan_col[j])
#log.info("%d %f %f"%(j,o,r))
overscan_col[j] = o
rdnoise[j] = r
else:
log.info(
"Subtracting average overscan for amplifier %s of camera %s" %
(amp, camera))
o, r = _overscan(raw_overscan_col)
overscan_col += o
rdnoise += r
rdnoise *= gain
median_rdnoise = np.median(rdnoise)
median_overscan = np.median(overscan_col)
log.info("Median rdnoise and overscan= %f %f" %
(median_rdnoise, median_overscan))
kk = parse_sec_keyword(header['CCDSEC' + amp])
for j in range(nrows):
readnoise[kk][j] = rdnoise[j]
header['OVERSCN' + amp] = (median_overscan, 'ADUs (gain not applied)')
if gain != 1:
rdnoise_message = 'electrons (gain is applied)'
gain_message = 'e/ADU (gain applied to image)'
else:
rdnoise_message = 'ADUs (gain not applied)'
gain_message = 'gain not applied to image'
header['OBSRDN' + amp] = (median_rdnoise, rdnoise_message)
header['GAIN' + amp] = (gain, gain_message)
#- Warn/error if measured readnoise is very different from expected if exists
if 'RDNOISE' + amp in header:
expected_readnoise = header['RDNOISE' + amp]
if median_rdnoise < 0.5 * expected_readnoise:
log.error(
'Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
elif median_rdnoise < 0.9 * expected_readnoise:
log.warning(
'Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
elif median_rdnoise > 2.0 * expected_readnoise:
log.error(
'Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
elif median_rdnoise > 1.2 * expected_readnoise:
log.warning(
'Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
#else:
# log.warning('Expected readnoise keyword {} missing'.format('RDNOISE'+amp))
log.info("Measured readnoise for AMP %s = %f" % (amp, median_rdnoise))
#- subtract overscan from data region and apply gain
jj = parse_sec_keyword(header['DATASEC' + amp])
data = rawimage[jj].copy()
# Subtract columns
for k in range(nrows):
data[k] -= overscan_col[k]
# And now the rows
if use_overscan_row:
# Savgol?
if use_savgol:
log.info("Using savgol")
collapse_oscan_row = np.zeros(overscan_row.shape[1])
for col in range(overscan_row.shape[1]):
o, _ = _overscan(overscan_row[:, col])
collapse_oscan_row[col] = o
oscan_row = _savgol_clipped(collapse_oscan_row, niter=0)
oimg_row = np.outer(np.ones(data.shape[0]), oscan_row)
data -= oimg_row
else:
o, r = _overscan(overscan_row)
data -= o
#- apply saturlev (defined in ADU), prior to multiplication by gain
saturated = (rawimage[jj] >= saturlev)
mask[kk][saturated] |= ccdmask.SATURATED
#- ADC to electrons
image[kk] = data * gain
if not nocrosstalk:
#- apply cross-talk
# the ccd looks like :
# C D
# A B
# for cross talk, we need a symmetric 4x4 flip_matrix
# of coordinates ABCD giving flip of both axis
# when computing crosstalk of
# A B C D
#
# A AA AB AC AD
# B BA BB BC BD
# C CA CB CC CD
# D DA DB DC BB
# orientation_matrix_defines change of orientation
#
fip_axis_0 = np.array([[1, 1, -1, -1], [1, 1, -1, -1], [-1, -1, 1, 1],
[-1, -1, 1, 1]])
fip_axis_1 = np.array([[1, -1, 1, -1], [-1, 1, -1, 1], [1, -1, 1, -1],
[-1, 1, -1, 1]])
for a1 in range(len(amp_ids)):
amp1 = amp_ids[a1]
ii1 = parse_sec_keyword(header['CCDSEC' + amp1])
a1flux = image[ii1]
#a1mask=mask[ii1]
for a2 in range(len(amp_ids)):
if a1 == a2:
continue
amp2 = amp_ids[a2]
if cfinder is None: continue
if not cfinder.haskey("CROSSTALK%s%s" % (amp1, amp2)): continue
crosstalk = cfinder.value("CROSSTALK%s%s" % (amp1, amp2))
if crosstalk == 0.: continue
log.info("Correct for crosstalk=%f from AMP %s into %s" %
(crosstalk, amp1, amp2))
a12flux = crosstalk * a1flux.copy()
#a12mask=a1mask.copy()
if fip_axis_0[a1, a2] == -1:
a12flux = a12flux[::-1]
#a12mask=a12mask[::-1]
if fip_axis_1[a1, a2] == -1:
a12flux = a12flux[:, ::-1]
#a12mask=a12mask[:,::-1]
ii2 = parse_sec_keyword(header['CCDSEC' + amp2])
image[ii2] -= a12flux
# mask[ii2] |= a12mask (not sure we really need to propagate the mask)
#- Poisson noise variance (prior to dark subtraction and prior to pixel flat field)
#- This is biasing, but that's what we have for now
poisson_var = image.clip(0)
#- subtract dark after multiplication by gain
if dark is not False:
log.info("subtracting dark for amp %s" % amp)
image -= dark
#- Correct for dark trails if any
if not nodarktrail and cfinder is not None:
for amp in amp_ids:
if cfinder.haskey("DARKTRAILAMP%s" % amp):
amplitude = cfinder.value("DARKTRAILAMP%s" % amp)
width = cfinder.value("DARKTRAILWIDTH%s" % amp)
ii = _parse_sec_keyword(header["CCDSEC" + amp])
log.info(
"Removing dark trails for amplifier %s with width=%3.1f and amplitude=%5.4f"
% (amp, width, amplitude))
correct_dark_trail(image,
ii,
left=((amp == "B") | (amp == "D")),
width=width,
amplitude=amplitude)
#- Divide by pixflat image
pixflat = get_calibration_image(cfinder, "PIXFLAT", pixflat)
if pixflat is not False:
if pixflat.shape != image.shape:
raise ValueError('shape mismatch pixflat {} != image {}'.format(
pixflat.shape, image.shape))
almost_zero = 0.001
if np.all(pixflat > almost_zero):
image /= pixflat
readnoise /= pixflat
poisson_var /= pixflat**2
else:
good = (pixflat > almost_zero)
image[good] /= pixflat[good]
readnoise[good] /= pixflat[good]
poisson_var[good] /= pixflat[good]**2
mask[~good] |= ccdmask.PIXFLATZERO
lowpixflat = (0 < pixflat) & (pixflat < 0.1)
if np.any(lowpixflat):
mask[lowpixflat] |= ccdmask.PIXFLATLOW
#- Inverse variance, estimated directly from the data (BEWARE: biased!)
var = poisson_var + readnoise**2
ivar = np.zeros(var.shape)
ivar[var > 0] = 1.0 / var[var > 0]
#- Ridiculously high readnoise is bad
mask[readnoise > 100] |= ccdmask.BADREADNOISE
if bkgsub:
bkg = _background(image, header)
image -= bkg
img = Image(image,
ivar=ivar,
mask=mask,
meta=header,
readnoise=readnoise,
camera=camera)
#- update img.mask to mask cosmic rays
if not nocosmic:
cosmics.reject_cosmic_rays(img,
nsig=cosmics_nsig,
cfudge=cosmics_cfudge,
c2fudge=cosmics_c2fudge)
if remove_scattered_light:
if psf_filename is None:
psf_filename = cfinder.findfile("PSF")
xyset = read_xytraceset(psf_filename)
img.pix -= model_scattered_light(img, xyset)
return img
def preproc(rawimage,
header,
primary_header,
bias=True,
dark=True,
pixflat=True,
mask=True,
bkgsub=False,
nocosmic=False,
cosmics_nsig=6,
cosmics_cfudge=3.,
cosmics_c2fudge=0.8,
ccd_calibration_filename=None,
nocrosstalk=False):
'''
preprocess image using metadata in header
image = ((rawimage-bias-overscan)*gain)/pixflat
Args:
rawimage : 2D numpy array directly from raw data file
header : dict-like metadata, e.g. from FITS header, with keywords
CAMERA, BIASSECx, DATASECx, CCDSECx
where x = A, B, C, D for each of the 4 amplifiers
(also supports old naming convention 1, 2, 3, 4).
primary_header: dict-like metadata fit keywords EXPTIME, DOSVER
DATE-OBS is also required if bias, pixflat, or mask=True
Optional bias, pixflat, and mask can each be:
False: don't apply that step
True: use default calibration data for that night
ndarray: use that array
filename (str or unicode): read HDU 0 and use that
Optional background subtraction with median filtering if bkgsub=True
Optional disabling of cosmic ray rejection if nocosmic=True
Optional tuning of cosmic ray rejection parameters:
cosmics_nsig: number of sigma above background required
cosmics_cfudge: number of sigma inconsistent with PSF required
cosmics_c2fudge: fudge factor applied to PSF
Returns Image object with member variables:
image : 2D preprocessed image in units of electrons per pixel
ivar : 2D inverse variance of image
mask : 2D mask of image (0=good)
readnoise : 2D per-pixel readnoise of image
meta : metadata dictionary
TODO: define what keywords are included
preprocessing includes the following steps:
- bias image subtraction
- overscan subtraction (from BIASSEC* keyword defined regions)
- readnoise estimation (from BIASSEC* keyword defined regions)
- gain correction (from GAIN* keywords)
- pixel flat correction
- cosmic ray masking
- propagation of input known bad pixel mask
- inverse variance estimation
Notes:
The bias image is subtracted before any other calculation to remove any
non-uniformities in the overscan regions prior to calculating overscan
levels and readnoise.
The readnoise is an image not just one number per amp, because the pixflat
image also affects the interpreted readnoise.
The inverse variance is estimated from the readnoise and the image itself,
and thus is biased.
'''
log = get_logger()
calibration_data = None
if ccd_calibration_filename is None:
srch_file = "data/ccd/ccd_calibration.yaml"
if not resource_exists('desispec', srch_file):
log.error(
"Cannot find CCD calibration file {:s}".format(srch_file))
else:
ccd_calibration_filename = resource_filename('desispec', srch_file)
if ccd_calibration_filename is not None and ccd_calibration_filename is not False:
calibration_data = read_ccd_calibration(header, primary_header,
ccd_calibration_filename)
#- Get path to calibration data
if "DESI_CCD_CALIBRATION_DATA" in os.environ:
calibration_data_path = os.environ["DESI_CCD_CALIBRATION_DATA"]
else:
calibration_data_path = None
#- TODO: Check for required keywords first
#- Subtract bias image
camera = header['CAMERA'].lower()
#- convert rawimage to float64 : this is the output format of read_image
rawimage = rawimage.astype(np.float64)
bias = get_calibration_image(calibration_data, calibration_data_path,
"BIAS", bias)
if bias is not False: #- it's an array
if bias.shape == rawimage.shape:
log.info("subtracting bias")
rawimage = rawimage - bias
else:
raise ValueError('shape mismatch bias {} != rawimage {}'.format(
bias.shape, rawimage.shape))
if calibration_data and "AMPLIFIERS" in calibration_data:
amp_ids = list(calibration_data["AMPLIFIERS"])
else:
amp_ids = ['A', 'B', 'C', 'D']
#- check whether it's indeed CCDSECx with x in ['A','B','C','D']
# or older version with x in ['1','2','3','4']
# we can remove this piece of code at later times
has_valid_keywords = True
for amp in amp_ids:
if not 'CCDSEC%s' % amp in header:
log.warning(
"No CCDSEC%s keyword in header , will look for alternative naming CCDSEC{1,2,3,4} ..."
% amp)
has_valid_keywords = False
break
if not has_valid_keywords:
amp_ids = ['1', '2', '3', '4']
for amp in ['1', '2', '3', '4']:
if not 'CCDSEC%s' % amp in header:
log.error("No CCDSEC%s keyword, exit" % amp)
raise KeyError("No CCDSEC%s keyword" % amp)
#- Output arrays
ny = 0
nx = 0
for amp in amp_ids:
yy, xx = _parse_sec_keyword(header['CCDSEC%s' % amp])
ny = max(ny, yy.stop)
nx = max(nx, xx.stop)
image = np.zeros((ny, nx))
readnoise = np.zeros_like(image)
#- Load mask
mask = get_calibration_image(calibration_data, calibration_data_path,
"MASK", mask)
if mask is False:
mask = np.zeros(image.shape, dtype=np.int32)
else:
if mask.shape != image.shape:
raise ValueError('shape mismatch mask {} != image {}'.format(
mask.shape, image.shape))
#- Load dark
dark = get_calibration_image(calibration_data, calibration_data_path,
"DARK", dark)
if dark is not False:
if dark.shape != image.shape:
log.error('shape mismatch dark {} != image {}'.format(
dark.shape, image.shape))
raise ValueError('shape mismatch dark {} != image {}'.format(
dark.shape, image.shape))
if calibration_data and "EXPTIMEKEY" in calibration_data:
exptime_key = calibration_data["EXPTIMEKEY"]
log.info("Using exposure time keyword %s for dark normalization" %
exptime_key)
else:
exptime_key = "EXPTIME"
exptime = primary_header[exptime_key]
log.info("Multiplying dark by exptime %f" % (exptime))
dark *= exptime
for amp in amp_ids:
ii = _parse_sec_keyword(header['BIASSEC' + amp])
#- Initial teststand data may be missing GAIN* keywords; don't crash
if 'GAIN' + amp in header:
gain = header['GAIN' + amp] #- gain = electrons / ADU
else:
if calibration_data and 'GAIN' + amp in calibration_data:
gain = float(calibration_data['GAIN' + amp])
log.info('Using GAIN{}={} from calibration data'.format(
amp, gain))
else:
gain = 1.0
log.warning(
'Missing keyword GAIN{} in header and nothing in calib data; using {}'
.format(amp, gain))
#- Add saturation level
if 'SATURLEV' + amp in header:
saturlev = header['SATURLEV' + amp] # in electrons
else:
if calibration_data and 'SATURLEV' + amp in calibration_data:
saturlev = float(calibration_data['SATURLEV' + amp])
log.info('Using SATURLEV{}={} from calibration data'.format(
amp, saturlev))
else:
saturlev = 200000
log.warning(
'Missing keyword SATURLEV{} in header and nothing in calib data; using 200000'
.format(amp, saturlev))
overscan_image = rawimage[ii].copy()
nrows = overscan_image.shape[0]
log.info("nrows in overscan=%d" % nrows)
overscan = np.zeros(nrows)
rdnoise = np.zeros(nrows)
overscan_per_row = True
if calibration_data and 'OVERSCAN' + amp in calibration_data and calibration_data[
"OVERSCAN" + amp].upper() == "PER_ROW":
log.info(
"Subtracting overscan per row for amplifier %s of camera %s" %
(amp, camera))
for j in range(nrows):
if np.isnan(np.sum(overscan_image[j])):
log.warning(
"NaN values in row %d of overscan of amplifier %s of camera %s"
% (j, amp, camera))
continue
o, r = _overscan(overscan_image[j])
#log.info("%d %f %f"%(j,o,r))
overscan[j] = o
rdnoise[j] = r
else:
log.info(
"Subtracting average overscan for amplifier %s of camera %s" %
(amp, camera))
o, r = _overscan(overscan_image)
overscan += o
rdnoise += r
rdnoise *= gain
median_rdnoise = np.median(rdnoise)
median_overscan = np.median(overscan)
log.info("Median rdnoise and overscan= %f %f" %
(median_rdnoise, median_overscan))
kk = _parse_sec_keyword(header['CCDSEC' + amp])
for j in range(nrows):
readnoise[kk][j] = rdnoise[j]
header['OVERSCN' + amp] = median_overscan
header['OBSRDN' + amp] = median_rdnoise
#- Warn/error if measured readnoise is very different from expected if exists
if 'RDNOISE' + amp in header:
expected_readnoise = header['RDNOISE' + amp]
if median_rdnoise < 0.5 * expected_readnoise:
log.error(
'Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
elif median_rdnoise < 0.9 * expected_readnoise:
log.warning(
'Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
elif median_rdnoise > 2.0 * expected_readnoise:
log.error(
'Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
elif median_rdnoise > 1.2 * expected_readnoise:
log.warning(
'Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
#else:
# log.warning('Expected readnoise keyword {} missing'.format('RDNOISE'+amp))
log.info("Measured readnoise for AMP %s = %f" % (amp, median_rdnoise))
#- subtract overscan from data region and apply gain
jj = _parse_sec_keyword(header['DATASEC' + amp])
data = rawimage[jj].copy()
for k in range(nrows):
data[k] -= overscan[k]
#- apply saturlev (defined in ADU), prior to multiplication by gain
saturated = (rawimage[jj] >= saturlev)
mask[kk][saturated] |= ccdmask.SATURATED
#- subtract dark prior to multiplication by gain
if dark is not False:
log.info("subtracting dark for amp %s" % amp)
data -= dark[kk]
image[kk] = data * gain
if not nocrosstalk:
#- apply cross-talk
# the ccd looks like :
# C D
# A B
# for cross talk, we need a symmetric 4x4 flip_matrix
# of coordinates ABCD giving flip of both axis
# when computing crosstalk of
# A B C D
#
# A AA AB AC AD
# B BA BB BC BD
# C CA CB CC CD
# D DA DB DC BB
# orientation_matrix_defines change of orientation
#
fip_axis_0 = np.array([[1, 1, -1, -1], [1, 1, -1, -1], [-1, -1, 1, 1],
[-1, -1, 1, 1]])
fip_axis_1 = np.array([[1, -1, 1, -1], [-1, 1, -1, 1], [1, -1, 1, -1],
[-1, 1, -1, 1]])
for a1 in range(len(amp_ids)):
amp1 = amp_ids[a1]
ii1 = _parse_sec_keyword(header['CCDSEC' + amp1])
a1flux = image[ii1]
#a1mask=mask[ii1]
for a2 in range(len(amp_ids)):
if a1 == a2:
continue
amp2 = amp_ids[a2]
if calibration_data is None: continue
if not "CROSSTALK%s%s" % (amp1, amp2) in calibration_data:
continue
crosstalk = calibration_data["CROSSTALK%s%s" % (amp1, amp2)]
if crosstalk == 0.: continue
log.info("Correct for crosstalk=%f from AMP %s into %s" %
(crosstalk, amp1, amp2))
a12flux = crosstalk * a1flux.copy()
#a12mask=a1mask.copy()
if fip_axis_0[a1, a2] == -1:
a12flux = a12flux[::-1]
#a12mask=a12mask[::-1]
if fip_axis_1[a1, a2] == -1:
a12flux = a12flux[:, ::-1]
#a12mask=a12mask[:,::-1]
ii2 = _parse_sec_keyword(header['CCDSEC' + amp2])
image[ii2] -= a12flux
# mask[ii2] |= a12mask (not sure we really need to propagate the mask)
#- Divide by pixflat image
pixflat = get_calibration_image(calibration_data, calibration_data_path,
"PIXFLAT", pixflat)
if pixflat is not False:
if pixflat.shape != image.shape:
raise ValueError('shape mismatch pixflat {} != image {}'.format(
pixflat.shape, image.shape))
if np.all(pixflat != 0.0):
image /= pixflat
readnoise /= pixflat
else:
good = (pixflat != 0.0)
image[good] /= pixflat[good]
readnoise[good] /= pixflat[good]
mask[~good] |= ccdmask.PIXFLATZERO
lowpixflat = (0 < pixflat) & (pixflat < 0.1)
if np.any(lowpixflat):
mask[lowpixflat] |= ccdmask.PIXFLATLOW
#- Inverse variance, estimated directly from the data (BEWARE: biased!)
var = image.clip(0) + readnoise**2
ivar = np.zeros(var.shape)
ivar[var > 0] = 1.0 / var[var > 0]
if bkgsub:
bkg = _background(image, header)
image -= bkg
img = Image(image,
ivar=ivar,
mask=mask,
meta=header,
readnoise=readnoise,
camera=camera)
#- update img.mask to mask cosmic rays
if not nocosmic:
cosmics.reject_cosmic_rays(img,
nsig=cosmics_nsig,
cfudge=cosmics_cfudge,
c2fudge=cosmics_c2fudge)
return img
def preproc(rawimage, header, bias=False, pixflat=False, mask=False):
'''
preprocess image using metadata in header
image = ((rawimage-bias-overscan)*gain)/pixflat
Args:
rawimage : 2D numpy array directly from raw data file
header : dict-like metadata, e.g. from FITS header, with keywords
CAMERA, BIASSECx, DATASECx, CCDSECx
where x = 1, 2, 3, 4 for each of the 4 amplifiers.
Optional bias, pixflat, and mask can each be:
False: don't apply that step
True: use default calibration data for that night
ndarray: use that array
filename (str or unicode): read HDU 0 and use that
DATE-OBS is required in header if bias, pixflat, or mask=True
Returns Image object with member variables:
image : 2D preprocessed image in units of electrons per pixel
ivar : 2D inverse variance of image
mask : 2D mask of image (0=good)
readnoise : 2D per-pixel readnoise of image
meta : metadata dictionary
TODO: define what keywords are included
preprocessing includes the following steps:
- bias image subtraction
- overscan subtraction (from BIASSEC* keyword defined regions)
- readnoise estimation (from BIASSEC* keyword defined regions)
- gain correction (from GAIN* keywords)
- pixel flat correction
- cosmic ray masking
- propagation of input known bad pixel mask
- inverse variance estimation
Notes:
The bias image is subtracted before any other calculation to remove any
non-uniformities in the overscan regions prior to calculating overscan
levels and readnoise.
The readnoise is an image not just one number per amp, because the pixflat
image also affects the interpreted readnoise.
The inverse variance is estimated from the readnoise and the image itself,
and thus is biased.
'''
#- TODO: Check for required keywords first
#- Subtract bias image
camera = header['CAMERA'].lower()
if bias is not False and bias is not None:
if bias is True:
#- use default bias file for this camera/night
dateobs = header['DATE-OBS']
bias = read_bias(camera=camera, dateobs=dateobs)
elif isinstance(bias, (str, unicode)):
#- treat as filename
bias = read_bias(filename=bias)
if bias.shape == rawimage.shape:
rawimage = rawimage - bias
else:
raise ValueError('shape mismatch bias {} != rawimage {}'.format(bias.shape, rawimage.shape))
#- Output arrays
yy, xx = _parse_sec_keyword(header['CCDSEC4']) #- 4 = upper right
image = np.zeros( (yy.stop, xx.stop) )
readnoise = np.zeros_like(image)
for amp in ['1', '2', '3', '4']:
ii = _parse_sec_keyword(header['BIASSEC'+amp])
#- Initial teststand data may be missing GAIN* keywords; don't crash
if 'GAIN'+amp in header:
gain = header['GAIN'+amp] #- gain = electrons / ADU
else:
log.error('Missing keyword GAIN{}; using 1.0'.format(amp))
gain = 1.0
overscan, rdnoise = _overscan(rawimage[ii])
rdnoise *= gain
kk = _parse_sec_keyword(header['CCDSEC'+amp])
readnoise[kk] = rdnoise
header['OVERSCN'+amp] = overscan
header['OBSRDN'+amp] = rdnoise
#- Warn/error if measured readnoise is very different from expected
if 'RDNOISE'+amp in header:
expected_readnoise = header['RDNOISE'+amp]
if rdnoise < 0.5*expected_readnoise:
log.error('Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}'.format(
amp, rdnoise, expected_readnoise))
elif rdnoise < 0.9*expected_readnoise:
log.warn('Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}'.format(
amp, rdnoise, expected_readnoise))
elif rdnoise > 2.0*expected_readnoise:
log.error('Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}'.format(
amp, rdnoise, expected_readnoise))
elif rdnoise > 1.2*expected_readnoise:
log.warn('Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}'.format(
amp, rdnoise, expected_readnoise))
else:
log.warn('Expected readnoise keyword {} missing'.format('RDNOISE'+amp))
#- subtract overscan from data region and apply gain
jj = _parse_sec_keyword(header['DATASEC'+amp])
data = rawimage[jj] - overscan
image[kk] = data*gain
#- Load mask
if mask is not False and mask is not None:
if mask is True:
dateobs = header['DATE-OBS']
mask = read_mask(camera=camera, dateobs=dateobs)
elif isinstance(mask, (str, unicode)):
mask = read_mask(filename=mask)
else:
mask = np.zeros(image.shape, dtype=np.int32)
if mask.shape != image.shape:
raise ValueError('shape mismatch mask {} != image {}'.format(mask.shape, image.shape))
#- Divide by pixflat image
if pixflat is not False and pixflat is not None:
if pixflat is True:
dateobs = header['DATE-OBS']
pixflat = read_pixflat(camera=camera, dateobs=dateobs)
elif isinstance(pixflat, (str, unicode)):
pixflat = read_pixflat(filename=pixflat)
if pixflat.shape != image.shape:
raise ValueError('shape mismatch pixflat {} != image {}'.format(pixflat.shape, image.shape))
if np.all(pixflat != 0.0):
image /= pixflat
readnoise /= pixflat
else:
good = (pixflat != 0.0)
image[good] /= pixflat[good]
readnoise[good] /= pixflat[good]
mask[~good] |= ccdmask.PIXFLATZERO
lowpixflat = (0 < pixflat) & (pixflat < 0.1)
if np.any(lowpixflat):
mask[lowpixflat] |= ccdmask.PIXFLATLOW
#- Inverse variance, estimated directly from the data (BEWARE: biased!)
var = image.clip(0) + readnoise**2
ivar = 1.0 / var
img = Image(image, ivar=ivar, mask=mask, meta=header, readnoise=readnoise, camera=camera)
#- update img.mask to mask cosmic rays
cosmics.reject_cosmic_rays(img)
return img
def preproc(rawimage, header, bias=False, pixflat=False, mask=False):
'''
preprocess image using metadata in header
image = ((rawimage-bias-overscan)*gain)/pixflat
Args:
rawimage : 2D numpy array directly from raw data file
header : dict-like metadata, e.g. from FITS header, with keywords
CAMERA, BIASSECx, DATASECx, CCDSECx
where x = 1, 2, 3, 4 for each of the 4 amplifiers.
Optional bias, pixflat, and mask can each be:
False: don't apply that step
True: use default calibration data for that night
ndarray: use that array
filename (str or unicode): read HDU 0 and use that
DATE-OBS is required in header if bias, pixflat, or mask=True
Returns Image object with member variables:
image : 2D preprocessed image in units of electrons per pixel
ivar : 2D inverse variance of image
mask : 2D mask of image (0=good)
readnoise : 2D per-pixel readnoise of image
meta : metadata dictionary
TODO: define what keywords are included
preprocessing includes the following steps:
- bias image subtraction
- overscan subtraction (from BIASSEC* keyword defined regions)
- readnoise estimation (from BIASSEC* keyword defined regions)
- gain correction (from GAIN* keywords)
- pixel flat correction
- cosmic ray masking
- propagation of input known bad pixel mask
- inverse variance estimation
Notes:
The bias image is subtracted before any other calculation to remove any
non-uniformities in the overscan regions prior to calculating overscan
levels and readnoise.
The readnoise is an image not just one number per amp, because the pixflat
image also affects the interpreted readnoise.
The inverse variance is estimated from the readnoise and the image itself,
and thus is biased.
'''
#- TODO: Check for required keywords first
#- Subtract bias image
camera = header['CAMERA'].lower()
if bias is not False and bias is not None:
if bias is True:
#- use default bias file for this camera/night
dateobs = header['DATE-OBS']
bias = read_bias(camera=camera, dateobs=dateobs)
elif isinstance(bias, (str, unicode)):
#- treat as filename
bias = read_bias(filename=bias)
if bias.shape == rawimage.shape:
rawimage = rawimage - bias
else:
raise ValueError('shape mismatch bias {} != rawimage {}'.format(
bias.shape, rawimage.shape))
#- Output arrays
yy, xx = _parse_sec_keyword(header['CCDSEC4']) #- 4 = upper right
image = np.zeros((yy.stop, xx.stop))
readnoise = np.zeros_like(image)
for amp in ['1', '2', '3', '4']:
ii = _parse_sec_keyword(header['BIASSEC' + amp])
#- Initial teststand data may be missing GAIN* keywords; don't crash
if 'GAIN' + amp in header:
gain = header['GAIN' + amp] #- gain = electrons / ADU
else:
log.error('Missing keyword GAIN{}; using 1.0'.format(amp))
gain = 1.0
overscan, rdnoise = _overscan(rawimage[ii])
rdnoise *= gain
kk = _parse_sec_keyword(header['CCDSEC' + amp])
readnoise[kk] = rdnoise
header['OVERSCN' + amp] = overscan
header['OBSRDN' + amp] = rdnoise
#- Warn/error if measured readnoise is very different from expected
if 'RDNOISE' + amp in header:
expected_readnoise = header['RDNOISE' + amp]
if rdnoise < 0.5 * expected_readnoise:
log.error(
'Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}'
.format(amp, rdnoise, expected_readnoise))
elif rdnoise < 0.9 * expected_readnoise:
log.warn(
'Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}'
.format(amp, rdnoise, expected_readnoise))
elif rdnoise > 2.0 * expected_readnoise:
log.error(
'Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}'
.format(amp, rdnoise, expected_readnoise))
elif rdnoise > 1.2 * expected_readnoise:
log.warn(
'Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}'
.format(amp, rdnoise, expected_readnoise))
else:
log.warn('Expected readnoise keyword {} missing'.format('RDNOISE' +
amp))
#- subtract overscan from data region and apply gain
jj = _parse_sec_keyword(header['DATASEC' + amp])
data = rawimage[jj] - overscan
image[kk] = data * gain
#- Load mask
if mask is not False and mask is not None:
if mask is True:
dateobs = header['DATE-OBS']
mask = read_mask(camera=camera, dateobs=dateobs)
elif isinstance(mask, (str, unicode)):
mask = read_mask(filename=mask)
else:
mask = np.zeros(image.shape, dtype=np.int32)
if mask.shape != image.shape:
raise ValueError('shape mismatch mask {} != image {}'.format(
mask.shape, image.shape))
#- Divide by pixflat image
if pixflat is not False and pixflat is not None:
if pixflat is True:
dateobs = header['DATE-OBS']
pixflat = read_pixflat(camera=camera, dateobs=dateobs)
elif isinstance(pixflat, (str, unicode)):
pixflat = read_pixflat(filename=pixflat)
if pixflat.shape != image.shape:
raise ValueError('shape mismatch pixflat {} != image {}'.format(
pixflat.shape, image.shape))
if np.all(pixflat != 0.0):
image /= pixflat
readnoise /= pixflat
else:
good = (pixflat != 0.0)
image[good] /= pixflat[good]
readnoise[good] /= pixflat[good]
mask[~good] |= ccdmask.PIXFLATZERO
lowpixflat = (0 < pixflat) & (pixflat < 0.1)
if np.any(lowpixflat):
mask[lowpixflat] |= ccdmask.PIXFLATLOW
#- Inverse variance, estimated directly from the data (BEWARE: biased!)
var = image.clip(0) + readnoise**2
ivar = 1.0 / var
img = Image(image,
ivar=ivar,
mask=mask,
meta=header,
readnoise=readnoise,
camera=camera)
#- update img.mask to mask cosmic rays
cosmics.reject_cosmic_rays(img)
return img
def preproc(rawimage, header, primary_header, bias=True, dark=True, pixflat=True, mask=True, bkgsub=False, nocosmic=False, cosmics_nsig=6, cosmics_cfudge=3., cosmics_c2fudge=0.5,ccd_calibration_filename=None, nocrosstalk=False, nogain=False):
'''
preprocess image using metadata in header
image = ((rawimage-bias-overscan)*gain)/pixflat
Args:
rawimage : 2D numpy array directly from raw data file
header : dict-like metadata, e.g. from FITS header, with keywords
CAMERA, BIASSECx, DATASECx, CCDSECx
where x = A, B, C, D for each of the 4 amplifiers
(also supports old naming convention 1, 2, 3, 4).
primary_header: dict-like metadata fit keywords EXPTIME, DOSVER
DATE-OBS is also required if bias, pixflat, or mask=True
Optional bias, pixflat, and mask can each be:
False: don't apply that step
True: use default calibration data for that night
ndarray: use that array
filename (str or unicode): read HDU 0 and use that
Optional background subtraction with median filtering if bkgsub=True
Optional disabling of cosmic ray rejection if nocosmic=True
Optional tuning of cosmic ray rejection parameters:
cosmics_nsig: number of sigma above background required
cosmics_cfudge: number of sigma inconsistent with PSF required
cosmics_c2fudge: fudge factor applied to PSF
Returns Image object with member variables:
image : 2D preprocessed image in units of electrons per pixel
ivar : 2D inverse variance of image
mask : 2D mask of image (0=good)
readnoise : 2D per-pixel readnoise of image
meta : metadata dictionary
TODO: define what keywords are included
preprocessing includes the following steps:
- bias image subtraction
- overscan subtraction (from BIASSEC* keyword defined regions)
- readnoise estimation (from BIASSEC* keyword defined regions)
- gain correction (from GAIN* keywords)
- pixel flat correction
- cosmic ray masking
- propagation of input known bad pixel mask
- inverse variance estimation
Notes:
The bias image is subtracted before any other calculation to remove any
non-uniformities in the overscan regions prior to calculating overscan
levels and readnoise.
The readnoise is an image not just one number per amp, because the pixflat
image also affects the interpreted readnoise.
The inverse variance is estimated from the readnoise and the image itself,
and thus is biased.
'''
log=get_logger()
header = header.copy()
cfinder = None
if ccd_calibration_filename is not False :
cfinder = CalibFinder([header, primary_header], yaml_file=ccd_calibration_filename)
#- TODO: Check for required keywords first
#- Subtract bias image
camera = header['CAMERA'].lower()
#- convert rawimage to float64 : this is the output format of read_image
rawimage = rawimage.astype(np.float64)
bias = get_calibration_image(cfinder,"BIAS",bias)
if bias is not False : #- it's an array
if bias.shape == rawimage.shape :
log.info("subtracting bias")
rawimage = rawimage - bias
else:
raise ValueError('shape mismatch bias {} != rawimage {}'.format(bias.shape, rawimage.shape))
if cfinder and cfinder.haskey("AMPLIFIERS") :
amp_ids=list(cfinder.value("AMPLIFIERS"))
else :
amp_ids=['A','B','C','D']
#- check whether it's indeed CCDSECx with x in ['A','B','C','D']
# or older version with x in ['1','2','3','4']
# we can remove this piece of code at later times
has_valid_keywords = True
for amp in amp_ids :
if not 'CCDSEC%s'%amp in header :
log.warning("No CCDSEC%s keyword in header , will look for alternative naming CCDSEC{1,2,3,4} ..."%amp)
has_valid_keywords = False
break
if not has_valid_keywords :
amp_ids=['1','2','3','4']
for amp in ['1','2','3','4'] :
if not 'CCDSEC%s'%amp in header :
log.error("No CCDSEC%s keyword, exit"%amp)
raise KeyError("No CCDSEC%s keyword"%amp)
#- Output arrays
ny=0
nx=0
for amp in amp_ids :
yy, xx = _parse_sec_keyword(header['CCDSEC%s'%amp])
ny=max(ny,yy.stop)
nx=max(nx,xx.stop)
image = np.zeros( (ny,nx) )
readnoise = np.zeros_like(image)
#- Load mask
mask = get_calibration_image(cfinder,"MASK",mask)
if mask is False :
mask = np.zeros(image.shape, dtype=np.int32)
else :
if mask.shape != image.shape :
raise ValueError('shape mismatch mask {} != image {}'.format(mask.shape, image.shape))
#- Load dark
dark = get_calibration_image(cfinder,"DARK",dark)
if dark is not False :
if dark.shape != image.shape :
log.error('shape mismatch dark {} != image {}'.format(dark.shape, image.shape))
raise ValueError('shape mismatch dark {} != image {}'.format(dark.shape, image.shape))
if cfinder and cfinder.haskey("EXPTIMEKEY") :
exptime_key=cfinder.value("EXPTIMEKEY")
log.info("Using exposure time keyword %s for dark normalization"%exptime_key)
else :
exptime_key="EXPTIME"
exptime = primary_header[exptime_key]
log.info("Multiplying dark by exptime %f"%(exptime))
dark *= exptime
for amp in amp_ids :
ii = _parse_sec_keyword(header['BIASSEC'+amp])
if nogain :
gain = 1.
else :
#- Initial teststand data may be missing GAIN* keywords; don't crash
if 'GAIN'+amp in header:
gain = header['GAIN'+amp] #- gain = electrons / ADU
else:
if cfinder and cfinder.haskey('GAIN'+amp) :
gain = float(cfinder.value('GAIN'+amp))
log.info('Using GAIN{}={} from calibration data'.format(amp,gain))
else :
gain = 1.0
log.warning('Missing keyword GAIN{} in header and nothing in calib data; using {}'.format(amp,gain))
#- Add saturation level
if 'SATURLEV'+amp in header:
saturlev = header['SATURLEV'+amp] # in electrons
else:
if cfinder and cfinder.haskey('SATURLEV'+amp) :
saturlev = float(cfinder.value('SATURLEV'+amp))
log.info('Using SATURLEV{}={} from calibration data'.format(amp,saturlev))
else :
saturlev = 200000
log.warning('Missing keyword SATURLEV{} in header and nothing in calib data; using 200000'.format(amp,saturlev))
overscan_image = rawimage[ii].copy()
nrows=overscan_image.shape[0]
log.info("nrows in overscan=%d"%nrows)
overscan = np.zeros(nrows)
rdnoise = np.zeros(nrows)
overscan_per_row = True
if cfinder and cfinder.haskey('OVERSCAN'+amp) and cfinder.value("OVERSCAN"+amp).upper()=="PER_ROW" :
log.info("Subtracting overscan per row for amplifier %s of camera %s"%(amp,camera))
for j in range(nrows) :
if np.isnan(np.sum(overscan_image[j])) :
log.warning("NaN values in row %d of overscan of amplifier %s of camera %s"%(j,amp,camera))
continue
o,r = _overscan(overscan_image[j])
#log.info("%d %f %f"%(j,o,r))
overscan[j]=o
rdnoise[j]=r
else :
log.info("Subtracting average overscan for amplifier %s of camera %s"%(amp,camera))
o,r = _overscan(overscan_image)
overscan += o
rdnoise += r
rdnoise *= gain
median_rdnoise = np.median(rdnoise)
median_overscan = np.median(overscan)
log.info("Median rdnoise and overscan= %f %f"%(median_rdnoise,median_overscan))
kk = _parse_sec_keyword(header['CCDSEC'+amp])
for j in range(nrows) :
readnoise[kk][j] = rdnoise[j]
header['OVERSCN'+amp] = (median_overscan,'ADUs (gain not applied)')
if gain != 1 :
rdnoise_message = 'electrons (gain is applied)'
gain_message = 'e/ADU (gain applied to image)'
else :
rdnoise_message = 'ADUs (gain not applied)'
gain_message = 'gain not applied to image'
header['OBSRDN'+amp] = (median_rdnoise,rdnoise_message)
header['GAIN'+amp] = (gain,gain_message)
#- Warn/error if measured readnoise is very different from expected if exists
if 'RDNOISE'+amp in header:
expected_readnoise = header['RDNOISE'+amp]
if median_rdnoise < 0.5*expected_readnoise:
log.error('Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}'.format(
amp, median_rdnoise, expected_readnoise))
elif median_rdnoise < 0.9*expected_readnoise:
log.warning('Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}'.format(
amp, median_rdnoise, expected_readnoise))
elif median_rdnoise > 2.0*expected_readnoise:
log.error('Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}'.format(
amp, median_rdnoise, expected_readnoise))
elif median_rdnoise > 1.2*expected_readnoise:
log.warning('Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}'.format(
amp, median_rdnoise, expected_readnoise))
#else:
# log.warning('Expected readnoise keyword {} missing'.format('RDNOISE'+amp))
log.info("Measured readnoise for AMP %s = %f"%(amp,median_rdnoise))
#- subtract overscan from data region and apply gain
jj = _parse_sec_keyword(header['DATASEC'+amp])
data = rawimage[jj].copy()
for k in range(nrows) :
data[k] -= overscan[k]
#- apply saturlev (defined in ADU), prior to multiplication by gain
saturated = (rawimage[jj]>=saturlev)
mask[kk][saturated] |= ccdmask.SATURATED
#- subtract dark prior to multiplication by gain
if dark is not False :
log.info("subtracting dark for amp %s"%amp)
data -= dark[kk]
image[kk] = data*gain
if not nocrosstalk :
#- apply cross-talk
# the ccd looks like :
# C D
# A B
# for cross talk, we need a symmetric 4x4 flip_matrix
# of coordinates ABCD giving flip of both axis
# when computing crosstalk of
# A B C D
#
# A AA AB AC AD
# B BA BB BC BD
# C CA CB CC CD
# D DA DB DC BB
# orientation_matrix_defines change of orientation
#
fip_axis_0= np.array([[1,1,-1,-1],
[1,1,-1,-1],
[-1,-1,1,1],
[-1,-1,1,1]])
fip_axis_1= np.array([[1,-1,1,-1],
[-1,1,-1,1],
[1,-1,1,-1],
[-1,1,-1,1]])
for a1 in range(len(amp_ids)) :
amp1=amp_ids[a1]
ii1 = _parse_sec_keyword(header['CCDSEC'+amp1])
a1flux=image[ii1]
#a1mask=mask[ii1]
for a2 in range(len(amp_ids)) :
if a1==a2 :
continue
amp2=amp_ids[a2]
if cfinder is None : continue
if not cfinder.haskey("CROSSTALK%s%s"%(amp1,amp2)) : continue
crosstalk=cfinder.value("CROSSTALK%s%s"%(amp1,amp2))
if crosstalk==0. : continue
log.info("Correct for crosstalk=%f from AMP %s into %s"%(crosstalk,amp1,amp2))
a12flux=crosstalk*a1flux.copy()
#a12mask=a1mask.copy()
if fip_axis_0[a1,a2]==-1 :
a12flux=a12flux[::-1]
#a12mask=a12mask[::-1]
if fip_axis_1[a1,a2]==-1 :
a12flux=a12flux[:,::-1]
#a12mask=a12mask[:,::-1]
ii2 = _parse_sec_keyword(header['CCDSEC'+amp2])
image[ii2] -= a12flux
# mask[ii2] |= a12mask (not sure we really need to propagate the mask)
#- Divide by pixflat image
pixflat = get_calibration_image(cfinder,"PIXFLAT",pixflat)
if pixflat is not False :
if pixflat.shape != image.shape:
raise ValueError('shape mismatch pixflat {} != image {}'.format(pixflat.shape, image.shape))
if np.all(pixflat != 0.0):
image /= pixflat
readnoise /= pixflat
else:
good = (pixflat != 0.0)
image[good] /= pixflat[good]
readnoise[good] /= pixflat[good]
mask[~good] |= ccdmask.PIXFLATZERO
lowpixflat = (0 < pixflat) & (pixflat < 0.1)
if np.any(lowpixflat):
mask[lowpixflat] |= ccdmask.PIXFLATLOW
#- Inverse variance, estimated directly from the data (BEWARE: biased!)
var = image.clip(0) + readnoise**2
ivar = np.zeros(var.shape)
ivar[var>0] = 1.0 / var[var>0]
if bkgsub :
bkg = _background(image,header)
image -= bkg
img = Image(image, ivar=ivar, mask=mask, meta=header, readnoise=readnoise, camera=camera)
#- update img.mask to mask cosmic rays
if not nocosmic :
cosmics.reject_cosmic_rays(img,nsig=cosmics_nsig,cfudge=cosmics_cfudge,c2fudge=cosmics_c2fudge)
return img