def image_data(self, image_name):
"""Return a DESImage object for a configured image
:Parameters:
-`image_name`: the type of image to return
@returns: the object of class DESImage
"""
# If we already have the data, return it
if image_name in self._image_data:
im = self._image_data[image_name]
else:
# If we don't already have the data, load it
# Get the class of the image we are loading
try:
image_class = self._image_types[image_name]
except KeyError:
image_class = DESImage
fname = self.config.get(self.config_section, image_name)
im = image_class.load(fname)
logger.info('Reading %s image from %s', image_name, fname)
self._image_data[image_name] = im
return im
def step_run(cls, image, config):
"""Customized execution for application of the BPM
:Parameters:
- `image`: the DESImage on which to operate
- `config`: the configuration from which to get other parameters
"""
if config.has_option(cls.step_name, 'bpm'):
bpm_fname = config.get(cls.step_name, 'bpm')
logger.info('reading BPM from %s' % bpm_fname)
bpm_im = DESBPMImage.load(bpm_fname)
else:
bpm_im = None
if config.has_option(cls.step_name, 'saturate'):
saturate = config.getboolean(cls.step_name, 'saturate')
else:
saturate = DEFAULT_SATURATE
if config.has_option(cls.step_name, 'clear'):
clear = config.getboolean(cls.step_name, 'clear')
else:
clear = DEFAULT_CLEAR
ret_code = cls.__call__(image, bpm_im, saturate, clear)
return ret_code
def clip(data, model, nsigma=4):
diff = data - model
avg, var, _ = clippedMean(diff, nsigma, maxSample=1000000)
logger.info('Clipped mean and RMS sky residual: {:f} +- {:f}'.format(
avg, np.sqrt(var)))
diff = np.abs(diff - avg) > nsigma * np.sqrt(var)
out = np.where(diff, model + avg, data)
return out, np.count_nonzero(diff)
def null_sci(self, input_image):
null_mask_sci = parse_badpix_mask(self.config.get(self.config_section, 'null_mask_sci'))
if null_mask_sci != 0:
logger.info('Nulling science image from null_mask_bits')
kill = np.array(self.sci.mask & null_mask_sci, dtype=bool)
self.sci.data[kill] = 0.0
else:
logger.info('Science image was not null')
def mask_cti(image, CTI, ctiDict, verbose=0):
"""Function to mask the region affected by a lightbulb"""
if ctiDict['isCTI']:
sec = section2slice(image['DATASEC' + CTI['amp']])
image.mask[sec] |= BADPIX_BADAMP
logger.info(' CTI: mask applied to image')
return image
def update_wcs_header(self, input_image, verbose=False):
# Get optional config file, first we try to get them as boolean, then as strings
headfile = get_safe_boolean('headfile', self.config, self.config_section)
hdupcfg = get_safe_boolean('hdupcfg', self.config, self.config_section)
# Update the header if both headfile and hdupcfg are present
if headfile and hdupcfg:
logger.info("Will update image header with scamp .head file %s", headfile)
self.sci = updateWCS.run_update(self.sci, headfile=headfile, hdupcfg=hdupcfg, verbose=verbose)
def __call__(cls, image, null_mask, resaturate):
"""Create or update the mask plane of an image
:Parameters:
- `image`: the DESImage to operate upon. Mask plane is created if absent
- `null_mask`: Integer or list of BADPIX bit names that, when set in mask image,
will put weight=0 for that pixel.
- `resaturate`: if True, set data for every pixel with BADPIX_SATURATE set
to a value above the SATURATE keyword
"""
if image.mask is None:
raise NullWeightsError('Mask is missing in image')
if null_mask != 0:
logger.info('Nulling weight image from mask bits')
if image.weight is None and image.variance is None:
raise NullWeightsError('Weight is missing in image')
weight = image.get_weight()
kill = np.array(image.mask & null_mask, dtype=bool)
weight[kill] = 0.
image['HISTORY'] = time.asctime(time.localtime()) + \
' Null weights with mask 0x{:04X}'.format(null_mask)
logger.debug('Finished nulling weight image')
if resaturate:
logger.info('Re-saturating pixels from mask bits')
sat = np.array(image.mask & maskbits.BADPIX_SATURATE, dtype=bool)
try:
saturation_level = image['SATURATE']
except (ValueError, KeyError):
# If there is no SATURATE, try taking max of amps
maxsat = 0.
try:
for amp in decaminfo.amps:
maxsat = max(maxsat, image['SATURAT' + amp])
except:
logger.error('SATURATx header keywords not found')
raise NullWeightsError(
'SATURATx header keywords not found')
saturation_level = maxsat
logger.warning(
'Taking SATURATE as max of single-amp SATURATx values')
image.data[sat] = 1.01 * saturation_level
image['HISTORY'] = time.asctime(time.localtime()) + \
' Set saturated pixels to {:.0f}'.format(saturation_level)
logger.debug('Finished nulling weight image')
ret_code = 0
return ret_code
def __call__(cls, images):
"""Replace data values with HDU numbers
:Parameters:
- `images`: the array of images
Applies the correction "in place"
"""
logger.info('Filling in data with HDU numbers')
c_call_status = fpnumber_c(images.cstruct)
logger.info('Finished')
return c_call_status
def step_run(cls, image, config):
"""Customized execution for check and masking of CTI
:Parameters:
- `image`: the DESImage on which to operate
# - `config`: the configuration from which to get other parameters
"""
logger.info('CTI check %s' % image)
ret_code = cls.__call__(image)
return ret_code
def __call__(cls, image):
"""Convert pixel values from ADU to electrons, including weight or variance
image and critical keywords.
:Parameters:
- `image`: the DESImage for pixel values to be converted
Applies the correction "in place"
"""
logger.info('Gain Correcting Image')
saturate = 0.
gains = []
for amp in decaminfo.amps:
sec = section2slice(image['DATASEC' + amp])
gain = image['GAIN' + amp]
gains.append(gain)
image.data[sec] *= gain
# Adjust the weight or variance image if present:
if image.weight is not None:
image.weight[sec] *= 1. / (gain * gain)
if image.variance is not None:
image.variance[sec] *= gain * gain
# Adjust keywords
image['GAIN' + amp] = image['GAIN' + amp] / gain
image['SATURAT' + amp] = image['SATURAT' + amp] * gain
saturate = max(saturate, image['SATURAT' + amp])
# Scale the SKYVAR if it's already here
kw = 'SKYVAR' + amp
if kw in image.header.keys():
image[kw] = image[kw] * gain * gain
# The FLATMED will keep track of rescalings *after* gain:
image['FLATMED' + amp] = 1.
# The SATURATE keyword is assigned to maximum of the two amps.
image['SATURATE'] = saturate
# Some other keywords that we will adjust crudely with mean gain
# if they are present:
gain = np.mean(gains)
for kw in ('SKYBRITE', 'SKYSIGMA'):
if kw in image.header.keys():
image[kw] = image[kw] * gain
# One other keyword to adjust:
image['BUNIT'] = 'electrons'
logger.debug('Finished applying Gain Correction')
ret_code = 0
return ret_code
def step_run(cls, image, config):
"""Customized execution for application of the gain correction
:Parameters:
- `image`: the DESImage on which to operate
- `config`: the configuration from which to get other parameters
"""
logger.info('Gain correction will be applied to %s', image)
ret_code = cls.__call__(image)
return ret_code
def __call__(cls, in_filenames, out_filename, mask_value, invalid):
"""
Produce compressed image of sky background for entire exposure by
combining minisky FITS images for single CCDs. Missing input minisky
will only generate a warning.
:Parameters:
- `in_filenames`: list of filenames of single-chip sky images
- `out_filename`: filename for the output combined sky image
- `mask_value`: value inserted into sky pixels with no data
- `invalid`: list of detpos values for CCDs to be left out of sky image
"""
logger.info('Combining sky')
out = None
# Insert each input image into the output image
for f in in_filenames:
try:
small = DESDataImage.load(f)
except (ValueError, IOError):
# A missing minisky file is not fatal:
logger.warning('SkyCombine could not load minisky ' + f)
continue
if small['DETPOS'].strip() in invalid:
# Skip any unwanted CCDs
continue
blocksize = small['BLOCKSIZ']
if out is None:
out = skyinfo.MiniDecam(blocksize, mask_value, invalid)
out.copy_header_info(small, cls.propagate, require=False)
if blocksize != out.blocksize:
raise skyinfo.SkyError('Mismatched blocksizes for SkyCombine')
out.fill(small.data, small['DETPOS'].strip())
# Issue warnings for mismatches of data, but skip if
# quantities are not in headers
for k in ('BAND', 'NITE', 'EXPNUM'):
try:
v1 = out[k]
v2 = small[k]
if v1 != v2:
logger.warning('Mismatched {:s} in file {:s}: {} vs {}'.\
format(k, f, v1, v2))
except:
pass
out.save(out_filename)
logger.debug('Finished sky combination')
ret_code = 0
return ret_code
def step_run(cls, image, config):
"""Customized execution for application of the BPM
:Parameters:
- `image`: the DESImage on which to operate
- `config`: the configuration from which to get other parameters
"""
fname_lincor = config.get(cls.step_name, 'lincor')
logger.info('Linearity correction will be applied to %s', image)
ret_code = cls.__call__(image, fname_lincor)
return ret_code
def step_run(cls, image, config):
"""Customized execution for masking columns based on the mask
:Parameters:
- `image`: the DESImage on which to operate
- `config`: the configuration from which to get other parameters
"""
bpm_fname = config.get(cls.step_name, 'bpm')
logger.info('reading BPM from %s' % bpm_fname)
bpm_im = DESBPMImage.load(bpm_fname)
ret_code = cls.__call__(image, bpm_im)
return ret_code
def __call__(cls, image, bpm_im):
"""Apply a bad pixel mask to a DES image
:Parameters:
- `image`: the DESImage to which the BPM is to be applied
- `bpm_im`: the DESImage with the bad pixel mask
Applies the correction "in place"
"""
logger.info('Applying BPM')
ret_code = obpm_c(image.cstruct, bpm_im.cstruct)
logger.debug('Finished applying BPM')
return ret_code
def step_run(cls, image, config):
"""Customized execution for sky subtraction
:Parameters:
- `config`: the configuration from which to get other parameters
"""
bpm_fname = config.get(cls.step_name, 'bpm')
logger.info('reading BPM from %s' % bpm_fname)
bpm_im = DESBPMImage.load(bpm_fname)
ret_code = cls.__call__(image, bpm_im)
return ret_code
def __call__(cls, image):
"""
This is currently written with instance of CTI (Charge Transfer Inefficiency) in mind (that occuring
for CCD=41 during DES Y6). It may be generalizable if further cases occur (but not all the parts have
yet been written with a generalized case in mind). When CTI is detected the entire amplifier in
question will be masked with BADPIX_BADAMP
"""
# A simple dictionary with parameters for the only known case of CTI
# Currently explist is set to encompass DES Y6 (20180815 and beyond (expnum>765533)
# This could be tightened to a range as no CTI has been detected after November 2018 but
# it has not yet been systematicall watched for.
CTI = {41: {'amp': 'B', 'explist': '765533-'}}
if image['CCDNUM'] in CTI:
#
# Currently can re-use the function developed for lightbulb checking
#
check_for_light = lb.check_lightbulb_explist(
image['EXPNUM'], CTI[image['CCDNUM']]['explist'])
if check_for_light:
logger.info(
' CTI: Expnum={:d}, CCDNUM={:d}, in proscribed range checking for CTI'
.format(image['EXPNUM'], image['CCDNUM']))
ctiDict = cti.check_cti(image, CTI[image['CCDNUM']], verbose=1)
#
# Current criterion:
# Looks for horizontal striping in image (with large deficits in counts that are not
# associated with an edge-bleed.
# Examines auto-correlation for lags in the x-direction at 5, 7, and 15 pixel offsets
# and compares to lags obtained from measurments in the diaganol direction.
# Looks for evidence of excessive power in the ratio between x-direction and diagnol sets
# that indicative that charge is bleeding in the x-direction.
#
if ctiDict['isCTI']:
image = cti.mask_cti(image,
CTI[image['CCDNUM']],
ctiDict,
verbose=1)
logger.info(
' CTI: Detected CTI for Exp={:d}, CCD={:d}, Amp={:s}'.
format(image['EXPNUM'], image['CCDNUM'],
CTI[image['CCDNUM']]['amp']))
image.write_key(
'DES_CTI', 'Masked DATASEC{:s}'.format(
CTI[image['CCDNUM']]['amp']))
logger.debug('Finished checking and applying mask CTI')
ret_code = 0
return ret_code
def __call__(cls, image, skyfilename, blocksize, bitmask):
"""Produce compressed image of sky background
:Parameters:
- `image`: the DESImage to be compressed.
- `skyfilename`: filename for the output compressed sky image
- `blocksize`: side length of squares in which medians are taken
- `bitmask`: Bitmask that will be or'ed with mask plane of image (if
any) to mark pixels to be ignored in calculating block
median.
"""
logger.info('Compressing sky')
# Raise exception if image is not a multiple of blocksize (or reshape fails)
if ((image.data.shape[1] % blocksize != 0)
or (image.data.shape[0] % blocksize != 0)):
raise skyinfo.SkyError(
'blocksize {:d} does not evenly divide image ({:d}x{:d})'.
format(blocksize, image.data.shape[0], image.data.shape[1]))
exit(1)
nx = int(image.data.shape[1] / blocksize)
ny = int(image.data.shape[0] / blocksize)
# Apply bit mask to the mask plane if any. Superpixels
# with no unmasked pixels will be filled with value -1
if image.mask is None:
sky = np.median(image.data.reshape(ny, blocksize, nx, blocksize)\
.swapaxes(1, 2)\
.reshape(ny, nx, blocksize * blocksize), axis=2)
else:
data = np.ma.array(image.data,
mask=(image.mask & bitmask),
fill_value=-1.)
sky = np.ma.median(data.reshape(ny, blocksize, nx, blocksize)\
.swapaxes(1, 2)\
.reshape(ny, nx, blocksize * blocksize), axis=2)
sky = np.ma.getdata(sky)
# Create HDU for output image, add some header info, save output to file
outimage = DESDataImage(sky)
outimage['BLOCKSIZ'] = blocksize
outimage.copy_header_info(image, cls.propagate, require=False)
## ?? catch exception from write error below?
outimage.save(skyfilename)
logger.debug('Finished sky compression')
ret_code = 0
return ret_code
def step_run(cls, image, config):
"""Customized execution for application of the Bias
:Parameters:
- `image`: the DESImage on which to operate
- `bias`: the bias image to apply
"""
bias_fname = config.get(cls.step_name, 'bias')
logger.info('reading Bias from %s' % bias_fname)
bias_im = DESImage.load(bias_fname)
ret_code = cls.__call__(image, bias_im)
return ret_code
def step_run(cls, image, config):
"""Customized execution for application of the Bias
:Parameters:
- `image`: the DESImage on which to operate
- `flat`: the bias image to apply
"""
flat_fname = config.get(cls.step_name, 'flat')
logger.info('Reading flat correction from %s' % flat_fname)
flat_im = DESImage.load(flat_fname)
ret_code = cls.__call__(image, flat_im)
return ret_code
def step_run(cls, image, config):
"""Customized execution for addition of a weight plane.
:Parameters:
- `image`: the DESImage on which to operate
- `config`: the configuration from which to get other parameters
"""
logger.info('Weight will be added to %s' % image)
flat_fname = config.get(cls.step_name, 'flat')
logger.info('Reading flat correction from %s' % flat_fname)
flat = DESImage.load(flat_fname)
ret_code = cls.__call__(image, flat)
return ret_code
def step_run(cls, image, config):
"""Customized execution for taking difference between an image and a comparison
:Parameters:
- `image`: the DESImage on which to operate
- `comp`: the comparison image (to be subtracted)
"""
comp_fname = config.get(cls.step_name, 'comp')
logger.info('reading Comparison image from %s', comp_fname)
comp_im = DESImage.load(comp_fname)
ret_code = cls.__call__(image, comp_im)
return ret_code
def __call__(cls, image, bias_im):
"""Apply a bias correction to an image
:Parameters:
- `image`: the DESImage to apply a bias correction
- `bias_im`: the bias correction image to apply
Applies the correction "in place." Also creates BAND and NITE
keywords if they are not present.
"""
logger.info('Applying Bias')
# Check that bias and data are from same CCD
try:
items_must_match(image, bias_im, 'CCDNUM')
except:
return 1
image.data -= bias_im.data
# If we have two weight images, add variance of the bias to the image's
if (image.weight is not None or image.variance is not None):
if bias_im.weight is not None:
var = image.get_variance()
var += 1. / bias_im.weight
elif bias_im.variance is not None:
var = image.get_variance()
var += bias_im.variance
logger.debug('Finished applying Bias')
if bias_im.sourcefile is None:
image.write_key('BIASFIL',
'UNKNOWN',
comment='Bias correction file')
else:
image.write_key('BIASFIL',
path.basename(bias_im.sourcefile),
comment='Bias correction file')
# Also create the BAND and NITE keywords if they are not present
try:
image['BAND']
except:
image['BAND'] = decaminfo.get_band(image['FILTER'])
try:
image['NITE']
except:
image['NITE'] = decaminfo.get_nite(image['DATE-OBS'])
ret_code = 0
return ret_code
def __call__(cls, image, normfactor, ampborder):
"""Apply a flat field correction to an image
:Parameters:
- `image`: input image (image to be normalized)
- `normval`: normalization value to use (with respect to SCALMEAN keyword)
- `ampborder`: area around periphery of AmpRegion to use when calculating statistics
Applies the correction to each input and writes a separate output file.
"""
logger.info('Normalizing Flat Image')
scalmean = image['SCALMEAN']
nfactor = scalmean / normfactor
nfactor2 = nfactor * nfactor
logger.info('SCALMEAN=%.2f NORMFACTOR=%.2f NORMALIZATION=%.5f',
scalmean, normfactor, nfactor)
image['NORMFACT'] = normfactor
#
image.data *= nfactor
if image.weight is not None:
image.weight *= nfactor2
elif image.variance is not None:
image.variance /= nfactor2
#
# Create keywords that reflect the median value of the flat on each amp.
#
for amp in decaminfo.amps:
datasecn = scan_fits_section(image, 'DATASEC' + amp)
datasecn[0] += ampborder
datasecn[1] -= ampborder
datasecn[2] += ampborder
datasecn[3] -= ampborder
image['FLATMED' + amp] = np.median(
image.data[datasecn[2]:datasecn[3] + 1,
datasecn[0]:datasecn[1] + 1])
logger.debug('Finished applying normalization to Flat')
ret_code = 0
return ret_code
def step_run(cls, image, config):
"""Customized execution for application of the BPM
:Parameters:
- `image`: the DESImage on which to operate
- `config`: the configuration from which to get other parameters
"""
# import pdb; pdb.set_trace()
# overscan_apply = config.getboolean(cls.step_name, 'overscan')
overscan_sample = config.getint(cls.step_name, 'overscansample')
overscan_function = config.getint(cls.step_name, 'overscanfunction')
overscan_order = config.getint(cls.step_name, 'overscanorder')
overscan_trim = config.getint(cls.step_name, 'overscantrim')
logger.info('Ovescan will be applied to %s' % image)
ret_code = cls.__call__(image, overscan_sample, overscan_function,
overscan_order, overscan_trim)
return ret_code
def image_data(self, image_name):
"""Return a DESFocalPlaneImages object for the configured images
:Parameters:
-`image_name`: the type of image to return
@returns: the object of class DESFocalPlaneImage
"""
# If we already have the data, return it
if image_name in self._image_data:
im = self._image_data[image_name]
else:
# If we don't already have the data, load it
fname = self.config.get(self.config_section, image_name)
im = DESFocalPlaneImages.load(fname)
logger.info('Reading %s image from %s' % (image_name, fname))
self._image_data[image_name] = im
return im
def run(cls, config):
"""Customized execution for sky combination. Note there is NO input image nor output
:Parameters:
- `config`: the configuration from which to get other parameters
"""
if config.has_option(cls.step_name, 'maskvalue'):
mask_value = config.getfloat(cls.step_name, 'maskvalue')
else:
mask_value = skyinfo.DEFAULT_MASK_VALUE
if config.has_option(cls.step_name, 'invalid'):
baddet = config.get(cls.step_name, 'invalid')
else:
baddet = skyinfo.DEFAULT_IGNORE
invalid = baddet.split(',')
if config.has_option(cls.step_name, 'ccdnums'):
ccdranges = config.get(cls.step_name, 'ccdnums')
else:
ccdranges = skyinfo.DEFAULT_CCDNUMS
ccdnumlist = skyinfo.parse_ranges(ccdranges)
if config.has_option(cls.step_name, 'miniskylist'):
miniskylist = config.get(cls.step_name, 'miniskylist')
in_filenames = filelist_to_list(miniskylist)
else:
if config.has_option(cls.step_name, 'miniskyfiles'):
miniskyfiles = config.get(cls.step_name, 'miniskyfiles')
else:
miniskyfiles = skyinfo.DEFAULT_MINISKY_FILES
in_filenames = []
for i in ccdnumlist:
# in_filenames.append(miniskyfiles.format(i))
in_filenames.append(miniskyfiles % i)
out_filename = config.get(cls.step_name, 'outfilename')
logger.info('Sky combine output to %s', out_filename)
ret_code = cls.__call__(in_filenames, out_filename, mask_value, invalid)
return ret_code
def __call__(cls, in_filename, ref_filename, out_filename, edge=None):
"""
Compare two compressed DES images, report size of deviations.
:Parameters:
- `in_filename`: the compressed DES image ("mini-sky" format) to check
- `ref_filename`: the reference image to compare to, must have same compression as in_filename
- `out_filename`: output image showing fractional residuals of input to reference after matching
normalizations. The header of the output image will also contain keywords RMS
and WORST giving the rms and maximum fractional deviations, and a keyword FACTOR
giving the overall flux factor used to normalize them.
- 'edge': number of compressed pixels along each CCD edge to ignore in calculating stats
"""
logger.info('Comparing compressed image' + in_filename +
" to reference " + ref_filename)
indata = skyinfo.MiniDecam.load(in_filename)
ref = skyinfo.MiniDecam.load(ref_filename)
if indata.blocksize != ref.blocksize or indata.invalid != ref.invalid or indata.halfS7 != ref.halfS7:
raise skyinfo.SkyError(
"Input and reference are not matching compressions of DECam")
resid = indata.vector() / ref.vector()
if edge is not None and edge > 0:
stats = resid[indata.edges(edge).vector() == 0]
else:
stats = np.array(resid)
factor = np.median(stats)
resid /= factor
resid -= 1.
indata.fill_from(resid)
stats /= factor
stats -= 1.
rms = np.std(stats)
worst = np.max(np.abs(stats))
indata.header['FACTOR'] = factor
indata.header['RMS'] = rms
indata.header['WORST'] = worst
if edge is not None and edge > 0:
indata.header['EDGE'] = edge
indata.save(out_filename)
logger.info('Normalization factor: %f', factor)
logger.info('RMS deviation: %f', rms)
logger.info('Worst deviation: %f', worst)
# Create a one-line binary fits table to hold the coefficients
logger.debug('Finished image comparison')
ret_code = 0
return ret_code
def step_run(cls, image, config):
"""Customized execution for application of the Flat
:Parameters:
- `image`: the DESImage on which to operate
- `flat`: the bias image to apply
"""
flat_fname = config.get(cls.step_name, 'flat')
logger.info('Reading flat correction from %s' % flat_fname)
flat_im = DESImage.load(flat_fname)
# At present the only way to acquire gains is when function is run through
# tandem operation with gain_correct. In the absence of having relative gains
# an empty dictionary is passed here.
rel_gain_for_flat = {}
ret_code = cls.__call__(image, flat_im, rel_gain_for_flat)
return ret_code
def step_run(cls, config):
"""Customized execution for sky combination. Note there is NO input image nor output
:Parameters:
- `config`: the configuration from which to get other parameters
"""
if config.has_option(cls.step_name, 'clipsigma'):
clip_sigma = config.getfloat(cls.step_name, 'clipsigma')
else:
clip_sigma = skyinfo.DEFAULT_CLIP_SIGMA
in_filename = config.get(cls.step_name, 'infilename')
out_filename = config.get(cls.step_name, 'outfilename')
pc_filename = config.get(cls.step_name, 'pcfilename')
logger.info('Sky fitting output to %s', out_filename)
ret_code = cls.__call__(in_filename, out_filename, pc_filename,
clip_sigma)
return ret_code