def __call__(self):
"""
Run row_interp and null_weights in one step, we run the tasks
by calling step_run in each class
"""
t0 = time.time()
# Get the science image
input_image = self.config.get(self.config_section, 'in')
self.sci = DESImage.load(input_image)
# Run null_weights
t1 = time.time()
logger.info("Running null_weights on: %s", input_image)
null_weights.step_run(self.sci, self.config)
logger.info("Time NullWeights : %s", elapsed_time(t1))
# Run row_interp
t2 = time.time()
logger.info("Running row_interp on: %s", input_image)
row_interp.step_run(self.sci, self.config)
logger.info("Time RowInterp : %s", elapsed_time(t2))
# Write out the image
output_image = self.config.get(self.config_section, 'out')
self.sci.save(output_image)
logger.info("Wrote new file: %s", output_image)
logger.info("Time Total: %s", elapsed_time(t0))
return 0
def __call__(self):
"""
Run row_zipper and null_weights in one step, we run the tasks
by calling step_run in each class
"""
t0 = time.time()
# Check if we want special multi-epoch weighting, and which bits we want to 'save'
me_wgt_keepmask = get_safe_boolean('me_wgt_keepmask', self.config, self.config_section)
# Get verbose
try:
verbose = self.config.get(self.config_section, 'verbose')
except:
verbose = False
# Get the science image
input_image = self.config.get(self.config_section, 'in')
self.sci = DESImage.load(input_image)
# In case a streak table is provided -- we proceed with the extra STREAK maskinh
streak_file = self.config.get(self.config_section, 'streak_file')
if os.path.exists(streak_file):
add_width = self.config.getfloat(self.config_section, 'add_width')
add_length = self.config.getfloat(self.config_section, 'add_length')
max_extrapolate = self.config.getfloat(self.config_section, 'max_extrapolate')
self.streakMask(streak_file,
addWidth=add_width,
addLength=add_length,
maxExtrapolate=max_extrapolate)
# Add TILENAME and TILEID to sci header (optional) if required
self.update_sci_header(input_image)
# Update the header wcs if both headfile and hdupcfg are present (optional)
self.update_wcs_header(input_image, verbose=verbose)
# Check if want to create the custon weight for SWArp/SExtractor combination
if me_wgt_keepmask:
self.custom_weight(input_image)
# Run null_weights
t1 = time.time()
logger.info("Running null_weights on: %s", input_image)
null_weights.step_run(self.sci, self.config)
logger.info("Time NullWeights : %s", elapsed_time(t1))
# Run row_zipper
t2 = time.time()
logger.info("Running row_zipper on: %s", input_image)
row_zipper.step_run(self.sci, self.config)
logger.info("Time ZipperInterp : %s", elapsed_time(t2))
# Null the sci image only if null_mask_sci !=0
self.null_sci(input_image)
output_image = self.config.get(self.config_section, 'out')
# Special write out
if me_wgt_keepmask:
self.custom_write(output_image)
else:
self.sci.save(output_image)
logger.info("Wrote new file: %s", output_image)
logger.info("Time Total: %s", elapsed_time(t0))
return 0
def __call__(cls, image, fit_filename, pc_filename, weight, dome,
skymodel_filename):
"""
Subtract sky from image using previous principal-components fit. Optionally
build weight image from fitted sky or all counts, in which case the dome flat
is needed and proper gain values are expected in the image header.
:Parameters:
- `image`: DESImage that has been flattened with dome already and fit
- `fit_filename`: filename with the coefficients from minisky fitting. Sky
subtraction is skipped if this is None.
- `pc_filename`: filename for the stored full-res sky principal components
- `weight`: 'none' to skip weights, 'sky' to calculate weight at sky level,
'all' to use all counts
- `dome`: DESImage for the dome flat, needed if weight is not 'none'.
- `skymodel_filename`: optional output filename for 'sky'
"""
if weight == 'sky' and fit_filename is None:
raise SkyError(
'Cannot make sky-only weight map without doing sky subtraction'
)
if fit_filename is not None:
logger.info('Subtracting sky')
mini = skyinfo.MiniDecam.load(fit_filename)
templates = skyinfo.SkyPC.load(pc_filename)
if templates.detpos != image['DETPOS']:
# Quit if we don't have the right CCD to subtract
logger.error(
'Image DETPOS {:s} does not match sky template {:s}'.
format(templates.detpos, image['DETPOS']))
return 1
try:
image['BAND']
except:
image['BAND'] = decaminfo.get_band(image['FILTER'])
try:
items_must_match(image, mini.header, 'BAND', 'EXPNUM')
items_must_match(image, templates.header, 'BAND')
# ??? Could check that template and image use same dome flat
except:
return 1
sky = templates.sky(mini.coeffs)
image.data -= sky
image.write_key('SKYSBFIL',
path.basename(pc_filename),
comment='Sky subtraction template file')
for i, c in enumerate(mini.coeffs):
image.write_key('SKYPC{:>02d}'.format(i),
c,
comment='Sky template coefficient')
logger.info('Finished sky subtraction')
#
# Optionally write the sky model that was subtracted from the image.
#
if skymodel_filename is not None:
# Create HDU for output skymodel, add some header info, save output to file
logger.info('Optional output of skymodel requested')
skymodel_image = DESDataImage(sky)
skymodel_image.write_key(
'SKYSBFIL',
path.basename(pc_filename),
comment='Sky subtraction template file')
for i, c in enumerate(mini.coeffs):
skymodel_image.write_key(
'SKYPC{:>02d}'.format(i),
c,
comment='Sky template coefficient')
skymodel_image.write_key('BAND', image['BAND'], comment='Band')
skymodel_image.write_key('EXPNUM',
image['EXPNUM'],
comment='Exposure Number')
skymodel_image.write_key('CCDNUM',
image['CCDNUM'],
comment='CCD Number')
skymodel_image.write_key('NITE',
image['NITE'],
comment='Night')
# skymodel_image.copy_header_info(image, cls.propagate, require=False)
## ?? catch exception from write error below?
skymodel_image.save(skymodel_filename)
else:
sky = None
if weight == 'none':
do_weight = False
sky_weight = False
elif weight == 'sky':
do_weight = True
sky_weight = True
elif weight == 'all':
do_weight = True
sky_weight = False
else:
raise SkyError('Invalid weight value: ' + weight)
if do_weight:
if dome is None:
raise SkyError(
'sky_subtract needs dome flat when making weights')
if sky_weight:
logger.info('Constructing weight image from sky image')
data = sky
else:
logger.info('Constructing weight image from all counts')
if sky is None:
# If we did not subtract a sky, the image data gives total counts
data = image.data
else:
# Add sky back in to get total counts
data = image.data + sky
if image.weight is not None or image.variance is not None:
image.weight = None
image.variance = None
logger.warning('Overwriting existing weight image')
"""
We assume in constructing the weight (=inverse variance) image that
the input image here has been divided by the dome flat already, and that
its GAIN[AB] keywords are correct for a pixel that has been divided
by the FLATMED[AB] of the flat image. So the number of *electrons* that
were read in a pixel whose current value=sky is
e = sky * (dome/FLATMED) * GAIN
The variance has three parts: read noise, and sky Poisson noise, and
multiplicative errors from noise in the flat field.
The read noise variance, in electrons, is
Var = RDNOISE^2
...and the shot noise from sky was, in electrons,
Var = sky * (dome/FLATMED) * GAIN
This means the total variance in the image, in its present form, is
Var = (RDNOISE * FLATMED / dome / GAIN)^2 + (FLATMED/GAIN)*sky/dome
We can also add the uncertainty propagated from shot noise in the dome flat,
if the dome image has a weight or variance. In which case we would add
Var += var(dome) * sky^2 / dome^2
(remembering that sky has already been divided by the dome).
If sky_weight = False, we can substitute the image data for sky in the above
calculations.
"""
# Transform the sky image into a variance image
var = np.array(data, dtype=weight_dtype)
for amp in decaminfo.amps:
sec = section2slice(image['DATASEC' + amp])
invgain = (image['FLATMED' + amp] /
image['GAIN' + amp]) / dome.data[sec]
var[sec] += image['RDNOISE' + amp]**2 * invgain
var[sec] *= invgain
# Add noise from the dome flat shot noise, if present
if dome.weight is not None:
var += data * data / (dome.weight * dome.data * dome.data)
elif dome.variance is not None:
var += data * data * dome.variance / (dome.data * dome.data)
image.variance = var
# Now there are statistics desired for the output image header.
# First, the median variance at sky level on the two amps, SKYVAR[AB]
meds = []
for amp in decaminfo.amps:
sec = section2slice(image['DATASEC' + amp])
v = np.median(var[sec][::4, ::4])
image.write_key(
'SKYVAR' + amp,
v,
comment='Median noise variance at sky level, amp ' + amp)
meds.append(v)
# SKYSIGMA is overall average noise level
image.write_key('SKYSIGMA',
np.sqrt(np.mean(meds)),
comment='RMS noise at sky level')
# SKYBRITE is a measure of sky brightness. Use the sky image if we've got it, else
# use the data
if sky is None:
skybrite = np.median(data[::4, ::4])
else:
skybrite = np.median(sky[::2, ::2])
image.write_key('SKYBRITE',
skybrite,
comment='Median sky brightness')
logger.debug('Finished weight construction')
# Run null_mask or resaturate if requested in the command-line
if cls.do_step('null_mask') or cls.do_step('resaturate'):
logger.info("Running null_weights")
# We need to fix the step_name if we want to call 'step_run'
null_weights.__class__.step_name = config_section
#null_weights.__class__.step_name = cls.config_section
null_weights.step_run(image, cls.config)
ret_code = 0
return ret_code
def __call__(self):
"""Do image-by-image pixel level corrections
"""
# All the code here, asside from one call for each step, should
# be assiciated with shoveling data between steps. Everything else should
# take inside the code for its respective step.
# Get the science image
self.sci = DESImage.load(self.config.get('pixcorrect_im', 'in'))
# Bias subtraction
if self.do_step('bias'):
self._check_return(bias_correct(self.sci, self.bias))
self.clean_im('bias')
# Linearization
if self.do_step('lincor'):
lincor_fname = self.config.get('pixcorrect_im', 'lincor')
self._check_return(linearity_correct(self.sci, lincor_fname))
# Make the mask plane and mark saturated pixels. Note that flags
# are set to mark saturated pixels and keep any previously existing mask bits.
if self.do_step('bpm'):
self._check_return(
make_mask(self.sci, self.bpm, saturate=True, clear=False))
if self.do_step('gain'):
self._check_return(gain_correct(self.sci))
# If done with the BPM; let python reclaim the memory
if not self.do_step('fixcols'):
self.clean_im('bpm')
# Fix columns
if self.do_step('fixcols'):
self._check_return(fix_columns(self.sci, self.bpm))
self.clean_im('bpm')
# B/F correction
if self.do_step('bf'):
bf_fname = self.config.get('pixcorrect_im', 'bf')
self._check_return(
bf_correct(self.sci, bf_fname, bfinfo.DEFAULT_BFMASK))
# Flat field
if self.do_step('flat'):
self._check_return(flat_correct(self.sci, self.flat))
if not self.do_step('sky'):
self.clean_im('flat')
# LightBulb
if self.do_step('lightbulb'):
self._check_return(lightbulb(self.sci))
# CTI Check
if self.do_step('cticheck'):
self._check_return(cticheck(self.sci))
# Make mini-sky image
if self.do_step('mini'):
mini = self.config.get('pixcorrect_im', 'mini')
blocksize = self.config.getint('pixcorrect_im', 'blocksize')
self._check_return(
sky_compress(self.sci, mini, blocksize,
skyinfo.DEFAULT_SKYMASK))
# Subtract sky and make weight plane - forcing option to do "sky-only" weight
if self.do_step('sky'):
sky_fname = self.config.get('pixcorrect_im', 'sky')
fit_fname = self.config.get('pixcorrect_im', 'skyfit')
self._check_return(
sky_subtract(self.sci, fit_fname, sky_fname, 'sky', self.flat))
if not self.do_step('addweight'):
self.clean_im('flat')
# Star flatten
if self.do_step('starflat'):
self._check_return(starflat_correct(self.sci, self.starflat))
self.clean_im('starflat')
### Do add_weight before null_weight step, else it will overwrite the nulls
if self.do_step('addweight'):
self._check_return(add_weight(self.sci, self.flat))
self.clean_im('flat')
# This new call should take care of both --resaturate and --null_mask
if self.do_step('null_mask') or self.do_step('resaturate'):
# We need to fix the step_name if we want to call 'step_run'
null_weights.__class__.step_name = self.config_section
logger.info("Running null_weights")
self._check_return(null_weights.step_run(self.sci, self.config))
out_fname = self.config.get('pixcorrect_im', 'out')
self.sci.save(out_fname)
return 0
def __call__(self):
"""Do image-by-image pixel level corrections
"""
# All the code here, asside from one call for each step, should
# be assiciated with shoveling data between steps. Everything else should
# take inside the code for its respective step.
# Get the science image
self.sci = DESImage.load(self.config.get('pixcorrect_cp', 'in'))
# Bias subtraction
if self.do_step('bias'):
self._check_return(bias_correct(self.sci, self.bias))
self.clean_im('bias')
# Linearization
if self.do_step('lincor'):
lincor_fname = self.config.get('pixcorrect_cp', 'lincor')
self._check_return(linearity_correct(self.sci, lincor_fname))
# Make the mask plane and mark saturated pixels. Note that flags
# are set to mark saturated pixels and keep any previously existing mask bits.
if self.do_step('bpm'):
self._check_return(
make_mask(self.sci, self.bpm, saturate=True, clear=False))
flat_gaincorrect = self.config.getboolean('pixcorrect_cp',
'flat_gaincorrect')
# get gains ahead of time so that jump can be removed from a flat with no gain correction
gain_preserve = {}
if flat_gaincorrect:
tmp_gains = {}
avg_gain = 0.0
for amp in decaminfo.amps:
tmp_gains[amp] = self.sci['GAIN' + amp]
avg_gain = avg_gain + tmp_gains[amp]
for amp in decaminfo.amps:
gain_preserve[amp] = 2.0 * tmp_gains[amp] / avg_gain
# print avg_gain
# print gain_preserve
if self.do_step('gain'):
self._check_return(gain_correct(self.sci))
# B/F correction
if self.do_step('bf'):
bf_fname = self.config.get('pixcorrect_cp', 'bf')
self._check_return(
bf_correct(self.sci, bf_fname, bfinfo.DEFAULT_BFMASK))
# If done with the BPM; let python reclaim the memory
if not self.do_step('fixcol'):
self.clean_im('bpm')
# Flat field
if self.do_step('flat'):
# allow_mismatch = self.config.get('pixcorrect_cp','flat_gaincorrect')
print("flat_gaincorrect: ", flat_gaincorrect)
# for amp in decaminfo.amps:
# self.flat[gain_preserve[amp]['sec']]*=gain_preserve[amp]['cor']
self._check_return(
flat_correct_cp(self.sci, self.flat, gain_preserve))
if not self.do_step('sky'):
self.clean_im('flat')
# Fix columns
if self.do_step('fixcols'):
self._check_return(fix_columns(self.sci, self.bpm))
self.clean_im('bpm')
# Make mini-sky image
if self.do_step('mini'):
mini = self.config.get('pixcorrect_cp', 'mini')
blocksize = self.config.getint('pixcorrect_cp', 'blocksize')
self._check_return(
sky_compress(self.sci, mini, blocksize,
skyinfo.DEFAULT_SKYMASK))
# Subtract sky and make weight plane - forcing option to do "sky-only" weight
if self.do_step('sky'):
sky_fname = self.config.get('pixcorrect_cp', 'sky')
fit_fname = self.config.get('pixcorrect_cp', 'skyfit')
self._check_return(
sky_subtract(self.sci, fit_fname, sky_fname, 'sky', self.flat))
if not self.do_step('addweight'):
self.clean_im('flat')
# Star flatten
if self.do_step('starflat'):
self._check_return(starflat_correct(self.sci, self.starflat))
self.clean_im('starflat')
### Do add_weight before null_weight step, else it will overwrite the nulls
if self.do_step('addweight'):
self._check_return(add_weight(self.sci, self.flat))
self.clean_im('flat')
# This new call should take care of both --resaturate and --null_mask
if self.do_step('null_mask') or self.do_step('resaturate'):
# We need to fix the step_name if we want to call 'step_run'
null_weights.__class__.step_name = self.config_section
logger.info("Running null_weights")
self._check_return(null_weights.step_run(self.sci, self.config))
out_fname = self.config.get('pixcorrect_cp', 'out')
self.sci.save(out_fname)
return 0