def superflat(files,
bias_frame=None,
outfile='superflat.fits',
bitpix=None,
bias_subtract=True,
bias_method='row'):
"""
The superflat is created by bias-offset correcting the input files
and median-ing them together.
"""
# Get overscan region.
overscan = makeAmplifierGeometry(files[0]).serial_overscan
output_images = dict()
for amp in imutils.allAmps(files[0]):
images = []
for infile in files:
image = afwImage.ImageF(infile, imutils.dm_hdu(amp))
if bias_subtract:
if bias_frame:
bias_image = afwImage.ImageF(bias_frame,
imutils.dm_hdu(amp))
image = bias_subtracted_image(image, bias_image, overscan,
bias_method)
else:
image -= imutils.bias_image(im=image,
overscan=overscan,
bias_method=bias_method)
images.append(image)
if lsst.afw.__version__.startswith('12.0'):
images = afwImage.vectorImageF(images)
output_images[amp] = afwMath.statisticsStack(images, afwMath.MEDIAN)
imutils.writeFits(output_images, outfile, files[0])
return outfile
def make_superbias(self, butler, slot_data, **kwargs):
"""Stack the input data to make superbias frames
The superbias frames are stored as data members of this class
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
self._superbias_frame = None
mask_files = self.get_mask_files()
stat_type = self.config.stat
if stat_type is None:
stat_type = DEFAULT_STAT_TYPE
if stat_type == DEFAULT_STAT_TYPE:
output_file = self.tablefile_name() + '.fits'
else:
output_file = self.get_filename_from_format(
SUPERBIAS_STAT_FORMATTER, '.fits', **kwargs)
data_files = self.get_input_files(slot_data)
if not data_files:
return
makedir_safe(output_file)
if not self.config.skip:
out_data = self.extract(butler, slot_data)
if out_data is None:
self.log_warn_slot_msg(self.config, "extract() returned None.")
return
if butler is None:
template_file = data_files[0]
else:
template_file = get_filename_from_id(butler, data_files[0])
imutil.writeFits(out_data, output_file, template_file,
self.config.bitpix)
if butler is not None:
flip_data_in_place(output_file)
try:
self._superbias_frame = self.get_ccd(None, output_file, mask_files)
except Exception:
self._superbias_frame = None
def run(self, sensor_id, dark_files, mask_files, gains, bias_frame=None):
imutils.check_temperatures(dark_files,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
median_images = {}
for amp in imutils.allAmps(dark_files[0]):
median_images[amp] = imutils.fits_median(dark_files,
imutils.dm_hdu(amp))
medfile = os.path.join(self.config.output_dir,
'%s_median_dark_bp.fits' % sensor_id)
imutils.writeFits(median_images, medfile, dark_files[0])
ccd = MaskedCCD(medfile, mask_files=mask_files, bias_frame=bias_frame)
md = imutils.Metadata(dark_files[0], 1)
exptime = ccd.md.get('EXPTIME')
total_bright_pixels = 0
total_bright_columns = 0
if self.config.verbose:
self.log.info("Amp # bright pixels # bright columns")
#
# Write bright pixel and column counts to results file.
#
results_file = self.config.eotest_results_file
if results_file is None:
results_file = os.path.join(self.config.output_dir,
'%s_eotest_results.fits' % sensor_id)
results = EOTestResults(results_file, namps=len(ccd))
pixels = {}
columns = {}
for amp in ccd:
bright_pixels = BrightPixels(ccd, amp, exptime, gains[amp])
pixels[amp], columns[amp] = bright_pixels.find()
pix_count = len(pixels[amp])
col_count = len(columns[amp])
total_bright_pixels += pix_count
total_bright_columns += col_count
results.add_seg_result(amp, 'NUM_BRIGHT_PIXELS', pix_count)
results.add_seg_result(amp, 'NUM_BRIGHT_COLUMNS', col_count)
self.log.info("%2i %i %i" %
(amp, pix_count, col_count))
if self.config.verbose:
self.log.info("Total bright pixels: %i" % total_bright_pixels)
self.log.info("Total bright columns: %i" % total_bright_columns)
results.write(clobber=True)
# Generate the mask file based on the pixel and columns.
mask_file = os.path.join(self.config.output_dir,
'%s_bright_pixel_mask.fits' % sensor_id)
if os.path.isfile(mask_file):
os.remove(mask_file)
generate_mask(medfile,
mask_file,
self.config.mask_plane,
pixels=pixels,
columns=columns)
def make_image(config, slot_names, mean_frame_pattern='_mean_bias_image.fits'):
"""
Make the mosaic image of the entire raft, when illuminated by the CCOB
according to config. Returns:
- the raw image of the raft
- the corrected image where mean bias frame has been removed and gains applied
"""
file_list = sorted(find_files(config))
fits_files_dict = {slot_names[i] : file_list[i] for i in range(len(file_list))}
ccd_dict = {}
ccd_dict_wbias = {}
gains_dict = {}
for slot in slot_names:
mean_bias_file = slot + mean_frame_pattern
ccd_dict[slot] = sensorTest.MaskedCCD(fits_files_dict[slot])
outfile = os.path.join(config['tmp_dir'],'ccd' + slot + '.fits')
image={}
ccd_dict_wbias[slot]=sensorTest.MaskedCCD(fits_files_dict[slot],\
bias_frame=os.path.join(config['tmp_dir'],mean_bias_file))
outfile_wbias = os.path.join(config['tmp_dir'],'ccd' + slot + '_wbias.fits')
image_wbias={}
eotest_results_file = os.path.join(config['eo_data_path'],'{}_eotest_results.fits'.format(ccd_dict[slot].md('LSST_NUM')))
gains_dict[slot] = gains(eotest_results_file)
for amp in ccd_dict[slot]:
image[amp] = ccd_dict[slot].bias_subtracted_image(amp)
image[amp] *= gains_dict[slot][amp]
image_wbias[amp] = ccd_dict_wbias[slot].bias_subtracted_image(amp)
image_wbias[amp] *= gains_dict[slot][amp]
imutils.writeFits({amp: image_wbias[amp].getImage() for amp in ccd_dict_wbias[slot]},
outfile_wbias, fits_files_dict[slot])
imutils.writeFits({amp: image[amp].getImage() for amp in ccd_dict[slot]},
outfile, fits_files_dict[slot])
fits_files_dict_corr={slot : os.path.join(config['tmp_dir'],'ccd'+slot+'.fits') for slot in slot_names}
fits_files_dict_corr_wbias={slot : os.path.join(config['tmp_dir'],'ccd'+slot+'_wbias.fits') for slot in slot_names}
im_corr = raft.RaftMosaic(fits_files_dict_corr, bias_subtract=False)
im_corr_wbias = raft.RaftMosaic(fits_files_dict_corr_wbias, bias_subtract=False)
im_raw = raft.RaftMosaic(fits_files_dict, bias_subtract=False)
return im_raw, im_corr, im_corr_wbias
def make_superdark(self, butler, slot_data, **kwargs):
"""Stack the input data to make superflat frames
The superdarks are stored as data members of this class
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
mask_files = self.get_mask_files()
output_file = self.tablefile_name() + '.fits'
if not slot_data['DARK']:
return
makedir_safe(output_file)
if not self.config.skip:
sdark = self.extract(butler, slot_data)
if butler is None:
template_file = slot_data['DARK'][0]
else:
template_file = get_filename_from_id(butler,
slot_data['DARK'][0])
imutil.writeFits(sdark, output_file, template_file,
self.config.bitpix)
if butler is not None:
flip_data_in_place(output_file)
self._superdark_frame = self.get_ccd(None, output_file, mask_files)
def calibrated_stack(infiles,
outfile,
bias_frame=None,
dark_frame=None,
linearity_correction=None,
bitpix=32):
ccds = [
MaskedCCD(infile,
bias_frame=bias_frame,
dark_frame=dark_frame,
linearity_correction=linearity_correction)
for infile in infiles
]
all_amps = imutils.allAmps(infiles[0])
amp_images = {}
for amp in all_amps:
amp_ims = [ccd.bias_subtracted_image(amp) for ccd in ccds]
amp_images[amp] = imutils.stack(amp_ims).getImage()
imutils.writeFits(amp_images, outfile, infiles[0], bitpix=bitpix)
def run(self, sensor_id, pre_flat_darks, flat, post_flat_darks, mask_files,
gains):
darks = list(pre_flat_darks) + list(post_flat_darks)
imutils.check_temperatures(darks,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
# Check that pre-flat dark frames all have the same exposure time
md = imutils.Metadata(pre_flat_darks[0], 1)
exptime = md.get('EXPTIME')
for item in pre_flat_darks[1:]:
md = imutils.Metadata(item, 1)
if exptime != md.get('EXPTIME'):
raise RuntimeError("Exposure times of pre-flat darks differ.")
# Make a median image of the preflat darks
median_images = {}
for amp in imutils.allAmps(darks[0]):
median_images[amp] = imutils.fits_median(pre_flat_darks,
imutils.dm_hdu(amp))
medfile = os.path.join(self.config.output_dir,
'%s_persistence_dark_median.fits' % sensor_id)
imutils.writeFits(median_images, medfile, darks[0])
ccd = MaskedCCD(medfile, mask_files=mask_files)
# Define the sub-region for assessing the deferred charge.
# This is the same bounding box for all segments, so use amp=1.
image = ccd.unbiased_and_trimmed_image(1)
xllc = ((image.getWidth() - self.config.region_size) / 2. -
self.config.region_x_offset)
yllc = ((image.getHeight() - self.config.region_size) / 2. -
self.config.region_y_offset)
imaging_reg = afwGeom.Box2I(
afwGeom.Point2I(int(xllc), int(yllc)),
afwGeom.Extent2I(self.config.region_size, self.config.region_size))
overscan = ccd.amp_geom.serial_overscan
# Compute reference dark current for each segment.
dc_ref = {}
for amp in ccd:
mi = imutils.unbias_and_trim(ccd[amp], overscan, imaging_reg)
dc_ref[amp] = afwMath.makeStatistics(mi, afwMath.MEDIAN,
ccd.stat_ctrl).getValue()
dc_ref[amp] *= gains[amp] / exptime
# Extract reference time for computing the time dependence
# of the deferred charge as the observation time + exposure time
# from the saturated flat.
tref = readout_time(flat)
# Loop over post-flat darks, compute median e-/pixel in
# subregion, subtract dc_ref*exptime, persist, and report the
# deferred charge vs time (using MJD-OBS + EXPTIME) for each amp.
deferred_charges = []
times = []
for dark in post_flat_darks:
ccd = MaskedCCD(dark, mask_files=mask_files)
dt = readout_time(dark) - tref
times.append(dt.sec)
exptime = ccd.md.get('EXPTIME')
charge = {}
for amp in ccd:
mi = imutils.unbias_and_trim(ccd[amp], overscan, imaging_reg)
estimators = afwMath.MEDIAN | afwMath.STDEV
stats = afwMath.makeStatistics(mi, estimators, ccd.stat_ctrl)
value = (stats.getValue(afwMath.MEDIAN) * gains[amp] -
dc_ref[amp] * exptime)
stdev = (stats.getValue(afwMath.STDEV) * gains[amp] -
dc_ref[amp] * exptime)
charge[amp] = (value, stdev)
deferred_charges.append(charge)
if self.config.verbose:
for amp in ccd:
self.log.info("amp: %i" % amp)
for i, time in enumerate(times):
self.log.info("%.1f %e %e" %
(time, deferred_charges[i][amp][0],
deferred_charges[i][amp][1]))
outfile = os.path.join(self.config.output_dir,
'%s_persistence.fits' % sensor_id)
self.write(times, deferred_charges, outfile, clobber=True)
def run(self, sensor_id, dark_files, mask_files, gains, bias_frame=None):
imutils.check_temperatures(dark_files,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
median_images = {}
md = imutils.Metadata(dark_files[0], 1)
for amp in imutils.allAmps(dark_files[0]):
median_images[amp] = imutils.fits_median(dark_files,
imutils.dm_hdu(amp))
medfile = os.path.join(self.config.output_dir,
'%s_median_dark_current.fits' % sensor_id)
imutils.writeFits(median_images, medfile, dark_files[0])
ccd = MaskedCCD(medfile, mask_files=mask_files, bias_frame=bias_frame)
dark95s = {}
exptime = md.get('EXPTIME')
if self.config.verbose:
self.log.info("Amp 95 percentile median")
dark_curr_pixels = []
dark_curr_pixels_per_amp = {}
for amp in ccd:
imaging_region = ccd.amp_geom.imaging
overscan = ccd.amp_geom.serial_overscan
image = imutils.unbias_and_trim(ccd[amp].getImage(), overscan,
imaging_region)
mask = imutils.trim(ccd[amp].getMask(), imaging_region)
imarr = image.getArray()
mskarr = mask.getArray()
pixels = imarr.reshape(1, imarr.shape[0] * imarr.shape[1])[0]
masked = mskarr.reshape(1, mskarr.shape[0] * mskarr.shape[1])[0]
unmasked = [
pixels[i] for i in range(len(pixels)) if masked[i] == 0
]
unmasked.sort()
unmasked = np.array(unmasked) * gains[amp] / exptime
dark_curr_pixels_per_amp[amp] = unmasked
dark_curr_pixels.extend(unmasked)
try:
dark95s[amp] = unmasked[int(len(unmasked) * 0.95)]
median = unmasked[len(unmasked) / 2]
except IndexError as eobj:
print str(eobj)
dark95s[amp] = -1.
median = -1.
if self.config.verbose:
self.log.info("%2i %.2e %.2e" %
(amp, dark95s[amp], median))
#
# Compute 95th percentile dark current for CCD as a whole.
#
dark_curr_pixels = sorted(dark_curr_pixels)
darkcurr95 = dark_curr_pixels[int(len(dark_curr_pixels) * 0.95)]
dark95mean = np.mean(dark95s.values())
if self.config.verbose:
#self.log.info("CCD: mean 95 percentile value = %s" % dark95mean)
self.log.info("CCD-wide 95 percentile value = %s" % darkcurr95)
#
# Update header of dark current median image file with dark
# files used and dark95 values, and write dark95 values to the
# eotest results file.
#
results_file = self.config.eotest_results_file
if results_file is None:
results_file = os.path.join(self.config.output_dir,
'%s_eotest_results.fits' % sensor_id)
results = EOTestResults(results_file, namps=len(ccd))
output = fits.open(medfile)
for i, dark in enumerate(dark_files):
output[0].header['DARK%02i' % i] = os.path.basename(dark)
# Write overall dark current 95th percentile
results.output['AMPLIFIER_RESULTS'].header['DARK95'] = darkcurr95
for amp in ccd:
output[0].header['DARK95%s' %
imutils.channelIds[amp]] = dark95s[amp]
results.add_seg_result(amp, 'DARK_CURRENT_95', dark95s[amp])
fitsWriteto(output, medfile, clobber=True, checksum=True)
results.write(clobber=True)
return dark_curr_pixels_per_amp, dark95s