def superflat(files,
bias_frame=None,
outfile='superflat.fits',
bitpix=-32,
bias_subtract=True):
"""
The superflat is created by bias-offset correcting the input files
and median-ing them together.
"""
# Get overscan region.
overscan = makeAmplifierGeometry(files[0]).serial_overscan
# Use the first file as a template for the fits output.
output = fits.open(files[0])
for amp in imutils.allAmps(files[0]):
images = afwImage.vectorImageF()
for infile in files:
image = afwImage.ImageF(infile, imutils.dm_hdu(amp))
if bias_subtract:
if bias_frame:
bias_image = afwImage.ImageF(bias_frame,
imutils.dm_hdu(amp))
image = bias_subtracted_image(image, bias_image, overscan)
else:
image -= imutils.bias_image(image,
overscan,
statistic=np.median)
images.push_back(image)
median_image = afwMath.statisticsStack(images, afwMath.MEDIAN)
output[amp].data = median_image.getArray()
if bitpix is not None:
imutils.set_bitpix(output[amp], bitpix)
fitsWriteto(output, outfile, clobber=True)
return outfile
def superflat(files,
bias_frame=None,
outfile='superflat.fits',
bitpix=None,
bias_subtract=True,
bias_method='row'):
"""
The superflat is created by bias-offset correcting the input files
and median-ing them together.
"""
# Get overscan region.
overscan = makeAmplifierGeometry(files[0]).serial_overscan
output_images = dict()
for amp in imutils.allAmps(files[0]):
images = []
for infile in files:
image = afwImage.ImageF(infile, imutils.dm_hdu(amp))
if bias_subtract:
if bias_frame:
bias_image = afwImage.ImageF(bias_frame,
imutils.dm_hdu(amp))
image = bias_subtracted_image(image, bias_image, overscan,
bias_method)
else:
image -= imutils.bias_image(im=image,
overscan=overscan,
bias_method=bias_method)
images.append(image)
if lsst.afw.__version__.startswith('12.0'):
images = afwImage.vectorImageF(images)
output_images[amp] = afwMath.statisticsStack(images, afwMath.MEDIAN)
imutils.writeFits(output_images, outfile, files[0])
return outfile
def __init__(self,
imfile,
mask_files=(),
bias_frame=None,
applyMasks=True,
linearity_correction=None):
super(MaskedCCD, self).__init__()
self.imfile = imfile
self.md = imutils.Metadata(imfile)
self.amp_geom = makeAmplifierGeometry(imfile)
all_amps = imutils.allAmps(imfile)
for amp in all_amps:
image = afwImage.ImageF(imfile, imutils.dm_hdu(amp))
mask = afwImage_Mask(image.getDimensions())
self[amp] = afwImage.MaskedImageF(image, mask)
self._added_mask_types = []
for mask_file in mask_files:
self.add_masks(mask_file)
self.stat_ctrl = afwMath.StatisticsControl()
if mask_files:
self.setAllMasks()
if bias_frame is not None:
self.bias_frame = MaskedCCD(bias_frame)
else:
self.bias_frame = None
self._applyMasks = applyMasks
self._linearity_correction = linearity_correction
def add_mask_files(mask_files, outfile, overwrite=True):
amp_list = imutils.allAmps(mask_files[0])
masks = dict([(amp, afwImage_Mask(mask_files[0], imutils.dm_hdu(amp)))
for amp in amp_list])
for mask_file in mask_files[1:]:
for amp in masks:
masks[amp] |= afwImage_Mask(mask_file, imutils.dm_hdu(amp))
output = fits.HDUList()
output.append(fits.PrimaryHDU())
output[0].header['MASKTYPE'] = 'SUMMED_MASKS'
fitsWriteto(output, outfile, overwrite=overwrite)
for amp in masks:
md = dafBase.PropertySet()
md.set('EXTNAME', 'SEGMENT%s' % imutils.channelIds[amp])
masks[amp].writeFits(outfile, md, 'a')
return masks
def run(self, sensor_id, dark_files, mask_files, gains, bias_frame=None):
imutils.check_temperatures(dark_files,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
median_images = {}
for amp in imutils.allAmps(dark_files[0]):
median_images[amp] = imutils.fits_median(dark_files,
imutils.dm_hdu(amp))
medfile = os.path.join(self.config.output_dir,
'%s_median_dark_bp.fits' % sensor_id)
imutils.writeFits(median_images, medfile, dark_files[0])
ccd = MaskedCCD(medfile, mask_files=mask_files, bias_frame=bias_frame)
md = imutils.Metadata(dark_files[0], 1)
exptime = ccd.md.get('EXPTIME')
total_bright_pixels = 0
total_bright_columns = 0
if self.config.verbose:
self.log.info("Amp # bright pixels # bright columns")
#
# Write bright pixel and column counts to results file.
#
results_file = self.config.eotest_results_file
if results_file is None:
results_file = os.path.join(self.config.output_dir,
'%s_eotest_results.fits' % sensor_id)
results = EOTestResults(results_file, namps=len(ccd))
pixels = {}
columns = {}
for amp in ccd:
bright_pixels = BrightPixels(ccd, amp, exptime, gains[amp])
pixels[amp], columns[amp] = bright_pixels.find()
pix_count = len(pixels[amp])
col_count = len(columns[amp])
total_bright_pixels += pix_count
total_bright_columns += col_count
results.add_seg_result(amp, 'NUM_BRIGHT_PIXELS', pix_count)
results.add_seg_result(amp, 'NUM_BRIGHT_COLUMNS', col_count)
self.log.info("%2i %i %i" %
(amp, pix_count, col_count))
if self.config.verbose:
self.log.info("Total bright pixels: %i" % total_bright_pixels)
self.log.info("Total bright columns: %i" % total_bright_columns)
results.write(clobber=True)
# Generate the mask file based on the pixel and columns.
mask_file = os.path.join(self.config.output_dir,
'%s_bright_pixel_mask.fits' % sensor_id)
if os.path.isfile(mask_file):
os.remove(mask_file)
generate_mask(medfile,
mask_file,
self.config.mask_plane,
pixels=pixels,
columns=columns)
def add_masks(self, mask_file):
"""
Add a masks from a mask file by or-ing with existing masks.
"""
md = imutils.Metadata(mask_file)
self._added_mask_types.append(md('MASKTYPE'))
for amp in self:
curr_mask = self[amp].getMask()
curr_mask |= afwImage_Mask(mask_file, imutils.dm_hdu(amp))
def get_overscans(infile, oscan_indices=None):
"Return the overscan data as numpy arrays."
if oscan_indices is None:
y0, y1, x0, x1 = get_oscan_indices(infile)
else:
y0, y1, x0, x1 = oscan_indices
overscans = dict()
for amp in imutils.allAmps(infile):
oscan_data = afw_image.ImageF(infile, imutils.dm_hdu(amp)).getArray()
overscans[amp] = copy.deepcopy(oscan_data[y0:y1, x0:x1])
return overscans
def __call__(self, fitsfile, amps):
tot_dark_ccd = 0
tot_dark_per_amp = []
pix_per_amp = []
col_per_amp = []
for amp in amps:
try:
tot, pixels = self.dark_pix(fitsfile, imutils.dm_hdu(amp))
tot_dark_ccd += tot
tot_dark_per_amp.append(tot)
pix_per_amp.append(pixels)
except:
print "Failed dark pixels for hdu ", amp, " ", \
imutils.dm_hdu(amp)
traceback.print_exc(file=sys.stdout)
continue
return tot_dark_ccd, tot_dark_per_amp, pix_per_amp, col_per_amp
def extract_unmasked_pixels(ccd, amp, gain, correction_image=None):
subimage = ccd.unbiased_and_trimmed_image(amp)
imarr = subimage.getImage().getArray()
if correction_image is not None:
correction = afwImage.ImageF(correction_image, imutils.dm_hdu(amp))
correction = correction.Factory(correction, ccd.amp_geom.imaging)
imarr *= correction.getArray()
maskarr = subimage.getMask().getArray()
if imarr.shape != maskarr.shape:
raise RuntimeError("image and mask subarray shapes differ")
indx = np.where(maskarr == 0)
return [x * gain for x in imarr[indx].flat]
def get_mean_overscans(infiles, oscan_indices=None):
"Compute the mean of the overscans of the list of files"
if oscan_indices is None:
y0, y1, x0, x1 = get_oscan_indices(infiles[0])
else:
y0, y1, x0, x1 = oscan_indices
mean_overscans = defaultdict(list)
for infile in infiles:
for amp in imutils.allAmps(infiles[0]):
oscan_data \
= afw_image.ImageF(infile, imutils.dm_hdu(amp)).getArray()
mean_overscans[amp].append(copy.deepcopy(oscan_data[y0:y1, x0:x1]))
for amp, images in mean_overscans.items():
mean_overscans[amp] = sum(images) / float(len(images))
return mean_overscans
def ctesim(infile, pcti=0, scti=0, verbose=False):
input = fits.open(infile)
amps = [
i for i in range(1, len(input))
if input[i].name.upper().startswith('SEGMENT')
]
if not isinstance(pcti, dict):
pcti = {amp: pcti for amp in amps}
if not isinstance(scti, dict):
scti = {amp: scti for amp in amps}
segments = {}
for amp in amps:
if verbose:
print("ctesim: working on amp", amp)
image = afwImage.ImageF(infile, imutils.dm_hdu(amp))
geom = makeAmplifierGeometry(infile)
#
# Temporarily remove readout bias median.
#
bias_med = imutils.median(image.Factory(image, geom.serial_overscan))
image -= bias_med
imarr = image.getArray()
outimage = afwImage.ImageF(image, True)
outarr = outimage.getArray()
if pcti[amp] != 0:
pcte_matrix = cte_matrix(imarr.shape[0], pcti[amp])
for col in range(0, imarr.shape[1]):
outarr[:, col] = np.dot(pcte_matrix, imarr[:, col])
if scti[amp] != 0:
scte_matrix = cte_matrix(imarr.shape[1], scti[amp])
for row in range(0, imarr.shape[0]):
outarr[row, :] = np.dot(scte_matrix, outarr[row, :])
#
# Restore readout bias
#
outarr += bias_med
segments[amp] = outimage
return fitsFile(segments, input)
def apply_cte(infile, pcti=None, scti=None, verbose=False,
amps=tuple(range(1, 17))):
"""
Function to apply distinct levels of parallel cte and/or serial cte
on each amplifier in an input sensor image.
"""
if pcti is None:
pcti = dict([(amp, 0) for amp in amps])
if scti is None:
scti = dict([(amp, 0) for amp in amps])
segments = {}
for amp in amps:
if verbose:
print("apply_cte: working on amp", amp)
image = afwImage.ImageF(infile, imutils.dm_hdu(amp))
geom = sensorTest.makeAmplifierGeometry(infile)
outimage = eotest_utils.ImageTools.applyCTI(image, geom.serial_overscan,
pcti[amp], scti[amp],
verbose)
segments[amp] = outimage
return fitsFile(segments, fits.open(infile))
def run(self, sensor_id, pre_flat_darks, flat, post_flat_darks, mask_files,
gains):
darks = list(pre_flat_darks) + list(post_flat_darks)
imutils.check_temperatures(darks,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
# Check that pre-flat dark frames all have the same exposure time
md = imutils.Metadata(pre_flat_darks[0], 1)
exptime = md.get('EXPTIME')
for item in pre_flat_darks[1:]:
md = imutils.Metadata(item, 1)
if exptime != md.get('EXPTIME'):
raise RuntimeError("Exposure times of pre-flat darks differ.")
# Make a median image of the preflat darks
median_images = {}
for amp in imutils.allAmps(darks[0]):
median_images[amp] = imutils.fits_median(pre_flat_darks,
imutils.dm_hdu(amp))
medfile = os.path.join(self.config.output_dir,
'%s_persistence_dark_median.fits' % sensor_id)
imutils.writeFits(median_images, medfile, darks[0])
ccd = MaskedCCD(medfile, mask_files=mask_files)
# Define the sub-region for assessing the deferred charge.
# This is the same bounding box for all segments, so use amp=1.
image = ccd.unbiased_and_trimmed_image(1)
xllc = ((image.getWidth() - self.config.region_size) / 2. -
self.config.region_x_offset)
yllc = ((image.getHeight() - self.config.region_size) / 2. -
self.config.region_y_offset)
imaging_reg = afwGeom.Box2I(
afwGeom.Point2I(int(xllc), int(yllc)),
afwGeom.Extent2I(self.config.region_size, self.config.region_size))
overscan = ccd.amp_geom.serial_overscan
# Compute reference dark current for each segment.
dc_ref = {}
for amp in ccd:
mi = imutils.unbias_and_trim(ccd[amp], overscan, imaging_reg)
dc_ref[amp] = afwMath.makeStatistics(mi, afwMath.MEDIAN,
ccd.stat_ctrl).getValue()
dc_ref[amp] *= gains[amp] / exptime
# Extract reference time for computing the time dependence
# of the deferred charge as the observation time + exposure time
# from the saturated flat.
tref = readout_time(flat)
# Loop over post-flat darks, compute median e-/pixel in
# subregion, subtract dc_ref*exptime, persist, and report the
# deferred charge vs time (using MJD-OBS + EXPTIME) for each amp.
deferred_charges = []
times = []
for dark in post_flat_darks:
ccd = MaskedCCD(dark, mask_files=mask_files)
dt = readout_time(dark) - tref
times.append(dt.sec)
exptime = ccd.md.get('EXPTIME')
charge = {}
for amp in ccd:
mi = imutils.unbias_and_trim(ccd[amp], overscan, imaging_reg)
estimators = afwMath.MEDIAN | afwMath.STDEV
stats = afwMath.makeStatistics(mi, estimators, ccd.stat_ctrl)
value = (stats.getValue(afwMath.MEDIAN) * gains[amp] -
dc_ref[amp] * exptime)
stdev = (stats.getValue(afwMath.STDEV) * gains[amp] -
dc_ref[amp] * exptime)
charge[amp] = (value, stdev)
deferred_charges.append(charge)
if self.config.verbose:
for amp in ccd:
self.log.info("amp: %i" % amp)
for i, time in enumerate(times):
self.log.info("%.1f %e %e" %
(time, deferred_charges[i][amp][0],
deferred_charges[i][amp][1]))
outfile = os.path.join(self.config.output_dir,
'%s_persistence.fits' % sensor_id)
self.write(times, deferred_charges, outfile, clobber=True)
fratio = np.mean(imarr1 / imarr2)
imarr2 *= fratio
# Calculate the mean value of the flat field images.
fmean = (np.mean(imarr1) + np.mean(imarr2)) / 2.
# Calculate the variance of the flat difference image.
fvar = np.var(imarr1 - imarr2) / 2.
gains.append(fvar / fmean)
gain = 1. / np.median(gains) # gain in Ne/DN
return gain, im1, im2
if __name__ == '__main__':
import os
from .sim_tools import simulateFlat
file1 = 'test_flat1.fits'
file2 = 'test_flat2.fits'
hdus = 2
simulateFlat(file1, 4000, 5, hdus=hdus)
simulateFlat(file2, 4000, 5, hdus=hdus)
for amp in range(1, hdus + 1):
image1 = afwImage.ImageF(file1, imutils.dm_hdu(amp))
image2 = afwImage.ImageF(file2, imutils.dm_hdu(amp))
gain, im1, im2 = flat_gain(image1, image2, count=1000)
print(amp, gain)
os.remove(file1)
os.remove(file2)
def run(self, sensor_id, dark_files, mask_files, gains, bias_frame=None):
imutils.check_temperatures(dark_files,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
median_images = {}
md = imutils.Metadata(dark_files[0], 1)
for amp in imutils.allAmps(dark_files[0]):
median_images[amp] = imutils.fits_median(dark_files,
imutils.dm_hdu(amp))
medfile = os.path.join(self.config.output_dir,
'%s_median_dark_current.fits' % sensor_id)
imutils.writeFits(median_images, medfile, dark_files[0])
ccd = MaskedCCD(medfile, mask_files=mask_files, bias_frame=bias_frame)
dark95s = {}
exptime = md.get('EXPTIME')
if self.config.verbose:
self.log.info("Amp 95 percentile median")
dark_curr_pixels = []
dark_curr_pixels_per_amp = {}
for amp in ccd:
imaging_region = ccd.amp_geom.imaging
overscan = ccd.amp_geom.serial_overscan
image = imutils.unbias_and_trim(ccd[amp].getImage(), overscan,
imaging_region)
mask = imutils.trim(ccd[amp].getMask(), imaging_region)
imarr = image.getArray()
mskarr = mask.getArray()
pixels = imarr.reshape(1, imarr.shape[0] * imarr.shape[1])[0]
masked = mskarr.reshape(1, mskarr.shape[0] * mskarr.shape[1])[0]
unmasked = [
pixels[i] for i in range(len(pixels)) if masked[i] == 0
]
unmasked.sort()
unmasked = np.array(unmasked) * gains[amp] / exptime
dark_curr_pixels_per_amp[amp] = unmasked
dark_curr_pixels.extend(unmasked)
try:
dark95s[amp] = unmasked[int(len(unmasked) * 0.95)]
median = unmasked[len(unmasked) / 2]
except IndexError as eobj:
print str(eobj)
dark95s[amp] = -1.
median = -1.
if self.config.verbose:
self.log.info("%2i %.2e %.2e" %
(amp, dark95s[amp], median))
#
# Compute 95th percentile dark current for CCD as a whole.
#
dark_curr_pixels = sorted(dark_curr_pixels)
darkcurr95 = dark_curr_pixels[int(len(dark_curr_pixels) * 0.95)]
dark95mean = np.mean(dark95s.values())
if self.config.verbose:
#self.log.info("CCD: mean 95 percentile value = %s" % dark95mean)
self.log.info("CCD-wide 95 percentile value = %s" % darkcurr95)
#
# Update header of dark current median image file with dark
# files used and dark95 values, and write dark95 values to the
# eotest results file.
#
results_file = self.config.eotest_results_file
if results_file is None:
results_file = os.path.join(self.config.output_dir,
'%s_eotest_results.fits' % sensor_id)
results = EOTestResults(results_file, namps=len(ccd))
output = fits.open(medfile)
for i, dark in enumerate(dark_files):
output[0].header['DARK%02i' % i] = os.path.basename(dark)
# Write overall dark current 95th percentile
results.output['AMPLIFIER_RESULTS'].header['DARK95'] = darkcurr95
for amp in ccd:
output[0].header['DARK95%s' %
imutils.channelIds[amp]] = dark95s[amp]
results.add_seg_result(amp, 'DARK_CURRENT_95', dark95s[amp])
fitsWriteto(output, medfile, clobber=True, checksum=True)
results.write(clobber=True)
return dark_curr_pixels_per_amp, dark95s
def _correction_images(self, ccd, correction_image):
images = OrderedDict()
for amp in ccd:
correction = afwImage.ImageF(correction_image, imutils.dm_hdu(amp))
images[amp] = correction.Factory(correction, ccd.amp_geom.imaging)
return images
