def test_bias_image(self):
ccd = MaskedCCD(self.image_file)
overscan = makeAmplifierGeometry(self.image_file)
ccd_mean = MaskedCCD(self.mean_image_file)
overscan_mean = makeAmplifierGeometry(self.mean_image_file)
for amp in ccd:
for method in ['mean', 'row', 'func', 'spline']:
if method == 'mean':
my_bias_image = imutils.bias_image(
ccd_mean[amp],
overscan_mean.serial_overscan,
bias_method=method,
**self.kwargs)
fracdiff = ((self.mean_bias_image.getArray() -
my_bias_image.getArray()) /
self.mean_bias_image.getArray())
self.assertTrue(max(np.abs(fracdiff.flat)) < 1.5e-3)
else:
my_bias_image = imutils.bias_image(
ccd[amp],
overscan.serial_overscan,
bias_method=method,
**self.kwargs)
fracdiff = ((self.bias_image.getArray() -
my_bias_image.getArray()) /
self.bias_image.getArray())
self.assertTrue(max(np.abs(fracdiff.flat)) < 1e-6)
my_bias_image = imutils.bias_image(ccd[amp],
self.amp_geom.serial_overscan)
fracdiff = (
(self.bias_image.getArray() - my_bias_image.getArray()) /
self.bias_image.getArray())
self.assertTrue(max(np.abs(fracdiff.flat)) < 1e-6)
def test_bias_image(self):
ccd = MaskedCCD(self.image_file)
for amp in ccd:
my_bias_image = ccd.bias_image(amp)
fracdiff = (
(self.bias_image.getArray() - my_bias_image.getArray()) /
self.bias_image.getArray())
self.assertTrue(max(np.abs(fracdiff.flat)) < 1e-6)
def test_unbias_and_trim(self):
ccd = MaskedCCD(self.image_file)
for amp in ccd:
#
# Test of corresponding MaskedCCD method.
#
image = ccd.unbiased_and_trimmed_image(amp)
imarr = image.getImage().getArray()
self.assertTrue(max(np.abs(imarr.flat)) < 1e-6)
def test_unbias_and_trim(self):
ccd = MaskedCCD(self.image_file)
for amp in ccd:
image = imutils.unbias_and_trim(ccd[amp],
self.amp_geom.serial_overscan,
self.amp_geom.imaging)
imarr = image.getImage().getArray()
self.assertTrue(max(np.abs(imarr.flat)) < 1e-6)
#
# Test of corresponding MaskedCCD method.
#
image = ccd.unbiased_and_trimmed_image(amp)
imarr = image.getImage().getArray()
self.assertTrue(max(np.abs(imarr.flat)) < 1e-6)
def run(self,
sensor_id,
infiles,
gains,
bias_frame=None,
linearity_correction=None):
## Calculate mean row for each flat file
minflux = self.config.minflux
maxflux = self.config.maxflux
num_oscan_pixels = self.config.num_oscan_pixels
fitter = OverscanFit(num_oscan_pixels=num_oscan_pixels,
minflux=minflux,
maxflux=maxflux)
for i, infile in enumerate(infiles):
if self.config.verbose:
self.log.info("Processing {0}".format(infile))
ccd = MaskedCCD(infile,
bias_frame=bias_frame,
linearity_correction=linearity_correction)
fitter.process_image(ccd, gains)
output_dir = self.config.output_dir
if self.config.output_file is None:
output_file = os.path.join(
output_dir, '{0}_overscan_results.fits'.format(sensor_id))
else:
output_file = os.path.join(output_dir, self.config.output_file)
if self.config.verbose:
self.log.info("writing to {0}".format(output_file))
fitter.write_results(outfile=output_file)
return output_file
def setUpClass(cls):
ccd = sim_tools.CCD(exptime=cls.exptime, gain=cls.gain)
for amp in ccd.segments:
imarr = ccd.segments[amp].image.getArray()
imarr[cls.ymin:cls.ymax, cls.xmin:cls.xmax] += cls.signal
ccd.writeto(cls.mask_image)
cls.mask_files = []
for mask_plane, bit in cls.mpd.items():
mask_file = 'mask_file_%s.fits' % mask_plane
cls.mask_files.append(mask_file)
masked_ccd = MaskedCCD(cls.mask_image)
pixels, columns = {}, {}
for amp in masked_ccd:
bp = BrightPixels(masked_ccd,
amp,
cls.exptime,
cls.gain,
mask_plane=mask_plane,
ethresh=cls.signal / 2.)
pixels[amp], columns[amp] = bp.find()
generate_mask(cls.mask_image,
mask_file,
mask_plane,
pixels=pixels,
columns=columns)
cls.summed_mask_file = 'summed_mask_file.fits'
add_mask_files(cls.mask_files, cls.summed_mask_file)
def test_stack(self):
ccd = MaskedCCD(self.image_file)
overscan = makeAmplifierGeometry(self.image_file)
for method in ['mean', 'row', 'func']:
corrected = []
for image in ccd.values():
corrected.append(
imutils.unbias_and_trim(image,
overscan.serial_overscan,
bias_method=method,
**self.kwargs).getImage())
stacked = imutils.stack(corrected)
imarr = stacked.getArray()
if method == 'mean':
self.assertTrue(max(np.abs(imarr.flat)) < 2)
else:
self.assertTrue(max(np.abs(imarr.flat)) < 1e-6)
def test_detector_crosstalk(self):
ccd = MaskedCCD(self.xtalk_file)
ratios = crosstalk.detector_crosstalk(ccd, self.aggressor)
for amp in ratios:
if amp != self.aggressor:
self.assertTrue(
abs(ratios[amp][0] -
self.xtalk_frac[amp]) < ratios[amp][1])
def test_bias_image(self):
ccd = MaskedCCD(self.image_file)
for amp in ccd:
my_bias_image = imutils.bias_image(ccd[amp],
self.amp_geom.serial_overscan)
fracdiff = (
(self.bias_image.getArray() - my_bias_image.getArray()) /
self.bias_image.getArray())
self.assertTrue(max(np.abs(fracdiff.flat)) < 1e-6)
def test_find_column_defects(self):
ccd = MaskedCCD(self.sflat_file)
for amp in ccd:
dp = DarkPixels(ccd, amp, frac_thresh=0.8, colthresh=100)
results = dp.find()
columns = sorted(self.columns[amp])
self.assertEqual(len(columns), self.ncols)
self.assertEqual(len(results[1]), self.ncols)
self.assertEqual(columns, results[1])
def get_ccd_from_id(butler, data_id, mask_files, **kwargs):
"""Get a CCD image from a data_id
If we are using `Butler` then this will take a
data_id `dict` ojbect and return an `ExposureF` object
If we are not using `Butler` (i.e., if bulter is `None`)
then this will take a filename and return a `MaskedCCD` object
Parameters
----------
butler : `Butler` or `None`
Data Butler
data_id : `dict` or `str`
Data identier
mask_files : `list`
List of data_ids for the files to construct the pixel mask
Keywords
--------
bias_frame : `ExposureF` or `MaskedCCD` or `None`
Object with the bias data
masked_ccd : `bool`
Use bulter only to get filename, return as MaskedCCD object
Returns
-------
ccd : `ExposureF` or `MaskedCCD`
CCD data object
"""
bias_frame = kwargs.get('bias_frame', None)
if butler is None:
exposure = MaskedCCD(str(data_id),
mask_files=mask_files,
bias_frame=bias_frame)
elif kwargs.get('masked_ccd', False):
filepath = butler.get('raw_filename', data_id)[0][0:-3]
exposure = MaskedCCD(str(filepath),
mask_files=mask_files,
bias_frame=bias_frame)
else:
exposure = butler.get('raw', data_id)
apply_masks(butler, exposure, mask_files)
return exposure
def test_find_pixel_defects(self):
ccd = MaskedCCD(self.sflat_file)
for amp in ccd:
dp = DarkPixels(ccd, amp, frac_thresh=0.8, colthresh=100)
results = dp.find()
pixels = np.array(np.where(self.pixels[amp] == 1))
pixels = pixels.transpose()
pixels = [(x, y) for y, x in pixels]
pixels.sort()
self.assertEqual(pixels, results[0])
def test_bias_row(self):
ccd = MaskedCCD(self.image_file)
overscan = makeAmplifierGeometry(self.image_file)
for amp in ccd:
# Unmasked image
br_i = imutils.bias_row(ccd[amp].getImage(),
overscan.serial_overscan)
# Masked image
br_m = imutils.bias_row(ccd[amp], overscan.serial_overscan)
for ii in range(2022):
self.assertEqual(br_i(ii), br_m(ii))
def test_find_column_defects(self):
ccd = MaskedCCD(self.dark_file)
for amp in ccd:
bp = BrightPixels(ccd,
amp,
ccd.md.get('EXPTIME'),
self.gain,
ethresh=self.emin / 2.,
colthresh=100)
results = bp.find()
columns = sorted(self.columns[amp])
self.assertEqual(columns, results[1])
def test_unbias_and_trim(self):
ccd = MaskedCCD(self.image_file)
overscan = makeAmplifierGeometry(self.image_file)
for amp in ccd:
for method in ['mean', 'row', 'func', 'spline']:
image = imutils.unbias_and_trim(ccd[amp],
overscan.serial_overscan,
bias_method=method,
**self.kwargs)
imarr = image.getImage().getArray()
if method == 'mean':
self.assertTrue(max(np.abs(imarr.flat)) < 2)
else:
self.assertTrue(max(np.abs(imarr.flat)) < 1e-6)
#
# Test of corresponding MaskedCCD method.
#
image = ccd.unbiased_and_trimmed_image(
amp, overscan.serial_overscan, **self.kwargs)
imarr = image.getImage().getArray()
self.assertTrue(max(np.abs(imarr.flat)) < 1e-6)
def test_01_nonlinearity_task(self):
"""Run the NonlinearityTask"""
task = NonlinearityTask()
sensorid = 'RTM-022'
task.run(sensorid,
_detrespfile,
outputfile=self.fits_file,
plotfile=None)
nlc = NonlinearityCorrection.create_from_fits_file(self.fits_file)
ccd_1 = MaskedCCD(_flat_frame, bias_frame=_bias_frame)
ccd_2 = MaskedCCD(_flat_frame,
bias_frame=_bias_frame,
linearity_correction=nlc)
img_1 = ccd_1.unbiased_and_trimmed_image(1)
img_2 = ccd_2.unbiased_and_trimmed_image(1)
mean_1 = afwMath.makeStatistics(img_1, afwMath.MEAN,
ccd_1.stat_ctrl).getValue()
mean_2 = afwMath.makeStatistics(img_2, afwMath.MEAN,
ccd_2.stat_ctrl).getValue()
self.assertAlmostEqual(mean_1, 40.724625956352185)
self.assertAlmostEqual(mean_2, 40.724101151614555)
def test_bias_spline(self):
ccd = MaskedCCD(self.image_file)
overscan = makeAmplifierGeometry(self.image_file)
for amp in ccd:
# Unmasked image
bs_i = imutils.bias_spline(ccd[amp].getImage(),
overscan.serial_overscan)
# Masked image
bs_m = imutils.bias_spline(ccd[amp], overscan.serial_overscan)
ny = len(bs_i)
for ii in range(ny):
self.assertEqual(interpolate.splev(ii, bs_i),
interpolate.splev(ii, bs_m))
def test_find_pixel_defects(self):
ccd1 = MaskedCCD(self.flat_file1)
ccd2 = MaskedCCD(self.flat_file2)
for amp in ccd1:
xcorr, xcorr_err = crossCorrelate(
ccd1.unbiased_and_trimmed_image(amp),
ccd2.unbiased_and_trimmed_image(amp), self.maxLag,
self.nSigmaClip, self.backgroundBinSize)
self.assertTrue(1 > xcorr[0][1] and 1 > xcorr[1][0])
def test_find_pixel_defects(self):
ccd = MaskedCCD(self.dark_file)
for amp in ccd:
bp = BrightPixels(ccd,
amp,
ccd.md.get('EXPTIME'),
self.gain,
ethresh=self.emin / 2.,
colthresh=100)
results = bp.find()
pixels = np.array(np.where(self.pixels[amp] == 1))
pixels = pixels.transpose()
pixels = [(x, y) for y, x in pixels]
pixels.sort()
self.assertEqual(pixels, results[0])
def test_bias_func(self):
bias_func = BiasFunc(self.bias_slope, self.bias_intercept)
ccd = MaskedCCD(self.image_file)
for amp in ccd:
bf_i = imutils.bias_func(ccd[amp].getImage(),
self.amp_geom.serial_overscan)
bf_m = imutils.bias_func(ccd[amp], self.amp_geom.serial_overscan)
for y in range(2022):
self.assertEqual(bf_i(y), bf_m(y))
self.assertAlmostEqual(bias_func(y), bf_m(y))
# Test that row-by-row median operates.
row_bias = imutils.bias_func(ccd[amp],
self.amp_geom.serial_overscan,
fit_order=-1)
for y in range(2022):
self.assertAlmostEqual(bf_i(y), row_bias(y), places=6)
def test_add_masks(self):
ccd = MaskedCCD(self.mask_image)
ccd.add_masks(self.summed_mask_file)
total_signal = (self.signal * (self.ymax - self.ymin) *
(self.xmax - self.xmin))
ny, nx = ccd[ccd.keys()[0]].getImage().getArray().shape
for amp in ccd:
self.assertEqual(sum(ccd[amp].getImage().getArray().flat),
total_signal)
stat_ctrl = ccd.setMask('BAD')
for amp in ccd:
stats = afwMath.makeStatistics(ccd[amp], afwMath.MEAN, stat_ctrl)
self.assertEqual(0, stats.getValue(afwMath.MEAN))
stat_ctrl = ccd.setMask(clear=True)
for amp in ccd:
stats = afwMath.makeStatistics(ccd[amp], afwMath.MEAN, stat_ctrl)
self.assertAlmostEqual(float(total_signal) / (nx * ny),
stats.getValue(afwMath.MEAN),
places=10)
def test_bias_frame_subtraction(self):
bias_file = 'bias_image.fits'
def bias_func_x(x):
return 100 * (np.sin(x / 509. * 2. * np.pi) + 1)
bias_func_y = BiasFunc(self.bias_slope, self.bias_intercept)
bias_ccd = sim_tools.CCD(exptime=self.exptime, gain=self.gain)
for amp in bias_ccd.segments:
imarr = bias_ccd.segments[amp].image.getArray()
yvals = np.arange(0, imarr.shape[0], dtype=np.float)
for x in range(imarr.shape[1]):
imarr[:, x] += bias_func_x(x) + bias_func_y(yvals)
bias_ccd.writeto(bias_file)
image_file = 'image_file.fits'
signal_level = 1000.
image_ccd = sim_tools.CCD(exptime=self.exptime, gain=self.gain)
for amp in image_ccd.segments:
imarr = image_ccd.segments[amp].image.getArray()
yvals = np.arange(0, imarr.shape[0], dtype=np.float)
for x in range(imarr.shape[1]):
imarr[:, x] +=\
bias_func_x(x) + bias_func_y(yvals)
imaging_section = image_ccd.segments[amp].image.Factory(
image_ccd.segments[amp].image, self.amp_geom.imaging)
imaging_section += signal_level
image_ccd.writeto(image_file)
ccd = MaskedCCD(image_file, bias_frame=bias_file)
for amp in ccd:
image = ccd.unbiased_and_trimmed_image(amp)
self.assertAlmostEqual(
max(np.abs(image.getImage().getArray().ravel())) /
signal_level, 1, 6)
ccd = MaskedCCD(image_file)
for amp in ccd:
image = ccd.unbiased_and_trimmed_image(amp)
self.assertTrue(
max(np.abs(image.getImage().getArray().ravel())) /
signal_level > 1.01)
os.remove(bias_file)
os.remove(image_file)
def test_linearity_correction(self):
"""Linearity correction test."""
def linearity_correction(amp, flux):
"""Set the correction to be the amp number times the flux."""
return amp * flux
np.random.seed(1000)
ccd = MaskedCCD(self.image_file)
ccd_corr = MaskedCCD(self.image_file,
linearity_correction=linearity_correction)
for amp in ccd:
# Check that the ratio of the corrected to uncorrected
# flux is (almost) equal to the amp number.
ratio = np.median(
ccd_corr.unbiased_and_trimmed_image(
amp).getImage().array) / np.median(
ccd.unbiased_and_trimmed_image(amp).getImage().array)
self.assertAlmostEqual(amp, ratio, places=5)
def main(sensor_id, infile, output_dir='./', cti=None, do_trapping=False,
do_electronics=False):
processed_file = os.path.join(output_dir,
'{0}_processed.fits'.format(sensor_id))
corrected_file = os.path.join(output_dir,
'{0}_corrected.fits'.format(sensor_id))
## CTI parameters
if cti is None:
do_cti = False
cti = 0.0
else:
do_cti = True
cti_dict = {amp : cti for amp in range(1, 17)}
## Trapping parameters
if do_trapping:
# traps = [LinearTrap(4.0, 0.4, 1, 0.08),
# LogisticTrap(1000.0, 0.4, 1, 60000., 0.0002)]
traps = [LinearTrap(4.0, 0.4, 1, 0.08),
LogisticTrap(40.0, 0.4, 1, 17500., 0.001)]
else:
traps = None
traps_dict = {amp : traps for amp in range(1, 17)}
## Electronics parameters
if do_electronics:
scale = 0.0002
decay_time = 2.5
output_amps = {amp : FloatingOutputAmplifier(1.0, scale, decay_time) for amp in range(1, 17)}
else:
output_amps = {amp : BaseOutputAmplifier(1.0) for amp in range(1, 17)}
## Process infile
imsim = ImageSimulator.from_image_fits(infile, output_amps,
cti=cti_dict, traps=traps_dict)
imarr_results = imsim.image_readout(infile, outfile=processed_file)
ccd = MaskedCCD(processed_file)
hdulist = fits.HDUList()
with fits.open(infile) as template:
hdulist.append(template[0])
hdulist[0].header['ORIGFILE'] = hdulist[0].header['FILENAME']
hdulist[0].header['FILENAME'] = corrected_file
for amp in range(1, 17):
imarr = ccd.bias_subtracted_image(amp).getImage().getArray()
## Electronics Correction
if do_electronics:
E = electronics_operator(imarr, scale, decay_time, num_previous_pixels=15)
corrected_imarr = imarr - E
else:
corrected_imarr = imarr
## Trap Correction
if do_trapping:
T = trap_operator(imarr, *traps)
corrected_imarr = corrected_imarr - (1-cti)*T
else:
pass
## CTI Correction
if do_cti:
Dinv_cti = cti_inverse_operator(cti, imarr.shape[1])
for i in range(imarr.shape[0]):
corrected_imarr[i, :] = Dinv_cti*corrected_imarr[i, :]
else:
pass
hdulist.append(fits.ImageHDU(data=corrected_imarr,
header=template[amp].header))
with warnings.catch_warnings():
for warning in (UserWarning, AstropyWarning,
AstropyUserWarning):
warnings.filterwarnings('ignore', category=warning,
append=True)
fitsWriteto(hdulist, corrected_file, overwrite=True)
def test_setMask(self):
ccd = MaskedCCD(self.mask_image)
for mp, bit in self.mpd.items():
sctrl = ccd.setMask(mask_name=mp, clear=True)
self.assertEqual(sctrl.getAndMask(), 2**bit)
def main(sensor_id,
infile,
main_dir,
gain_file=None,
output_dir='./',
no_bias=False):
## Get existing parameter results
param_file = join(main_dir, '{0}_parameter_results.fits'.format(sensor_id))
param_results = OverscanParameterResults.from_fits(param_file)
cti_results = param_results.cti_results
drift_scales = param_results.drift_scales
decay_times = param_results.decay_times
trap_files = [
join(main_dir, '{0}_amp{1}_trap.pkl'.format(sensor_id, i))
for i in range(1, 17)
]
## Get gains
if gain_file is not None:
with open(gain_file, 'rb') as f:
gain_results = pickle.load(f)
gains = gain_results.get_amp_gains(sensor_id)
else:
gains = {i: 1.0 for i in range(1, 17)}
## Output filename
base = splitext(os.path.basename(infile))[0]
outfile = join(output_dir, '{0}_corrected.fits'.format(base))
## Include bias frame for calibration
if no_bias:
bias_frame = None
else:
bias_frame = join(main_dir, '{0}_superbias.fits'.format(sensor_id))
ccd = MaskedCCD(infile, bias_frame=bias_frame)
hdulist = fits.HDUList()
with fits.open(infile) as template:
hdulist.append(template[0])
## Perform correction amp by amp
for amp in range(1, 17):
imarr = ccd.bias_subtracted_image(
amp).getImage().getArray() * gains[amp]
## Electronics Correction
if drift_scales[amp] > 0.:
Linv = electronics_inverse_operator(imarr,
drift_scales[amp],
decay_times[amp],
num_previous_pixels=15)
corrected_imarr = imarr - Linv
else:
corrected_imarr = imarr
## Trap Correction
spltrap = pickle.load(open(trap_files[amp], 'rb'))
Tinv = trap_inverse_operator(corrected_imarr, spltrap)
corrected_imarr = corrected_imarr - (1 - cti_results[amp]) * Tinv
## Reassemble HDUList
hdulist.append(
fits.ImageHDU(data=corrected_imarr / gains[amp],
header=template[amp].header))
with warnings.catch_warnings():
for warning in (UserWarning, AstropyWarning,
AstropyUserWarning):
warnings.filterwarnings('ignore',
category=warning,
append=True)
fitsWriteto(hdulist, outfile, overwrite=True)
