def test_bias_image(self): ccd = MaskedCCD(self.image_file) overscan = makeAmplifierGeometry(self.image_file) ccd_mean = MaskedCCD(self.mean_image_file) overscan_mean = makeAmplifierGeometry(self.mean_image_file) for amp in ccd: for method in ['mean', 'row', 'func', 'spline']: if method == 'mean': my_bias_image = imutils.bias_image( ccd_mean[amp], overscan_mean.serial_overscan, bias_method=method, **self.kwargs) fracdiff = ((self.mean_bias_image.getArray() - my_bias_image.getArray()) / self.mean_bias_image.getArray()) self.assertTrue(max(np.abs(fracdiff.flat)) < 1.5e-3) else: my_bias_image = imutils.bias_image( ccd[amp], overscan.serial_overscan, bias_method=method, **self.kwargs) fracdiff = ((self.bias_image.getArray() - my_bias_image.getArray()) / self.bias_image.getArray()) self.assertTrue(max(np.abs(fracdiff.flat)) < 1e-6) my_bias_image = imutils.bias_image(ccd[amp], self.amp_geom.serial_overscan) fracdiff = ( (self.bias_image.getArray() - my_bias_image.getArray()) / self.bias_image.getArray()) self.assertTrue(max(np.abs(fracdiff.flat)) < 1e-6)
def test_bias_image(self): ccd = MaskedCCD(self.image_file) for amp in ccd: my_bias_image = ccd.bias_image(amp) fracdiff = ( (self.bias_image.getArray() - my_bias_image.getArray()) / self.bias_image.getArray()) self.assertTrue(max(np.abs(fracdiff.flat)) < 1e-6)
def test_unbias_and_trim(self): ccd = MaskedCCD(self.image_file) for amp in ccd: # # Test of corresponding MaskedCCD method. # image = ccd.unbiased_and_trimmed_image(amp) imarr = image.getImage().getArray() self.assertTrue(max(np.abs(imarr.flat)) < 1e-6)
def test_unbias_and_trim(self): ccd = MaskedCCD(self.image_file) for amp in ccd: image = imutils.unbias_and_trim(ccd[amp], self.amp_geom.serial_overscan, self.amp_geom.imaging) imarr = image.getImage().getArray() self.assertTrue(max(np.abs(imarr.flat)) < 1e-6) # # Test of corresponding MaskedCCD method. # image = ccd.unbiased_and_trimmed_image(amp) imarr = image.getImage().getArray() self.assertTrue(max(np.abs(imarr.flat)) < 1e-6)
def run(self, sensor_id, infiles, gains, bias_frame=None, linearity_correction=None): ## Calculate mean row for each flat file minflux = self.config.minflux maxflux = self.config.maxflux num_oscan_pixels = self.config.num_oscan_pixels fitter = OverscanFit(num_oscan_pixels=num_oscan_pixels, minflux=minflux, maxflux=maxflux) for i, infile in enumerate(infiles): if self.config.verbose: self.log.info("Processing {0}".format(infile)) ccd = MaskedCCD(infile, bias_frame=bias_frame, linearity_correction=linearity_correction) fitter.process_image(ccd, gains) output_dir = self.config.output_dir if self.config.output_file is None: output_file = os.path.join( output_dir, '{0}_overscan_results.fits'.format(sensor_id)) else: output_file = os.path.join(output_dir, self.config.output_file) if self.config.verbose: self.log.info("writing to {0}".format(output_file)) fitter.write_results(outfile=output_file) return output_file
def setUpClass(cls): ccd = sim_tools.CCD(exptime=cls.exptime, gain=cls.gain) for amp in ccd.segments: imarr = ccd.segments[amp].image.getArray() imarr[cls.ymin:cls.ymax, cls.xmin:cls.xmax] += cls.signal ccd.writeto(cls.mask_image) cls.mask_files = [] for mask_plane, bit in cls.mpd.items(): mask_file = 'mask_file_%s.fits' % mask_plane cls.mask_files.append(mask_file) masked_ccd = MaskedCCD(cls.mask_image) pixels, columns = {}, {} for amp in masked_ccd: bp = BrightPixels(masked_ccd, amp, cls.exptime, cls.gain, mask_plane=mask_plane, ethresh=cls.signal / 2.) pixels[amp], columns[amp] = bp.find() generate_mask(cls.mask_image, mask_file, mask_plane, pixels=pixels, columns=columns) cls.summed_mask_file = 'summed_mask_file.fits' add_mask_files(cls.mask_files, cls.summed_mask_file)
def test_stack(self): ccd = MaskedCCD(self.image_file) overscan = makeAmplifierGeometry(self.image_file) for method in ['mean', 'row', 'func']: corrected = [] for image in ccd.values(): corrected.append( imutils.unbias_and_trim(image, overscan.serial_overscan, bias_method=method, **self.kwargs).getImage()) stacked = imutils.stack(corrected) imarr = stacked.getArray() if method == 'mean': self.assertTrue(max(np.abs(imarr.flat)) < 2) else: self.assertTrue(max(np.abs(imarr.flat)) < 1e-6)
def test_detector_crosstalk(self): ccd = MaskedCCD(self.xtalk_file) ratios = crosstalk.detector_crosstalk(ccd, self.aggressor) for amp in ratios: if amp != self.aggressor: self.assertTrue( abs(ratios[amp][0] - self.xtalk_frac[amp]) < ratios[amp][1])
def test_bias_image(self): ccd = MaskedCCD(self.image_file) for amp in ccd: my_bias_image = imutils.bias_image(ccd[amp], self.amp_geom.serial_overscan) fracdiff = ( (self.bias_image.getArray() - my_bias_image.getArray()) / self.bias_image.getArray()) self.assertTrue(max(np.abs(fracdiff.flat)) < 1e-6)
def test_find_column_defects(self): ccd = MaskedCCD(self.sflat_file) for amp in ccd: dp = DarkPixels(ccd, amp, frac_thresh=0.8, colthresh=100) results = dp.find() columns = sorted(self.columns[amp]) self.assertEqual(len(columns), self.ncols) self.assertEqual(len(results[1]), self.ncols) self.assertEqual(columns, results[1])
def get_ccd_from_id(butler, data_id, mask_files, **kwargs): """Get a CCD image from a data_id If we are using `Butler` then this will take a data_id `dict` ojbect and return an `ExposureF` object If we are not using `Butler` (i.e., if bulter is `None`) then this will take a filename and return a `MaskedCCD` object Parameters ---------- butler : `Butler` or `None` Data Butler data_id : `dict` or `str` Data identier mask_files : `list` List of data_ids for the files to construct the pixel mask Keywords -------- bias_frame : `ExposureF` or `MaskedCCD` or `None` Object with the bias data masked_ccd : `bool` Use bulter only to get filename, return as MaskedCCD object Returns ------- ccd : `ExposureF` or `MaskedCCD` CCD data object """ bias_frame = kwargs.get('bias_frame', None) if butler is None: exposure = MaskedCCD(str(data_id), mask_files=mask_files, bias_frame=bias_frame) elif kwargs.get('masked_ccd', False): filepath = butler.get('raw_filename', data_id)[0][0:-3] exposure = MaskedCCD(str(filepath), mask_files=mask_files, bias_frame=bias_frame) else: exposure = butler.get('raw', data_id) apply_masks(butler, exposure, mask_files) return exposure
def test_find_pixel_defects(self): ccd = MaskedCCD(self.sflat_file) for amp in ccd: dp = DarkPixels(ccd, amp, frac_thresh=0.8, colthresh=100) results = dp.find() pixels = np.array(np.where(self.pixels[amp] == 1)) pixels = pixels.transpose() pixels = [(x, y) for y, x in pixels] pixels.sort() self.assertEqual(pixels, results[0])
def test_bias_row(self): ccd = MaskedCCD(self.image_file) overscan = makeAmplifierGeometry(self.image_file) for amp in ccd: # Unmasked image br_i = imutils.bias_row(ccd[amp].getImage(), overscan.serial_overscan) # Masked image br_m = imutils.bias_row(ccd[amp], overscan.serial_overscan) for ii in range(2022): self.assertEqual(br_i(ii), br_m(ii))
def test_find_column_defects(self): ccd = MaskedCCD(self.dark_file) for amp in ccd: bp = BrightPixels(ccd, amp, ccd.md.get('EXPTIME'), self.gain, ethresh=self.emin / 2., colthresh=100) results = bp.find() columns = sorted(self.columns[amp]) self.assertEqual(columns, results[1])
def test_unbias_and_trim(self): ccd = MaskedCCD(self.image_file) overscan = makeAmplifierGeometry(self.image_file) for amp in ccd: for method in ['mean', 'row', 'func', 'spline']: image = imutils.unbias_and_trim(ccd[amp], overscan.serial_overscan, bias_method=method, **self.kwargs) imarr = image.getImage().getArray() if method == 'mean': self.assertTrue(max(np.abs(imarr.flat)) < 2) else: self.assertTrue(max(np.abs(imarr.flat)) < 1e-6) # # Test of corresponding MaskedCCD method. # image = ccd.unbiased_and_trimmed_image( amp, overscan.serial_overscan, **self.kwargs) imarr = image.getImage().getArray() self.assertTrue(max(np.abs(imarr.flat)) < 1e-6)
def test_01_nonlinearity_task(self): """Run the NonlinearityTask""" task = NonlinearityTask() sensorid = 'RTM-022' task.run(sensorid, _detrespfile, outputfile=self.fits_file, plotfile=None) nlc = NonlinearityCorrection.create_from_fits_file(self.fits_file) ccd_1 = MaskedCCD(_flat_frame, bias_frame=_bias_frame) ccd_2 = MaskedCCD(_flat_frame, bias_frame=_bias_frame, linearity_correction=nlc) img_1 = ccd_1.unbiased_and_trimmed_image(1) img_2 = ccd_2.unbiased_and_trimmed_image(1) mean_1 = afwMath.makeStatistics(img_1, afwMath.MEAN, ccd_1.stat_ctrl).getValue() mean_2 = afwMath.makeStatistics(img_2, afwMath.MEAN, ccd_2.stat_ctrl).getValue() self.assertAlmostEqual(mean_1, 40.724625956352185) self.assertAlmostEqual(mean_2, 40.724101151614555)
def test_bias_spline(self): ccd = MaskedCCD(self.image_file) overscan = makeAmplifierGeometry(self.image_file) for amp in ccd: # Unmasked image bs_i = imutils.bias_spline(ccd[amp].getImage(), overscan.serial_overscan) # Masked image bs_m = imutils.bias_spline(ccd[amp], overscan.serial_overscan) ny = len(bs_i) for ii in range(ny): self.assertEqual(interpolate.splev(ii, bs_i), interpolate.splev(ii, bs_m))
def test_find_pixel_defects(self): ccd1 = MaskedCCD(self.flat_file1) ccd2 = MaskedCCD(self.flat_file2) for amp in ccd1: xcorr, xcorr_err = crossCorrelate( ccd1.unbiased_and_trimmed_image(amp), ccd2.unbiased_and_trimmed_image(amp), self.maxLag, self.nSigmaClip, self.backgroundBinSize) self.assertTrue(1 > xcorr[0][1] and 1 > xcorr[1][0])
def test_find_pixel_defects(self): ccd = MaskedCCD(self.dark_file) for amp in ccd: bp = BrightPixels(ccd, amp, ccd.md.get('EXPTIME'), self.gain, ethresh=self.emin / 2., colthresh=100) results = bp.find() pixels = np.array(np.where(self.pixels[amp] == 1)) pixels = pixels.transpose() pixels = [(x, y) for y, x in pixels] pixels.sort() self.assertEqual(pixels, results[0])
def test_bias_func(self): bias_func = BiasFunc(self.bias_slope, self.bias_intercept) ccd = MaskedCCD(self.image_file) for amp in ccd: bf_i = imutils.bias_func(ccd[amp].getImage(), self.amp_geom.serial_overscan) bf_m = imutils.bias_func(ccd[amp], self.amp_geom.serial_overscan) for y in range(2022): self.assertEqual(bf_i(y), bf_m(y)) self.assertAlmostEqual(bias_func(y), bf_m(y)) # Test that row-by-row median operates. row_bias = imutils.bias_func(ccd[amp], self.amp_geom.serial_overscan, fit_order=-1) for y in range(2022): self.assertAlmostEqual(bf_i(y), row_bias(y), places=6)
def test_add_masks(self): ccd = MaskedCCD(self.mask_image) ccd.add_masks(self.summed_mask_file) total_signal = (self.signal * (self.ymax - self.ymin) * (self.xmax - self.xmin)) ny, nx = ccd[ccd.keys()[0]].getImage().getArray().shape for amp in ccd: self.assertEqual(sum(ccd[amp].getImage().getArray().flat), total_signal) stat_ctrl = ccd.setMask('BAD') for amp in ccd: stats = afwMath.makeStatistics(ccd[amp], afwMath.MEAN, stat_ctrl) self.assertEqual(0, stats.getValue(afwMath.MEAN)) stat_ctrl = ccd.setMask(clear=True) for amp in ccd: stats = afwMath.makeStatistics(ccd[amp], afwMath.MEAN, stat_ctrl) self.assertAlmostEqual(float(total_signal) / (nx * ny), stats.getValue(afwMath.MEAN), places=10)
def test_bias_frame_subtraction(self): bias_file = 'bias_image.fits' def bias_func_x(x): return 100 * (np.sin(x / 509. * 2. * np.pi) + 1) bias_func_y = BiasFunc(self.bias_slope, self.bias_intercept) bias_ccd = sim_tools.CCD(exptime=self.exptime, gain=self.gain) for amp in bias_ccd.segments: imarr = bias_ccd.segments[amp].image.getArray() yvals = np.arange(0, imarr.shape[0], dtype=np.float) for x in range(imarr.shape[1]): imarr[:, x] += bias_func_x(x) + bias_func_y(yvals) bias_ccd.writeto(bias_file) image_file = 'image_file.fits' signal_level = 1000. image_ccd = sim_tools.CCD(exptime=self.exptime, gain=self.gain) for amp in image_ccd.segments: imarr = image_ccd.segments[amp].image.getArray() yvals = np.arange(0, imarr.shape[0], dtype=np.float) for x in range(imarr.shape[1]): imarr[:, x] +=\ bias_func_x(x) + bias_func_y(yvals) imaging_section = image_ccd.segments[amp].image.Factory( image_ccd.segments[amp].image, self.amp_geom.imaging) imaging_section += signal_level image_ccd.writeto(image_file) ccd = MaskedCCD(image_file, bias_frame=bias_file) for amp in ccd: image = ccd.unbiased_and_trimmed_image(amp) self.assertAlmostEqual( max(np.abs(image.getImage().getArray().ravel())) / signal_level, 1, 6) ccd = MaskedCCD(image_file) for amp in ccd: image = ccd.unbiased_and_trimmed_image(amp) self.assertTrue( max(np.abs(image.getImage().getArray().ravel())) / signal_level > 1.01) os.remove(bias_file) os.remove(image_file)
def test_linearity_correction(self): """Linearity correction test.""" def linearity_correction(amp, flux): """Set the correction to be the amp number times the flux.""" return amp * flux np.random.seed(1000) ccd = MaskedCCD(self.image_file) ccd_corr = MaskedCCD(self.image_file, linearity_correction=linearity_correction) for amp in ccd: # Check that the ratio of the corrected to uncorrected # flux is (almost) equal to the amp number. ratio = np.median( ccd_corr.unbiased_and_trimmed_image( amp).getImage().array) / np.median( ccd.unbiased_and_trimmed_image(amp).getImage().array) self.assertAlmostEqual(amp, ratio, places=5)
def main(sensor_id, infile, output_dir='./', cti=None, do_trapping=False, do_electronics=False): processed_file = os.path.join(output_dir, '{0}_processed.fits'.format(sensor_id)) corrected_file = os.path.join(output_dir, '{0}_corrected.fits'.format(sensor_id)) ## CTI parameters if cti is None: do_cti = False cti = 0.0 else: do_cti = True cti_dict = {amp : cti for amp in range(1, 17)} ## Trapping parameters if do_trapping: # traps = [LinearTrap(4.0, 0.4, 1, 0.08), # LogisticTrap(1000.0, 0.4, 1, 60000., 0.0002)] traps = [LinearTrap(4.0, 0.4, 1, 0.08), LogisticTrap(40.0, 0.4, 1, 17500., 0.001)] else: traps = None traps_dict = {amp : traps for amp in range(1, 17)} ## Electronics parameters if do_electronics: scale = 0.0002 decay_time = 2.5 output_amps = {amp : FloatingOutputAmplifier(1.0, scale, decay_time) for amp in range(1, 17)} else: output_amps = {amp : BaseOutputAmplifier(1.0) for amp in range(1, 17)} ## Process infile imsim = ImageSimulator.from_image_fits(infile, output_amps, cti=cti_dict, traps=traps_dict) imarr_results = imsim.image_readout(infile, outfile=processed_file) ccd = MaskedCCD(processed_file) hdulist = fits.HDUList() with fits.open(infile) as template: hdulist.append(template[0]) hdulist[0].header['ORIGFILE'] = hdulist[0].header['FILENAME'] hdulist[0].header['FILENAME'] = corrected_file for amp in range(1, 17): imarr = ccd.bias_subtracted_image(amp).getImage().getArray() ## Electronics Correction if do_electronics: E = electronics_operator(imarr, scale, decay_time, num_previous_pixels=15) corrected_imarr = imarr - E else: corrected_imarr = imarr ## Trap Correction if do_trapping: T = trap_operator(imarr, *traps) corrected_imarr = corrected_imarr - (1-cti)*T else: pass ## CTI Correction if do_cti: Dinv_cti = cti_inverse_operator(cti, imarr.shape[1]) for i in range(imarr.shape[0]): corrected_imarr[i, :] = Dinv_cti*corrected_imarr[i, :] else: pass hdulist.append(fits.ImageHDU(data=corrected_imarr, header=template[amp].header)) with warnings.catch_warnings(): for warning in (UserWarning, AstropyWarning, AstropyUserWarning): warnings.filterwarnings('ignore', category=warning, append=True) fitsWriteto(hdulist, corrected_file, overwrite=True)
def test_setMask(self): ccd = MaskedCCD(self.mask_image) for mp, bit in self.mpd.items(): sctrl = ccd.setMask(mask_name=mp, clear=True) self.assertEqual(sctrl.getAndMask(), 2**bit)
def main(sensor_id, infile, main_dir, gain_file=None, output_dir='./', no_bias=False): ## Get existing parameter results param_file = join(main_dir, '{0}_parameter_results.fits'.format(sensor_id)) param_results = OverscanParameterResults.from_fits(param_file) cti_results = param_results.cti_results drift_scales = param_results.drift_scales decay_times = param_results.decay_times trap_files = [ join(main_dir, '{0}_amp{1}_trap.pkl'.format(sensor_id, i)) for i in range(1, 17) ] ## Get gains if gain_file is not None: with open(gain_file, 'rb') as f: gain_results = pickle.load(f) gains = gain_results.get_amp_gains(sensor_id) else: gains = {i: 1.0 for i in range(1, 17)} ## Output filename base = splitext(os.path.basename(infile))[0] outfile = join(output_dir, '{0}_corrected.fits'.format(base)) ## Include bias frame for calibration if no_bias: bias_frame = None else: bias_frame = join(main_dir, '{0}_superbias.fits'.format(sensor_id)) ccd = MaskedCCD(infile, bias_frame=bias_frame) hdulist = fits.HDUList() with fits.open(infile) as template: hdulist.append(template[0]) ## Perform correction amp by amp for amp in range(1, 17): imarr = ccd.bias_subtracted_image( amp).getImage().getArray() * gains[amp] ## Electronics Correction if drift_scales[amp] > 0.: Linv = electronics_inverse_operator(imarr, drift_scales[amp], decay_times[amp], num_previous_pixels=15) corrected_imarr = imarr - Linv else: corrected_imarr = imarr ## Trap Correction spltrap = pickle.load(open(trap_files[amp], 'rb')) Tinv = trap_inverse_operator(corrected_imarr, spltrap) corrected_imarr = corrected_imarr - (1 - cti_results[amp]) * Tinv ## Reassemble HDUList hdulist.append( fits.ImageHDU(data=corrected_imarr / gains[amp], header=template[amp].header)) with warnings.catch_warnings(): for warning in (UserWarning, AstropyWarning, AstropyUserWarning): warnings.filterwarnings('ignore', category=warning, append=True) fitsWriteto(hdulist, outfile, overwrite=True)