def superflat(files, bias_frame=None, outfile='superflat.fits', bitpix=-32, bias_subtract=True): """ The superflat is created by bias-offset correcting the input files and median-ing them together. """ # Get overscan region. overscan = makeAmplifierGeometry(files[0]).serial_overscan # Use the first file as a template for the fits output. output = fits.open(files[0]) for amp in imutils.allAmps(files[0]): images = afwImage.vectorImageF() for infile in files: image = afwImage.ImageF(infile, imutils.dm_hdu(amp)) if bias_subtract: if bias_frame: bias_image = afwImage.ImageF(bias_frame, imutils.dm_hdu(amp)) image = bias_subtracted_image(image, bias_image, overscan) else: image -= imutils.bias_image(image, overscan, statistic=np.median) images.push_back(image) median_image = afwMath.statisticsStack(images, afwMath.MEDIAN) output[amp].data = median_image.getArray() if bitpix is not None: imutils.set_bitpix(output[amp], bitpix) fitsWriteto(output, outfile, clobber=True) return outfile
def superflat(files, bias_frame=None, outfile='superflat.fits', bitpix=None, bias_subtract=True, bias_method='row'): """ The superflat is created by bias-offset correcting the input files and median-ing them together. """ # Get overscan region. overscan = makeAmplifierGeometry(files[0]).serial_overscan output_images = dict() for amp in imutils.allAmps(files[0]): images = [] for infile in files: image = afwImage.ImageF(infile, imutils.dm_hdu(amp)) if bias_subtract: if bias_frame: bias_image = afwImage.ImageF(bias_frame, imutils.dm_hdu(amp)) image = bias_subtracted_image(image, bias_image, overscan, bias_method) else: image -= imutils.bias_image(im=image, overscan=overscan, bias_method=bias_method) images.append(image) if lsst.afw.__version__.startswith('12.0'): images = afwImage.vectorImageF(images) output_images[amp] = afwMath.statisticsStack(images, afwMath.MEDIAN) imutils.writeFits(output_images, outfile, files[0]) return outfile
def __init__(self, imfile, mask_files=(), bias_frame=None, applyMasks=True, linearity_correction=None): super(MaskedCCD, self).__init__() self.imfile = imfile self.md = imutils.Metadata(imfile) self.amp_geom = makeAmplifierGeometry(imfile) all_amps = imutils.allAmps(imfile) for amp in all_amps: image = afwImage.ImageF(imfile, imutils.dm_hdu(amp)) mask = afwImage_Mask(image.getDimensions()) self[amp] = afwImage.MaskedImageF(image, mask) self._added_mask_types = [] for mask_file in mask_files: self.add_masks(mask_file) self.stat_ctrl = afwMath.StatisticsControl() if mask_files: self.setAllMasks() if bias_frame is not None: self.bias_frame = MaskedCCD(bias_frame) else: self.bias_frame = None self._applyMasks = applyMasks self._linearity_correction = linearity_correction
def add_mask_files(mask_files, outfile, overwrite=True): amp_list = imutils.allAmps(mask_files[0]) masks = dict([(amp, afwImage_Mask(mask_files[0], imutils.dm_hdu(amp))) for amp in amp_list]) for mask_file in mask_files[1:]: for amp in masks: masks[amp] |= afwImage_Mask(mask_file, imutils.dm_hdu(amp)) output = fits.HDUList() output.append(fits.PrimaryHDU()) output[0].header['MASKTYPE'] = 'SUMMED_MASKS' fitsWriteto(output, outfile, overwrite=overwrite) for amp in masks: md = dafBase.PropertySet() md.set('EXTNAME', 'SEGMENT%s' % imutils.channelIds[amp]) masks[amp].writeFits(outfile, md, 'a') return masks
def run(self, sensor_id, dark_files, mask_files, gains, bias_frame=None): imutils.check_temperatures(dark_files, self.config.temp_set_point_tol, setpoint=self.config.temp_set_point, warn_only=True) median_images = {} for amp in imutils.allAmps(dark_files[0]): median_images[amp] = imutils.fits_median(dark_files, imutils.dm_hdu(amp)) medfile = os.path.join(self.config.output_dir, '%s_median_dark_bp.fits' % sensor_id) imutils.writeFits(median_images, medfile, dark_files[0]) ccd = MaskedCCD(medfile, mask_files=mask_files, bias_frame=bias_frame) md = imutils.Metadata(dark_files[0], 1) exptime = ccd.md.get('EXPTIME') total_bright_pixels = 0 total_bright_columns = 0 if self.config.verbose: self.log.info("Amp # bright pixels # bright columns") # # Write bright pixel and column counts to results file. # results_file = self.config.eotest_results_file if results_file is None: results_file = os.path.join(self.config.output_dir, '%s_eotest_results.fits' % sensor_id) results = EOTestResults(results_file, namps=len(ccd)) pixels = {} columns = {} for amp in ccd: bright_pixels = BrightPixels(ccd, amp, exptime, gains[amp]) pixels[amp], columns[amp] = bright_pixels.find() pix_count = len(pixels[amp]) col_count = len(columns[amp]) total_bright_pixels += pix_count total_bright_columns += col_count results.add_seg_result(amp, 'NUM_BRIGHT_PIXELS', pix_count) results.add_seg_result(amp, 'NUM_BRIGHT_COLUMNS', col_count) self.log.info("%2i %i %i" % (amp, pix_count, col_count)) if self.config.verbose: self.log.info("Total bright pixels: %i" % total_bright_pixels) self.log.info("Total bright columns: %i" % total_bright_columns) results.write(clobber=True) # Generate the mask file based on the pixel and columns. mask_file = os.path.join(self.config.output_dir, '%s_bright_pixel_mask.fits' % sensor_id) if os.path.isfile(mask_file): os.remove(mask_file) generate_mask(medfile, mask_file, self.config.mask_plane, pixels=pixels, columns=columns)
def add_masks(self, mask_file): """ Add a masks from a mask file by or-ing with existing masks. """ md = imutils.Metadata(mask_file) self._added_mask_types.append(md('MASKTYPE')) for amp in self: curr_mask = self[amp].getMask() curr_mask |= afwImage_Mask(mask_file, imutils.dm_hdu(amp))
def get_overscans(infile, oscan_indices=None): "Return the overscan data as numpy arrays." if oscan_indices is None: y0, y1, x0, x1 = get_oscan_indices(infile) else: y0, y1, x0, x1 = oscan_indices overscans = dict() for amp in imutils.allAmps(infile): oscan_data = afw_image.ImageF(infile, imutils.dm_hdu(amp)).getArray() overscans[amp] = copy.deepcopy(oscan_data[y0:y1, x0:x1]) return overscans
def __call__(self, fitsfile, amps): tot_dark_ccd = 0 tot_dark_per_amp = [] pix_per_amp = [] col_per_amp = [] for amp in amps: try: tot, pixels = self.dark_pix(fitsfile, imutils.dm_hdu(amp)) tot_dark_ccd += tot tot_dark_per_amp.append(tot) pix_per_amp.append(pixels) except: print "Failed dark pixels for hdu ", amp, " ", \ imutils.dm_hdu(amp) traceback.print_exc(file=sys.stdout) continue return tot_dark_ccd, tot_dark_per_amp, pix_per_amp, col_per_amp
def extract_unmasked_pixels(ccd, amp, gain, correction_image=None): subimage = ccd.unbiased_and_trimmed_image(amp) imarr = subimage.getImage().getArray() if correction_image is not None: correction = afwImage.ImageF(correction_image, imutils.dm_hdu(amp)) correction = correction.Factory(correction, ccd.amp_geom.imaging) imarr *= correction.getArray() maskarr = subimage.getMask().getArray() if imarr.shape != maskarr.shape: raise RuntimeError("image and mask subarray shapes differ") indx = np.where(maskarr == 0) return [x * gain for x in imarr[indx].flat]
def get_mean_overscans(infiles, oscan_indices=None): "Compute the mean of the overscans of the list of files" if oscan_indices is None: y0, y1, x0, x1 = get_oscan_indices(infiles[0]) else: y0, y1, x0, x1 = oscan_indices mean_overscans = defaultdict(list) for infile in infiles: for amp in imutils.allAmps(infiles[0]): oscan_data \ = afw_image.ImageF(infile, imutils.dm_hdu(amp)).getArray() mean_overscans[amp].append(copy.deepcopy(oscan_data[y0:y1, x0:x1])) for amp, images in mean_overscans.items(): mean_overscans[amp] = sum(images) / float(len(images)) return mean_overscans
def ctesim(infile, pcti=0, scti=0, verbose=False): input = fits.open(infile) amps = [ i for i in range(1, len(input)) if input[i].name.upper().startswith('SEGMENT') ] if not isinstance(pcti, dict): pcti = {amp: pcti for amp in amps} if not isinstance(scti, dict): scti = {amp: scti for amp in amps} segments = {} for amp in amps: if verbose: print("ctesim: working on amp", amp) image = afwImage.ImageF(infile, imutils.dm_hdu(amp)) geom = makeAmplifierGeometry(infile) # # Temporarily remove readout bias median. # bias_med = imutils.median(image.Factory(image, geom.serial_overscan)) image -= bias_med imarr = image.getArray() outimage = afwImage.ImageF(image, True) outarr = outimage.getArray() if pcti[amp] != 0: pcte_matrix = cte_matrix(imarr.shape[0], pcti[amp]) for col in range(0, imarr.shape[1]): outarr[:, col] = np.dot(pcte_matrix, imarr[:, col]) if scti[amp] != 0: scte_matrix = cte_matrix(imarr.shape[1], scti[amp]) for row in range(0, imarr.shape[0]): outarr[row, :] = np.dot(scte_matrix, outarr[row, :]) # # Restore readout bias # outarr += bias_med segments[amp] = outimage return fitsFile(segments, input)
def apply_cte(infile, pcti=None, scti=None, verbose=False, amps=tuple(range(1, 17))): """ Function to apply distinct levels of parallel cte and/or serial cte on each amplifier in an input sensor image. """ if pcti is None: pcti = dict([(amp, 0) for amp in amps]) if scti is None: scti = dict([(amp, 0) for amp in amps]) segments = {} for amp in amps: if verbose: print("apply_cte: working on amp", amp) image = afwImage.ImageF(infile, imutils.dm_hdu(amp)) geom = sensorTest.makeAmplifierGeometry(infile) outimage = eotest_utils.ImageTools.applyCTI(image, geom.serial_overscan, pcti[amp], scti[amp], verbose) segments[amp] = outimage return fitsFile(segments, fits.open(infile))
def run(self, sensor_id, pre_flat_darks, flat, post_flat_darks, mask_files, gains): darks = list(pre_flat_darks) + list(post_flat_darks) imutils.check_temperatures(darks, self.config.temp_set_point_tol, setpoint=self.config.temp_set_point, warn_only=True) # Check that pre-flat dark frames all have the same exposure time md = imutils.Metadata(pre_flat_darks[0], 1) exptime = md.get('EXPTIME') for item in pre_flat_darks[1:]: md = imutils.Metadata(item, 1) if exptime != md.get('EXPTIME'): raise RuntimeError("Exposure times of pre-flat darks differ.") # Make a median image of the preflat darks median_images = {} for amp in imutils.allAmps(darks[0]): median_images[amp] = imutils.fits_median(pre_flat_darks, imutils.dm_hdu(amp)) medfile = os.path.join(self.config.output_dir, '%s_persistence_dark_median.fits' % sensor_id) imutils.writeFits(median_images, medfile, darks[0]) ccd = MaskedCCD(medfile, mask_files=mask_files) # Define the sub-region for assessing the deferred charge. # This is the same bounding box for all segments, so use amp=1. image = ccd.unbiased_and_trimmed_image(1) xllc = ((image.getWidth() - self.config.region_size) / 2. - self.config.region_x_offset) yllc = ((image.getHeight() - self.config.region_size) / 2. - self.config.region_y_offset) imaging_reg = afwGeom.Box2I( afwGeom.Point2I(int(xllc), int(yllc)), afwGeom.Extent2I(self.config.region_size, self.config.region_size)) overscan = ccd.amp_geom.serial_overscan # Compute reference dark current for each segment. dc_ref = {} for amp in ccd: mi = imutils.unbias_and_trim(ccd[amp], overscan, imaging_reg) dc_ref[amp] = afwMath.makeStatistics(mi, afwMath.MEDIAN, ccd.stat_ctrl).getValue() dc_ref[amp] *= gains[amp] / exptime # Extract reference time for computing the time dependence # of the deferred charge as the observation time + exposure time # from the saturated flat. tref = readout_time(flat) # Loop over post-flat darks, compute median e-/pixel in # subregion, subtract dc_ref*exptime, persist, and report the # deferred charge vs time (using MJD-OBS + EXPTIME) for each amp. deferred_charges = [] times = [] for dark in post_flat_darks: ccd = MaskedCCD(dark, mask_files=mask_files) dt = readout_time(dark) - tref times.append(dt.sec) exptime = ccd.md.get('EXPTIME') charge = {} for amp in ccd: mi = imutils.unbias_and_trim(ccd[amp], overscan, imaging_reg) estimators = afwMath.MEDIAN | afwMath.STDEV stats = afwMath.makeStatistics(mi, estimators, ccd.stat_ctrl) value = (stats.getValue(afwMath.MEDIAN) * gains[amp] - dc_ref[amp] * exptime) stdev = (stats.getValue(afwMath.STDEV) * gains[amp] - dc_ref[amp] * exptime) charge[amp] = (value, stdev) deferred_charges.append(charge) if self.config.verbose: for amp in ccd: self.log.info("amp: %i" % amp) for i, time in enumerate(times): self.log.info("%.1f %e %e" % (time, deferred_charges[i][amp][0], deferred_charges[i][amp][1])) outfile = os.path.join(self.config.output_dir, '%s_persistence.fits' % sensor_id) self.write(times, deferred_charges, outfile, clobber=True)
fratio = np.mean(imarr1 / imarr2) imarr2 *= fratio # Calculate the mean value of the flat field images. fmean = (np.mean(imarr1) + np.mean(imarr2)) / 2. # Calculate the variance of the flat difference image. fvar = np.var(imarr1 - imarr2) / 2. gains.append(fvar / fmean) gain = 1. / np.median(gains) # gain in Ne/DN return gain, im1, im2 if __name__ == '__main__': import os from .sim_tools import simulateFlat file1 = 'test_flat1.fits' file2 = 'test_flat2.fits' hdus = 2 simulateFlat(file1, 4000, 5, hdus=hdus) simulateFlat(file2, 4000, 5, hdus=hdus) for amp in range(1, hdus + 1): image1 = afwImage.ImageF(file1, imutils.dm_hdu(amp)) image2 = afwImage.ImageF(file2, imutils.dm_hdu(amp)) gain, im1, im2 = flat_gain(image1, image2, count=1000) print(amp, gain) os.remove(file1) os.remove(file2)
def run(self, sensor_id, dark_files, mask_files, gains, bias_frame=None): imutils.check_temperatures(dark_files, self.config.temp_set_point_tol, setpoint=self.config.temp_set_point, warn_only=True) median_images = {} md = imutils.Metadata(dark_files[0], 1) for amp in imutils.allAmps(dark_files[0]): median_images[amp] = imutils.fits_median(dark_files, imutils.dm_hdu(amp)) medfile = os.path.join(self.config.output_dir, '%s_median_dark_current.fits' % sensor_id) imutils.writeFits(median_images, medfile, dark_files[0]) ccd = MaskedCCD(medfile, mask_files=mask_files, bias_frame=bias_frame) dark95s = {} exptime = md.get('EXPTIME') if self.config.verbose: self.log.info("Amp 95 percentile median") dark_curr_pixels = [] dark_curr_pixels_per_amp = {} for amp in ccd: imaging_region = ccd.amp_geom.imaging overscan = ccd.amp_geom.serial_overscan image = imutils.unbias_and_trim(ccd[amp].getImage(), overscan, imaging_region) mask = imutils.trim(ccd[amp].getMask(), imaging_region) imarr = image.getArray() mskarr = mask.getArray() pixels = imarr.reshape(1, imarr.shape[0] * imarr.shape[1])[0] masked = mskarr.reshape(1, mskarr.shape[0] * mskarr.shape[1])[0] unmasked = [ pixels[i] for i in range(len(pixels)) if masked[i] == 0 ] unmasked.sort() unmasked = np.array(unmasked) * gains[amp] / exptime dark_curr_pixels_per_amp[amp] = unmasked dark_curr_pixels.extend(unmasked) try: dark95s[amp] = unmasked[int(len(unmasked) * 0.95)] median = unmasked[len(unmasked) / 2] except IndexError as eobj: print str(eobj) dark95s[amp] = -1. median = -1. if self.config.verbose: self.log.info("%2i %.2e %.2e" % (amp, dark95s[amp], median)) # # Compute 95th percentile dark current for CCD as a whole. # dark_curr_pixels = sorted(dark_curr_pixels) darkcurr95 = dark_curr_pixels[int(len(dark_curr_pixels) * 0.95)] dark95mean = np.mean(dark95s.values()) if self.config.verbose: #self.log.info("CCD: mean 95 percentile value = %s" % dark95mean) self.log.info("CCD-wide 95 percentile value = %s" % darkcurr95) # # Update header of dark current median image file with dark # files used and dark95 values, and write dark95 values to the # eotest results file. # results_file = self.config.eotest_results_file if results_file is None: results_file = os.path.join(self.config.output_dir, '%s_eotest_results.fits' % sensor_id) results = EOTestResults(results_file, namps=len(ccd)) output = fits.open(medfile) for i, dark in enumerate(dark_files): output[0].header['DARK%02i' % i] = os.path.basename(dark) # Write overall dark current 95th percentile results.output['AMPLIFIER_RESULTS'].header['DARK95'] = darkcurr95 for amp in ccd: output[0].header['DARK95%s' % imutils.channelIds[amp]] = dark95s[amp] results.add_seg_result(amp, 'DARK_CURRENT_95', dark95s[amp]) fitsWriteto(output, medfile, clobber=True, checksum=True) results.write(clobber=True) return dark_curr_pixels_per_amp, dark95s
def _correction_images(self, ccd, correction_image): images = OrderedDict() for amp in ccd: correction = afwImage.ImageF(correction_image, imutils.dm_hdu(amp)) images[amp] = correction.Factory(correction, ccd.amp_geom.imaging) return images