def write_bias_subtracted_MEF(self, outfile, gains=None, overwrite=True): """ Write a bias-subtracted MEF file with the same format as the original raw FITS file. Parameters ---------- outfile: str Output filename. gains: dict [None] Gains to apply to pixel data. If None, then pixel values are written as ADUs. overwrite: bool [True] Flag to overwrite an existing output file. """ hdulist = fits.HDUList() with fits.open(self.imfile) as template: hdulist.append(template[0]) hdulist[0].header['ORIGFILE'] = hdulist[0].header['FILENAME'] hdulist[0].header['FILENAME'] = outfile for amp in self: imarr = self.bias_subtracted_image(amp).getImage().getArray() if gains is not None: imarr *= gains[amp] hdulist.append( fits.CompImageHDU(data=imarr, header=template[amp].header)) with warnings.catch_warnings(): for warning in (UserWarning, AstropyWarning, AstropyUserWarning): warnings.filterwarnings('ignore', category=warning, append=True) fitsWriteto(hdulist, outfile, overwrite=True)
def writeto(self, outfile, pars=None, bitpix=-32, obs_time=None): ccd_segments = [self.segments[amp] for amp in self.segments] output = fitsFile(ccd_segments) if pars is not None: output[0].header['CCDGAIN'] = pars.system_gain output[0].header['BIASLVL'] = pars.bias_level output[0].header['CCDNOISE'] = pars.bias_sigma output[0].header['RDNOISE'] = pars.read_noise output[0].header['DARKCURR'] = pars.dark_current for key, value in self.md.items(): output[0].header[key] = value if bitpix > 0: my_round = np.round else: # # Delete any BSCALE and BZERO entries, since we are # writing image data as floats. # try: del output[amp].header['BSCALE'] del output[amp].header['BZERO'] except KeyError: pass my_round = lambda x: x for hdu in output[1:-2]: hdu.data = np.array(my_round(hdu.data), dtype=self.dtypes[bitpix]) if obs_time is None: # Compute the start of the observation from the current time # minus the exposure time. obs_time = utcnow(dt=-output[0].header['EXPTIME']) output[0].header['DATE-OBS'] = obs_time.isot output[0].header['DATE'] = obs_time.isot output[0].header.set('MJD-OBS', value=float('%.5f' % obs_time.mjd)) fitsWriteto(output, outfile, clobber=True, checksum=True)
def write(self, outfile, overwrite=True): """ Write the results as a FITS binary table. """ nrows = sum([len(self[amp]) for amp in self]) output = fits.HDUList() output.append(fits.PrimaryHDU()) colnames = ['AMPLIFIER', 'XPOS', 'YPOS', 'TRAP_SIZE', 'A0', 'A1'] formats = 'IIIIEE' units = ['None', 'pixel', 'pixel', 'electrons', 'ADU', 'ADU'] columns = ([np.zeros(nrows, dtype=int)] * 4 + [np.zeros(nrows, dtype=float)] * 2) hdu = fitsTableFactory([ fits.Column(name=colname, format=format, unit=unit, array=column) for colname, format, unit, column in zip( colnames, formats, units, columns) ]) hdu.name = 'TRAPS' output.append(hdu) row = 0 for amp in self: for xpos, ypos, trap_size, a0, a1 in self[amp]: output['TRAPS'].data[row]['AMPLIFIER'] = amp output['TRAPS'].data[row]['XPOS'] = xpos output['TRAPS'].data[row]['YPOS'] = ypos output['TRAPS'].data[row]['TRAP_SIZE'] = trap_size output['TRAPS'].data[row]['A0'] = a0 output['TRAPS'].data[row]['A1'] = a1 row += 1 fitsWriteto(output, outfile, overwrite=overwrite)
def write(self, outfile=None, clobber=True): """ Write or update the output file. """ if outfile is None: outfile = self.infile fitsWriteto(self.output, outfile, clobber=clobber)
def test_raft_image_mosaic(self): """ Test of raft-level mosaicking code. """ infiles = sorted( glob.glob( os.path.join(_root_dir, 'S??', '*_lambda_flat_1000_*.fits'))) infiles = OrderedDict([(filename.split('/')[-2], filename) for filename in infiles]) test_files = dict() step = 100 level = step for slot, infile in list(infiles.items()): outfile = '%s_test_image_%05i.fits' % (slot, level) with fits.open(infile) as hdu_list: for hdu in hdu_list[1:17]: hdu.data = np.ones(hdu.data.shape, dtype=np.float32) * level level += step fitsWriteto(hdu_list, outfile, overwrite=True) test_files[slot] = outfile raft_mosaic = raftTest.RaftMosaic(test_files, bias_subtract=False) raft_mosaic.plot(title='Test pattern') plt.savefig(self.outfile)
def write(self, results, outfile, clobber=True): colnames = ['WAVELENGTH', 'STDEV', 'MEAN'] formats = 'IEE' my_types = dict((("I", np.int), ("E", np.float))) columns = [np.zeros(len(results), dtype=my_types[fmt]) for fmt in formats] units = ['nm', 'rms e-', 'e-'] hdu = fitsTableFactory([fits.Column(name=colnames[i], format=formats[i], unit=units[i], array=columns[i]) for i in range(len(colnames))]) hdu.name = 'PRNU_RESULTS' for i, wl in enumerate(results.keys()): hdu.data.field('WAVELENGTH')[i] = wl hdu.data.field('STDEV')[i] = results[wl][0] hdu.data.field('MEAN')[i] =results[wl][1] if os.path.isfile(outfile): output = fits.open(outfile) else: output = fits.HDUList() output.append(fits.PrimaryHDU()) try: output[hdu.name] = hdu except KeyError: output.append(hdu) fitsWriteto(output, outfile, clobber=clobber)
def superflat(files, bias_frame=None, outfile='superflat.fits', bitpix=-32, bias_subtract=True): """ The superflat is created by bias-offset correcting the input files and median-ing them together. """ # Get overscan region. overscan = makeAmplifierGeometry(files[0]).serial_overscan # Use the first file as a template for the fits output. output = fits.open(files[0]) for amp in imutils.allAmps(files[0]): images = afwImage.vectorImageF() for infile in files: image = afwImage.ImageF(infile, imutils.dm_hdu(amp)) if bias_subtract: if bias_frame: bias_image = afwImage.ImageF(bias_frame, imutils.dm_hdu(amp)) image = bias_subtracted_image(image, bias_image, overscan) else: image -= imutils.bias_image(image, overscan, statistic=np.median) images.push_back(image) median_image = afwMath.statisticsStack(images, afwMath.MEDIAN) output[amp].data = median_image.getArray() if bitpix is not None: imutils.set_bitpix(output[amp], bitpix) fitsWriteto(output, outfile, clobber=True) return outfile
def write(self, times, deferred_charges, outfile, clobber=True): colnames = ['TIME'] columns = [times] units = ['s'] for amp in deferred_charges[0]: colnames.extend(['MEDIAN%02i' % amp, 'STDEV%02i' % amp]) columns.extend([ np.array( [deferred_charges[i][amp][0] for i in range(len(times))]), np.array( [deferred_charges[i][amp][1] for i in range(len(times))]) ]) units.extend(['e-/pixel', 'rms e-/pixel']) formats = ['E'] * len(columns) HDUList = fits.HDUList() HDUList.append(fits.PrimaryHDU()) HDUList.append( fitsTableFactory([ fits.Column(name=colname, format=format, unit=unit, array=column) for colname, format, unit, column in zip( colnames, formats, units, columns) ])) HDUList[-1].name = 'IMAGE_PERSISTENCE_CURVES' HDUList[0].header['NAMPS'] = len(deferred_charges[0]) fitsWriteto(HDUList, outfile, clobber=clobber)
def write_test_image(outfile, emin=10, dark_curr=2e-3, exptime=10, gain=5, ccdtemp=-100, bias_level=1e2, bias_sigma=4, ncols=2, npix=100): ccd = sim_tools.CCD(exptime=exptime, gain=gain, ccdtemp=ccdtemp) ccd.add_bias(bias_level, bias_sigma) ccd.add_dark_current(level=dark_curr) # # Simulation code sets dark pixel values by nsig*seg.sigma(), # but we want to set using e- per sec, so need to compute # equivalent nsig. # nsig = (emin * exptime / gain) / ccd.segments[imutils.allAmps[0]].sigma() # columns = ccd.generate_dark_cols(ncols) ccd.add_dark_cols(columns, nsig=nsig) pixels = ccd.generate_dark_pix(npix) ccd.add_dark_pix(pixels, nsig=nsig) fitsWriteto(ccd, outfile)
def write_fits(self, outfile=None, clobber=True): if outfile is None: outfile = self.filename else: self.filename = outfile output = fits.HDUList() output.append(fits.PrimaryHDU(data=self.matrix)) fitsWriteto(output, outfile, clobber=clobber)
def writeFits(ccd_segments, outfile, clobber=True): output = fitsFile(ccd_segments) if clobber: try: os.remove(outfile) except OSError: pass fitsWriteto(output, outfile, clobber=clobber, checksum=True) return outfile
def writeFits(ccd_segments, outfile, overwrite=True): output = fitsFile(ccd_segments) if overwrite: try: os.remove(outfile) except OSError: pass fitsWriteto(output, outfile, overwrite=overwrite, checksum=True) return outfile
def run(self, sensor_id, infiles, mask_files, gains, binsize=1, bias_frame=None): outfile = os.path.join(self.config.output_dir, '%s_ptc.fits' % sensor_id) all_amps = imutils.allAmps(infiles[0]) ptc_stats = dict([(amp, ([], [])) for amp in all_amps]) exposure = [] file1s = sorted([item for item in infiles if item.find('flat1') != -1]) for flat1 in file1s: flat2 = find_flat2(flat1) if self.config.verbose: self.log.info("processing %s" % flat1) exposure.append(exptime(flat1)) ccd1 = MaskedCCD(flat1, mask_files=mask_files, bias_frame=bias_frame) ccd2 = MaskedCCD(flat2, mask_files=mask_files, bias_frame=bias_frame) for amp in ccd1: results = flat_pair_stats(ccd1, ccd2, amp, mask_files=mask_files, bias_frame=bias_frame) ptc_stats[amp][0].append(results.flat_mean) ptc_stats[amp][1].append(results.flat_var) self._fit_curves(ptc_stats, sensor_id) output = fits.HDUList() output.append(fits.PrimaryHDU()) colnames = ['EXPOSURE'] units = ['seconds'] columns = [np.array(exposure, dtype=np.float)] for amp in all_amps: colnames.extend(['AMP%02i_MEAN' % amp, 'AMP%02i_VAR' % amp]) units.extend(['ADU', 'ADU**2']) columns.extend([ np.array(ptc_stats[amp][0], dtype=np.float), np.array(ptc_stats[amp][1], dtype=np.float) ]) formats = 'E' * len(colnames) fits_cols = [ fits.Column(name=colnames[i], format=formats[i], unit=units[i], array=columns[i]) for i in range(len(columns)) ] output.append(fitsTableFactory(fits_cols)) output[-1].name = 'PTC_STATS' output[0].header['NAMPS'] = len(all_amps) fitsWriteto(output, outfile, clobber=True)
def extract_det_response(self): outfile = os.path.join( self.config.output_dir, '%s_det_response_linearity.fits' % self.sensor_id) file1s = sorted([ item for item in self.infiles if item.find('flat1') != -1 or item.find('linearity_flat') != -1 ]) if self.config.verbose: self.log.info("writing to %s" % outfile) self._create_detresp_fits_output(len(file1s), files1[0]) for row, file1 in enumerate(file1s): if self.config.verbose: self.log.info("processing %s" % file1) try: file2 = find_flat2(file1) except IndexError: # Just use flat1 again since only average is taken and # FPN subtraction isn't needed. file2 = file1 flat1 = MaskedCCD(file1, mask_files=self.mask_files, bias_frame=self.bias_frame, linearity_correction=self.linearity_correction) flat2 = MaskedCCD(file2, mask_files=self.mask_files, bias_frame=self.bias_frame, linearity_correction=self.linearity_correction) if flat1.md.get('EXPTIME') != flat2.md.get('EXPTIME'): raise RuntimeError( "Exposure times do not match for:\n%s\n%s\n" % (file1, file2)) if (flat1.md.get('MONDIODE') != 0 and flat2.md.get('MONDIODE') != 0): flux = abs( flat1.md.get('EXPTIME') * flat1.md.get('MONDIODE') + flat2.md.get('EXPTIME') * flat2.md.get('MONDIODE')) / 2. else: flux = flat1.md.get('EXPTIME') self.output[-1].data.field('FLUX')[row] = flux for amp in flat1: # Convert to e- and write out for each segment. signal = pair_mean(flat1, flat2, amp) * self.gains[amp] self.output[-1].data.field('AMP%02i_SIGNAL' % amp)[row] = signal self.output[0].header['NAMPS'] = len(flat1) fitsWriteto(self.output, outfile, overwrite=True) return outfile
def add_mask_files(mask_files, outfile, overwrite=True): amp_list = imutils.allAmps(mask_files[0]) masks = dict([(amp, afwImage_Mask(mask_files[0], imutils.dm_hdu(amp))) for amp in amp_list]) for mask_file in mask_files[1:]: for amp in masks: masks[amp] |= afwImage_Mask(mask_file, imutils.dm_hdu(amp)) output = fits.HDUList() output.append(fits.PrimaryHDU()) output[0].header['MASKTYPE'] = 'SUMMED_MASKS' fitsWriteto(output, outfile, overwrite=overwrite) for amp in masks: md = dafBase.PropertySet() md.set('EXTNAME', 'SEGMENT%s' % imutils.channelIds[amp]) masks[amp].writeFits(outfile, md, 'a') return masks
def write_eotest_output(self, BFResults, sensor_id, meanidx=0): """Write the correlation curves to a FITS file for plotting, and the BF results to the eotest results file.""" outfile = os.path.join(self.config.output_dir, '%s_bf.fits' % sensor_id) output = fits.HDUList() output.append(fits.PrimaryHDU()) colnames = [] units = [] columns = [] for amp in BFResults: colnames.extend(['AMP%02i_XCORR' % amp, 'AMP%02i_XCORR_ERR' % amp, 'AMP%02i_YCORR' % amp, 'AMP%02i_YCORR_ERR' % amp, 'AMP%02i_MEAN' % amp]) units.extend( ['Unitless', 'Unitless', 'Unitless', 'Unitless', 'ADU']) columns.extend([np.array(BFResults[amp][0], dtype=np.float), np.array(BFResults[amp][1], dtype=np.float), np.array(BFResults[amp][2], dtype=np.float), np.array(BFResults[amp][3], dtype=np.float), np.array(BFResults[amp][4], dtype=np.float)]) formats = 'E'*len(colnames) fits_cols = [fits.Column(name=colnames[i], format=formats[i], unit=units[i], array=columns[i]) for i in range(len(columns))] output.append(fitsTools.fitsTableFactory(fits_cols)) output[-1].name = 'BF_STATS' output[0].header['NAMPS'] = len(BFResults) fitsTools.fitsWriteto(output, outfile, clobber=True) # Output a file of the coefficients at a given mean, given # as the index of the exposure in the list. results_file = self.config.eotest_results_file if results_file is None: results_file = os.path.join(self.config.output_dir, '%s_eotest_results.fits' % sensor_id) results = EOTestResults(results_file, namps=len(BFResults)) for amp in BFResults: results.add_seg_result(amp, 'BF_XCORR', BFResults[amp][0][meanidx]) results.add_seg_result(amp, 'BF_XCORR_ERR', BFResults[amp][1][meanidx]) results.add_seg_result(amp, 'BF_YCORR', BFResults[amp][2][meanidx]) results.add_seg_result(amp, 'BF_YCORR_ERR', BFResults[amp][3][meanidx]) results.add_seg_result(amp, 'BF_MEAN', BFResults[amp][4][meanidx]) results.write(clobber=True)
def __call__(self, infile, outfile, overwrite=True): if self.verbose: print("processing", infile) # Recompute the amplifier geometry using NAXIS[12] from # first image extension of input file. self.geom.compute_geometry(fitsfile=infile) self.input = fits.open(infile) self.output = fits.open(infile, do_not_scale_image_data=True) prototypes = sensorTest.fits_headers.fits_headers() # Primary HDU try: self.output[0].header.set('MJD', self.output[0].header['JD'] - 2400000.5) except KeyError: from astropy.time import Time t = Time(self.output[0].header['DATE-OBS']) self.output[0].header.set('MJD', t.jd - 2400000.5) self.output[0].header.set('FILENAME', os.path.basename(infile)) self.output[0].header.set('SEQNUM', self._seqnum(infile)) self.output[0].header.set('HEADVER', self._headver()) self.output[0].header.set('LSST_NUM', self.args.sensor_id) self.output[0].header.set('CCD_MANU', self.args.vendor) self._update_keywords(0) # TEST_COND and CCD_COND extensions self._update_extension('TEST_COND', prototypes['TEST_COND']) self._update_extension('CCD_COND', prototypes['CCD_COND']) # Image extensions self._update_amp_keywords() # Special handling for monitoring diode current in BNL data. try: self.luts[0]['MONDIODE'] except KeyError: self._set_bnl_mondiode_keyword_value() fitsWriteto(self.output, outfile, overwrite=overwrite, checksum=True, output_verify='fix')
def write_fits_tables(self, outfile, clobber=True): amps = self.qe.keys() colnames = ['WAVELENGTH'] colnames.extend(['AMP%02i' % i for i in amps]) colnames.append('DEVICE_MEAN') columns = [self.wlarrs[1]] columns.extend([self.qe[i] for i in amps]) columns.append(self.ccd_qe) formats = ["E"] * len(columns) units = ["nm"] units.extend(["e-/photon %"] * (len(columns) - 1)) fits_cols = lambda coldata: [ fits.Column(name=colname, format=format, unit=unit, array=column) for colname, format, unit, column in coldata ] HDUList = fits.HDUList() HDUList.append(fits.PrimaryHDU()) HDUList.append( fitsTableFactory(fits_cols(zip(colnames, formats, units, columns)))) HDUList[-1].name = 'QE_CURVES' columns = [self.ccd_qe_band.keys()] columns.extend([np.array(self.qe_band[amp].values()) for amp in amps]) columns.append(np.array(self.ccd_qe_band.values())) colnames[0] = 'BAND' formats[0] = '2A' units[0] = None HDUList.append( fitsTableFactory(fits_cols(zip(colnames, formats, units, columns)))) HDUList[-1].name = 'QE_BANDS' HDUList[0].header['NAMPS'] = len(amps) fitsWriteto(HDUList, outfile, clobber=clobber)
def _write_read_noise_dists(outfile, Ntot, Nsys, gains, bias, sysnoise): output = fits.HDUList() output.append(fits.PrimaryHDU()) output[0].header['BIASFILE'] = bias output[0].header['SYSNFILE'] = str(sysnoise) for amp in Ntot: sigtot, sigsys = Ntot[amp], Nsys[amp] nread_col = fits.Column(name="TOTAL_NOISE", format="E", unit="e- rms", array=sigtot) nsys_col = fits.Column(name="SYSTEM_NOISE", format="E", unit="e- rms", array=sigsys) output.append(fitsTableFactory((nread_col, nsys_col))) output[amp].name = "SEGMENT%s" % imutils.channelIds[amp] output[0].header["GAIN%s" % imutils.channelIds[amp]] = gains[amp] output[0].header["SIGTOT%s" % imutils.channelIds[amp]] = \ imutils.median(sigtot) output[0].header["SIGSYS%s" % imutils.channelIds[amp]] = \ imutils.median(sigsys) fitsWriteto(output, outfile, clobber=True)
def write_to_fits(self, fits_file): """Write this object to a FITS file""" output = fits.HDUList() output.append(fits.PrimaryHDU()) col_prof_x = fits.Column(name='prof_x', format='%iE' % self._nxbins, unit='ADU', array=self._prof_x) col_prof_y = fits.Column(name='prof_y_corr', format='%iE' % self._nxbins, unit='ADU', array=self._prof_y) col_prof_yerr = fits.Column(name='prof_yerr', format='%iE' % self._nxbins, unit='ADU', array=self._prof_yerr) fits_cols = [col_prof_x, col_prof_y, col_prof_yerr] hdu = fitsTableFactory(fits_cols) hdu.name = 'nonlin' output.append(hdu) fitsWriteto(output, fits_file, overwrite=True)
def extract_det_response(self, use_exptime): max_pd_frac_dev = self.max_pd_frac_dev outfile = os.path.join(self.config.output_dir, '%s_det_response.fits' % self.sensor_id) file1s = sorted( [item for item in self.infiles if item.find('flat1') != -1]) if self.config.verbose: self.log.info("writing to %s" % outfile) self._create_detresp_fits_output(len(file1s)) for row, file1 in enumerate(file1s): try: file2 = find_flat2(file1) except IndexError: # Just use flat1 again since only average is taken and # FPN subtraction isn't needed. file2 = file1 if self.config.verbose: self.log.info( "processing\n %s as flat1 and\n %s as flat2" % (file1, file2)) flat1 = MaskedCCD(file1, mask_files=self.mask_files, bias_frame=self.bias_frame) flat2 = MaskedCCD(file2, mask_files=self.mask_files, bias_frame=self.bias_frame) pd1 = flat1.md.get('MONDIODE') pd2 = flat2.md.get('MONDIODE') exptime1 = flat1.md.get('EXPTIME') exptime2 = flat2.md.get('EXPTIME') if exptime1 != exptime2: raise RuntimeError( "Exposure times do not match for:\n%s\n%s\n" % (file1, file2)) if (use_exptime or isinstance(pd1, str) or isinstance(pd2, str) or pd1 == 0 or pd2 == 0): flux = exptime1 else: flux = abs(pd1 * exptime1 + pd2 * exptime2) / 2. if np.abs((pd1 - pd2) / ((pd1 + pd2) / 2.)) > max_pd_frac_dev: self.log.info( "Skipping %s and %s since MONDIODE values do not agree to %.1f%%" % (file1, file2, max_pd_frac_dev * 100.)) continue if self.config.verbose: self.log.info(' row = %s' % row) self.log.info(' pd1, pd2 = %s, %s' % (pd1, pd2)) self.log.info(' exptime1, exptime2 = %s, %s ' % (exptime1, exptime2)) self.log.info(' flux = %s' % flux) self.log.info(' flux/exptime = %s' % (flux / exptime1, )) self.output[-1].data.field('FLUX')[row] = flux for amp in flat1: # Convert to e- and write out for each segment. signal = pair_mean(flat1, flat2, amp) * self.gains[amp] self.output[-1].data.field('AMP%02i_SIGNAL' % amp)[row] = signal self.output[0].header['NAMPS'] = len(flat1) fitsWriteto(self.output, outfile, clobber=True) return outfile
amps=tuple(range(1, 17))): """ Function to apply distinct levels of parallel cte and/or serial cte on each amplifier in an input sensor image. """ if pcti is None: pcti = dict([(amp, 0) for amp in amps]) if scti is None: scti = dict([(amp, 0) for amp in amps]) segments = {} for amp in amps: if verbose: print("apply_cte: working on amp", amp) image = afwImage.ImageF(infile, imutils.dm_hdu(amp)) geom = sensorTest.makeAmplifierGeometry(infile) outimage = eotest_utils.ImageTools.applyCTI(image, geom.serial_overscan, pcti[amp], scti[amp], verbose) segments[amp] = outimage return fitsFile(segments, fits.open(infile)) infiles = sorted(glob.glob('sim_data/000-01_fe55_fe55*.fits')) nfmin = 0 nfmax = min(len(infiles), 25) for infile_ in infiles[nfmin:nfmax]: foo = apply_cte(infile_, pcti=pcti_, scti=scti_, verbose=True) tokens = os.path.basename(infile_).split('_') outfile = '_'.join(tuple(tokens[:-1] + ['scti.fits'])) print(outfile) fitsWriteto(foo, outfile, clobber=True)
def main(sensor_id, infile, main_dir, gain_file=None, output_dir='./', no_bias=False): ## Get existing parameter results param_file = join(main_dir, '{0}_parameter_results.fits'.format(sensor_id)) param_results = OverscanParameterResults.from_fits(param_file) cti_results = param_results.cti_results drift_scales = param_results.drift_scales decay_times = param_results.decay_times trap_files = [ join(main_dir, '{0}_amp{1}_trap.pkl'.format(sensor_id, i)) for i in range(1, 17) ] ## Get gains if gain_file is not None: with open(gain_file, 'rb') as f: gain_results = pickle.load(f) gains = gain_results.get_amp_gains(sensor_id) else: gains = {i: 1.0 for i in range(1, 17)} ## Output filename base = splitext(os.path.basename(infile))[0] outfile = join(output_dir, '{0}_corrected.fits'.format(base)) ## Include bias frame for calibration if no_bias: bias_frame = None else: bias_frame = join(main_dir, '{0}_superbias.fits'.format(sensor_id)) ccd = MaskedCCD(infile, bias_frame=bias_frame) hdulist = fits.HDUList() with fits.open(infile) as template: hdulist.append(template[0]) ## Perform correction amp by amp for amp in range(1, 17): imarr = ccd.bias_subtracted_image( amp).getImage().getArray() * gains[amp] ## Electronics Correction if drift_scales[amp] > 0.: Linv = electronics_inverse_operator(imarr, drift_scales[amp], decay_times[amp], num_previous_pixels=15) corrected_imarr = imarr - Linv else: corrected_imarr = imarr ## Trap Correction spltrap = pickle.load(open(trap_files[amp], 'rb')) Tinv = trap_inverse_operator(corrected_imarr, spltrap) corrected_imarr = corrected_imarr - (1 - cti_results[amp]) * Tinv ## Reassemble HDUList hdulist.append( fits.ImageHDU(data=corrected_imarr / gains[amp], header=template[amp].header)) with warnings.catch_warnings(): for warning in (UserWarning, AstropyWarning, AstropyUserWarning): warnings.filterwarnings('ignore', category=warning, append=True) fitsWriteto(hdulist, outfile, overwrite=True)
def write_results(self, outfile): """Export results as a FITs file.""" for amp in range(1, 17): extname = 'Amp{0:02d}'.format(amp) nrows1 = len(self.flux[amp]) ncols = len(self.meanrow[amp][0]) meanrow_col = fits.Column('MEANROW', format='{0}E'.format(ncols), unit='e-', array=self.meanrow[amp]) flux_col = fits.Column('FLUX', format='E', unit='e-', array=self.flux[amp]) flux_std_col = fits.Column('FLUX_STD', format='E', unit='e-', array=self.flux_std[amp]) noise_col = fits.Column('NOISE', format='E', unit='e-', array=self.noise[amp]) signal_col = fits.Column('SIGNAL', format='E', unit='e-', array=self.signal[amp]) signal_std_col = fits.Column('SIGNAL_STD', format='E', unit='e-', array=self.signal_std[amp]) tau_col = fits.Column('TAU', format='E', unit='None', array=self.tau[amp]) tau_std_col = fits.Column('TAU_STD', format='E', unit='None', array=self.tau_std[amp]) cti_col = fits.Column('CTI', format='E', unit='None', array=self.cti[amp]) cti_std_col = fits.Column('CTI_STD', format='E', unit='None', array=self.cti_std[amp]) try: # # Append new rows if HDU for this segment already exists # table_hdu = self.output[extname] row0 = table_hdu.header['NAXIS2'] nrows = row0 + nrows1 table_hdu = fitsTableFactory(table_hdu.data, nrows=nrows) table_hdu.data['MEANROW'][row0:] = meanrow_col table_hdu.data['FLUX'][row0:] = flux_col table_hdu.data['FLUX_STD'][row0:] = flux_std_col table_hdu.data['NOISE'][row0:] = noise_col table_hdu.data['SIGNAL'][row0:] = signal_col table_hdu.data['SIGNAL_STD'][row0:] = signal_std_col table_hdu.data['TAU'][row0:] = tau_col table_hdu.data['TAU_STD'][row0:] = tau_std_col table_hdu.data['CTI'][row0:] = cti_col table_hdu.data['CTI_STD'][row0:] = cti_std_col table_hdu.name = extname self.output[extname] = table_hdu except KeyError: self.output.append( fitsTableFactory([ meanrow_col, flux_col, flux_std_col, noise_col, signal_col, signal_std_col, tau_col, tau_std_col, cti_col, cti_std_col ])) self.output[-1].name = extname self.output[0].header['NAMPS'] = 16 fitsWriteto(self.output, outfile, overwrite=True, checksum=True)
def rolloff_mask(infile, outfile, mask_plane='ROLLOFF_DEFECTS', tmp_mask_image=None, outer_edge_width=10, bloom_stop_width=5, signal=10, cleanup=True): """ This function creates a file containing masks for each segment to mask pixels affected the edge roll-off and midline blooming stop distortions. infile: Input file to mask. outfile: The name of the file to contain the masks. outer_edge_width: This is the width in pixels of the masked region along the sides closest to the sensor edges of the imaging region of each segment. bloom_stop_width: The width in pixels of the masked region of the imaging region of each segment, on either side of the central blooming stop implant. signal: This is the artificial signal written to the mask image so that the afwDetect code can generate the footprints used define the masks. It basically just needs to be some positive (non-zero) number. """ # # Create an empty set of frames to fill with an image of the mask # from which we will generate the masks. The exposure time and # system gain need to be set to unity so that the BrightPixels.py # code can interpret DN directly as e- per second. # amp_geom = makeAmplifierGeometry(infile) gain = 1 exptime = 1 all_amps = imutils.allAmps(infile) ccd = CCD(exptime=exptime, gain=gain, geometry=amp_geom, amps=all_amps) # # Write the output file with a primary HDU so that the DMstack code # can append only image extensions (and not write to the PHDU). # warnings.filterwarnings('ignore', category=fits.verify.VerifyWarning, append=True) hdulist = fits.HDUList() hdulist.append(fits.PrimaryHDU()) with fits.open(infile) as fd: hdulist[0].header.update(fd[0].header) # Use the mask_plane value ('ROLLOFF_DEFECTS') to distinguish # this file from other mask files. hdulist[0].header['MASKTYPE'] = mask_plane fitsWriteto(hdulist, outfile, overwrite=True) # # Amplifiers 1 (AMP10), 8 (AMP17), 9 (AMP07) and 16 (AMP00) are # along the perimeter, but have different edge rolloff pixels # depending on the vendor. # # These amps have edge roll-off adjacent to the prescan. amps = (8, 16) if amp_geom.vendor == 'E2V' else (8, 9) xmin = amp_geom.imaging.getMinX() xmax = xmin + outer_edge_width for amp in amps: if amp not in all_amps: continue imarr = ccd.segments[amp].image.getArray() imarr[:, xmin:xmax] += signal # # These amps have edge roll-off adjacent to the serial overscan: # amps = (1, 9) if amp_geom.vendor == 'E2V' else (1, 16) xmax = amp_geom.imaging.getMaxX() + 1 xmin = xmax - outer_edge_width for amp in amps: if amp not in all_amps: continue imarr = ccd.segments[amp].image.getArray() imarr[:, xmin:xmax] += signal # # Loop over all amps, set signal in perimeter and around blooming # stop. # ymax = amp_geom.imaging.getMaxY() + 1 ymin = ymax - bloom_stop_width for i, amp in enumerate(ccd.segments): image = ccd.segments[amp].image imarr = image.getArray() # # Set signal in row direction along perimeter. # xmin = amp_geom.imaging.getMinX() xmax = amp_geom.imaging.getMaxX() imarr[0:outer_edge_width, xmin:xmax] += signal if amp_geom.amp_loc == amp_loc['E2V']: #if True: # # Set signal around blooming stop # imarr[ymin:ymax, :] += signal # # Write the images of the mask regions to the FITS file. # if tmp_mask_image is None: tmp_mask_image = tempfile.mkstemp(suffix='.fits', dir='.')[-1] fitsWriteto(ccd, tmp_mask_image) # # Use BrightPixels code to detect the mask regions and write the mask file. # try: mask = afwImage.Mask(image.getDimensions()) except AttributeError: mask = afwImage.MaskU(image.getDimensions()) mask.addMaskPlane(mask_plane) maskedCCD = MaskedCCD(tmp_mask_image) pixels, columns = {}, {} for amp in maskedCCD: bright_pixels = BrightPixels(maskedCCD, amp, exptime, gain, ethresh=signal/2., mask_plane=mask_plane) pixels[amp], columns[amp] = bright_pixels.find() generate_mask(infile, outfile, mask_plane, pixels=pixels, columns=columns) if cleanup: os.remove(tmp_mask_image)
def generate_mask(infile, outfile, mask_plane, pixels=None, columns=None, temp_mask_image=None): """ Generate a mask file for the specified pixels and columns. The amplifier geometry will be taken from infile. """ # Insert artificial signal into specified pixels and columns for # each segment. exptime = 1 gain = 1 geometry = makeAmplifierGeometry(infile) ccd = CCD(exptime=exptime, gain=gain, geometry=geometry) # Account for prescan in x-coordinate since we want to mask # columns the full segment. prescan = geometry.prescan.getWidth() signal = 10 if pixels is None: pixels = {} for amp in pixels: imarr = ccd.segments[amp].image.getArray() for ix, iy in pixels[amp]: imarr[iy][ix + prescan] = signal if columns is None: columns = {} for amp in columns: imarr = ccd.segments[amp].image.getArray() for ix in columns[amp]: imarr[:, ix + prescan] = signal if temp_mask_image is None: temp_mask_image = tempfile.mkstemp(suffix='.fits', dir='.')[-1] fitsWriteto(ccd, temp_mask_image) # Use the afw code to create a mask file. afwImage.MaskU.addMaskPlane(mask_plane) # Create a primary HDU with the input file primary HDU metadata, # updated with the mask type info. hdulist = fits.HDUList() hdulist.append(fits.open(infile)[0]) hdulist[0].header['MASKTYPE'] = mask_plane fitsWriteto(hdulist, outfile, clobber=True) # Loop over segments in the temporary file and add all pixels to # the mask with the inserted signal. maskedCCD = MaskedCCD(temp_mask_image) for amp in maskedCCD: threshold = afwDetect.Threshold(signal / 2. * exptime / gain) fp_set = afwDetect.FootprintSet(maskedCCD[amp], threshold) mask = afwImage.MaskU(maskedCCD[amp].getDimensions()) fp_set.setMask(mask, mask_plane) md = dafBase.PropertySet() md.set('EXTNAME', 'SEGMENT%s' % imutils.channelIds[amp]) md.set('DETSIZE', geometry[amp]['DETSIZE']) md.set('DETSEC', geometry[amp]['DETSEC']) md.set('DATASEC', geometry[amp]['DATASEC']) mask.writeFits(outfile, md, 'a') os.remove(temp_mask_image)
def main(sensor_id, infile, output_dir='./', cti=None, do_trapping=False, do_electronics=False): processed_file = os.path.join(output_dir, '{0}_processed.fits'.format(sensor_id)) corrected_file = os.path.join(output_dir, '{0}_corrected.fits'.format(sensor_id)) ## CTI parameters if cti is None: do_cti = False cti = 0.0 else: do_cti = True cti_dict = {amp : cti for amp in range(1, 17)} ## Trapping parameters if do_trapping: # traps = [LinearTrap(4.0, 0.4, 1, 0.08), # LogisticTrap(1000.0, 0.4, 1, 60000., 0.0002)] traps = [LinearTrap(4.0, 0.4, 1, 0.08), LogisticTrap(40.0, 0.4, 1, 17500., 0.001)] else: traps = None traps_dict = {amp : traps for amp in range(1, 17)} ## Electronics parameters if do_electronics: scale = 0.0002 decay_time = 2.5 output_amps = {amp : FloatingOutputAmplifier(1.0, scale, decay_time) for amp in range(1, 17)} else: output_amps = {amp : BaseOutputAmplifier(1.0) for amp in range(1, 17)} ## Process infile imsim = ImageSimulator.from_image_fits(infile, output_amps, cti=cti_dict, traps=traps_dict) imarr_results = imsim.image_readout(infile, outfile=processed_file) ccd = MaskedCCD(processed_file) hdulist = fits.HDUList() with fits.open(infile) as template: hdulist.append(template[0]) hdulist[0].header['ORIGFILE'] = hdulist[0].header['FILENAME'] hdulist[0].header['FILENAME'] = corrected_file for amp in range(1, 17): imarr = ccd.bias_subtracted_image(amp).getImage().getArray() ## Electronics Correction if do_electronics: E = electronics_operator(imarr, scale, decay_time, num_previous_pixels=15) corrected_imarr = imarr - E else: corrected_imarr = imarr ## Trap Correction if do_trapping: T = trap_operator(imarr, *traps) corrected_imarr = corrected_imarr - (1-cti)*T else: pass ## CTI Correction if do_cti: Dinv_cti = cti_inverse_operator(cti, imarr.shape[1]) for i in range(imarr.shape[0]): corrected_imarr[i, :] = Dinv_cti*corrected_imarr[i, :] else: pass hdulist.append(fits.ImageHDU(data=corrected_imarr, header=template[amp].header)) with warnings.catch_warnings(): for warning in (UserWarning, AstropyWarning, AstropyUserWarning): warnings.filterwarnings('ignore', category=warning, append=True) fitsWriteto(hdulist, corrected_file, overwrite=True)
def generate_mask(infile, outfile, mask_plane, pixels=None, columns=None, temp_mask_image=None): """ Generate a mask file for the specified pixels and columns. The amplifier geometry will be taken from infile. """ # Insert artificial signal into specified pixels and columns for # each segment. exptime = 1 gain = 1 geometry = makeAmplifierGeometry(infile) amps = imutils.allAmps(infile) ccd = CCD(exptime=exptime, gain=gain, geometry=geometry, amps=amps) # Account for prescan in x-coordinate since we want to mask # columns the full segment. prescan = geometry.prescan.getWidth() signal = 10 if pixels is None: pixels = {} for amp in pixels: imarr = ccd.segments[amp].image.getArray() for ix, iy in pixels[amp]: imarr[iy][ix + prescan] = signal if columns is None: columns = {} for amp in columns: imarr = ccd.segments[amp].image.getArray() for ix in columns[amp]: imarr[:, ix + prescan] = signal if temp_mask_image is None: temp_mask_image = tempfile.mkstemp(suffix='.fits', dir='.')[-1] fitsWriteto(ccd, temp_mask_image) # Use the afw code to create a mask file. try: afwImage.Mask.addMaskPlane(mask_plane) except AttributeError: afwImage.MaskU.addMaskPlane(mask_plane) # Loop over segments in the temporary file and add all pixels to # the mask with the inserted signal. with fits.open(infile) as hdus: hdus[0].header['MASKTYPE'] = mask_plane hdus[0].header['FILENAME'] = outfile maskedCCD = MaskedCCD(temp_mask_image) for amp in maskedCCD: threshold = afwDetect.Threshold(signal / 2. * exptime / gain) fp_set = afwDetect.FootprintSet(maskedCCD[amp], threshold) try: mask = afwImage.Mask(maskedCCD[amp].getDimensions()) except AttributeError: mask = afwImage.MaskU(maskedCCD[amp].getDimensions()) fp_set.setMask(mask, mask_plane) hdus[amp].data = mask.array # add mask plane keywords for key, value in mask.getMaskPlaneDict().items(): hdus[amp].header['MP_' + key] = value hdus.writeto(outfile, overwrite=True) os.remove(temp_mask_image)
def extract_det_response(self, use_exptime): max_pd_frac_dev = self.max_pd_frac_dev outfile = os.path.join(self.config.output_dir, '%s_det_response.fits' % self.sensor_id) file1s = sorted( [item for item in self.infiles if item.find('flat1') != -1]) if self.config.verbose: self.log.info("writing to %s" % outfile) self._create_detresp_fits_output(len(file1s), file1s[0]) for row, file1 in enumerate(file1s): try: file2 = self.find_flat2(file1) except IndexError: # Just use flat1 again since only average is taken and # FPN subtraction isn't needed. file2 = file1 if self.config.verbose: self.log.info("processing\n %s as flat1 and\n %s as flat2", file1, file2) flat1 = MaskedCCD(file1, mask_files=self.mask_files, bias_frame=self.bias_frame, linearity_correction=self.linearity_correction) flat2 = MaskedCCD(file2, mask_files=self.mask_files, bias_frame=self.bias_frame, linearity_correction=self.linearity_correction) seqnum = flat1.md.get('SEQNUM') exptime1 = flat1.md.get('EXPTIME') exptime2 = flat2.md.get('EXPTIME') if exptime1 != exptime2: raise RuntimeError( "Exposure times do not match for:\n%s\n%s\n" % (file1, file2)) if self.mondiode_func is None: pd1 = flat1.md.get('MONDIODE') pd2 = flat2.md.get('MONDIODE') else: try: pd1 = self.mondiode_func(file1, exptime1) pd2 = self.mondiode_func(file2, exptime2) except KeyError as eobj: self.log.info( "KeyError exception computing pd current:\n" + str(eobj)) continue if use_exptime: flux = exptime1 else: flux = abs(pd1 * exptime1 + pd2 * exptime2) / 2. if np.abs((pd1 - pd2) / ((pd1 + pd2) / 2.)) > max_pd_frac_dev: self.log.info( "Skipping %s and %s since MONDIODE values " "do not agree to %.1f%%", file1, file2, max_pd_frac_dev * 100.) continue if self.config.verbose: self.log.info(' row = %s', row) self.log.info(' pd1, pd2 = %s, %s', pd1, pd2) self.log.info(' exptime1, exptime2 = %s, %s ', exptime1, exptime2) self.log.info(' flux = %s', flux) self.log.info(' flux/exptime = %s', flux / exptime1) self.output[-1].data.field('FLUX')[row] = flux for amp in flat1: # Convert to e- and write out for each segment. signal, sig1, sig2 \ = pair_mean(flat1, flat2, amp)*self.gains[amp] colname = 'AMP%02i_SIGNAL' % amp self.output[-1].data.field(colname)[row] = signal self.output[-1].data.field('FLAT1_' + colname)[row] = sig1 self.output[-1].data.field('FLAT2_' + colname)[row] = sig2 self.output[-1].data.field('SEQNUM')[row] = seqnum self.output[0].header['NAMPS'] = len(flat1) fitsWriteto(self.output, outfile, overwrite=True) return outfile
def write_results(self, outfile='fe55_psf_params.fits'): self.output[0].header['NAMPS'] = len(self.amp_set) fitsWriteto(self.output, outfile, clobber=True, checksum=True)
def run(self, sensor_id, dark_files, mask_files, gains, bias_frame=None): imutils.check_temperatures(dark_files, self.config.temp_set_point_tol, setpoint=self.config.temp_set_point, warn_only=True) median_images = {} md = imutils.Metadata(dark_files[0], 1) for amp in imutils.allAmps(dark_files[0]): median_images[amp] = imutils.fits_median(dark_files, imutils.dm_hdu(amp)) medfile = os.path.join(self.config.output_dir, '%s_median_dark_current.fits' % sensor_id) imutils.writeFits(median_images, medfile, dark_files[0]) ccd = MaskedCCD(medfile, mask_files=mask_files, bias_frame=bias_frame) dark95s = {} exptime = md.get('EXPTIME') if self.config.verbose: self.log.info("Amp 95 percentile median") dark_curr_pixels = [] dark_curr_pixels_per_amp = {} for amp in ccd: imaging_region = ccd.amp_geom.imaging overscan = ccd.amp_geom.serial_overscan image = imutils.unbias_and_trim(ccd[amp].getImage(), overscan, imaging_region) mask = imutils.trim(ccd[amp].getMask(), imaging_region) imarr = image.getArray() mskarr = mask.getArray() pixels = imarr.reshape(1, imarr.shape[0] * imarr.shape[1])[0] masked = mskarr.reshape(1, mskarr.shape[0] * mskarr.shape[1])[0] unmasked = [ pixels[i] for i in range(len(pixels)) if masked[i] == 0 ] unmasked.sort() unmasked = np.array(unmasked) * gains[amp] / exptime dark_curr_pixels_per_amp[amp] = unmasked dark_curr_pixels.extend(unmasked) try: dark95s[amp] = unmasked[int(len(unmasked) * 0.95)] median = unmasked[len(unmasked) / 2] except IndexError as eobj: print str(eobj) dark95s[amp] = -1. median = -1. if self.config.verbose: self.log.info("%2i %.2e %.2e" % (amp, dark95s[amp], median)) # # Compute 95th percentile dark current for CCD as a whole. # dark_curr_pixels = sorted(dark_curr_pixels) darkcurr95 = dark_curr_pixels[int(len(dark_curr_pixels) * 0.95)] dark95mean = np.mean(dark95s.values()) if self.config.verbose: #self.log.info("CCD: mean 95 percentile value = %s" % dark95mean) self.log.info("CCD-wide 95 percentile value = %s" % darkcurr95) # # Update header of dark current median image file with dark # files used and dark95 values, and write dark95 values to the # eotest results file. # results_file = self.config.eotest_results_file if results_file is None: results_file = os.path.join(self.config.output_dir, '%s_eotest_results.fits' % sensor_id) results = EOTestResults(results_file, namps=len(ccd)) output = fits.open(medfile) for i, dark in enumerate(dark_files): output[0].header['DARK%02i' % i] = os.path.basename(dark) # Write overall dark current 95th percentile results.output['AMPLIFIER_RESULTS'].header['DARK95'] = darkcurr95 for amp in ccd: output[0].header['DARK95%s' % imutils.channelIds[amp]] = dark95s[amp] results.add_seg_result(amp, 'DARK_CURRENT_95', dark95s[amp]) fitsWriteto(output, medfile, clobber=True, checksum=True) results.write(clobber=True) return dark_curr_pixels_per_amp, dark95s