def beam_recons(led_names, ref_slot='11', ref_amp=4, ref_pix_x=1000, ref_pix_y=256, npix_for_avg=30, npix_beam_image=300): mean_bias_file = ref_slot + '_mean_bias_image_RTM-006.fits' recons = {} for led in led_names: config = u.load_ccob_config( '/home/combet/ccob-wb/ccob_config_RTM-006.yaml') config['led_name'] = led config['path'] = os.path.join(config['path'], config['led_name']) flist = sorted(u.find_files(config, slot=ref_slot)) ccd_dict = sensorTest.MaskedCCD(flist[0], bias_frame=os.path.join( config['tmp_dir'], mean_bias_file)) eotest_results_file = os.path.join( config['eo_data_path'], '{}_eotest_results.fits'.format(ccd_dict.md('LSST_NUM'))) gains_dict = u.gains(eotest_results_file) nodes = {} nodes['xarr'] = [] nodes['yarr'] = [] nodes['val'] = [] for i, f in enumerate(sorted(flist)): nodes['xarr'].append( float(os.path.basename(f).split('_')[3].split('x')[1])) nodes['yarr'].append( float(os.path.basename(f).split('_')[4].split('y')[1])) ccd_dict = sensorTest.MaskedCCD(f, bias_frame=os.path.join( config['tmp_dir'], mean_bias_file)) image = ccd_dict.unbiased_and_trimmed_image(ref_amp) image *= gains_dict[ref_amp] arr = image.getArrays() nodes['val'].append( np.mean( (arr[0][ref_pix_x - npix_for_avg / 2:ref_pix_x + npix_for_avg / 2, ref_pix_y - npix_for_avg / 2:ref_pix_y + npix_for_avg / 2]))) f_interp = interpolate.interp2d(np.unique(nodes['xarr']), np.unique(nodes['yarr']), nodes['val'], kind='cubic') xarr2 = np.linspace(min(nodes['xarr']), max(nodes['xarr']), npix_beam_image) yarr2 = np.linspace(min(nodes['yarr']), max(nodes['yarr']), npix_beam_image) tmp = f_inerp(xarr2, yarr2) recons[led] = tmp / max(tmp.flatten()) return recons
def persistence_task(run, det_name, bias_files, superbias_frame, mask_files): """Single sensor execution of the persistence analysis.""" file_prefix = make_file_prefix(run, det_name) data = defaultdict(list) ccd = None for bias_file in bias_files: ccd = sensorTest.MaskedCCD(bias_file, mask_files=mask_files, bias_frame=superbias_frame) tseqnum = ccd.md.get('TSEQNUM') for amp in ccd: amp_image = ccd.unbiased_and_trimmed_image(amp) stats = afwMath.makeStatistics(amp_image, afwMath.MEAN | afwMath.STDEV, ccd.stat_ctrl) data['tseqnum'].append(tseqnum) data['amp'].append(amp) data['mean_signal'].append(stats.getValue(afwMath.MEAN)) data['stdev'].append(stats.getValue(afwMath.STDEV)) df = pd.DataFrame(data=data) outfile = f'{file_prefix}_persistence_data.pickle' df.to_pickle(outfile) fig = plt.figure() for amp in ccd: my_df = df.query(f'amp == {amp}') plt.scatter(my_df['tseqnum'], my_df['mean_signal'], s=2, label=f'{amp}') xmax = 1.2*(np.max(df['tseqnum']) - np.min(df['tseqnum'])) \ + np.min(df['tseqnum']) axis = plt.axis() plt.xlim(axis[0], xmax) plt.legend(fontsize='x-small') plt.xlabel('test sequence number') plt.ylabel('mean residual signal (ADU)') plt.title(f'{file_prefix} persistence test') plt.savefig(f'{file_prefix}_persistence.png') plt.close()
def make_image(config, slot_names, mean_frame_pattern='_mean_bias_image.fits'): """ Make the mosaic image of the entire raft, when illuminated by the CCOB according to config. Returns: - the raw image of the raft - the corrected image where mean bias frame has been removed and gains applied """ file_list = sorted(find_files(config)) fits_files_dict = {slot_names[i] : file_list[i] for i in range(len(file_list))} ccd_dict = {} ccd_dict_wbias = {} gains_dict = {} for slot in slot_names: mean_bias_file = slot + mean_frame_pattern ccd_dict[slot] = sensorTest.MaskedCCD(fits_files_dict[slot]) outfile = os.path.join(config['tmp_dir'],'ccd' + slot + '.fits') image={} ccd_dict_wbias[slot]=sensorTest.MaskedCCD(fits_files_dict[slot],\ bias_frame=os.path.join(config['tmp_dir'],mean_bias_file)) outfile_wbias = os.path.join(config['tmp_dir'],'ccd' + slot + '_wbias.fits') image_wbias={} eotest_results_file = os.path.join(config['eo_data_path'],'{}_eotest_results.fits'.format(ccd_dict[slot].md('LSST_NUM'))) gains_dict[slot] = gains(eotest_results_file) for amp in ccd_dict[slot]: image[amp] = ccd_dict[slot].bias_subtracted_image(amp) image[amp] *= gains_dict[slot][amp] image_wbias[amp] = ccd_dict_wbias[slot].bias_subtracted_image(amp) image_wbias[amp] *= gains_dict[slot][amp] imutils.writeFits({amp: image_wbias[amp].getImage() for amp in ccd_dict_wbias[slot]}, outfile_wbias, fits_files_dict[slot]) imutils.writeFits({amp: image[amp].getImage() for amp in ccd_dict[slot]}, outfile, fits_files_dict[slot]) fits_files_dict_corr={slot : os.path.join(config['tmp_dir'],'ccd'+slot+'.fits') for slot in slot_names} fits_files_dict_corr_wbias={slot : os.path.join(config['tmp_dir'],'ccd'+slot+'_wbias.fits') for slot in slot_names} im_corr = raft.RaftMosaic(fits_files_dict_corr, bias_subtract=False) im_corr_wbias = raft.RaftMosaic(fits_files_dict_corr_wbias, bias_subtract=False) im_raw = raft.RaftMosaic(fits_files_dict, bias_subtract=False) return im_raw, im_corr, im_corr_wbias
def __init__(self, mask_file): """ Parameters ---------- mask_file: str Filename of a mask file generated with the eotest code. """ self.ccd = sensorTest.MaskedCCD(mask_file)
def __init__(self, fits_files, gains=None, bias_subtract=True, nx=12700, ny=12700, nx_segments=8, ny_segments=2, segment_processor=None): """ Constructor. Parameters ---------- fits_files : dict Dictionary of single sensor FITS files, keyed by raft slot name. These files should conform to LCA-13501. gains : dict, optional Dictionary (keyed by slot name) of dictionaries (one per FITS file) of system gain values for each amp. Default: None (i.e., do not apply gain correction). bias_subtract : bool, optional Flag do to a bias subtraction based on the serial overscan. Default: True nx : int, optional Number of pixels in the x (serial) direction. Default: 12700 ny : int, optional Number of pixels in the y (parallel) direction. Default: 12700 nx_segments : int, optional Number of segments in the x (serial) direction. Default: 8 ny_segments : int, optional Number of pixels in the y (parallel) direction. Default: 2 segment_processor : function, optional Function to apply to pixel data in each segment. If None (default), then set do the standard bias subtraction and gain correction. """ self.fits_files = fits_files self.raft_name = fits.open( fits_files.values()[0])[0].header['RAFTNAME'] self.wl = fits.open(fits_files.values()[0])[0].header['MONOWL'] self.image_array = np.zeros((nx, ny), dtype=np.float32) self.nx = nx self.ny = ny self.nx_segments = nx_segments self.ny_segments = ny_segments self.segment_processor = segment_processor self._amp_coords = defaultdict(dict) if gains is None: # Assume unit gain for all amplifiers. unit_gains = dict([(i, 1) for i in range(1, 17)]) gains = dict([(slot, unit_gains) for slot in fits_files]) for slot, filename in fits_files.items(): print("processing", os.path.basename(filename)) ccd = sensorTest.MaskedCCD(filename) hdu_list = fits.open(filename) for amp, hdu in zip(ccd, hdu_list[1:]): self._set_segment(slot, ccd, amp, hdu, gains[slot][amp], bias_subtract)
def test_EPERTask_run_parallel(self): "Test the EPERTask.run method for the parallel direction." ccd = sensorTest.MaskedCCD(self.fits_file) task = sensorTest.EPERTask() task.config.direction = 'p' task.config.cti = True task.config.verbose = self.verbose cti, bias_ests = task.run(self.fits_file, 1, range(1, 17), self.overscans) nrows = ccd.amp_geom.imaging.getHeight() cti_expected = (float(self.overscans * self.overscan_value) / float(self.imaging_value) / float(nrows)) for amp in ccd: self.assertEqual(cti[amp].value, cti_expected)
def test_EPERTask_run_serial(self): "Test the EPERTask.run method for the serial direction." ccd = sensorTest.MaskedCCD(self.fits_file) task = sensorTest.EPERTask() task.config.direction = 's' task.config.cti = True task.config.verbose = self.verbose cti, bias_ests = task.run(self.fits_file, 1, range(1, 17), self.overscans) ncols = (ccd.amp_geom.prescan.getWidth() + ccd.amp_geom.imaging.getWidth()) cti_expected = (float(self.overscans * self.overscan_value) / float(self.imaging_value) / float(ncols)) for amp in ccd: self.assertEqual(cti[amp].value, cti_expected)
def diff_image_arrays(flat1_file, flat2_file, bias_frame, buffer=10): """ Compute the difference images for each amp, using the Astier weighting scheme to account for somewhat different exposure times, and return a dict of the image arrays, keyed by amp. """ image_arrays = dict() ccd1 = sensorTest.MaskedCCD(flat1_file, bias_frame=bias_frame) ccd2 = sensorTest.MaskedCCD(flat2_file, bias_frame=bias_frame) imaging_bbox = ccd1.amp_geom.imaging imaging_bbox.grow(-buffer) for amp in ccd1: image1 = ccd1.unbiased_and_trimmed_image(amp, imaging=imaging_bbox) image2 = ccd2.unbiased_and_trimmed_image(amp, imaging=imaging_bbox) mean1 = afwMath.makeStatistics(image1, afwMath.MEAN, ccd1.stat_ctrl)\ .getValue() mean2 = afwMath.makeStatistics(image2, afwMath.MEAN, ccd1.stat_ctrl)\ .getValue() fmean = (mean1 + mean2) / 2. image1 *= mean2 / fmean image2 *= mean1 / fmean image1 -= image2 image_arrays[amp] = copy.deepcopy(image1.getImage().getArray()) return image_arrays
def test_get_median_signal_levels(self): """ Test the get_median_signal_levels function on the flat and bias frames. """ flat = sensorTest.MaskedCCD(self.flat_file) overscan = flat.amp_geom.serial_overscan imaging = flat.amp_geom.imaging imaging_signals \ = aliveness_utils.get_median_signal_levels(flat, imaging) oscan_signals \ = aliveness_utils.get_median_signal_levels(flat, overscan) for amp in oscan_signals: signal = imaging_signals[amp] - oscan_signals[amp] self.assertGreater(signal, 0.9*self.nominal_signal) self.assertLess(signal, 1.1*self.nominal_signal)
def make_fits(QE_map, amp_coord, outfile, template_file): """ Saves the mosaicked QE image into a fits file, using the default format Parameters ---------- outfile: string Name of the output fits file into which the data will be saved template_file: string Name of the file to be used as template in the writeFits function """ amp_dict = {} for amp_pos in amp_coord.keys(): amp = amp_coord[amp_pos]['amp'] xmin = amp_coord[amp_pos]['xmin'] xmax = amp_coord[amp_pos]['xmax'] ymin = amp_coord[amp_pos]['ymin'] ymax = amp_coord[amp_pos]['ymax'] flipx = amp_coord[amp_pos]['flipx'] flipy = amp_coord[amp_pos]['flipy'] detsec = amp_coord[amp_pos]['detsec'] datasec = amp_coord[amp_pos]['datasec'] foo = QE_map[ymin:ymax, xmin:xmax] if flipx: amp_dict[amp] = foo[:, ::-1] elif flipy: amp_dict[amp] = foo[::-1, :] else: amp_dict[amp] = foo ccd_dict = sensorTest.MaskedCCD(template_file) shape = ccd_dict[1].getImage().getArray().shape amp_dict_w_overscan = {} for amp in amp_dict: arr = np.zeros(shape) arr[datasec['ymin'] - 1:datasec['ymax'], datasec['xmin'] - 1:datasec['xmax']] = amp_dict[amp] amp_dict_w_overscan[amp] = arr u.writeFits_from_dict(amp_dict_w_overscan, outfile, template_file) #,bitpix=-32)
def test_subimage_parallel(self): "Test the SubImage class for the parallel direction." ccd = sensorTest.MaskedCCD(self.fits_file) task = sensorTest.EPERTask() task.config.direction = 'p' task.config.verbose = self.verbose for amp in ccd: subimage = SubImage(ccd, amp, self.overscans, task) lastrow = ccd.amp_geom.imaging.getMaxY() ncols = ccd.amp_geom.imaging.getWidth() last_imaging_row = subimage(lastrow).getImage().getArray().flatten() self.assertEqual(ncols, len(last_imaging_row)) self.assertEqual(ncols*self.imaging_value, sum(last_imaging_row)) for irow in range(1, self.overscans+1): overscan_row = subimage(lastrow + irow) self.assertEqual(ncols*self.overscan_value, sum(overscan_row.getImage().getArray().flatten()))
def test_subimage_serial(self): "Test the SubImage class for the serial direction." ccd = sensorTest.MaskedCCD(self.fits_file) task = sensorTest.EPERTask() task.config.direction = 's' task.config.verbose = self.verbose for amp in ccd: subimage = SubImage(ccd, amp, self.overscans, task) lastcol = ccd.amp_geom.imaging.getMaxX() nrows = ccd.amp_geom.imaging.getHeight() last_imaging_col = subimage(lastcol).getImage().getArray().flatten() self.assertEqual(nrows, len(last_imaging_col)) self.assertEqual(nrows*self.imaging_value, sum(last_imaging_col)) for icol in range(1, self.overscans+1): overscan_col = subimage(lastcol + icol) self.assertEqual(nrows*self.overscan_value, sum(overscan_col.getImage().getArray().flatten()))
def make_mask(med_file, gains, outfile, ethresh=0.1, colthresh=20, mask_plane='BAD'): """ Create bright pixel mask from a medianed single sensor dark frame. Parameters ---------- med_file: str Filename of the medianded dark frame. gains: dict Gains (e-/ADU) of the sixteen amplifiers in the CCD, keyed by amp. outfile: str Output filename of the mask file. ethresh: float, optional Threshold in e- per second per pixel for identifying a bright pixel defect. Default: 0.1 colthresh: float, optional Threshold in e- per second per column for identifying a bright column. Default: 20 mask_plane: str, optional Name of the mask plane. Default: 'BAD' Returns ------- IsMasked object """ ccd = sensorTest.MaskedCCD(med_file) exptime = ccd.md.get('EXPTIME') pixels, columns = {}, {} for amp in ccd: bright_pixels \ = sensorTest.BrightPixels(ccd, amp, exptime, gains[amp], ethresh=ethresh, colthresh=colthresh) pixels[amp], columns[amp] = bright_pixels.find() sensorTest.generate_mask(med_file, outfile, mask_plane, pixels=pixels, columns=columns) return IsMasked(outfile)
def bias_stability_task(run, det_name, bias_files, nsigma=10): """Compute amp-wise bias stability time histories and serial profiles.""" raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) data = defaultdict(list) fig = plt.figure(figsize=(16, 16)) xlabel_amps = (13, 14, 15, 16) ylabel_amps = (1, 5, 9, 13) ax = {amp: fig.add_subplot(4, 4, amp) for amp in range(1, 17)} for bias_file in bias_files: with fits.open(bias_file) as hdus: temps = dict() for i in range(1, 10): key = f'TEMP{i}' if key in hdus['REB_COND'].header: temps[key] = hdus['REB_COND'].header[key] ccd = sensorTest.MaskedCCD(bias_file) for amp in ccd: # Retrieve the per row overscan subtracted imaging section. amp_image = ccd.unbiased_and_trimmed_image(amp) # Plot the median of each column versus serial pixel number. imarr = amp_image.getImage().array ax[amp].plot(range(imarr.shape[1]), np.median(imarr, axis=0)) # Compute 10-sigma clipped mean and stdev mean, stdev = image_stats(amp_image, nsigma=nsigma) data['raft'].append(raft) data['slot'].append(slot) data['tseqnum'].append(ccd.md.get('TSEQNUM')) for key, value in temps.items(): data[key].append(value) data['MJD'].append(ccd.md.get('MJD-OBS')) data['amp'].append(amp) data['mean'].append(mean) data['stdev'].append(stdev) plt.suptitle(f'{det_name}, Run {run}\nmedian signal (ADU) vs column') plt.tight_layout(rect=(0, 0, 1, 0.95)) for amp in ccd: ax[amp].annotate(f'amp {amp}', (0.5, 0.95), xycoords='axes fraction', ha='center') plt.savefig(f'{file_prefix}_bias_serial_profiles.png') df = pd.DataFrame(data=data) df.to_pickle(f'{file_prefix}_bias_frame_stats.pickle')
def load_ccd(self, led_name='red'): """ Load and generate mosaic image from a given sensor illuminated by the CCOB. The path to the data is provided in the self.config_file_data file. Parameters ---------- led_name: choice of the CCOB LED. Either one of ['nm960','nm850','nm750,'red','blue,'uv'] """ #config_file_data = '../ccob_config_RTM-006.yaml' # config_data = u.load_ccob_config(self.config_file_data) # config_beam = u.load_ccob_config(self.config_file_beam) self.config_data['led_name'] = led_name slot = self.beam.properties['ref_slot'] file_list=sorted(u.find_files(self.config_data, slot=slot)) mean_slot_file = slot+'_mean_ccob_image.fits' imutils.fits_mean_file(file_list, os.path.join(self.config_data['tmp_dir'],mean_slot_file)) fits_file = os.path.join(self.config_data['tmp_dir'],mean_slot_file) gains_dict={} ccd_dict={} #bias_frames = glob.glob(os.path.join(config['path'], slot+'_bias*')) #mean_bias_file = slot+'_mean_bias_image_RTM-006_new.fits' #imutils.fits_mean_file(bias_frames, os.path.join(config['tmp_dir'],mean_bias_file)) #ccd_dict = sensorTest.MaskedCCD(fits_file, bias_frame=os.path.join(self.config_data['tmp_dir'],mean_bias_file)) superbias_frame = make_superbias_frame(self.config_data, slot=slot) ccd_dict = sensorTest.MaskedCCD(fits_file, bias_frame=os.path.join(self.config_data['tmp_dir'],superbias_frame)) eotest_results_file = os.path.join(self.config_data['eo_data_path'],\ '{}_eotest_results.fits'.format(ccd_dict.md('LSST_NUM'))) gains_dict = u.gains(eotest_results_file) self.ccd = {} self.ccd['mosaic'], self.ccd['amp_coord'] = u.make_ccd_2d_array(fits_file, gains=gains_dict) self.ccd['xpos_ccob'] = self.config_data['xpos'] self.ccd['ypos_ccob'] = self.config_data['ypos'] return self.ccd
def recons(self, ref_slot='11', ref_amp=4, ref_pix_x=1000, ref_pix_y=256, npix_for_avg=30): self.properties["ref_slot"] = ref_slot self.properties["ref_amp"] = ref_amp self.properties["ref_pix_x"] = int(ref_pix_x) self.properties["ref_pix_y"] = int(ref_pix_y) self.properties["npix_for_avg"] = int(npix_for_avg) recons = {} led = self.config['led_name'] print(led) flist = sorted(u.find_files(self.config, slot=ref_slot)) nodes = {} nodes['xarr'] = [] nodes['yarr'] = [] nodes['val'] = [] for i, f in enumerate(sorted(flist)): nodes['xarr'].append( float(os.path.basename(f).split('_')[3].split('x')[1])) nodes['yarr'].append( float(os.path.basename(f).split('_')[4].split('y')[1])) ccd_dict = sensorTest.MaskedCCD(f) image = ccd_dict.unbiased_and_trimmed_image(ref_amp) arr = image.getArrays() nodes['val'].append(np.mean((arr[0][int(ref_pix_x-npix_for_avg/2):int(ref_pix_x+npix_for_avg/2),\ int(ref_pix_y-npix_for_avg/2):int(ref_pix_y+npix_for_avg/2)]))) f_interp = interpolate.interp2d(np.unique(nodes['xarr']), np.unique(nodes['yarr']), nodes['val'], kind='cubic') self.beam_image['nodes'] = nodes self.beam_image['f_interp'] = f_interp
def test_generate_mask(self): "Test the generated mask for expected masked and unmasked pixels." sensorTest.generate_mask(self.template_file, self.mask_file, mask_plane='TRAPS', pixels=self.pixels, columns=self.columns, temp_mask_image='my_temp_mask_file.fits') ccd = sensorTest.MaskedCCD(self.mask_file) for amp in self.pixels: image = imutils.trim(ccd[amp].getImage(), ccd.amp_geom.imaging) imarr = image.getArray() for ix, iy in self.pixels[amp]: self.assertNotEqual(0, imarr[iy][ix]) for xoffset, yoffset in ((-1, -1), (-1, 0), (-1, 1), (0, -1), (0, 1), (1, -1), (1, 0), (1, 1)): self.assertEqual(0, imarr[iy+yoffset][ix+xoffset]) for amp in self.columns: image = imutils.trim(ccd[amp].getImage(), ccd.amp_geom.imaging) imarr = image.getArray() for ix in self.columns[amp]: self.assertNotEqual(0, imarr[0][ix]) self.assertEqual(imarr[0][ix]*imarr.shape[0], sum(imarr[:, ix]))
ccs_setup_class=CcsRaftSetup) # Perform noise analysis of last bias frame taken. raft_id = siteUtils.getUnitId() run_number = siteUtils.getRunNumber() raft = camera_components.Raft.create_from_etrav(raft_id) results_files = dict() bias_files = dict() bbox = None for slot, sensor_id in raft.items(): # glob the bias files for each sensor. Use the last one in the sequence. bias_files[slot] = sorted( glob.glob('%s/%s_conn_bias_*.fits' % (slot, sensor_id)))[-2:] ccd = sensorTest.MaskedCCD(bias_files[slot][-1]) if bbox is None: bbox = ccd.amp_geom.serial_overscan bbox.grow(-10) outfile = '%s_eotest_results.fits' % sensor_id results = sensorTest.EOTestResults(outfile) for amp in ccd: image = ccd[amp].getImage() oscan = image.Factory(image, bbox) read_noise \ = afw_math.makeStatistics(oscan, afw_math.STDEVCLIP).getValue() results.add_seg_result(amp, 'READ_NOISE', read_noise) results.write() results_files[slot] = outfile
def raft_level_oscan_correlations(bias_files, buffer=10, title='', vrange=None, stretch=viz.LinearStretch, figsize=(8, 8)): """ Compute the correlation coefficients between the overscan pixels of the 144 amplifiers in raft. Parameters ---------- bias_files: dict Dictionary of bias image files, indexed by sensor slot id. buffer: int [10] Buffer region around perimeter of serial overscan region to avoid when computing the correlation coefficients. title: str [''] Plot title. vrange: (float, float) [None] Minimum and maximum values for color scale range. If None, then the range of the central 98th percentile of the absolute value of the data is used. stretch: astropy.visualization.BaseStretch [LinearStretch] Stretch to use for the color scale. Returns ------- (matplotlib.figure.Figure, np.array): The figure containing the plot and the numpy array containing the correlation coefficients. """ slots = 'S00 S01 S02 S10 S11 S12 S20 S21 S22'.split() overscans = [] ccd0 = sensorTest.MaskedCCD(list(bias_files.values())[0]) bbox = ccd0.amp_geom.serial_overscan bbox.grow(-buffer) for slot in slots: if slot not in bias_files: for amp in ccd0: overscans.append(np.zeros((bbox.getHeight(), bbox.getWidth()))) else: ccd = sensorTest.MaskedCCD(bias_files[slot]) for amp in ccd: image = ccd[amp].getImage() overscans.append(image.Factory(image, bbox).getArray()) namps = len(overscans) data = np.array([ np.corrcoef(overscans[i[0]].ravel(), overscans[i[1]].ravel())[0, 1] for i in itertools.product(range(namps), range(namps)) ]) data = data.reshape((namps, namps)) fig = plt.figure(figsize=figsize) ax = fig.add_subplot(111) ax.set_title(title, fontsize='medium') interval = viz.PercentileInterval(98.) if vrange is None: vrange = interval.get_limits(np.abs(data.ravel())) norm = ImageNormalize(vmin=vrange[0], vmax=vrange[1], stretch=stretch()) image = ax.imshow(data, interpolation='none', norm=norm) plt.colorbar(image) set_ticks(ax, slots, amps=16) return fig, data
def normed_mean_response_vscol(sflat_file): """ For an input .fits file, calculates the normalized sigma clipped mean flux vs. Col# for a group of Rows returns two arrays for the top and bottom section of the CCD """ amc = sensorTest.MaskedCCD(sflat_file) amps = imutils.allAmps(sflat_file) ncol = amc.amp_geom.nx sensor_type = amc.amp_geom.vendor.lower() imaging = amc.amp_geom.imaging # use 200 rows close to the amplifier row_lo = 10 row_hi = 210 # top row averow_top = np.zeros(ncol*8) for i_amp in range(1, 8+1): # Segments 10-17 anamp = imutils.trim(amc[i_amp], imaging=imaging) anamp_im = anamp.getImage() anamp_arr = anamp_im.getArray() # use a robust mean anamp_meanbyrow, _, _ \ = stats.sigma_clipped_stats(anamp_arr[row_lo:row_hi, :], axis=0) # normalize nmean_byrow = anamp_meanbyrow/np.median(anamp_meanbyrow) lopix = 0 + (i_amp-1)*ncol hipix = ncol + (i_amp-1)*ncol averow_top[lopix:hipix] = np.flip(nmean_byrow) # bot row averow_bot = np.zeros((ncol*8)) for j_amp in range(16, 8, -1): if j_amp not in amps: continue # Segments 00-07 # i_amp goes from 1 to 8, in order of increasing Yccs i_amp = 17 - j_amp anamp = imutils.trim(amc[j_amp], imaging=imaging) anamp_im = anamp.getImage() anamp_arr = anamp_im.getArray() # use a robust mean anamp_meanbyrow, _, _ \ = stats.sigma_clipped_stats(anamp_arr[row_lo:row_hi, :], axis=0) # normalize nmean_byrow = anamp_meanbyrow/np.median(anamp_meanbyrow) lopix = 0 + (i_amp-1)*ncol hipix = ncol + (i_amp-1)*ncol if sensor_type == 'e2v': averow_bot[lopix:hipix] = nmean_byrow elif sensor_type == 'itl': averow_bot[lopix:hipix] = np.flip(nmean_byrow) # analyze the gaps between amplifiers for Divisidero Tearing, and # find the max(abs) deviation in the +-2 columns at the boundaries max_divisidero_tearing = [] # 14 entries per CCD for k in range(1, 7+1): collo = ncol*k - 2 # 2nd to last column in Amplifier max_divisidero = np.max(np.abs(averow_top[collo:collo+4] - 1.0)) # +-2 columns max_divisidero_tearing.append(max_divisidero) for k in range(1, 7+1): if k + 8 not in amps: continue collo = ncol*k - 2 # 2nd to last column in Amplifier max_divisidero = np.max(np.abs(averow_bot[collo:collo+4] - 1.0)) # +-2 columns max_divisidero_tearing.append(max_divisidero) return averow_top, averow_bot, max_divisidero_tearing
def __init__(self, fits_files, gains=None, bias_subtract=True, nx=12700, ny=12700, nx_segments=8, ny_segments=2, segment_processor=None, bias_frames=None, dark_currents=None): """ Parameters ---------- fits_files : dict Dictionary of single sensor FITS files, keyed by raft slot name. These files should conform to LCA-13501. gains : dict [None] Dictionary (keyed by slot name) of dictionaries (one per FITS file) of system gain values for each amp. If None, then do not apply gain correction. bias_subtract : bool [True] Flag do to a bias subtraction based on the serial overscan or provided bias frame. nx : int [12700] Number of pixels in the x (serial) direction. ny : int, [12700] Number of pixels in the y (parallel) direction. nx_segments : int [8] Number of segments in the x (serial) direction. ny_segments : int [2] Number of pixels in the y (parallel) direction. segment_processor : function [None] Function to apply to pixel data in each segment. If None, then set do the standard bias subtraction and gain correction. bias_frames : dict [None] Dictionary of single sensor bias frames, keyed by raft slot. If None, then just do the bias level subtraction using overscan region. dark_currents : dict [None] Dictionary of dictionaries of dark current values per amp in e-/s, keyed by raft slot and by amp number. If None, then dark current subtraction is not applied. """ self.fits_files = fits_files with fits.open(list(fits_files.values())[0]) as hdu_list: self.raft_name = hdu_list[0].header['RAFTNAME'] try: self.wl = hdu_list[0].header['MONOWL'] except KeyError: wl = 0 self.image_array = np.zeros((nx, ny), dtype=np.float32) self.nx = nx self.ny = ny self.nx_segments = nx_segments self.ny_segments = ny_segments self.segment_processor = segment_processor self._amp_coords = defaultdict(dict) if gains is None: # Assume unit gain for all amplifiers. unit_gains = dict([(i, 1) for i in range(1, 17)]) gains = dict([(slot, unit_gains) for slot in fits_files]) for slot, filename in list(fits_files.items()): #print("processing", os.path.basename(filename)) bias_frame = bias_frames[slot] if bias_frames is not None else None ccd = sensorTest.MaskedCCD(filename, bias_frame=bias_frame) if dark_currents is not None: try: dark_time = ccd.md.get('DARKTIME') except: dark_time = ccd.md.get('EXPTIME') with fits.open(filename) as hdu_list: for amp, hdu in zip(ccd, hdu_list[1:]): if dark_currents: dark_correction = dark_time * dark_currents[slot][amp] else: dark_correction = 0 self._set_segment(slot, ccd, amp, hdu, gains[slot][amp], bias_subtract, dark_correction)
def read_multibunch(self, config, dirlist=None, silent=False): """ Reads the data from a bunch of reference pixels after a CCOB scan and fills in self.raw_data and self.properties Parameters ---------- config : dict Contains all required information to reconstruct the beam dirlist : list List of directories containing the data silent : boolean If True, track the progress of the beam reconstruction """ self.properties["ref_raft"] = ref_raft = 'R22' if ( 'ref_raft' not in config) else config['ref_raft'] self.properties["ref_slot"] = ref_slot = 'S11' if ( 'ref_slot' not in config) else config['ref_slot'] self.properties["ref_amp"] = ref_amps = [5] if ( 'ref_amps' not in config) else config['ref_amps'] self.properties["ref_pix_x"] = ref_pix_x = 1000 if ( 'ref_pix_x' not in config) else int(config['ref_pix_x']) self.properties["ref_pix_y"] = ref_pix_y = 256 if ( 'ref_pix_y' not in config) else int(config['ref_pix_y']) self.properties["npix_for_avg"] = npix_for_avg = 30 if ( 'npix_for_avg' not in config) else int(config['npix_for_avg']) biasfile = None if ('biasfile' not in config) else config['biasfile'] outdir = './' if ('tmpdir' not in config) else config['tmpdir'] recons = {} led = self.config['led_name'] dirlist = sorted(dirlist) if 'xarr' not in self.raw_data: self.raw_data['xarr'] = [] if 'yarr' not in self.raw_data: self.raw_data['yarr'] = [] if 'val' not in self.raw_data: self.raw_data['val'] = {} if 'pd_value' not in self.raw_data: self.raw_data['pd_value'] = [] for i, d in enumerate(dirlist): dd = os.path.basename(d) xcurr = float(dd.split('_')[2]) ycurr = float(dd.split('_')[3]) l = list(zip(self.raw_data['xarr'], self.raw_data['yarr'])) # print(i) if (xcurr, ycurr) in l: # print('here') continue self.raw_data['xarr'].append(xcurr) self.raw_data['yarr'].append(ycurr) f = glob.glob( os.path.join(d, "*" + ref_raft + "*" + ref_slot + '*')) bb = fits.open(f[0]) xh = np.round(bb[0].header['BOTXCAM'], 1) yh = np.round(bb[0].header['BOTYCAM'], 1) # self.raw_data['xarr'].append(np.round(bb[0].header['BOTX'],1)) # self.raw_data['yarr'].append(np.round(bb[0].header['BOTY'],1)) self.raw_data['pd_value'].append(bb[0].header['CCOBADC']) bb.close() ccd_dict = sensorTest.MaskedCCD(f[0], bias_frame=biasfile) for amp in ref_amps: image = ccd_dict.unbiased_and_trimmed_image(amp) arr = image.getArrays() val = np.mean((arr[0][int(ref_pix_x-npix_for_avg/2):int(ref_pix_x+npix_for_avg/2),\ int(ref_pix_y-npix_for_avg/2):int(ref_pix_y+npix_for_avg/2)])) if amp in self.raw_data['val']: self.raw_data['val'][amp].append(val) else: self.raw_data['val'][amp] = [val] if not silent: #print(os.path.basename(f[0])) print(amp, self.raw_data['xarr'][-1], xh, self.raw_data['yarr'][-1], yh, val) #print(self.raw_data['val']) # Reordering the data x = self.raw_data['xarr'] y = self.raw_data['yarr'] val = self.raw_data['val'] pd = self.raw_data['pd_value'] newx = [x for x, _, _, _ in sorted(zip(x, y, val[ref_amps[0]], pd))] newy = [y for _, y, _, _ in sorted(zip(x, y, val[ref_amps[0]], pd))] newpd = [pd for _, _, _, pd in sorted(zip(x, y, val[ref_amps[0]], pd))] newval = {} for amp in ref_amps: newval[amp] = [ val for _, _, val, _ in sorted(zip(x, y, val[amp], pd)) ] self.raw_data['xarr'] = newx self.raw_data['yarr'] = newy self.raw_data['val'] = newval self.raw_data['pd_value'] = newpd if outdir is not None: for amp in ref_amps: outf = os.path.join( outdir, 'beam_raw_' + led + '_' + ref_raft + '_' + ref_slot + '_' + str(amp) + '_' + str(ref_pix_x) + '_' + str(ref_pix_y) + '.txt') np.savetxt(outf, np.array([newx, newy, newval[amp], newpd]).T, delimiter=' ')
dir='.', suffix='.fits').name imutils.fits_median_file(darks, med_file, bitpix=-32) mask_file = tempfile.NamedTemporaryFile(prefix='tmp_mask_', dir='.', suffix='.fits').name is_masked = make_mask(med_file, gains, mask_file) fp_id, x0, y0, pixel_values = [], [], [], [] index = -1 exptime = 0 num_pix = 0 for dark in darks: print "processing", dark ccd = sensorTest.MaskedCCD(dark, mask_files=(mask_file, )) exptime += ccd.md.get('EXPTIME') for amp in ccd.keys(): image = ccd[amp] image -= bg_image(ccd, amp) image = image.Factory(image, ccd.amp_geom.imaging) num_pix += np.prod(image.getImage().getArray().shape) flags = afw_math.MEDIAN | afw_math.STDEVCLIP stats = afw_math.makeStatistics(image, flags, ccd.stat_ctrl) median = stats.getValue(afw_math.MEDIAN) stdev = stats.getValue(afw_math.STDEVCLIP) threshold = afw_detect.Threshold(median + args.nsig * stdev) fp_set = afw_detect.FootprintSet(image, threshold) for fp in fp_set.getFootprints():
def make_ccd_2d_array(infile, biasfile=None, gains=None): ''' Generate a 2d array of a sensor image (trimmed, bias subtracted) from an input fits file and a gain dictionary. Function adapted from sensorTest.plot_flat() ''' ccd = sensorTest.MaskedCCD(infile, bias_frame=biasfile) foo = fits.open(infile) datasec = parse_geom_kwd(foo[1].header['DATASEC']) # Specialize to science sensor or wavefront sensor geometries. nx_segments = 8 ny_segments = len(ccd) // nx_segments nx = nx_segments * (datasec['xmax'] - datasec['xmin'] + 1) ny = ny_segments * (datasec['ymax'] - datasec['ymin'] + 1) mosaic = np.zeros((ny, nx), dtype=np.float) amp_coords = {} for ypos in range(ny_segments): for xpos in range(nx_segments): amp = ypos * nx_segments + xpos + 1 # # Determine subarray boundaries in the mosaicked image array # from DETSEC keywords for each segment. detsec = parse_geom_kwd(foo[amp].header['DETSEC']) datasec = parse_geom_kwd(foo[amp].header['DATASEC']) xmin = nx - max(detsec['xmin'], detsec['xmax']) xmax = nx - min(detsec['xmin'], detsec['xmax']) + 1 ymin = ny - max(detsec['ymin'], detsec['ymax']) ymax = ny - min(detsec['ymin'], detsec['ymax']) + 1 # # # Extract the bias-subtracted masked image for this segment. segment_image = ccd.unbiased_and_trimmed_image(amp) subarr = segment_image.getImage().getArray() # # Determine flips in x- and y-directions in order to # get the (1, 1) pixel in the lower right corner. flipx = False flipy = False if detsec['xmax'] > detsec['xmin']: # flip in x-direction subarr = subarr[:, ::-1] flipx = True if detsec['ymax'] > detsec['ymin']: # flip in y-direction subarr = subarr[::-1, :] flipy = True # # Convert from ADU to e- if gains is not None: subarr *= gains[amp] # # Save coordinates of segment for later use amp_coords[(xpos, ypos)] = { 'amp': amp, 'segment': foo[amp].header['EXTNAME'], 'xmin': xmin, 'xmax': xmax, 'ymin': ymin, 'ymax': ymax, 'flipx': flipx, 'flipy': flipy, 'detsec': detsec, 'datasec': datasec } # Set the subarray in the mosaic. mosaic[ymin:ymax, xmin:xmax] = subarr return mosaic, amp_coords
def fit_superflats(datapath, sensor_id): filepath = lambda level: \ os.path.join(datapath, '%s_superflat_%s.fits' % (sensor_id, level)) ccd_low = sensorTest.MaskedCCD(filepath('low')) ccd_high = sensorTest.MaskedCCD(filepath('high')) TrailedChargeResults = namedtuple('TrailedChargeResults', ['fit_results', 'plots']) fit_results = FitResults(sensor_id) my_plots = MultiPanelPlot(4, 4, figsize=(12, 12)) for amp in ccd_high: tc_low = TrailedCharge(ccd_low, amp, lastskip=4) tc_high = TrailedCharge(ccd_high, amp, lastskip=4) # Fix bias levels to last column value tc_low.bias_per_pixel = tc_low.oscan_values[-1] / tc_low.nrows tc_high.bias_per_pixel = tc_high.oscan_values[-1] / tc_high.nrows # Initial parameter estimates cti_0 = tc_high.cti() tau = 1. / np.log( (tc_high.oscan_values[1] - tc_high.oscan_values[-1]) / (tc_high.oscan_values[2] - tc_high.oscan_values[-1])) q0 = ((tc_high.oscan_values[1] / tc_high.nrows - tc_high.bias_per_pixel) / np.exp(-2 / tau) / tc_high.q_lastcol) p0 = q0, tau, cti_0 bounds = ((0, None), (0, None), (0, None)) result_low = scipy.optimize.minimize(tc_low, p0, method='L-BFGS-B', bounds=bounds) fit_results.add_results('low', amp, result_low) result_high = scipy.optimize.minimize(tc_high, p0, method='L-BFGS-B', bounds=bounds) fit_results.add_results('high', amp, result_high) tc_joint = MultiObjectiveFunctions([tc_low, tc_high]) result = scipy.optimize.minimize(tc_joint, p0, method='L-BFGS-B', bounds=bounds) fit_results.add_results('joint', amp, result) my_plots.add_subplot() tc_low.plot_fit(result_low.x, label='low flux-only', linewidth=1) tc_high.plot_fit(result_high.x, color='red', label='high flux-only', linewidth=1) tc_low.plot_model(result.x, color='green', marker='--', label='joint') tc_high.plot_model(result.x, color='green', marker='--') plt.annotate('Amp %i' % amp, (0.1, 0.9), xycoords='axes fraction', size='x-small', horizontalalignment='left') if tc_high.oscan_values[0] / tc_low.oscan_values[0] > 5: plt.yscale('log') axis_range = list(plt.axis()) axis_range[:2] = 0.5, axis_range[1] + 0.5 plt.axis(axis_range) plt.legend(loc=0, fontsize='x-small') my_plots.set_title(sensor_id) my_plots.set_xlabel('overscan pixel') my_plots.set_ylabel('ADU / pixel') return TrailedChargeResults(fit_results, my_plots)