def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) for key, val in data.items(): data[key] = val.replace(self.config.outsuffix, self.config.insuffix) # Define the set of columns to keep and remove # keep_cols = [] # remove_cols = [] outtable = vstack_tables(data, tablename='tmplsuffix_stats') dtables = TableDict() dtables.add_datatable('tmplsuffix_sum', outtable) dtables.make_datatable('runs', dict(runs=sorted(data.keys()))) return dtables
def extract(self, butler, data, **kwargs): """Extract the summary data from the PTC analyses Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) if butler is not None: self.log.warn("Ignoring butler") if not self.config.skip: outtable = vstack_tables(data, tablename='ptc_stats') dtables = TableDict() dtables.add_datatable('ptc_sum', outtable) dtables.make_datatable('runs', dict(runs=sorted(data.keys()))) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` output data tables """ self.safe_update(**kwargs) for key, val in data.items(): print(key, val) data[key] = val.replace(self.config.filekey, self.config.infilekey) # Define the set of columns to keep and remove # keep_cols = [] # remove_cols = [] outtable = vstack_tables(data, tablename='eo_results') dtables = TableDict() dtables.add_datatable('eo_results_run', outtable) dtables.make_datatable('runs', dict(runs=[self.config.run])) return dtables
def extract(self, butler, data, **kwargs): """Make a summary table Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) if butler is not None: self.log.warn("Ignoring butler") for key, val in data.items(): data[key] = val.replace('_sum.fits', '_stats.fits') outtable = vstack_tables(data, tablename='biasosstack_stats') dtables = TableDict() dtables.add_datatable('biasosstack_sum', outtable) dtables.make_datatable('runs', dict(runs=sorted(data.keys()))) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) run_dict = dict(runs=[], rafts=[]) for key, val in data.items(): run_dict['runs'].append(key[4:]) run_dict['rafts'].append(key[0:3]) data[key] = val.replace(self.config.filekey, self.config.infilekey) # Define the set of columns to keep and remove # keep_cols = [] # remove_cols = [] outtable = vstack_tables(data, tablename='dark_current') dtables = TableDict() dtables.add_datatable('stats', outtable) dtables.make_datatable('runs', run_dict) return dtables
def extract(self, butler, data, **kwargs): """Make a summary table of the superbias statistics Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) if butler is not None: self.log.warn("Ignoring butler") run_dict = dict(runs=[], rafts=[]) for key, val in sorted(data.items()): run_dict['runs'].append(key[4:]) run_dict['rafts'].append(key[0:3]) data[key] = val.replace(self.config.filekey, self.config.infilekey) outtable = vstack_tables(data, tablename='stats') dtables = TableDict() dtables.add_datatable('stats', outtable) dtables.make_datatable('runs', run_dict) return dtables
def extract(self, butler, data, **kwargs): """Stack the overscan region from all the amps on a sensor to look for coherent read noise Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) bias_files = data['BIAS'] bias_type = self.get_bias_algo() mask_files = self.get_mask_files() superbias_frame = self.get_superbias_frame(mask_files=mask_files) self.log_info_slot_msg(self.config, "%i files" % len(bias_files)) stack_arrays = {} nfiles = len(bias_files) for ifile, bias_file in enumerate(bias_files): if ifile % 10 == 0: self.log_progress(" %i" % ifile) ccd = self.get_ccd(butler, bias_file, mask_files) if ifile == 0: dim_array_dict = get_dimension_arrays_from_ccd(ccd) for key, val in dim_array_dict.items(): stack_arrays[key] = np.zeros((nfiles, 16, len(val))) stack_by_amps(stack_arrays, ccd, ifile=ifile, bias_type=bias_type, superbias_frame=superbias_frame) self.log_progress("Done!") stackdata_dict = convert_stack_arrays_to_dict(stack_arrays, dim_array_dict, nfiles) dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, bias_files)) for key, val in stackdata_dict.items(): dtables.make_datatable('stack-%s' % key, val) return dtables
def extract(self, butler, data, **kwargs): """Plot the row-wise and col-wise struture in a series of bias frames Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) slot = self.config.slot bias_files = data['BIAS'] mask_files = self.get_mask_files() superbias_frame = self.get_superbias_frame(mask_files=mask_files) self.log_info_slot_msg(self.config, "%i files" % len(bias_files)) biasstruct_data = {} for ifile, bias_file in enumerate(bias_files): if ifile % 10 == 0: self.log_progress(" %i" % ifile) ccd = self.get_ccd(butler, bias_file, mask_files) if ifile == 0: dim_array_dict = get_dimension_arrays_from_ccd(ccd) for key, val in dim_array_dict.items(): biasstruct_data[key] = {key: val} self.get_ccd_data(ccd, biasstruct_data, slot=slot, ifile=ifile, nfiles_used=len(bias_files), superbias_frame=superbias_frame) self.log_progress("Done!") if not biasstruct_data: return None dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, bias_files)) for key, val in biasstruct_data.items(): dtables.make_datatable('biasst-%s' % key, val) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) gains = np.ones((17)) ppump_files = data['PPUMP'] print(ppump_files) ppump_file = ppump_files[0] mask_files = self.get_mask_files() self.log_info_slot_msg(self.config, "") # This is a dictionary of dictionaries to store all the # data you extract from the ppump_files data_dict = {} # Analysis goes here, you should fill data_dict with data extracted # by the analysis # ccd = self.get_ccd(butler, ppump_file, mask_files, masked_ccd=True) my_traps = Traps(ccd, gains, cycles=self.config.cycles, C2_thresh=self.config.C2_thresh, C3_thresh=self.config.C3_thresh, nx=self.config.bkg_nx, ny=self.config.bkg_ny, edge_rolloff=self.config.edge_rolloff) data_dict = my_traps.make_data_dict() self.log_progress("Done!") dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, ppump_files)) dtables.make_datatable('traps', data_dict) return dtables
def extract(self, butler, data, **kwargs): """Extract the row-wise and col-wise struture in a superbias frame Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) slot = self.config.slot if butler is not None: self.log.warn("Ignoring butler") if data is not None: self.log.warn("Ignoring data") mask_files = self.get_mask_files() superbias = self.get_superbias_frame(mask_files=mask_files) if superbias is None: return None self.log_info_slot_msg(self.config, "") biasstruct_data = {} dim_array_dict = get_dimension_arrays_from_ccd(superbias) for key, val in dim_array_dict.items(): biasstruct_data[key] = {key: val} self.get_ccd_data(superbias, biasstruct_data, slot=slot, bias_type=None, std=self.config.std, superbias_frame=None) self.log_progress("Done!") dtables = TableDict() dtables.make_datatable('files', make_file_dict(None, [slot])) for key, val in biasstruct_data.items(): dtables.make_datatable('biasst-%s' % key, val) return dtables
def extract(self, butler, data, **kwargs): """Extract the bias as function of row Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) bias_files = data['BIAS'] if not bias_files: return None mask_files = self.get_mask_files() self.log_info_slot_msg(self.config, "%i files" % len(bias_files)) biasval_data = {} for ifile, bias_file in enumerate(bias_files): if ifile % 10 == 0: self.log_progress(" %i" % ifile) ccd = self.get_ccd(butler, bias_file, mask_files) if ifile == 0: dims = get_dims_from_ccd(ccd) xrow_s = np.linspace(0, dims['nrow_s'] - 1, dims['nrow_s']) self.get_ccd_data(ccd, biasval_data, ifile=ifile, nfiles=len(bias_files)) #Need to truncate the row array to match the data a_row = biasval_data[sorted(biasval_data.keys())[0]] biasval_data['row_s'] = xrow_s[0:len(a_row)] self.log_progress("Done!") dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, bias_files)) dtables.make_datatable('biasval', biasval_data) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) # You should expand this to include space for the data you want to extract data_dict = dict(slot=[], amp=[]) sys.stdout.write("Working on 9 slots: ") sys.stdout.flush() for islot, slot in enumerate(ALL_SLOTS): sys.stdout.write(" %s" % slot) sys.stdout.flush() basename = data[slot] datapath = basename.replace(self.config.outsuffix, self.config.insuffix) dtables = TableDict(datapath) print(dtables) for amp in range(16): # Here you can get the data out for each amp and append it to the # data_dict data_dict['slot'].append(islot) data_dict['amp'].append(amp) sys.stdout.write(".\n") sys.stdout.flush() outtables = TableDict() outtables.make_datatable("tmplsuffix", data_dict) return outtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) # You should expand this to include space for the data you want to extract data_dict = dict(amp=[], full_well=[], max_dev=[]) self.log_info_raft_msg(self.config, "") basename = data[0] datapath = basename.replace(self.config.filekey, self.config.infilekey) detresp = DetectorResponse(datapath, hdu_name='flat') for amp in range(1, 17): # Here you can get the data out for each amp and append it to the # data_dict data_dict['amp'].append(amp) linearity_data = detresp.linearity(amp) full_well_data = detresp.full_well(amp) data_dict['full_well'].append(full_well_data[0]) data_dict['max_dev'].append(linearity_data[0]) self.log_progress("Done!") outtables = TableDict() outtables.make_datatable("flat_lin", data_dict) return outtables
def extract(self, butler, data, **kwargs): """Extract the correlations between the serial overscan for each amp on a raft Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) slots = ALL_SLOTS overscans = [] for slot in slots: bias_files = data[slot]['BIAS'] mask_files = self.get_mask_files(slot=slot) superbias_frame = self.get_superbias_frame(mask_files, slot=slot) ccd = self.get_ccd(butler, bias_files[0], []) overscans += self.get_ccd_data(butler, ccd, superbias_frame=superbias_frame) namps = len(overscans) if self.config.covar: data = np.array([np.cov(overscans[i[0]].ravel(), overscans[i[1]].ravel())[0, 1] for i in itertools.product(range(namps), range(namps))]) else: data = np.array([np.corrcoef(overscans[i[0]].ravel(), overscans[i[1]].ravel())[0, 1] for i in itertools.product(range(namps), range(namps))]) data = data.reshape(namps, namps) dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, bias_files)) dtables.make_datatable('correl', dict(correl=data)) return dtables
def extract(self, butler, data, **kwargs): """Extract the utliers in the superdark frames for the raft Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) self._mask_file_dict = {} self._sdark_file_dict = {} if butler is not None: self.log.warn("Ignoring butler") self.set_local_data(butler, data, **kwargs) if not self._sdark_file_dict: self.log.warn("No files for %s, skipping" % (self.config.raft)) return None self._sdark_images, ccd_dict = extract_raft_unbiased_images( self._sdark_file_dict, mask_dict=self._mask_file_dict) self._sdark_arrays = extract_raft_imaging_data(self._sdark_images, ccd_dict) out_data = outlier_raft_dict(self._sdark_arrays, 0., 25.) fp_dict = build_defect_dict(self._sdark_images, fp_type='bright', abs_thresh=50) dtables = TableDict() dtables.make_datatable('defects', fp_dict) dtables.make_datatable('outliers', out_data) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) # Get the slot, if needed #slot = self.config.slot tmpl_files = data['TMPL'] # Get the superbias frame, if needed #mask_files = self.get_mask_files() #superbias_frame = self.get_superbias_frame(mask_files) self.log_info_slot_msg(self.config, "") # This is a dictionary of dictionaries to store all the # data you extract from the tmpl_files data_dict = {} # Analysis goes here, you should fill data_dict with data extracted # by the analysis # self.log_progress("Done!") dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, tmpl_files)) for key, val in data_dict.items(): dtables.make_datatable(key, val) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) # You should expand this to include space for the data you want to extract data_dict = dict(slot=[], amp=[]) self.log_info_raft_msg(self.config, "") for islot, slot in enumerate(ALL_SLOTS): self.log_progress(" %s" % slot) # get the data from the input table #basename = data[slot] #datapath = basename.replace(self.config.outsuffix, self.config.insuffix) #dtables = TableDict(datapath) for amp in range(16): # Here you can get the data out for each amp and append it to the # data_dict data_dict['slot'].append(islot) data_dict['amp'].append(amp) self.log_progress("Done!") outtables = TableDict() outtables.make_datatable("tmplsuffix", data_dict) return outtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` output data tables """ self.safe_update(**kwargs) slots = ALL_SLOTS # This is a dictionary of dictionaries to store all the # data you extract from the tmpl_files data_dict = {} for slot in slots: #Get the datafiles, and the correpsonding superbias files tmpl_files = data[slot]['TMPL'] print(tmpl_files) #mask_files = self.get_mask_files(slot=slot) #superbias_frame = self.get_superbias_frame(mask_files, slot=slot) # Analysis goes here, you should fill data_dict with data extracted # by the analysis # dtables = TableDict() for key, val in data_dict.items(): dtables.make_datatable(key, val) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` output data tables """ self.safe_update(**kwargs) slots = ALL_SLOTS # This is a dictionary of dictionaries to store all the # data you extract from the base_files data_dict = dict(slot=np.ndarray((144))) for islot, slot in enumerate(slots): infile = data[slot] try: dtables = TableDict(infile, ['amplifier_results']) except FileNotFoundError: return None table = dtables['amplifier_results'] data_dict['slot'][16 * islot:16 * (islot + 1)] = islot for key in table.keys(): if key not in data_dict: data_dict[key] = np.ndarray((144)) data_dict[key][16 * islot:16 * (islot + 1)] = table[key] dtables = TableDict() dtables.make_datatable('eo_results', data_dict) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) self.log_info_slot_msg(self.config, "") pd_calib_file = self.get_pd_calib_file() gains = np.ones((17)) basename = data[0] qe_data = QE_Data() qe_data.read_fits(basename) qe_data.incidentPower(pd_calib_file) qe_data.calculate_QE(gains) qe_curves_data_dict = qe_data.make_qe_curves_data_dict() bands_data_dict = qe_data.make_bands_data_dict() dtables = TableDict() dtables.make_datatable('qe_curves', qe_curves_data_dict) dtables.make_datatable('bands', bands_data_dict) return dtables
def extract(self, butler, data, **kwargs): """Extract the outliers in the superflat frames for the raft Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) self._mask_file_dict = {} self._sflat_file_dict_l = {} self._sflat_file_dict_h = {} self._sflat_file_dict_r = {} if butler is not None: self.log.warn("Ignoring butler") self.set_local_data(butler, data, **kwargs) if not self._sflat_file_dict_l: self.log.warn("No files for %s, skipping" % (self.config.raft)) return None self._sflat_images_h, ccd_dict = extract_raft_unbiased_images( self._sflat_file_dict_h, mask_dict=self._mask_file_dict) self._sflat_array_l = extract_raft_array_dict( self._sflat_file_dict_l, mask_dict=self._mask_file_dict) self._sflat_array_h = extract_raft_imaging_data( self._sflat_images_h, ccd_dict) self._sflat_array_r = extract_raft_array_dict( self._sflat_file_dict_r, mask_dict=self._mask_file_dict) out_data_l = outlier_raft_dict(self._sflat_array_l, 1000., 300.) out_data_h = outlier_raft_dict(self._sflat_array_h, 50000., 15000.) out_data_r = outlier_raft_dict(self._sflat_array_r, 0.019, 0.002) fp_dict = SuperflatRaftTask.build_defect_dict(self._sflat_images_h, frac_thresh=0.9) dtables = TableDict() dtables.make_datatable('defects', fp_dict) dtables.make_datatable('outliers_l', out_data_l) dtables.make_datatable('outliers_h', out_data_h) dtables.make_datatable('outliers_r', out_data_r) return dtables
def extract(self, butler, data, **kwargs): """Compare frames to superflat Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- sflat_l : `dict` Dictionary keyed by amp of the low exposure superflats sflat_h : `dict` Dictionary keyed by amp of the high exposure superflats ratio_images : `dict` Dictionary keyed by amp of the low/high ratio images """ self.safe_update(**kwargs) mask_files = self.get_mask_files() superbias_frame = self.get_superbias_frame(mask_files) bias_type = self.get_bias_algo() sflat_files = data['SFLAT'] if not sflat_files: self.log_warn_slot_msg(self.config, "No superflat files") return None self.log_info_slot_msg(self.config, "%i files" % (len(sflat_files))) # This is a dictionary of dictionaries to store all the # data you extract from the flat_files data_dict = dict(FLUX=[], EXPTIME=[], MONDIODE=[]) for i in range(1, 17): data_dict['AMP%02i_FLAT_MEAN' % i] = [] data_dict['AMP%02i_FLAT_MEDIAN' % i] = [] data_dict['AMP%02i_FLAT_VAR' % i] = [] data_dict['AMP%02i_FLAT_ROWMEAN' % i] = [] data_dict['AMP%02i_FLAT_COLMEAN' % i] = [] for ifile, sflat_file in enumerate(sflat_files): if ifile % 10 == 0: self.log_progress(" %i" % ifile) sflat = self.get_ccd(butler, sflat_file, mask_files) exp_time = get_exposure_time(sflat) mondiode = get_monodiode_val_from_data_id(sflat_file, exp_time, self.config.teststand, butler) if mondiode is None: continue flux = exp_time * mondiode data_dict['EXPTIME'].append(exp_time) data_dict['MONDIODE'].append(mondiode) data_dict['FLUX'].append(flux) ccd_ims = unbiased_ccd_image_dict(sflat, bias=bias_type, superbias_frame=superbias_frame, trim='imaging') amps = get_amp_list(sflat) for amp in amps: image = ccd_ims[amp] fstats = self.get_stats(image) data_dict['AMP%02i_FLAT_MEAN' % amp].append(fstats[0]) data_dict['AMP%02i_FLAT_MEDIAN' % amp].append(fstats[1]) data_dict['AMP%02i_FLAT_VAR' % amp].append(fstats[2]) data_dict['AMP%02i_FLAT_ROWMEAN' % amp].append(image.image.array.mean(0)) data_dict['AMP%02i_FLAT_COLMEAN' % amp].append(image.image.array.mean(1)) self.log_progress("Done!") primary_hdu = fits.PrimaryHDU() primary_hdu.header['NAMPS'] = 16 dtables = TableDict(primary=primary_hdu) dtables.make_datatable('files', make_file_dict(butler, sflat_files)) dtables.make_datatable('stability', data_dict) return dtables
def extract(self, butler, data, **kwargs): """Extract the gains and widths from the f355 clusters Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) if butler is not None: self.log.warn("Ignoring butler") use_all = self.config.use_all data_dict = dict(kalpha_peak=[], kalpha_sigma=[], ncluster=[], ngood=[], gain=[], gain_error=[], fit_xmin=[], fit_xmax=[], fit_pars=[], fit_nbins=[], sigmax_median=[], sigmay_median=[], slot=[], amp=[]) self.log_info_raft_msg(self.config, "") for islot, slot in enumerate(ALL_SLOTS): self.log_progress(" %s" % slot) basename = data[slot] dtables = TableDict(basename) for amp in range(16): table = dtables['amp%02i' % (amp + 1)] if use_all: mask = np.ones((len(table)), bool) else: mask = (np.fabs(table['XPOS'] - table['XPEAK']) < 1)*\ (np.fabs(table['YPOS'] - table['YPEAK']) < 1) tablevals = table[mask]['DN'] gainfitter = Fe55GainFitter(tablevals) try: kalpha_peak, kalpha_sigma = gainfitter.fit(bins=100) gain = gainfitter.gain gain_error = gainfitter.gain_error pars = gainfitter.pars except Exception: kalpha_peak, kalpha_sigma = (np.nan, np.nan) gain = np.nan gain_error = np.nan pars = np.nan * np.ones((4)) data_dict['kalpha_peak'].append(kalpha_peak) data_dict['kalpha_sigma'].append(kalpha_sigma) data_dict['gain'].append(gain) data_dict['gain_error'].append(gain_error) xra = gainfitter.xrange data_dict['ncluster'].append(mask.size) data_dict['ngood'].append(mask.sum()) if xra is None: data_dict['fit_xmin'].append(np.nan) data_dict['fit_xmax'].append(np.nan) else: data_dict['fit_xmin'].append(xra[0]) data_dict['fit_xmax'].append(xra[1]) data_dict['fit_pars'].append(pars) data_dict['fit_nbins'].append(100.) data_dict['sigmax_median'].append(np.median(table['SIGMAX'])) data_dict['sigmay_median'].append(np.median(table['SIGMAY'])) data_dict['slot'].append(islot) data_dict['amp'].append(amp) self.log_progress("Done!") outtables = TableDict() outtables.make_datatable("fe55_gain_stats", data_dict) return outtables
def extract(self, butler, data, **kwargs): """Extract the summary statistics about the fluctuations ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) if butler is not None: self.log.warn("Ignoring bulter") data_dict = dict(s_row_min_mean=[], s_row_min_median=[], s_row_min_std=[], s_row_min_min=[], s_row_min_max=[], s_row_max_mean=[], s_row_max_median=[], s_row_max_std=[], s_row_max_min=[], s_row_max_max=[], p_col_min_mean=[], p_col_min_median=[], p_col_min_std=[], p_col_min_min=[], p_col_min_max=[], p_col_max_mean=[], p_col_max_median=[], p_col_max_std=[], p_col_max_min=[], p_col_max_max=[], slot=[]) self.log_info_raft_msg(self.config, "") for islot, slot in enumerate(ALL_SLOTS): self.log_progress(" %s" % slot) basename = data[slot] datapath = basename.replace('_stats.fits', '.fits') try: dtables = TableDict(datapath, ['stack-row_s', 'stack-col_p']) table_s = dtables['stack-row_s'] table_p = dtables['stack-col_p'] except FileNotFoundError: self.log.error("Could not open %s\n" % datapath) table_s = None table_p = None if table_s is not None: tablevals_s_min = table_s['stack_mean'].min(0) tablevals_s_max = table_s['stack_mean'].max(0) else: tablevals_s_min = [0., 0.] tablevals_s_max = [0., 0.] if table_p is not None: tablevals_p_min = table_p['stack_mean'].min(0) tablevals_p_max = table_p['stack_mean'].max(0) else: tablevals_p_min = [0., 0.] tablevals_p_max = [0., 0.] data_dict['s_row_min_mean'].append(np.mean(tablevals_s_min)) data_dict['s_row_min_median'].append(np.median(tablevals_s_min)) data_dict['s_row_min_std'].append(np.std(tablevals_s_min)) data_dict['s_row_min_min'].append(np.min(tablevals_s_min)) data_dict['s_row_min_max'].append(np.max(tablevals_s_min)) data_dict['s_row_max_mean'].append(np.mean(tablevals_s_max)) data_dict['s_row_max_median'].append(np.median(tablevals_s_max)) data_dict['s_row_max_std'].append(np.std(tablevals_s_max)) data_dict['s_row_max_min'].append(np.min(tablevals_s_max)) data_dict['s_row_max_max'].append(np.max(tablevals_s_max)) data_dict['p_col_min_mean'].append(np.mean(tablevals_p_min)) data_dict['p_col_min_median'].append(np.median(tablevals_p_min)) data_dict['p_col_min_std'].append(np.std(tablevals_p_min)) data_dict['p_col_min_min'].append(np.min(tablevals_p_min)) data_dict['p_col_min_max'].append(np.max(tablevals_p_min)) data_dict['p_col_max_mean'].append(np.mean(tablevals_p_max)) data_dict['p_col_max_median'].append(np.median(tablevals_p_max)) data_dict['p_col_max_std'].append(np.std(tablevals_p_max)) data_dict['p_col_max_min'].append(np.min(tablevals_p_max)) data_dict['p_col_max_max'].append(np.max(tablevals_p_max)) data_dict['slot'].append(islot) self.log_progress("Done!") outtables = TableDict() outtables.make_datatable("biasosstack_stats", data_dict) return outtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) if self.config.teststand == 'ts8': flat1_files = data['FLAT1'] flat2_files = data['FLAT2'] elif self.config.teststand == 'bot': flat1_files = data['FLAT0'] flat2_files = data['FLAT1'] bias_type = self.get_bias_algo() mask_files = self.get_mask_files() superbias_frame = self.get_superbias_frame(mask_files) nlc = self.get_nonlinearirty_correction() #slot_idx = ALL_SLOTS.index(self.config.slot) self.log_info_slot_msg(self.config, "%i %i files" % (len(flat1_files), len(flat2_files))) # This is a dictionary of dictionaries to store all the # data you extract from the flat_files data_dict = dict(FLUX=[], EXPTIME=[], MONDIODE1=[], MONDIODE2=[], MONOCH_SLIT_B=[]) for i in range(1, 17): data_dict['AMP%02i_RATIO' % i] = [] data_dict['AMP%02i_MEAN' % i] = [] data_dict['AMP%02i_CORRMEAN' % i] = [] data_dict['AMP%02i_VAR' % i] = [] data_dict['AMP%02i_SIGNAL' % i] = [] data_dict['AMP%02i_MEAN1' % i] = [] data_dict['AMP%02i_MEAN2' % i] = [] # Analysis goes here, you should fill data_dict with data extracted # by the analysis # for ifile, (id_1, id_2) in enumerate(zip(flat1_files, flat2_files)): if ifile % 10 == 0: self.log_progress(" %i" % ifile) flat_1 = self.get_ccd(butler, id_1, mask_files) flat_2 = self.get_ccd(butler, id_2, mask_files) amps = get_amp_list(flat_1) exp_time_1 = get_exposure_time(flat_1) exp_time_2 = get_exposure_time(flat_2) if exp_time_1 != exp_time_2: self.log.warn("Exposure times do not match for:\n%s\n%s\n %0.3F %0.3F. Skipping Pair\n" % (id_1, id_2, exp_time_1, exp_time_2)) continue mondiode_1 = get_monodiode_val_from_data_id(id_1, exp_time_1, self.config.teststand, butler) mondiode_2 = get_monodiode_val_from_data_id(id_2, exp_time_2, self.config.teststand, butler) if mondiode_1 is None or mondiode_2 is None: self.log.warn("No monitoring data for:\n%s\n%s\n Skipping Pair\n" % (id_1, id_2)) continue flux = (exp_time_1 * mondiode_1 + exp_time_2 * mondiode_2)/2. data_dict['EXPTIME'].append(exp_time_1) data_dict['MONDIODE1'].append(mondiode_1) data_dict['MONDIODE2'].append(mondiode_2) data_dict['FLUX'].append(flux) try: data_dict['MONOCH_SLIT_B'].append(get_mono_slit_b(flat_1)) except KeyError: data_dict['MONOCH_SLIT_B'].append(0.) ccd_1_ims = unbiased_ccd_image_dict(flat_1, bias=bias_type, superbias_frame=superbias_frame, trim='imaging', nonlinearity=nlc) ccd_2_ims = unbiased_ccd_image_dict(flat_2, bias=bias_type, superbias_frame=superbias_frame, trim='imaging', nonlinearity=nlc) for i, amp in enumerate(amps): image_1 = ccd_1_ims[amp] image_2 = ccd_2_ims[amp] fstats = self.get_pair_stats(image_1, image_2) signal = fstats[1] #if gains is not None: # signal *= gains[slot_idx][i] data_dict['AMP%02i_RATIO' % (i+1)].append(fstats[0]) data_dict['AMP%02i_MEAN' % (i+1)].append(fstats[1]) data_dict['AMP%02i_CORRMEAN' % (i+1)].append(fstats[2]) data_dict['AMP%02i_VAR' % (i+1)].append(fstats[3]) data_dict['AMP%02i_SIGNAL' % (i+1)].append(signal) data_dict['AMP%02i_MEAN1' % (i+1)].append(fstats[4]) data_dict['AMP%02i_MEAN2' % (i+1)].append(fstats[5]) self.log_progress("Done!") primary_hdu = fits.PrimaryHDU() primary_hdu.header['NAMPS'] = 16 dtables = TableDict(primary=primary_hdu) dtables.make_datatable('files', make_file_dict(butler, flat1_files + flat2_files)) dtables.make_datatable('flat', data_dict) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) slot = self.config.slot bias_type = self.get_bias_algo() flat1_files = data['FLAT1'] mask_files = self.get_mask_files() superbias_frame = self.get_superbias_frame(mask_files) self.log_info_slot_msg(self.config, "%i files" % len(flat1_files)) sflat_table_file = self.get_filename_from_format( RAFT_SFLAT_TABLE_FORMATTER, "sflat.fits") sflat_tables = TableDict(sflat_table_file) # dictionary of dictionaries of lists of bounding # boxes, keyed by slot, amp bbox_dict = construct_bbox_dict(sflat_tables['defects']) slot_bbox_dict = bbox_dict[slot] slot_idx_dict = dict(S00=0, S01=1, S02=2, S10=3, S11=4, S12=5, S20=6, S21=7, S22=8) # This is a dictionary of dictionaries to store all the # data you extract from the flat1_files fp_dict = dict(slot=[], amp=[], x_corner=[], y_corner=[], x_size=[], y_size=[], med_flux_full=[], exptime=[], mondiode=[], amp_median=[], ratio_full=[]) for i in range(4): fp_dict['ratio_%i' % i] = [] fp_dict['med_flux_%i' % i] = [] fp_dict['npix_%i' % i] = [] fp_dict['npix_0p2_%i' % i] = [] # Analysis goes here, you should fill fp_dict with data extracted # by the analysis # for ifile, flat1_file in enumerate(flat1_files): if ifile % 10 == 0: self.log_progress(" %i" % ifile) ccd = self.get_ccd(butler, flat1_file, mask_files) # To be appended while looping over bounding boxes exptime = get_exposure_time(ccd) mondiode_val = get_mondiode_val(ccd) islot = slot_idx_dict[slot] unbiased_images = unbiased_ccd_image_dict( ccd, bias=bias_type, superbias_frame=superbias_frame) for iamp, (amp, image) in enumerate(unbiased_images.items()): bbox_list = slot_bbox_dict[iamp] amp_median = np.median(image.array) #fill fp_dict frac_thresh = kwargs.get('frac_thresh', 0.9) thresh_float = frac_thresh * amp_median thresh_0p2_float = (1. - (1. - frac_thresh) * 0.2) * amp_median max_bbox = min(len(bbox_list), 10) #print(i, amp, bbox_list[0:max_bbox]) #print(max_bbox) for bbox in bbox_list[0:max_bbox]: try: cutout = image[bbox].array except Exception: print("cutout failed", slot, amp, bbox) continue #print(ifile, bbox, cutout) ratio_full = np.mean(cutout) / amp_median cutout_median_full = np.median(cutout) peak_idx = cutout.argmax() peak_x = bbox.getMinX() + int(peak_idx % cutout.shape[1]) peak_y = bbox.getMinY() + int(peak_idx / cutout.shape[1]) peak = afwGeom.Point2I(peak_x, peak_y) extent = afwGeom.Extent2I(1, 1) bbox_expand = afwGeom.Box2I(peak, extent) #npix = np.array([1, 9, 25, 49]) npix_cumul = np.array([1, 8, 16, 24]) sums = np.zeros((4)) meds = np.zeros((4)) over_thresh = np.zeros((4), int) over_0p2_thresh = np.zeros((4), int) sums_cumul = np.zeros((4)) meds_cumul = np.zeros((4)) over_thresh_cumul = np.zeros((4), int) over_0p2_thresh_cumul = np.zeros((4), int) for j in range(4): try: cuttout_array = image[bbox_expand].array except LengthError: break sums[j] = cuttout_array.sum() meds[j] = np.median(cuttout_array) over_thresh[j] = (cuttout_array > thresh_float).sum() over_0p2_thresh[j] = (cuttout_array > thresh_0p2_float).sum() if j > 0: sums_cumul[j] = sums[j] - sums[j - 1] meds_cumul[j] = meds[j] - meds[j - 1] over_thresh_cumul[ j] = over_thresh[j] - over_thresh[j - 1] over_0p2_thresh_cumul[j] = over_0p2_thresh[ j] - over_0p2_thresh[j - 1] else: sums_cumul[j] = sums[j] meds_cumul[j] = meds[j] over_thresh_cumul[j] = over_thresh[j] over_0p2_thresh_cumul[j] = over_0p2_thresh[j] bbox_expand.grow(1) means_cumul = sums_cumul / (npix_cumul * amp_median) fp_dict['exptime'].append(exptime) fp_dict['mondiode'].append(mondiode_val) fp_dict['amp'].append(iamp) fp_dict['slot'].append(islot) fp_dict['amp_median'].append(amp_median) fp_dict['x_corner'].append(bbox.getMinX()) fp_dict['y_corner'].append(bbox.getMinY()) fp_dict['x_size'].append(bbox.getWidth()) fp_dict['y_size'].append(bbox.getHeight()) fp_dict['med_flux_full'].append(cutout_median_full) fp_dict['ratio_full'].append(ratio_full) for i in range(4): fp_dict['ratio_%i' % i].append(means_cumul[i]) fp_dict['med_flux_%i' % i].append(meds_cumul[i]) fp_dict['npix_%i' % i].append(over_thresh_cumul[i]) fp_dict['npix_0p2_%i' % i].append( over_0p2_thresh_cumul[i]) sys.stdout.write("!\n") sys.stdout.flush() dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, flat1_files)) dtables.make_datatable('footprints', fp_dict) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` output data tables """ self.safe_update(**kwargs) slots = self.config.slots if slots is None: slots = ALL_SLOTS if butler is not None: self.log.warn("Ignoring butler") dark_current_data = dict(median=[], stdev=[], mean=[], fit_mean=[], fit_width=[], fit_dof=[], fit_chi2=[], exptime=[], current=[], slot=[], amp=[]) self.log_info_raft_msg(self.config, "") for islot, slot in enumerate(slots): self.log_progress(" %s" % slot) mask_files = self.get_mask_files(slot=slot) superdark_file = data[slot] if not os.path.exists(superdark_file): self.log.warn(" %s does not exist, skipping" % superdark_file) continue superdark_frame = self.get_ccd(None, superdark_file, mask_files) exptime = get_exposure_time(superdark_frame) amps = get_amp_list(superdark_frame) for iamp, amp in enumerate(amps): regions = get_geom_regions(superdark_frame, amp) imaging = regions['imaging'] superdark_im = get_raw_image(superdark_frame, amp).image image_data = superdark_im[imaging].array median = np.median(image_data) stdev = np.std(image_data) hist_bins = np.linspace(median - 5 * stdev, median + 5 * stdev, 101) hist = np.histogram(image_data, bins=hist_bins) fit_result = gauss_fit(hist) fit_pars = fit_result[0] dark_current_data['median'].append(median) dark_current_data['stdev'].append(stdev) dark_current_data['mean'].append(np.mean(image_data)) dark_current_data['fit_mean'].append(fit_pars[1]) dark_current_data['fit_width'].append(fit_pars[2]) dark_current_data['fit_dof'].append(0.) dark_current_data['fit_chi2'].append(0.) dark_current_data['exptime'].append(exptime) dark_current_data['current'].append(median/exptime) dark_current_data['slot'].append(islot) dark_current_data['amp'].append(iamp) self.log_progress("Done!") dtables = TableDict() dtables.make_datatable('dark_current', dark_current_data) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) slot = self.config.slot qe_files = data['LAMBDA'] lams = [] for file in qe_files: lam = file.split('flat_')[1].split('_')[0] lams.append(lam) mask_files = self.get_mask_files() superbias_frame = self.get_superbias_frame(mask_files) bias_type = self.get_bias_algo() self.log_info_slot_msg(self.config, "%i files" % len(qe_files)) sflat_table_file = self.get_filename_from_format(RAFT_SFLAT_TABLE_FORMATTER, "sflat.fits") sflat_tables = TableDict(sflat_table_file) bbox_dict = construct_bbox_dict(sflat_tables['defects']) slot_bbox_dict = bbox_dict[slot] # This is a dictionary of dictionaries to store all the # data you extract from the qe_files data_dict = dict(SLOT=[], AMP=[], XCORNER=[], YCORNER=[], XSIZE=[], YSIZE=[]) for lam in lams: data_dict['FLUX_' + lam] = [] data_dict['MED_' + lam] = [] temp_list = [] # Analysis goes here, you should fill data_dict with data extracted # by the analysis # # For this slot, loop over qe files, loop over amplifiers, loop over dust spots in that amplifier, # record the flux and median for each dust spot # Then loop over dust spots CCDs, amplifiers, and QE files to accumulate the output dictionary ccd = self.get_ccd(butler, qe_files[0], mask_files) amps = get_amp_list(ccd) spot = 0 for ifile, qe_file in enumerate(qe_files): lam = qe_file.split('flat_')[1].split('_')[0] for i, amp in enumerate(amps): bbox_list = slot_bbox_dict[amp] if len(bbox_list) > 100: if i == 0: self.log.warn("Skipping slot:amp %s:%i with %i defects" % (slot, amp, len(bbox_list))) continue ccd = self.get_ccd(butler, qe_file, mask_files) regions = get_geom_regions(ccd, amp) serial_oscan = regions['serial_overscan'] imaging = regions['imaging'] img = get_raw_image(ccd, amp) if superbias_frame is not None: if butler is not None: superbias_im = get_raw_image(superbias_frame, amp) else: superbias_im = get_raw_image(superbias_frame, amp) else: superbias_im = None image = unbias_amp(img, serial_oscan, bias_type=bias_type, superbias_im=superbias_im, region=imaging) for bbox in bbox_list: if ifile == 0: data_dict['SLOT'].append(slot) #data_dict['LAM'].append(lam) data_dict['AMP'].append(amp) data_dict['XCORNER'].append(bbox.getBeginX()) data_dict['YCORNER'].append(bbox.getBeginY()) data_dict['XSIZE'].append(bbox.getWidth()) data_dict['YSIZE'].append(bbox.getHeight()) try: cutout = image[bbox] except Exception: pass # Here evaluate the 'flux' of the feature, relative to the median # value of the amplifier image. May also want to assemble bounding # box corners into a ds9 region file, CCD by CCD med = np.median(image.array) flux = np.sum(cutout.array) - bbox.getWidth()*bbox.getHeight()*med spot += 1 #temp_dict[spot].append((lam, flux, med)) temp_list.append((lam, flux, med)) for i, tmp_data in enumerate(temp_list): #data_dict['SLOT'].append(temp_dict['SLOT'][i]) #data_dict['AMP'].append(temp_dict['AMP'][i]) #lam = temp_dict['LAM'][i] lam, flux, med = tmp_data data_dict['FLUX_' + lam].append(flux) #temp_dict['FLUX'][i]) data_dict['MED_' + lam].append(med) #np.median(image.array)) sys.stdout.write("!\n") sys.stdout.flush() dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, qe_files)) #dtables.make_datatable('dust_color', data_dict) dtables.make_datatable('dust_color_hack', data_dict) return dtables
def extract(self, butler, data, **kwargs): """Extract the data from the overscan region to estimate the deffered charge Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) if butler is not None: raise ValueError( "FlatOverscanTask not implemented for Butlerized data") flat_files = data['FLAT1'] mask_files = self.get_mask_files() superbias_frame = self.get_superbias_frame(mask_files) self.log_info_slot_msg(self.config, "%i files" % len(flat_files)) #fitter = OverscanFit(num_oscan_pixels=self.config.num_oscan_pixels, # minflux=self.config.minflux, maxflux=self.config.maxflux) gains = self.get_gains() if gains is None: gains = np.ones((17)) # Analysis goes here, you should fill data_dict with data extracted # by the analysis for ifile, flat_id in enumerate(flat_files): if ifile % 10 == 0: self.log_progress(" %i" % ifile) flat = self.get_ccd(butler, flat_id, mask_files, bias_frame=superbias_frame) fitter.process_image(flat, gains) self.xmax = fitter.xmax_val self.log_progress("Done!") data_dict = fitter.build_output_dict() primary = fitter.output[0] primary.header['NAMPS'] = 16 dtables = TableDict(primary=primary) for key, val in data_dict.items(): dtables.make_datatable(key.lower(), val) dtables.make_datatable('files', make_file_dict(butler, flat_files)) return dtables
def extract(self, butler, data, **kwargs): """Extract the statistics from the superbias frames Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) if butler is not None: self.log.warn("Ignoring butler") stats_data = {} self.log_info_raft_msg(self.config, "") slot_list = self.config.slots if slot_list is None: slot_list = ALL_SLOTS for islot, slot in enumerate(slot_list): superbias_file = data[slot] if self.config.stat is not None: superbias_file = superbias_file.replace('_superbias_', '_%s_' % self.config.stat) if not os.path.exists(superbias_file): self.log.warn("Skipping missing file for %s:%s" % (self.config.raft, slot)) continue self.log_progress(" %s" % slot) mask_files = self.get_mask_files(slot=slot) try: superbias_frame = self.get_ccd(None, superbias_file, mask_files) except Exception: self.log.warn("Skipping %s:%s:%s" % (self.config.run, self.config.raft, slot)) superbias_frame = None self.get_superbias_stats(superbias_frame, stats_data, islot) self.log_progress("Done!") if not stats_data: return None stats_data['slot'] = np.arange(9) dtables = TableDict() dtables.make_datatable('files', make_file_dict(None, slot_list)) dtables.make_datatable('stats', stats_data) return dtables