def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) # You should expand this to include space for the data you want to extract data_dict = dict(slot=[], amp=[]) sys.stdout.write("Working on 9 slots: ") sys.stdout.flush() for islot, slot in enumerate(ALL_SLOTS): sys.stdout.write(" %s" % slot) sys.stdout.flush() basename = data[slot] datapath = basename.replace(self.config.outsuffix, self.config.insuffix) dtables = TableDict(datapath) print(dtables) for amp in range(16): # Here you can get the data out for each amp and append it to the # data_dict data_dict['slot'].append(islot) data_dict['amp'].append(amp) sys.stdout.write(".\n") sys.stdout.flush() outtables = TableDict() outtables.make_datatable("tmplsuffix", data_dict) return outtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) for key, val in data.items(): data[key] = val.replace(self.config.outsuffix, self.config.insuffix) # Define the set of columns to keep and remove # keep_cols = [] # remove_cols = [] outtable = vstack_tables(data, tablename='tmplsuffix_stats') dtables = TableDict() dtables.add_datatable('tmplsuffix_sum', outtable) dtables.make_datatable('runs', dict(runs=sorted(data.keys()))) return dtables
def extract(self, butler, data, **kwargs): """Extract the summary data from the PTC analyses Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) if butler is not None: self.log.warn("Ignoring butler") if not self.config.skip: outtable = vstack_tables(data, tablename='ptc_stats') dtables = TableDict() dtables.add_datatable('ptc_sum', outtable) dtables.make_datatable('runs', dict(runs=sorted(data.keys()))) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) run_dict = dict(runs=[], rafts=[]) for key, val in data.items(): run_dict['runs'].append(key[4:]) run_dict['rafts'].append(key[0:3]) data[key] = val.replace(self.config.filekey, self.config.infilekey) # Define the set of columns to keep and remove # keep_cols = [] # remove_cols = [] outtable = vstack_tables(data, tablename='dark_current') dtables = TableDict() dtables.add_datatable('stats', outtable) dtables.make_datatable('runs', run_dict) return dtables
def extract(self, butler, data, **kwargs): """Make a summary table of the superbias statistics Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) if butler is not None: self.log.warn("Ignoring butler") run_dict = dict(runs=[], rafts=[]) for key, val in sorted(data.items()): run_dict['runs'].append(key[4:]) run_dict['rafts'].append(key[0:3]) data[key] = val.replace(self.config.filekey, self.config.infilekey) outtable = vstack_tables(data, tablename='stats') dtables = TableDict() dtables.add_datatable('stats', outtable) dtables.make_datatable('runs', run_dict) return dtables
def extract(self, butler, data, **kwargs): """Make a summary table Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) if butler is not None: self.log.warn("Ignoring butler") for key, val in data.items(): data[key] = val.replace('_sum.fits', '_stats.fits') outtable = vstack_tables(data, tablename='biasosstack_stats') dtables = TableDict() dtables.add_datatable('biasosstack_sum', outtable) dtables.make_datatable('runs', dict(runs=sorted(data.keys()))) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` output data tables """ self.safe_update(**kwargs) for key, val in data.items(): print(key, val) data[key] = val.replace(self.config.filekey, self.config.infilekey) # Define the set of columns to keep and remove # keep_cols = [] # remove_cols = [] outtable = vstack_tables(data, tablename='eo_results') dtables = TableDict() dtables.add_datatable('eo_results_run', outtable) dtables.make_datatable('runs', dict(runs=[self.config.run])) return dtables
def extract(self, butler, data, **kwargs): """Stack the overscan region from all the amps on a sensor to look for coherent read noise Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) bias_files = data['BIAS'] bias_type = self.get_bias_algo() mask_files = self.get_mask_files() superbias_frame = self.get_superbias_frame(mask_files=mask_files) self.log_info_slot_msg(self.config, "%i files" % len(bias_files)) stack_arrays = {} nfiles = len(bias_files) for ifile, bias_file in enumerate(bias_files): if ifile % 10 == 0: self.log_progress(" %i" % ifile) ccd = self.get_ccd(butler, bias_file, mask_files) if ifile == 0: dim_array_dict = get_dimension_arrays_from_ccd(ccd) for key, val in dim_array_dict.items(): stack_arrays[key] = np.zeros((nfiles, 16, len(val))) stack_by_amps(stack_arrays, ccd, ifile=ifile, bias_type=bias_type, superbias_frame=superbias_frame) self.log_progress("Done!") stackdata_dict = convert_stack_arrays_to_dict(stack_arrays, dim_array_dict, nfiles) dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, bias_files)) for key, val in stackdata_dict.items(): dtables.make_datatable('stack-%s' % key, val) return dtables
def extract(self, butler, data, **kwargs): """Plot the row-wise and col-wise struture in a series of bias frames Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) slot = self.config.slot bias_files = data['BIAS'] mask_files = self.get_mask_files() superbias_frame = self.get_superbias_frame(mask_files=mask_files) self.log_info_slot_msg(self.config, "%i files" % len(bias_files)) biasstruct_data = {} for ifile, bias_file in enumerate(bias_files): if ifile % 10 == 0: self.log_progress(" %i" % ifile) ccd = self.get_ccd(butler, bias_file, mask_files) if ifile == 0: dim_array_dict = get_dimension_arrays_from_ccd(ccd) for key, val in dim_array_dict.items(): biasstruct_data[key] = {key: val} self.get_ccd_data(ccd, biasstruct_data, slot=slot, ifile=ifile, nfiles_used=len(bias_files), superbias_frame=superbias_frame) self.log_progress("Done!") if not biasstruct_data: return None dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, bias_files)) for key, val in biasstruct_data.items(): dtables.make_datatable('biasst-%s' % key, val) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) gains = np.ones((17)) ppump_files = data['PPUMP'] print(ppump_files) ppump_file = ppump_files[0] mask_files = self.get_mask_files() self.log_info_slot_msg(self.config, "") # This is a dictionary of dictionaries to store all the # data you extract from the ppump_files data_dict = {} # Analysis goes here, you should fill data_dict with data extracted # by the analysis # ccd = self.get_ccd(butler, ppump_file, mask_files, masked_ccd=True) my_traps = Traps(ccd, gains, cycles=self.config.cycles, C2_thresh=self.config.C2_thresh, C3_thresh=self.config.C3_thresh, nx=self.config.bkg_nx, ny=self.config.bkg_ny, edge_rolloff=self.config.edge_rolloff) data_dict = my_traps.make_data_dict() self.log_progress("Done!") dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, ppump_files)) dtables.make_datatable('traps', data_dict) return dtables
def extract(self, butler, data, **kwargs): """Extract the outliers in the superflat frames for the raft Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) self._mask_file_dict = {} self._sflat_file_dict_l = {} self._sflat_file_dict_h = {} self._sflat_file_dict_r = {} if butler is not None: self.log.warn("Ignoring butler") self.set_local_data(butler, data, **kwargs) if not self._sflat_file_dict_l: self.log.warn("No files for %s, skipping" % (self.config.raft)) return None self._sflat_images_h, ccd_dict = extract_raft_unbiased_images( self._sflat_file_dict_h, mask_dict=self._mask_file_dict) self._sflat_array_l = extract_raft_array_dict( self._sflat_file_dict_l, mask_dict=self._mask_file_dict) self._sflat_array_h = extract_raft_imaging_data( self._sflat_images_h, ccd_dict) self._sflat_array_r = extract_raft_array_dict( self._sflat_file_dict_r, mask_dict=self._mask_file_dict) out_data_l = outlier_raft_dict(self._sflat_array_l, 1000., 300.) out_data_h = outlier_raft_dict(self._sflat_array_h, 50000., 15000.) out_data_r = outlier_raft_dict(self._sflat_array_r, 0.019, 0.002) fp_dict = SuperflatRaftTask.build_defect_dict(self._sflat_images_h, frac_thresh=0.9) dtables = TableDict() dtables.make_datatable('defects', fp_dict) dtables.make_datatable('outliers_l', out_data_l) dtables.make_datatable('outliers_h', out_data_h) dtables.make_datatable('outliers_r', out_data_r) return dtables
def extract(self, butler, data, **kwargs): """Extract the row-wise and col-wise struture in a superbias frame Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) slot = self.config.slot if butler is not None: self.log.warn("Ignoring butler") if data is not None: self.log.warn("Ignoring data") mask_files = self.get_mask_files() superbias = self.get_superbias_frame(mask_files=mask_files) if superbias is None: return None self.log_info_slot_msg(self.config, "") biasstruct_data = {} dim_array_dict = get_dimension_arrays_from_ccd(superbias) for key, val in dim_array_dict.items(): biasstruct_data[key] = {key: val} self.get_ccd_data(superbias, biasstruct_data, slot=slot, bias_type=None, std=self.config.std, superbias_frame=None) self.log_progress("Done!") dtables = TableDict() dtables.make_datatable('files', make_file_dict(None, [slot])) for key, val in biasstruct_data.items(): dtables.make_datatable('biasst-%s' % key, val) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` output data tables """ self.safe_update(**kwargs) slots = ALL_SLOTS # This is a dictionary of dictionaries to store all the # data you extract from the base_files data_dict = dict(slot=np.ndarray((144))) for islot, slot in enumerate(slots): infile = data[slot] try: dtables = TableDict(infile, ['amplifier_results']) except FileNotFoundError: return None table = dtables['amplifier_results'] data_dict['slot'][16 * islot:16 * (islot + 1)] = islot for key in table.keys(): if key not in data_dict: data_dict[key] = np.ndarray((144)) data_dict[key][16 * islot:16 * (islot + 1)] = table[key] dtables = TableDict() dtables.make_datatable('eo_results', data_dict) return dtables
def extract(self, butler, data, **kwargs): """Extract the bias as function of row Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) bias_files = data['BIAS'] if not bias_files: return None mask_files = self.get_mask_files() self.log_info_slot_msg(self.config, "%i files" % len(bias_files)) biasval_data = {} for ifile, bias_file in enumerate(bias_files): if ifile % 10 == 0: self.log_progress(" %i" % ifile) ccd = self.get_ccd(butler, bias_file, mask_files) if ifile == 0: dims = get_dims_from_ccd(ccd) xrow_s = np.linspace(0, dims['nrow_s'] - 1, dims['nrow_s']) self.get_ccd_data(ccd, biasval_data, ifile=ifile, nfiles=len(bias_files)) #Need to truncate the row array to match the data a_row = biasval_data[sorted(biasval_data.keys())[0]] biasval_data['row_s'] = xrow_s[0:len(a_row)] self.log_progress("Done!") dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, bias_files)) dtables.make_datatable('biasval', biasval_data) return dtables
def get_data_table_names_from_file(fname): """Get a name of tables in a particular file Parameters ---------- fname : `str` File name Returns ------- keys : `list` Table names """ dtables = TableDict(fname) return dtables.keys()
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) # You should expand this to include space for the data you want to extract data_dict = dict(amp=[], full_well=[], max_dev=[]) self.log_info_raft_msg(self.config, "") basename = data[0] datapath = basename.replace(self.config.filekey, self.config.infilekey) detresp = DetectorResponse(datapath, hdu_name='flat') for amp in range(1, 17): # Here you can get the data out for each amp and append it to the # data_dict data_dict['amp'].append(amp) linearity_data = detresp.linearity(amp) full_well_data = detresp.full_well(amp) data_dict['full_well'].append(full_well_data[0]) data_dict['max_dev'].append(linearity_data[0]) self.log_progress("Done!") outtables = TableDict() outtables.make_datatable("flat_lin", data_dict) return outtables
def extract(self, butler, data, **kwargs): """Extract the utliers in the superdark frames for the raft Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) self._mask_file_dict = {} self._sdark_file_dict = {} if butler is not None: self.log.warn("Ignoring butler") self.set_local_data(butler, data, **kwargs) if not self._sdark_file_dict: self.log.warn("No files for %s, skipping" % (self.config.raft)) return None self._sdark_images, ccd_dict = extract_raft_unbiased_images( self._sdark_file_dict, mask_dict=self._mask_file_dict) self._sdark_arrays = extract_raft_imaging_data(self._sdark_images, ccd_dict) out_data = outlier_raft_dict(self._sdark_arrays, 0., 25.) fp_dict = build_defect_dict(self._sdark_images, fp_type='bright', abs_thresh=50) dtables = TableDict() dtables.make_datatable('defects', fp_dict) dtables.make_datatable('outliers', out_data) return dtables
def extract(self, butler, data, **kwargs): """Extract the correlations between the serial overscan for each amp on a raft Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) slots = ALL_SLOTS overscans = [] for slot in slots: bias_files = data[slot]['BIAS'] mask_files = self.get_mask_files(slot=slot) superbias_frame = self.get_superbias_frame(mask_files, slot=slot) ccd = self.get_ccd(butler, bias_files[0], []) overscans += self.get_ccd_data(butler, ccd, superbias_frame=superbias_frame) namps = len(overscans) if self.config.covar: data = np.array([np.cov(overscans[i[0]].ravel(), overscans[i[1]].ravel())[0, 1] for i in itertools.product(range(namps), range(namps))]) else: data = np.array([np.corrcoef(overscans[i[0]].ravel(), overscans[i[1]].ravel())[0, 1] for i in itertools.product(range(namps), range(namps))]) data = data.reshape(namps, namps) dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, bias_files)) dtables.make_datatable('correl', dict(correl=data)) return dtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) # You should expand this to include space for the data you want to extract data_dict = dict(slot=[], amp=[]) self.log_info_raft_msg(self.config, "") for islot, slot in enumerate(ALL_SLOTS): self.log_progress(" %s" % slot) # get the data from the input table #basename = data[slot] #datapath = basename.replace(self.config.outsuffix, self.config.insuffix) #dtables = TableDict(datapath) for amp in range(16): # Here you can get the data out for each amp and append it to the # data_dict data_dict['slot'].append(islot) data_dict['amp'].append(amp) self.log_progress("Done!") outtables = TableDict() outtables.make_datatable("tmplsuffix", data_dict) return outtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) # Get the slot, if needed #slot = self.config.slot tmpl_files = data['TMPL'] # Get the superbias frame, if needed #mask_files = self.get_mask_files() #superbias_frame = self.get_superbias_frame(mask_files) self.log_info_slot_msg(self.config, "") # This is a dictionary of dictionaries to store all the # data you extract from the tmpl_files data_dict = {} # Analysis goes here, you should fill data_dict with data extracted # by the analysis # self.log_progress("Done!") dtables = TableDict() dtables.make_datatable('files', make_file_dict(butler, tmpl_files)) for key, val in data_dict.items(): dtables.make_datatable(key, val) return dtables
def inspect_tablefile(self, filetype, **kwargs): """Print information about the contents of a particular type of file with data tables Parameters ---------- filetyple : `str` Type of file in `FileFormatDict` kwargs Passed to the class file name formatting function """ fname = self.get_filename(filetype, **kwargs) tdict = TableDict(fname) sys.stdout.write("File %s:\n" % fname) for key, val in tdict.items(): sys.stdout.write("Table %s:\n" % key) for col in val.columns: sys.stdout.write(" %s:\n" % col)
def get_task_table_dict(self, key, **kwargs): """Get the tables produceded by a particular task Parameters ---------- key : `str` Name associated to that `Task` kwargs Passed to the class file name formatting function Returns ------- tdict: `TableDict` The tables produced by the task/ configuration """ tfile = self.get_task_tablefile(key, **kwargs) + '.fits' tdict = TableDict(tfile) return tdict
def get_data_column_names_from_file(fname, tname): """Get a column names from a particular table Parameters ---------- fname : `str` File name tname : `str` Table name Returns ------- keys : `list` Table names """ dtables = TableDict(fname, [tname]) dtab = dtables[tname] return dtab.columns
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` output data tables """ self.safe_update(**kwargs) slots = ALL_SLOTS # This is a dictionary of dictionaries to store all the # data you extract from the tmpl_files data_dict = {} for slot in slots: #Get the datafiles, and the correpsonding superbias files tmpl_files = data[slot]['TMPL'] print(tmpl_files) #mask_files = self.get_mask_files(slot=slot) #superbias_frame = self.get_superbias_frame(mask_files, slot=slot) # Analysis goes here, you should fill data_dict with data extracted # by the analysis # dtables = TableDict() for key, val in data_dict.items(): dtables.make_datatable(key, val) return dtables
def get_data_column_from_file(fname, tname, cname): """Get a column from a particular table Parameters ---------- fname : `str` File name tname : `str` Table name cname : `str` Column name kwargs Passed to the class file name formatting function Returns ------- data : `np.array` The requested column data """ dtables = TableDict(fname, [tname]) return dtables[tname][cname]
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) self.log_info_slot_msg(self.config, "") pd_calib_file = self.get_pd_calib_file() gains = np.ones((17)) basename = data[0] qe_data = QE_Data() qe_data.read_fits(basename) qe_data.incidentPower(pd_calib_file) qe_data.calculate_QE(gains) qe_curves_data_dict = qe_data.make_qe_curves_data_dict() bands_data_dict = qe_data.make_bands_data_dict() dtables = TableDict() dtables.make_datatable('qe_curves', qe_curves_data_dict) dtables.make_datatable('bands', bands_data_dict) return dtables
def get_data_columns_from_file(fname, tname, clist): """Get a column from a particular table Parameters ---------- filetyple : `str` Type of file in `FileFormatDict` tname : `str` Table name clist : `list` Column names kwargs Passed to the class file name formatting function Returns ------- odict : `dict` Dictionary mapping column name to data """ dtables = TableDict(fname, [tname]) dtab = dtables[tname] return {key:dtab[key] for key in clist}
def extract(self, butler, data, **kwargs): """Extract the gains and widths from the f355 clusters Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) if butler is not None: self.log.warn("Ignoring butler") use_all = self.config.use_all data_dict = dict(kalpha_peak=[], kalpha_sigma=[], ncluster=[], ngood=[], gain=[], gain_error=[], fit_xmin=[], fit_xmax=[], fit_pars=[], fit_nbins=[], sigmax_median=[], sigmay_median=[], slot=[], amp=[]) self.log_info_raft_msg(self.config, "") for islot, slot in enumerate(ALL_SLOTS): self.log_progress(" %s" % slot) basename = data[slot] dtables = TableDict(basename) for amp in range(16): table = dtables['amp%02i' % (amp + 1)] if use_all: mask = np.ones((len(table)), bool) else: mask = (np.fabs(table['XPOS'] - table['XPEAK']) < 1)*\ (np.fabs(table['YPOS'] - table['YPEAK']) < 1) tablevals = table[mask]['DN'] gainfitter = Fe55GainFitter(tablevals) try: kalpha_peak, kalpha_sigma = gainfitter.fit(bins=100) gain = gainfitter.gain gain_error = gainfitter.gain_error pars = gainfitter.pars except Exception: kalpha_peak, kalpha_sigma = (np.nan, np.nan) gain = np.nan gain_error = np.nan pars = np.nan * np.ones((4)) data_dict['kalpha_peak'].append(kalpha_peak) data_dict['kalpha_sigma'].append(kalpha_sigma) data_dict['gain'].append(gain) data_dict['gain_error'].append(gain_error) xra = gainfitter.xrange data_dict['ncluster'].append(mask.size) data_dict['ngood'].append(mask.sum()) if xra is None: data_dict['fit_xmin'].append(np.nan) data_dict['fit_xmax'].append(np.nan) else: data_dict['fit_xmin'].append(xra[0]) data_dict['fit_xmax'].append(xra[1]) data_dict['fit_pars'].append(pars) data_dict['fit_nbins'].append(100.) data_dict['sigmax_median'].append(np.median(table['SIGMAX'])) data_dict['sigmay_median'].append(np.median(table['SIGMAY'])) data_dict['slot'].append(islot) data_dict['amp'].append(amp) self.log_progress("Done!") outtables = TableDict() outtables.make_datatable("fe55_gain_stats", data_dict) return outtables
def extract(self, butler, data, **kwargs): """Extract the summary statistics about the fluctuations ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` The resulting data """ self.safe_update(**kwargs) if butler is not None: self.log.warn("Ignoring bulter") data_dict = dict(s_row_min_mean=[], s_row_min_median=[], s_row_min_std=[], s_row_min_min=[], s_row_min_max=[], s_row_max_mean=[], s_row_max_median=[], s_row_max_std=[], s_row_max_min=[], s_row_max_max=[], p_col_min_mean=[], p_col_min_median=[], p_col_min_std=[], p_col_min_min=[], p_col_min_max=[], p_col_max_mean=[], p_col_max_median=[], p_col_max_std=[], p_col_max_min=[], p_col_max_max=[], slot=[]) self.log_info_raft_msg(self.config, "") for islot, slot in enumerate(ALL_SLOTS): self.log_progress(" %s" % slot) basename = data[slot] datapath = basename.replace('_stats.fits', '.fits') try: dtables = TableDict(datapath, ['stack-row_s', 'stack-col_p']) table_s = dtables['stack-row_s'] table_p = dtables['stack-col_p'] except FileNotFoundError: self.log.error("Could not open %s\n" % datapath) table_s = None table_p = None if table_s is not None: tablevals_s_min = table_s['stack_mean'].min(0) tablevals_s_max = table_s['stack_mean'].max(0) else: tablevals_s_min = [0., 0.] tablevals_s_max = [0., 0.] if table_p is not None: tablevals_p_min = table_p['stack_mean'].min(0) tablevals_p_max = table_p['stack_mean'].max(0) else: tablevals_p_min = [0., 0.] tablevals_p_max = [0., 0.] data_dict['s_row_min_mean'].append(np.mean(tablevals_s_min)) data_dict['s_row_min_median'].append(np.median(tablevals_s_min)) data_dict['s_row_min_std'].append(np.std(tablevals_s_min)) data_dict['s_row_min_min'].append(np.min(tablevals_s_min)) data_dict['s_row_min_max'].append(np.max(tablevals_s_min)) data_dict['s_row_max_mean'].append(np.mean(tablevals_s_max)) data_dict['s_row_max_median'].append(np.median(tablevals_s_max)) data_dict['s_row_max_std'].append(np.std(tablevals_s_max)) data_dict['s_row_max_min'].append(np.min(tablevals_s_max)) data_dict['s_row_max_max'].append(np.max(tablevals_s_max)) data_dict['p_col_min_mean'].append(np.mean(tablevals_p_min)) data_dict['p_col_min_median'].append(np.median(tablevals_p_min)) data_dict['p_col_min_std'].append(np.std(tablevals_p_min)) data_dict['p_col_min_min'].append(np.min(tablevals_p_min)) data_dict['p_col_min_max'].append(np.max(tablevals_p_min)) data_dict['p_col_max_mean'].append(np.mean(tablevals_p_max)) data_dict['p_col_max_median'].append(np.median(tablevals_p_max)) data_dict['p_col_max_std'].append(np.std(tablevals_p_max)) data_dict['p_col_max_min'].append(np.min(tablevals_p_max)) data_dict['p_col_max_max'].append(np.max(tablevals_p_max)) data_dict['slot'].append(islot) self.log_progress("Done!") outtables = TableDict() outtables.make_datatable("biasosstack_stats", data_dict) return outtables
def extract(self, butler, data, **kwargs): """Extract data Parameters ---------- butler : `Butler` The data butler data : `dict` Dictionary (or other structure) contain the input data kwargs Used to override default configuration Returns ------- dtables : `TableDict` Output data tables """ self.safe_update(**kwargs) if self.config.teststand == 'ts8': flat1_files = data['FLAT1'] flat2_files = data['FLAT2'] elif self.config.teststand == 'bot': flat1_files = data['FLAT0'] flat2_files = data['FLAT1'] bias_type = self.get_bias_algo() mask_files = self.get_mask_files() superbias_frame = self.get_superbias_frame(mask_files) nlc = self.get_nonlinearirty_correction() #slot_idx = ALL_SLOTS.index(self.config.slot) self.log_info_slot_msg(self.config, "%i %i files" % (len(flat1_files), len(flat2_files))) # This is a dictionary of dictionaries to store all the # data you extract from the flat_files data_dict = dict(FLUX=[], EXPTIME=[], MONDIODE1=[], MONDIODE2=[], MONOCH_SLIT_B=[]) for i in range(1, 17): data_dict['AMP%02i_RATIO' % i] = [] data_dict['AMP%02i_MEAN' % i] = [] data_dict['AMP%02i_CORRMEAN' % i] = [] data_dict['AMP%02i_VAR' % i] = [] data_dict['AMP%02i_SIGNAL' % i] = [] data_dict['AMP%02i_MEAN1' % i] = [] data_dict['AMP%02i_MEAN2' % i] = [] # Analysis goes here, you should fill data_dict with data extracted # by the analysis # for ifile, (id_1, id_2) in enumerate(zip(flat1_files, flat2_files)): if ifile % 10 == 0: self.log_progress(" %i" % ifile) flat_1 = self.get_ccd(butler, id_1, mask_files) flat_2 = self.get_ccd(butler, id_2, mask_files) amps = get_amp_list(flat_1) exp_time_1 = get_exposure_time(flat_1) exp_time_2 = get_exposure_time(flat_2) if exp_time_1 != exp_time_2: self.log.warn("Exposure times do not match for:\n%s\n%s\n %0.3F %0.3F. Skipping Pair\n" % (id_1, id_2, exp_time_1, exp_time_2)) continue mondiode_1 = get_monodiode_val_from_data_id(id_1, exp_time_1, self.config.teststand, butler) mondiode_2 = get_monodiode_val_from_data_id(id_2, exp_time_2, self.config.teststand, butler) if mondiode_1 is None or mondiode_2 is None: self.log.warn("No monitoring data for:\n%s\n%s\n Skipping Pair\n" % (id_1, id_2)) continue flux = (exp_time_1 * mondiode_1 + exp_time_2 * mondiode_2)/2. data_dict['EXPTIME'].append(exp_time_1) data_dict['MONDIODE1'].append(mondiode_1) data_dict['MONDIODE2'].append(mondiode_2) data_dict['FLUX'].append(flux) try: data_dict['MONOCH_SLIT_B'].append(get_mono_slit_b(flat_1)) except KeyError: data_dict['MONOCH_SLIT_B'].append(0.) ccd_1_ims = unbiased_ccd_image_dict(flat_1, bias=bias_type, superbias_frame=superbias_frame, trim='imaging', nonlinearity=nlc) ccd_2_ims = unbiased_ccd_image_dict(flat_2, bias=bias_type, superbias_frame=superbias_frame, trim='imaging', nonlinearity=nlc) for i, amp in enumerate(amps): image_1 = ccd_1_ims[amp] image_2 = ccd_2_ims[amp] fstats = self.get_pair_stats(image_1, image_2) signal = fstats[1] #if gains is not None: # signal *= gains[slot_idx][i] data_dict['AMP%02i_RATIO' % (i+1)].append(fstats[0]) data_dict['AMP%02i_MEAN' % (i+1)].append(fstats[1]) data_dict['AMP%02i_CORRMEAN' % (i+1)].append(fstats[2]) data_dict['AMP%02i_VAR' % (i+1)].append(fstats[3]) data_dict['AMP%02i_SIGNAL' % (i+1)].append(signal) data_dict['AMP%02i_MEAN1' % (i+1)].append(fstats[4]) data_dict['AMP%02i_MEAN2' % (i+1)].append(fstats[5]) self.log_progress("Done!") primary_hdu = fits.PrimaryHDU() primary_hdu.header['NAMPS'] = 16 dtables = TableDict(primary=primary_hdu) dtables.make_datatable('files', make_file_dict(butler, flat1_files + flat2_files)) dtables.make_datatable('flat', data_dict) return dtables