def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
for key, val in data.items():
data[key] = val.replace(self.config.outsuffix,
self.config.insuffix)
# Define the set of columns to keep and remove
# keep_cols = []
# remove_cols = []
outtable = vstack_tables(data, tablename='tmplsuffix_stats')
dtables = TableDict()
dtables.add_datatable('tmplsuffix_sum', outtable)
dtables.make_datatable('runs', dict(runs=sorted(data.keys())))
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the summary data from the PTC analyses
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
self.log.warn("Ignoring butler")
if not self.config.skip:
outtable = vstack_tables(data, tablename='ptc_stats')
dtables = TableDict()
dtables.add_datatable('ptc_sum', outtable)
dtables.make_datatable('runs', dict(runs=sorted(data.keys())))
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
output data tables
"""
self.safe_update(**kwargs)
for key, val in data.items():
print(key, val)
data[key] = val.replace(self.config.filekey, self.config.infilekey)
# Define the set of columns to keep and remove
# keep_cols = []
# remove_cols = []
outtable = vstack_tables(data, tablename='eo_results')
dtables = TableDict()
dtables.add_datatable('eo_results_run', outtable)
dtables.make_datatable('runs', dict(runs=[self.config.run]))
return dtables
def extract(self, butler, data, **kwargs):
"""Make a summary table
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
self.log.warn("Ignoring butler")
for key, val in data.items():
data[key] = val.replace('_sum.fits', '_stats.fits')
outtable = vstack_tables(data, tablename='biasosstack_stats')
dtables = TableDict()
dtables.add_datatable('biasosstack_sum', outtable)
dtables.make_datatable('runs', dict(runs=sorted(data.keys())))
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
run_dict = dict(runs=[], rafts=[])
for key, val in data.items():
run_dict['runs'].append(key[4:])
run_dict['rafts'].append(key[0:3])
data[key] = val.replace(self.config.filekey, self.config.infilekey)
# Define the set of columns to keep and remove
# keep_cols = []
# remove_cols = []
outtable = vstack_tables(data, tablename='dark_current')
dtables = TableDict()
dtables.add_datatable('stats', outtable)
dtables.make_datatable('runs', run_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Make a summary table of the superbias statistics
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
self.log.warn("Ignoring butler")
run_dict = dict(runs=[], rafts=[])
for key, val in sorted(data.items()):
run_dict['runs'].append(key[4:])
run_dict['rafts'].append(key[0:3])
data[key] = val.replace(self.config.filekey, self.config.infilekey)
outtable = vstack_tables(data, tablename='stats')
dtables = TableDict()
dtables.add_datatable('stats', outtable)
dtables.make_datatable('runs', run_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Stack the overscan region from all the amps on a sensor
to look for coherent read noise
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
bias_files = data['BIAS']
bias_type = self.get_bias_algo()
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files=mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(bias_files))
stack_arrays = {}
nfiles = len(bias_files)
for ifile, bias_file in enumerate(bias_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, bias_file, mask_files)
if ifile == 0:
dim_array_dict = get_dimension_arrays_from_ccd(ccd)
for key, val in dim_array_dict.items():
stack_arrays[key] = np.zeros((nfiles, 16, len(val)))
stack_by_amps(stack_arrays,
ccd,
ifile=ifile,
bias_type=bias_type,
superbias_frame=superbias_frame)
self.log_progress("Done!")
stackdata_dict = convert_stack_arrays_to_dict(stack_arrays,
dim_array_dict, nfiles)
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
for key, val in stackdata_dict.items():
dtables.make_datatable('stack-%s' % key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Plot the row-wise and col-wise struture
in a series of bias frames
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
slot = self.config.slot
bias_files = data['BIAS']
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files=mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(bias_files))
biasstruct_data = {}
for ifile, bias_file in enumerate(bias_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, bias_file, mask_files)
if ifile == 0:
dim_array_dict = get_dimension_arrays_from_ccd(ccd)
for key, val in dim_array_dict.items():
biasstruct_data[key] = {key: val}
self.get_ccd_data(ccd,
biasstruct_data,
slot=slot,
ifile=ifile,
nfiles_used=len(bias_files),
superbias_frame=superbias_frame)
self.log_progress("Done!")
if not biasstruct_data:
return None
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
for key, val in biasstruct_data.items():
dtables.make_datatable('biasst-%s' % key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
gains = np.ones((17))
ppump_files = data['PPUMP']
print(ppump_files)
ppump_file = ppump_files[0]
mask_files = self.get_mask_files()
self.log_info_slot_msg(self.config, "")
# This is a dictionary of dictionaries to store all the
# data you extract from the ppump_files
data_dict = {}
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
ccd = self.get_ccd(butler, ppump_file, mask_files, masked_ccd=True)
my_traps = Traps(ccd,
gains,
cycles=self.config.cycles,
C2_thresh=self.config.C2_thresh,
C3_thresh=self.config.C3_thresh,
nx=self.config.bkg_nx,
ny=self.config.bkg_ny,
edge_rolloff=self.config.edge_rolloff)
data_dict = my_traps.make_data_dict()
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, ppump_files))
dtables.make_datatable('traps', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the row-wise and col-wise struture in a superbias frame
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
slot = self.config.slot
if butler is not None:
self.log.warn("Ignoring butler")
if data is not None:
self.log.warn("Ignoring data")
mask_files = self.get_mask_files()
superbias = self.get_superbias_frame(mask_files=mask_files)
if superbias is None:
return None
self.log_info_slot_msg(self.config, "")
biasstruct_data = {}
dim_array_dict = get_dimension_arrays_from_ccd(superbias)
for key, val in dim_array_dict.items():
biasstruct_data[key] = {key: val}
self.get_ccd_data(superbias,
biasstruct_data,
slot=slot,
bias_type=None,
std=self.config.std,
superbias_frame=None)
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(None, [slot]))
for key, val in biasstruct_data.items():
dtables.make_datatable('biasst-%s' % key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the bias as function of row
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
bias_files = data['BIAS']
if not bias_files:
return None
mask_files = self.get_mask_files()
self.log_info_slot_msg(self.config, "%i files" % len(bias_files))
biasval_data = {}
for ifile, bias_file in enumerate(bias_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, bias_file, mask_files)
if ifile == 0:
dims = get_dims_from_ccd(ccd)
xrow_s = np.linspace(0, dims['nrow_s'] - 1, dims['nrow_s'])
self.get_ccd_data(ccd,
biasval_data,
ifile=ifile,
nfiles=len(bias_files))
#Need to truncate the row array to match the data
a_row = biasval_data[sorted(biasval_data.keys())[0]]
biasval_data['row_s'] = xrow_s[0:len(a_row)]
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
dtables.make_datatable('biasval', biasval_data)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
# You should expand this to include space for the data you want to extract
data_dict = dict(slot=[], amp=[])
sys.stdout.write("Working on 9 slots: ")
sys.stdout.flush()
for islot, slot in enumerate(ALL_SLOTS):
sys.stdout.write(" %s" % slot)
sys.stdout.flush()
basename = data[slot]
datapath = basename.replace(self.config.outsuffix,
self.config.insuffix)
dtables = TableDict(datapath)
print(dtables)
for amp in range(16):
# Here you can get the data out for each amp and append it to the
# data_dict
data_dict['slot'].append(islot)
data_dict['amp'].append(amp)
sys.stdout.write(".\n")
sys.stdout.flush()
outtables = TableDict()
outtables.make_datatable("tmplsuffix", data_dict)
return outtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
# You should expand this to include space for the data you want to extract
data_dict = dict(amp=[],
full_well=[],
max_dev=[])
self.log_info_raft_msg(self.config, "")
basename = data[0]
datapath = basename.replace(self.config.filekey, self.config.infilekey)
detresp = DetectorResponse(datapath, hdu_name='flat')
for amp in range(1, 17):
# Here you can get the data out for each amp and append it to the
# data_dict
data_dict['amp'].append(amp)
linearity_data = detresp.linearity(amp)
full_well_data = detresp.full_well(amp)
data_dict['full_well'].append(full_well_data[0])
data_dict['max_dev'].append(linearity_data[0])
self.log_progress("Done!")
outtables = TableDict()
outtables.make_datatable("flat_lin", data_dict)
return outtables
def extract(self, butler, data, **kwargs):
"""Extract the correlations between the serial overscan for each amp on a raft
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
slots = ALL_SLOTS
overscans = []
for slot in slots:
bias_files = data[slot]['BIAS']
mask_files = self.get_mask_files(slot=slot)
superbias_frame = self.get_superbias_frame(mask_files, slot=slot)
ccd = self.get_ccd(butler, bias_files[0], [])
overscans += self.get_ccd_data(butler, ccd, superbias_frame=superbias_frame)
namps = len(overscans)
if self.config.covar:
data = np.array([np.cov(overscans[i[0]].ravel(),
overscans[i[1]].ravel())[0, 1]
for i in itertools.product(range(namps), range(namps))])
else:
data = np.array([np.corrcoef(overscans[i[0]].ravel(),
overscans[i[1]].ravel())[0, 1]
for i in itertools.product(range(namps), range(namps))])
data = data.reshape(namps, namps)
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
dtables.make_datatable('correl', dict(correl=data))
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the utliers in the superdark frames for the raft
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
self._mask_file_dict = {}
self._sdark_file_dict = {}
if butler is not None:
self.log.warn("Ignoring butler")
self.set_local_data(butler, data, **kwargs)
if not self._sdark_file_dict:
self.log.warn("No files for %s, skipping" % (self.config.raft))
return None
self._sdark_images, ccd_dict = extract_raft_unbiased_images(
self._sdark_file_dict, mask_dict=self._mask_file_dict)
self._sdark_arrays = extract_raft_imaging_data(self._sdark_images,
ccd_dict)
out_data = outlier_raft_dict(self._sdark_arrays, 0., 25.)
fp_dict = build_defect_dict(self._sdark_images,
fp_type='bright',
abs_thresh=50)
dtables = TableDict()
dtables.make_datatable('defects', fp_dict)
dtables.make_datatable('outliers', out_data)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
# Get the slot, if needed
#slot = self.config.slot
tmpl_files = data['TMPL']
# Get the superbias frame, if needed
#mask_files = self.get_mask_files()
#superbias_frame = self.get_superbias_frame(mask_files)
self.log_info_slot_msg(self.config, "")
# This is a dictionary of dictionaries to store all the
# data you extract from the tmpl_files
data_dict = {}
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, tmpl_files))
for key, val in data_dict.items():
dtables.make_datatable(key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
# You should expand this to include space for the data you want to extract
data_dict = dict(slot=[], amp=[])
self.log_info_raft_msg(self.config, "")
for islot, slot in enumerate(ALL_SLOTS):
self.log_progress(" %s" % slot)
# get the data from the input table
#basename = data[slot]
#datapath = basename.replace(self.config.outsuffix, self.config.insuffix)
#dtables = TableDict(datapath)
for amp in range(16):
# Here you can get the data out for each amp and append it to the
# data_dict
data_dict['slot'].append(islot)
data_dict['amp'].append(amp)
self.log_progress("Done!")
outtables = TableDict()
outtables.make_datatable("tmplsuffix", data_dict)
return outtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
output data tables
"""
self.safe_update(**kwargs)
slots = ALL_SLOTS
# This is a dictionary of dictionaries to store all the
# data you extract from the tmpl_files
data_dict = {}
for slot in slots:
#Get the datafiles, and the correpsonding superbias files
tmpl_files = data[slot]['TMPL']
print(tmpl_files)
#mask_files = self.get_mask_files(slot=slot)
#superbias_frame = self.get_superbias_frame(mask_files, slot=slot)
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
dtables = TableDict()
for key, val in data_dict.items():
dtables.make_datatable(key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
output data tables
"""
self.safe_update(**kwargs)
slots = ALL_SLOTS
# This is a dictionary of dictionaries to store all the
# data you extract from the base_files
data_dict = dict(slot=np.ndarray((144)))
for islot, slot in enumerate(slots):
infile = data[slot]
try:
dtables = TableDict(infile, ['amplifier_results'])
except FileNotFoundError:
return None
table = dtables['amplifier_results']
data_dict['slot'][16 * islot:16 * (islot + 1)] = islot
for key in table.keys():
if key not in data_dict:
data_dict[key] = np.ndarray((144))
data_dict[key][16 * islot:16 * (islot + 1)] = table[key]
dtables = TableDict()
dtables.make_datatable('eo_results', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
self.log_info_slot_msg(self.config, "")
pd_calib_file = self.get_pd_calib_file()
gains = np.ones((17))
basename = data[0]
qe_data = QE_Data()
qe_data.read_fits(basename)
qe_data.incidentPower(pd_calib_file)
qe_data.calculate_QE(gains)
qe_curves_data_dict = qe_data.make_qe_curves_data_dict()
bands_data_dict = qe_data.make_bands_data_dict()
dtables = TableDict()
dtables.make_datatable('qe_curves', qe_curves_data_dict)
dtables.make_datatable('bands', bands_data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the outliers in the superflat frames for the raft
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
self._mask_file_dict = {}
self._sflat_file_dict_l = {}
self._sflat_file_dict_h = {}
self._sflat_file_dict_r = {}
if butler is not None:
self.log.warn("Ignoring butler")
self.set_local_data(butler, data, **kwargs)
if not self._sflat_file_dict_l:
self.log.warn("No files for %s, skipping" % (self.config.raft))
return None
self._sflat_images_h, ccd_dict = extract_raft_unbiased_images(
self._sflat_file_dict_h, mask_dict=self._mask_file_dict)
self._sflat_array_l = extract_raft_array_dict(
self._sflat_file_dict_l, mask_dict=self._mask_file_dict)
self._sflat_array_h = extract_raft_imaging_data(
self._sflat_images_h, ccd_dict)
self._sflat_array_r = extract_raft_array_dict(
self._sflat_file_dict_r, mask_dict=self._mask_file_dict)
out_data_l = outlier_raft_dict(self._sflat_array_l, 1000., 300.)
out_data_h = outlier_raft_dict(self._sflat_array_h, 50000., 15000.)
out_data_r = outlier_raft_dict(self._sflat_array_r, 0.019, 0.002)
fp_dict = SuperflatRaftTask.build_defect_dict(self._sflat_images_h,
frac_thresh=0.9)
dtables = TableDict()
dtables.make_datatable('defects', fp_dict)
dtables.make_datatable('outliers_l', out_data_l)
dtables.make_datatable('outliers_h', out_data_h)
dtables.make_datatable('outliers_r', out_data_r)
return dtables
def extract(self, butler, data, **kwargs):
"""Compare frames to superflat
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
sflat_l : `dict`
Dictionary keyed by amp of the low exposure superflats
sflat_h : `dict`
Dictionary keyed by amp of the high exposure superflats
ratio_images : `dict`
Dictionary keyed by amp of the low/high ratio images
"""
self.safe_update(**kwargs)
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
bias_type = self.get_bias_algo()
sflat_files = data['SFLAT']
if not sflat_files:
self.log_warn_slot_msg(self.config, "No superflat files")
return None
self.log_info_slot_msg(self.config, "%i files" % (len(sflat_files)))
# This is a dictionary of dictionaries to store all the
# data you extract from the flat_files
data_dict = dict(FLUX=[],
EXPTIME=[],
MONDIODE=[])
for i in range(1, 17):
data_dict['AMP%02i_FLAT_MEAN' % i] = []
data_dict['AMP%02i_FLAT_MEDIAN' % i] = []
data_dict['AMP%02i_FLAT_VAR' % i] = []
data_dict['AMP%02i_FLAT_ROWMEAN' % i] = []
data_dict['AMP%02i_FLAT_COLMEAN' % i] = []
for ifile, sflat_file in enumerate(sflat_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
sflat = self.get_ccd(butler, sflat_file, mask_files)
exp_time = get_exposure_time(sflat)
mondiode = get_monodiode_val_from_data_id(sflat_file, exp_time,
self.config.teststand, butler)
if mondiode is None:
continue
flux = exp_time * mondiode
data_dict['EXPTIME'].append(exp_time)
data_dict['MONDIODE'].append(mondiode)
data_dict['FLUX'].append(flux)
ccd_ims = unbiased_ccd_image_dict(sflat, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging')
amps = get_amp_list(sflat)
for amp in amps:
image = ccd_ims[amp]
fstats = self.get_stats(image)
data_dict['AMP%02i_FLAT_MEAN' % amp].append(fstats[0])
data_dict['AMP%02i_FLAT_MEDIAN' % amp].append(fstats[1])
data_dict['AMP%02i_FLAT_VAR' % amp].append(fstats[2])
data_dict['AMP%02i_FLAT_ROWMEAN' % amp].append(image.image.array.mean(0))
data_dict['AMP%02i_FLAT_COLMEAN' % amp].append(image.image.array.mean(1))
self.log_progress("Done!")
primary_hdu = fits.PrimaryHDU()
primary_hdu.header['NAMPS'] = 16
dtables = TableDict(primary=primary_hdu)
dtables.make_datatable('files', make_file_dict(butler, sflat_files))
dtables.make_datatable('stability', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the gains and widths from the f355 clusters
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
self.log.warn("Ignoring butler")
use_all = self.config.use_all
data_dict = dict(kalpha_peak=[],
kalpha_sigma=[],
ncluster=[],
ngood=[],
gain=[],
gain_error=[],
fit_xmin=[],
fit_xmax=[],
fit_pars=[],
fit_nbins=[],
sigmax_median=[],
sigmay_median=[],
slot=[],
amp=[])
self.log_info_raft_msg(self.config, "")
for islot, slot in enumerate(ALL_SLOTS):
self.log_progress(" %s" % slot)
basename = data[slot]
dtables = TableDict(basename)
for amp in range(16):
table = dtables['amp%02i' % (amp + 1)]
if use_all:
mask = np.ones((len(table)), bool)
else:
mask = (np.fabs(table['XPOS'] - table['XPEAK']) < 1)*\
(np.fabs(table['YPOS'] - table['YPEAK']) < 1)
tablevals = table[mask]['DN']
gainfitter = Fe55GainFitter(tablevals)
try:
kalpha_peak, kalpha_sigma = gainfitter.fit(bins=100)
gain = gainfitter.gain
gain_error = gainfitter.gain_error
pars = gainfitter.pars
except Exception:
kalpha_peak, kalpha_sigma = (np.nan, np.nan)
gain = np.nan
gain_error = np.nan
pars = np.nan * np.ones((4))
data_dict['kalpha_peak'].append(kalpha_peak)
data_dict['kalpha_sigma'].append(kalpha_sigma)
data_dict['gain'].append(gain)
data_dict['gain_error'].append(gain_error)
xra = gainfitter.xrange
data_dict['ncluster'].append(mask.size)
data_dict['ngood'].append(mask.sum())
if xra is None:
data_dict['fit_xmin'].append(np.nan)
data_dict['fit_xmax'].append(np.nan)
else:
data_dict['fit_xmin'].append(xra[0])
data_dict['fit_xmax'].append(xra[1])
data_dict['fit_pars'].append(pars)
data_dict['fit_nbins'].append(100.)
data_dict['sigmax_median'].append(np.median(table['SIGMAX']))
data_dict['sigmay_median'].append(np.median(table['SIGMAY']))
data_dict['slot'].append(islot)
data_dict['amp'].append(amp)
self.log_progress("Done!")
outtables = TableDict()
outtables.make_datatable("fe55_gain_stats", data_dict)
return outtables
def extract(self, butler, data, **kwargs):
"""Extract the summary statistics about the fluctuations
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
self.log.warn("Ignoring bulter")
data_dict = dict(s_row_min_mean=[],
s_row_min_median=[],
s_row_min_std=[],
s_row_min_min=[],
s_row_min_max=[],
s_row_max_mean=[],
s_row_max_median=[],
s_row_max_std=[],
s_row_max_min=[],
s_row_max_max=[],
p_col_min_mean=[],
p_col_min_median=[],
p_col_min_std=[],
p_col_min_min=[],
p_col_min_max=[],
p_col_max_mean=[],
p_col_max_median=[],
p_col_max_std=[],
p_col_max_min=[],
p_col_max_max=[],
slot=[])
self.log_info_raft_msg(self.config, "")
for islot, slot in enumerate(ALL_SLOTS):
self.log_progress(" %s" % slot)
basename = data[slot]
datapath = basename.replace('_stats.fits', '.fits')
try:
dtables = TableDict(datapath, ['stack-row_s', 'stack-col_p'])
table_s = dtables['stack-row_s']
table_p = dtables['stack-col_p']
except FileNotFoundError:
self.log.error("Could not open %s\n" % datapath)
table_s = None
table_p = None
if table_s is not None:
tablevals_s_min = table_s['stack_mean'].min(0)
tablevals_s_max = table_s['stack_mean'].max(0)
else:
tablevals_s_min = [0., 0.]
tablevals_s_max = [0., 0.]
if table_p is not None:
tablevals_p_min = table_p['stack_mean'].min(0)
tablevals_p_max = table_p['stack_mean'].max(0)
else:
tablevals_p_min = [0., 0.]
tablevals_p_max = [0., 0.]
data_dict['s_row_min_mean'].append(np.mean(tablevals_s_min))
data_dict['s_row_min_median'].append(np.median(tablevals_s_min))
data_dict['s_row_min_std'].append(np.std(tablevals_s_min))
data_dict['s_row_min_min'].append(np.min(tablevals_s_min))
data_dict['s_row_min_max'].append(np.max(tablevals_s_min))
data_dict['s_row_max_mean'].append(np.mean(tablevals_s_max))
data_dict['s_row_max_median'].append(np.median(tablevals_s_max))
data_dict['s_row_max_std'].append(np.std(tablevals_s_max))
data_dict['s_row_max_min'].append(np.min(tablevals_s_max))
data_dict['s_row_max_max'].append(np.max(tablevals_s_max))
data_dict['p_col_min_mean'].append(np.mean(tablevals_p_min))
data_dict['p_col_min_median'].append(np.median(tablevals_p_min))
data_dict['p_col_min_std'].append(np.std(tablevals_p_min))
data_dict['p_col_min_min'].append(np.min(tablevals_p_min))
data_dict['p_col_min_max'].append(np.max(tablevals_p_min))
data_dict['p_col_max_mean'].append(np.mean(tablevals_p_max))
data_dict['p_col_max_median'].append(np.median(tablevals_p_max))
data_dict['p_col_max_std'].append(np.std(tablevals_p_max))
data_dict['p_col_max_min'].append(np.min(tablevals_p_max))
data_dict['p_col_max_max'].append(np.max(tablevals_p_max))
data_dict['slot'].append(islot)
self.log_progress("Done!")
outtables = TableDict()
outtables.make_datatable("biasosstack_stats", data_dict)
return outtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
if self.config.teststand == 'ts8':
flat1_files = data['FLAT1']
flat2_files = data['FLAT2']
elif self.config.teststand == 'bot':
flat1_files = data['FLAT0']
flat2_files = data['FLAT1']
bias_type = self.get_bias_algo()
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
nlc = self.get_nonlinearirty_correction()
#slot_idx = ALL_SLOTS.index(self.config.slot)
self.log_info_slot_msg(self.config, "%i %i files" % (len(flat1_files), len(flat2_files)))
# This is a dictionary of dictionaries to store all the
# data you extract from the flat_files
data_dict = dict(FLUX=[],
EXPTIME=[],
MONDIODE1=[],
MONDIODE2=[],
MONOCH_SLIT_B=[])
for i in range(1, 17):
data_dict['AMP%02i_RATIO' % i] = []
data_dict['AMP%02i_MEAN' % i] = []
data_dict['AMP%02i_CORRMEAN' % i] = []
data_dict['AMP%02i_VAR' % i] = []
data_dict['AMP%02i_SIGNAL' % i] = []
data_dict['AMP%02i_MEAN1' % i] = []
data_dict['AMP%02i_MEAN2' % i] = []
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
for ifile, (id_1, id_2) in enumerate(zip(flat1_files, flat2_files)):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
flat_1 = self.get_ccd(butler, id_1, mask_files)
flat_2 = self.get_ccd(butler, id_2, mask_files)
amps = get_amp_list(flat_1)
exp_time_1 = get_exposure_time(flat_1)
exp_time_2 = get_exposure_time(flat_2)
if exp_time_1 != exp_time_2:
self.log.warn("Exposure times do not match for:\n%s\n%s\n %0.3F %0.3F. Skipping Pair\n"
% (id_1, id_2, exp_time_1, exp_time_2))
continue
mondiode_1 = get_monodiode_val_from_data_id(id_1, exp_time_1,
self.config.teststand, butler)
mondiode_2 = get_monodiode_val_from_data_id(id_2, exp_time_2,
self.config.teststand, butler)
if mondiode_1 is None or mondiode_2 is None:
self.log.warn("No monitoring data for:\n%s\n%s\n Skipping Pair\n"
% (id_1, id_2))
continue
flux = (exp_time_1 * mondiode_1 + exp_time_2 * mondiode_2)/2.
data_dict['EXPTIME'].append(exp_time_1)
data_dict['MONDIODE1'].append(mondiode_1)
data_dict['MONDIODE2'].append(mondiode_2)
data_dict['FLUX'].append(flux)
try:
data_dict['MONOCH_SLIT_B'].append(get_mono_slit_b(flat_1))
except KeyError:
data_dict['MONOCH_SLIT_B'].append(0.)
ccd_1_ims = unbiased_ccd_image_dict(flat_1, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging', nonlinearity=nlc)
ccd_2_ims = unbiased_ccd_image_dict(flat_2, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging', nonlinearity=nlc)
for i, amp in enumerate(amps):
image_1 = ccd_1_ims[amp]
image_2 = ccd_2_ims[amp]
fstats = self.get_pair_stats(image_1, image_2)
signal = fstats[1]
#if gains is not None:
# signal *= gains[slot_idx][i]
data_dict['AMP%02i_RATIO' % (i+1)].append(fstats[0])
data_dict['AMP%02i_MEAN' % (i+1)].append(fstats[1])
data_dict['AMP%02i_CORRMEAN' % (i+1)].append(fstats[2])
data_dict['AMP%02i_VAR' % (i+1)].append(fstats[3])
data_dict['AMP%02i_SIGNAL' % (i+1)].append(signal)
data_dict['AMP%02i_MEAN1' % (i+1)].append(fstats[4])
data_dict['AMP%02i_MEAN2' % (i+1)].append(fstats[5])
self.log_progress("Done!")
primary_hdu = fits.PrimaryHDU()
primary_hdu.header['NAMPS'] = 16
dtables = TableDict(primary=primary_hdu)
dtables.make_datatable('files', make_file_dict(butler, flat1_files + flat2_files))
dtables.make_datatable('flat', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
slot = self.config.slot
bias_type = self.get_bias_algo()
flat1_files = data['FLAT1']
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(flat1_files))
sflat_table_file = self.get_filename_from_format(
RAFT_SFLAT_TABLE_FORMATTER, "sflat.fits")
sflat_tables = TableDict(sflat_table_file)
# dictionary of dictionaries of lists of bounding
# boxes, keyed by slot, amp
bbox_dict = construct_bbox_dict(sflat_tables['defects'])
slot_bbox_dict = bbox_dict[slot]
slot_idx_dict = dict(S00=0,
S01=1,
S02=2,
S10=3,
S11=4,
S12=5,
S20=6,
S21=7,
S22=8)
# This is a dictionary of dictionaries to store all the
# data you extract from the flat1_files
fp_dict = dict(slot=[],
amp=[],
x_corner=[],
y_corner=[],
x_size=[],
y_size=[],
med_flux_full=[],
exptime=[],
mondiode=[],
amp_median=[],
ratio_full=[])
for i in range(4):
fp_dict['ratio_%i' % i] = []
fp_dict['med_flux_%i' % i] = []
fp_dict['npix_%i' % i] = []
fp_dict['npix_0p2_%i' % i] = []
# Analysis goes here, you should fill fp_dict with data extracted
# by the analysis
#
for ifile, flat1_file in enumerate(flat1_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, flat1_file, mask_files)
# To be appended while looping over bounding boxes
exptime = get_exposure_time(ccd)
mondiode_val = get_mondiode_val(ccd)
islot = slot_idx_dict[slot]
unbiased_images = unbiased_ccd_image_dict(
ccd, bias=bias_type, superbias_frame=superbias_frame)
for iamp, (amp, image) in enumerate(unbiased_images.items()):
bbox_list = slot_bbox_dict[iamp]
amp_median = np.median(image.array)
#fill fp_dict
frac_thresh = kwargs.get('frac_thresh', 0.9)
thresh_float = frac_thresh * amp_median
thresh_0p2_float = (1. - (1. - frac_thresh) * 0.2) * amp_median
max_bbox = min(len(bbox_list), 10)
#print(i, amp, bbox_list[0:max_bbox])
#print(max_bbox)
for bbox in bbox_list[0:max_bbox]:
try:
cutout = image[bbox].array
except Exception:
print("cutout failed", slot, amp, bbox)
continue
#print(ifile, bbox, cutout)
ratio_full = np.mean(cutout) / amp_median
cutout_median_full = np.median(cutout)
peak_idx = cutout.argmax()
peak_x = bbox.getMinX() + int(peak_idx % cutout.shape[1])
peak_y = bbox.getMinY() + int(peak_idx / cutout.shape[1])
peak = afwGeom.Point2I(peak_x, peak_y)
extent = afwGeom.Extent2I(1, 1)
bbox_expand = afwGeom.Box2I(peak, extent)
#npix = np.array([1, 9, 25, 49])
npix_cumul = np.array([1, 8, 16, 24])
sums = np.zeros((4))
meds = np.zeros((4))
over_thresh = np.zeros((4), int)
over_0p2_thresh = np.zeros((4), int)
sums_cumul = np.zeros((4))
meds_cumul = np.zeros((4))
over_thresh_cumul = np.zeros((4), int)
over_0p2_thresh_cumul = np.zeros((4), int)
for j in range(4):
try:
cuttout_array = image[bbox_expand].array
except LengthError:
break
sums[j] = cuttout_array.sum()
meds[j] = np.median(cuttout_array)
over_thresh[j] = (cuttout_array > thresh_float).sum()
over_0p2_thresh[j] = (cuttout_array >
thresh_0p2_float).sum()
if j > 0:
sums_cumul[j] = sums[j] - sums[j - 1]
meds_cumul[j] = meds[j] - meds[j - 1]
over_thresh_cumul[
j] = over_thresh[j] - over_thresh[j - 1]
over_0p2_thresh_cumul[j] = over_0p2_thresh[
j] - over_0p2_thresh[j - 1]
else:
sums_cumul[j] = sums[j]
meds_cumul[j] = meds[j]
over_thresh_cumul[j] = over_thresh[j]
over_0p2_thresh_cumul[j] = over_0p2_thresh[j]
bbox_expand.grow(1)
means_cumul = sums_cumul / (npix_cumul * amp_median)
fp_dict['exptime'].append(exptime)
fp_dict['mondiode'].append(mondiode_val)
fp_dict['amp'].append(iamp)
fp_dict['slot'].append(islot)
fp_dict['amp_median'].append(amp_median)
fp_dict['x_corner'].append(bbox.getMinX())
fp_dict['y_corner'].append(bbox.getMinY())
fp_dict['x_size'].append(bbox.getWidth())
fp_dict['y_size'].append(bbox.getHeight())
fp_dict['med_flux_full'].append(cutout_median_full)
fp_dict['ratio_full'].append(ratio_full)
for i in range(4):
fp_dict['ratio_%i' % i].append(means_cumul[i])
fp_dict['med_flux_%i' % i].append(meds_cumul[i])
fp_dict['npix_%i' % i].append(over_thresh_cumul[i])
fp_dict['npix_0p2_%i' % i].append(
over_0p2_thresh_cumul[i])
sys.stdout.write("!\n")
sys.stdout.flush()
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, flat1_files))
dtables.make_datatable('footprints', fp_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
output data tables
"""
self.safe_update(**kwargs)
slots = self.config.slots
if slots is None:
slots = ALL_SLOTS
if butler is not None:
self.log.warn("Ignoring butler")
dark_current_data = dict(median=[],
stdev=[],
mean=[],
fit_mean=[],
fit_width=[],
fit_dof=[],
fit_chi2=[],
exptime=[],
current=[],
slot=[],
amp=[])
self.log_info_raft_msg(self.config, "")
for islot, slot in enumerate(slots):
self.log_progress(" %s" % slot)
mask_files = self.get_mask_files(slot=slot)
superdark_file = data[slot]
if not os.path.exists(superdark_file):
self.log.warn(" %s does not exist, skipping" % superdark_file)
continue
superdark_frame = self.get_ccd(None, superdark_file, mask_files)
exptime = get_exposure_time(superdark_frame)
amps = get_amp_list(superdark_frame)
for iamp, amp in enumerate(amps):
regions = get_geom_regions(superdark_frame, amp)
imaging = regions['imaging']
superdark_im = get_raw_image(superdark_frame, amp).image
image_data = superdark_im[imaging].array
median = np.median(image_data)
stdev = np.std(image_data)
hist_bins = np.linspace(median - 5 * stdev, median + 5 * stdev, 101)
hist = np.histogram(image_data, bins=hist_bins)
fit_result = gauss_fit(hist)
fit_pars = fit_result[0]
dark_current_data['median'].append(median)
dark_current_data['stdev'].append(stdev)
dark_current_data['mean'].append(np.mean(image_data))
dark_current_data['fit_mean'].append(fit_pars[1])
dark_current_data['fit_width'].append(fit_pars[2])
dark_current_data['fit_dof'].append(0.)
dark_current_data['fit_chi2'].append(0.)
dark_current_data['exptime'].append(exptime)
dark_current_data['current'].append(median/exptime)
dark_current_data['slot'].append(islot)
dark_current_data['amp'].append(iamp)
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('dark_current', dark_current_data)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
slot = self.config.slot
qe_files = data['LAMBDA']
lams = []
for file in qe_files:
lam = file.split('flat_')[1].split('_')[0]
lams.append(lam)
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
bias_type = self.get_bias_algo()
self.log_info_slot_msg(self.config, "%i files" % len(qe_files))
sflat_table_file = self.get_filename_from_format(RAFT_SFLAT_TABLE_FORMATTER, "sflat.fits")
sflat_tables = TableDict(sflat_table_file)
bbox_dict = construct_bbox_dict(sflat_tables['defects'])
slot_bbox_dict = bbox_dict[slot]
# This is a dictionary of dictionaries to store all the
# data you extract from the qe_files
data_dict = dict(SLOT=[], AMP=[], XCORNER=[], YCORNER=[], XSIZE=[], YSIZE=[])
for lam in lams:
data_dict['FLUX_' + lam] = []
data_dict['MED_' + lam] = []
temp_list = []
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
# For this slot, loop over qe files, loop over amplifiers, loop over dust spots in that amplifier,
# record the flux and median for each dust spot
# Then loop over dust spots CCDs, amplifiers, and QE files to accumulate the output dictionary
ccd = self.get_ccd(butler, qe_files[0], mask_files)
amps = get_amp_list(ccd)
spot = 0
for ifile, qe_file in enumerate(qe_files):
lam = qe_file.split('flat_')[1].split('_')[0]
for i, amp in enumerate(amps):
bbox_list = slot_bbox_dict[amp]
if len(bbox_list) > 100:
if i == 0:
self.log.warn("Skipping slot:amp %s:%i with %i defects" % (slot, amp, len(bbox_list)))
continue
ccd = self.get_ccd(butler, qe_file, mask_files)
regions = get_geom_regions(ccd, amp)
serial_oscan = regions['serial_overscan']
imaging = regions['imaging']
img = get_raw_image(ccd, amp)
if superbias_frame is not None:
if butler is not None:
superbias_im = get_raw_image(superbias_frame, amp)
else:
superbias_im = get_raw_image(superbias_frame, amp)
else:
superbias_im = None
image = unbias_amp(img, serial_oscan, bias_type=bias_type,
superbias_im=superbias_im, region=imaging)
for bbox in bbox_list:
if ifile == 0:
data_dict['SLOT'].append(slot)
#data_dict['LAM'].append(lam)
data_dict['AMP'].append(amp)
data_dict['XCORNER'].append(bbox.getBeginX())
data_dict['YCORNER'].append(bbox.getBeginY())
data_dict['XSIZE'].append(bbox.getWidth())
data_dict['YSIZE'].append(bbox.getHeight())
try:
cutout = image[bbox]
except Exception:
pass
# Here evaluate the 'flux' of the feature, relative to the median
# value of the amplifier image. May also want to assemble bounding
# box corners into a ds9 region file, CCD by CCD
med = np.median(image.array)
flux = np.sum(cutout.array) - bbox.getWidth()*bbox.getHeight()*med
spot += 1
#temp_dict[spot].append((lam, flux, med))
temp_list.append((lam, flux, med))
for i, tmp_data in enumerate(temp_list):
#data_dict['SLOT'].append(temp_dict['SLOT'][i])
#data_dict['AMP'].append(temp_dict['AMP'][i])
#lam = temp_dict['LAM'][i]
lam, flux, med = tmp_data
data_dict['FLUX_' + lam].append(flux) #temp_dict['FLUX'][i])
data_dict['MED_' + lam].append(med) #np.median(image.array))
sys.stdout.write("!\n")
sys.stdout.flush()
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, qe_files))
#dtables.make_datatable('dust_color', data_dict)
dtables.make_datatable('dust_color_hack', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the data from the overscan region
to estimate the deffered charge
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
raise ValueError(
"FlatOverscanTask not implemented for Butlerized data")
flat_files = data['FLAT1']
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(flat_files))
#fitter = OverscanFit(num_oscan_pixels=self.config.num_oscan_pixels,
# minflux=self.config.minflux, maxflux=self.config.maxflux)
gains = self.get_gains()
if gains is None:
gains = np.ones((17))
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
for ifile, flat_id in enumerate(flat_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
flat = self.get_ccd(butler,
flat_id,
mask_files,
bias_frame=superbias_frame)
fitter.process_image(flat, gains)
self.xmax = fitter.xmax_val
self.log_progress("Done!")
data_dict = fitter.build_output_dict()
primary = fitter.output[0]
primary.header['NAMPS'] = 16
dtables = TableDict(primary=primary)
for key, val in data_dict.items():
dtables.make_datatable(key.lower(), val)
dtables.make_datatable('files', make_file_dict(butler, flat_files))
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the statistics from the superbias frames
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
self.log.warn("Ignoring butler")
stats_data = {}
self.log_info_raft_msg(self.config, "")
slot_list = self.config.slots
if slot_list is None:
slot_list = ALL_SLOTS
for islot, slot in enumerate(slot_list):
superbias_file = data[slot]
if self.config.stat is not None:
superbias_file = superbias_file.replace('_superbias_', '_%s_' % self.config.stat)
if not os.path.exists(superbias_file):
self.log.warn("Skipping missing file for %s:%s" % (self.config.raft, slot))
continue
self.log_progress(" %s" % slot)
mask_files = self.get_mask_files(slot=slot)
try:
superbias_frame = self.get_ccd(None, superbias_file, mask_files)
except Exception:
self.log.warn("Skipping %s:%s:%s" % (self.config.run, self.config.raft, slot))
superbias_frame = None
self.get_superbias_stats(superbias_frame, stats_data, islot)
self.log_progress("Done!")
if not stats_data:
return None
stats_data['slot'] = np.arange(9)
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(None, slot_list))
dtables.make_datatable('stats', stats_data)
return dtables