def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
# You should expand this to include space for the data you want to extract
data_dict = dict(slot=[], amp=[])
sys.stdout.write("Working on 9 slots: ")
sys.stdout.flush()
for islot, slot in enumerate(ALL_SLOTS):
sys.stdout.write(" %s" % slot)
sys.stdout.flush()
basename = data[slot]
datapath = basename.replace(self.config.outsuffix,
self.config.insuffix)
dtables = TableDict(datapath)
print(dtables)
for amp in range(16):
# Here you can get the data out for each amp and append it to the
# data_dict
data_dict['slot'].append(islot)
data_dict['amp'].append(amp)
sys.stdout.write(".\n")
sys.stdout.flush()
outtables = TableDict()
outtables.make_datatable("tmplsuffix", data_dict)
return outtables
def get_data_table_names_from_file(fname):
"""Get a name of tables in a particular file
Parameters
----------
fname : `str`
File name
Returns
-------
keys : `list`
Table names
"""
dtables = TableDict(fname)
return dtables.keys()
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
# You should expand this to include space for the data you want to extract
data_dict = dict(amp=[],
full_well=[],
max_dev=[])
self.log_info_raft_msg(self.config, "")
basename = data[0]
datapath = basename.replace(self.config.filekey, self.config.infilekey)
detresp = DetectorResponse(datapath, hdu_name='flat')
for amp in range(1, 17):
# Here you can get the data out for each amp and append it to the
# data_dict
data_dict['amp'].append(amp)
linearity_data = detresp.linearity(amp)
full_well_data = detresp.full_well(amp)
data_dict['full_well'].append(full_well_data[0])
data_dict['max_dev'].append(linearity_data[0])
self.log_progress("Done!")
outtables = TableDict()
outtables.make_datatable("flat_lin", data_dict)
return outtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
# You should expand this to include space for the data you want to extract
data_dict = dict(slot=[], amp=[])
self.log_info_raft_msg(self.config, "")
for islot, slot in enumerate(ALL_SLOTS):
self.log_progress(" %s" % slot)
# get the data from the input table
#basename = data[slot]
#datapath = basename.replace(self.config.outsuffix, self.config.insuffix)
#dtables = TableDict(datapath)
for amp in range(16):
# Here you can get the data out for each amp and append it to the
# data_dict
data_dict['slot'].append(islot)
data_dict['amp'].append(amp)
self.log_progress("Done!")
outtables = TableDict()
outtables.make_datatable("tmplsuffix", data_dict)
return outtables
def inspect_tablefile(self, filetype, **kwargs):
"""Print information about the contents of a particular
type of file with data tables
Parameters
----------
filetyple : `str`
Type of file in `FileFormatDict`
kwargs
Passed to the class file name formatting function
"""
fname = self.get_filename(filetype, **kwargs)
tdict = TableDict(fname)
sys.stdout.write("File %s:\n" % fname)
for key, val in tdict.items():
sys.stdout.write("Table %s:\n" % key)
for col in val.columns:
sys.stdout.write(" %s:\n" % col)
def extract(self, butler, data, **kwargs):
"""Extract the outliers in the superflat frames for the raft
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
self._mask_file_dict = {}
self._sflat_file_dict_l = {}
self._sflat_file_dict_h = {}
self._sflat_file_dict_r = {}
if butler is not None:
self.log.warn("Ignoring butler")
self.set_local_data(butler, data, **kwargs)
if not self._sflat_file_dict_l:
self.log.warn("No files for %s, skipping" % (self.config.raft))
return None
self._sflat_images_h, ccd_dict = extract_raft_unbiased_images(
self._sflat_file_dict_h, mask_dict=self._mask_file_dict)
self._sflat_array_l = extract_raft_array_dict(
self._sflat_file_dict_l, mask_dict=self._mask_file_dict)
self._sflat_array_h = extract_raft_imaging_data(
self._sflat_images_h, ccd_dict)
self._sflat_array_r = extract_raft_array_dict(
self._sflat_file_dict_r, mask_dict=self._mask_file_dict)
out_data_l = outlier_raft_dict(self._sflat_array_l, 1000., 300.)
out_data_h = outlier_raft_dict(self._sflat_array_h, 50000., 15000.)
out_data_r = outlier_raft_dict(self._sflat_array_r, 0.019, 0.002)
fp_dict = SuperflatRaftTask.build_defect_dict(self._sflat_images_h,
frac_thresh=0.9)
dtables = TableDict()
dtables.make_datatable('defects', fp_dict)
dtables.make_datatable('outliers_l', out_data_l)
dtables.make_datatable('outliers_h', out_data_h)
dtables.make_datatable('outliers_r', out_data_r)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
output data tables
"""
self.safe_update(**kwargs)
slots = ALL_SLOTS
# This is a dictionary of dictionaries to store all the
# data you extract from the tmpl_files
data_dict = {}
for slot in slots:
#Get the datafiles, and the correpsonding superbias files
tmpl_files = data[slot]['TMPL']
print(tmpl_files)
#mask_files = self.get_mask_files(slot=slot)
#superbias_frame = self.get_superbias_frame(mask_files, slot=slot)
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
dtables = TableDict()
for key, val in data_dict.items():
dtables.make_datatable(key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
for key, val in data.items():
data[key] = val.replace(self.config.outsuffix,
self.config.insuffix)
# Define the set of columns to keep and remove
# keep_cols = []
# remove_cols = []
outtable = vstack_tables(data, tablename='tmplsuffix_stats')
dtables = TableDict()
dtables.add_datatable('tmplsuffix_sum', outtable)
dtables.make_datatable('runs', dict(runs=sorted(data.keys())))
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the summary data from the PTC analyses
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
self.log.warn("Ignoring butler")
if not self.config.skip:
outtable = vstack_tables(data, tablename='ptc_stats')
dtables = TableDict()
dtables.add_datatable('ptc_sum', outtable)
dtables.make_datatable('runs', dict(runs=sorted(data.keys())))
return dtables
def extract(self, butler, data, **kwargs):
"""Make a summary table of the superbias statistics
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
self.log.warn("Ignoring butler")
run_dict = dict(runs=[], rafts=[])
for key, val in sorted(data.items()):
run_dict['runs'].append(key[4:])
run_dict['rafts'].append(key[0:3])
data[key] = val.replace(self.config.filekey, self.config.infilekey)
outtable = vstack_tables(data, tablename='stats')
dtables = TableDict()
dtables.add_datatable('stats', outtable)
dtables.make_datatable('runs', run_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
run_dict = dict(runs=[], rafts=[])
for key, val in data.items():
run_dict['runs'].append(key[4:])
run_dict['rafts'].append(key[0:3])
data[key] = val.replace(self.config.filekey, self.config.infilekey)
# Define the set of columns to keep and remove
# keep_cols = []
# remove_cols = []
outtable = vstack_tables(data, tablename='dark_current')
dtables = TableDict()
dtables.add_datatable('stats', outtable)
dtables.make_datatable('runs', run_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Make a summary table
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
self.log.warn("Ignoring butler")
for key, val in data.items():
data[key] = val.replace('_sum.fits', '_stats.fits')
outtable = vstack_tables(data, tablename='biasosstack_stats')
dtables = TableDict()
dtables.add_datatable('biasosstack_sum', outtable)
dtables.make_datatable('runs', dict(runs=sorted(data.keys())))
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
output data tables
"""
self.safe_update(**kwargs)
for key, val in data.items():
print(key, val)
data[key] = val.replace(self.config.filekey, self.config.infilekey)
# Define the set of columns to keep and remove
# keep_cols = []
# remove_cols = []
outtable = vstack_tables(data, tablename='eo_results')
dtables = TableDict()
dtables.add_datatable('eo_results_run', outtable)
dtables.make_datatable('runs', dict(runs=[self.config.run]))
return dtables
def extract(self, butler, data, **kwargs):
"""Stack the overscan region from all the amps on a sensor
to look for coherent read noise
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
bias_files = data['BIAS']
bias_type = self.get_bias_algo()
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files=mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(bias_files))
stack_arrays = {}
nfiles = len(bias_files)
for ifile, bias_file in enumerate(bias_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, bias_file, mask_files)
if ifile == 0:
dim_array_dict = get_dimension_arrays_from_ccd(ccd)
for key, val in dim_array_dict.items():
stack_arrays[key] = np.zeros((nfiles, 16, len(val)))
stack_by_amps(stack_arrays,
ccd,
ifile=ifile,
bias_type=bias_type,
superbias_frame=superbias_frame)
self.log_progress("Done!")
stackdata_dict = convert_stack_arrays_to_dict(stack_arrays,
dim_array_dict, nfiles)
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
for key, val in stackdata_dict.items():
dtables.make_datatable('stack-%s' % key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Plot the row-wise and col-wise struture
in a series of bias frames
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
slot = self.config.slot
bias_files = data['BIAS']
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files=mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(bias_files))
biasstruct_data = {}
for ifile, bias_file in enumerate(bias_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, bias_file, mask_files)
if ifile == 0:
dim_array_dict = get_dimension_arrays_from_ccd(ccd)
for key, val in dim_array_dict.items():
biasstruct_data[key] = {key: val}
self.get_ccd_data(ccd,
biasstruct_data,
slot=slot,
ifile=ifile,
nfiles_used=len(bias_files),
superbias_frame=superbias_frame)
self.log_progress("Done!")
if not biasstruct_data:
return None
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
for key, val in biasstruct_data.items():
dtables.make_datatable('biasst-%s' % key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
gains = np.ones((17))
ppump_files = data['PPUMP']
print(ppump_files)
ppump_file = ppump_files[0]
mask_files = self.get_mask_files()
self.log_info_slot_msg(self.config, "")
# This is a dictionary of dictionaries to store all the
# data you extract from the ppump_files
data_dict = {}
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
ccd = self.get_ccd(butler, ppump_file, mask_files, masked_ccd=True)
my_traps = Traps(ccd,
gains,
cycles=self.config.cycles,
C2_thresh=self.config.C2_thresh,
C3_thresh=self.config.C3_thresh,
nx=self.config.bkg_nx,
ny=self.config.bkg_ny,
edge_rolloff=self.config.edge_rolloff)
data_dict = my_traps.make_data_dict()
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, ppump_files))
dtables.make_datatable('traps', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the row-wise and col-wise struture in a superbias frame
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
slot = self.config.slot
if butler is not None:
self.log.warn("Ignoring butler")
if data is not None:
self.log.warn("Ignoring data")
mask_files = self.get_mask_files()
superbias = self.get_superbias_frame(mask_files=mask_files)
if superbias is None:
return None
self.log_info_slot_msg(self.config, "")
biasstruct_data = {}
dim_array_dict = get_dimension_arrays_from_ccd(superbias)
for key, val in dim_array_dict.items():
biasstruct_data[key] = {key: val}
self.get_ccd_data(superbias,
biasstruct_data,
slot=slot,
bias_type=None,
std=self.config.std,
superbias_frame=None)
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(None, [slot]))
for key, val in biasstruct_data.items():
dtables.make_datatable('biasst-%s' % key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the bias as function of row
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
bias_files = data['BIAS']
if not bias_files:
return None
mask_files = self.get_mask_files()
self.log_info_slot_msg(self.config, "%i files" % len(bias_files))
biasval_data = {}
for ifile, bias_file in enumerate(bias_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, bias_file, mask_files)
if ifile == 0:
dims = get_dims_from_ccd(ccd)
xrow_s = np.linspace(0, dims['nrow_s'] - 1, dims['nrow_s'])
self.get_ccd_data(ccd,
biasval_data,
ifile=ifile,
nfiles=len(bias_files))
#Need to truncate the row array to match the data
a_row = biasval_data[sorted(biasval_data.keys())[0]]
biasval_data['row_s'] = xrow_s[0:len(a_row)]
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
dtables.make_datatable('biasval', biasval_data)
return dtables
def get_task_table_dict(self, key, **kwargs):
"""Get the tables produceded by a particular task
Parameters
----------
key : `str`
Name associated to that `Task`
kwargs
Passed to the class file name formatting function
Returns
-------
tdict: `TableDict`
The tables produced by the task/ configuration
"""
tfile = self.get_task_tablefile(key, **kwargs) + '.fits'
tdict = TableDict(tfile)
return tdict
def get_data_column_names_from_file(fname, tname):
"""Get a column names from a particular table
Parameters
----------
fname : `str`
File name
tname : `str`
Table name
Returns
-------
keys : `list`
Table names
"""
dtables = TableDict(fname, [tname])
dtab = dtables[tname]
return dtab.columns
def get_data_column_from_file(fname, tname, cname):
"""Get a column from a particular table
Parameters
----------
fname : `str`
File name
tname : `str`
Table name
cname : `str`
Column name
kwargs
Passed to the class file name formatting function
Returns
-------
data : `np.array`
The requested column data
"""
dtables = TableDict(fname, [tname])
return dtables[tname][cname]
def extract(self, butler, data, **kwargs):
"""Extract the correlations between the serial overscan for each amp on a raft
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
slots = ALL_SLOTS
overscans = []
for slot in slots:
bias_files = data[slot]['BIAS']
mask_files = self.get_mask_files(slot=slot)
superbias_frame = self.get_superbias_frame(mask_files, slot=slot)
ccd = self.get_ccd(butler, bias_files[0], [])
overscans += self.get_ccd_data(butler, ccd, superbias_frame=superbias_frame)
namps = len(overscans)
if self.config.covar:
data = np.array([np.cov(overscans[i[0]].ravel(),
overscans[i[1]].ravel())[0, 1]
for i in itertools.product(range(namps), range(namps))])
else:
data = np.array([np.corrcoef(overscans[i[0]].ravel(),
overscans[i[1]].ravel())[0, 1]
for i in itertools.product(range(namps), range(namps))])
data = data.reshape(namps, namps)
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
dtables.make_datatable('correl', dict(correl=data))
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the utliers in the superdark frames for the raft
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
self._mask_file_dict = {}
self._sdark_file_dict = {}
if butler is not None:
self.log.warn("Ignoring butler")
self.set_local_data(butler, data, **kwargs)
if not self._sdark_file_dict:
self.log.warn("No files for %s, skipping" % (self.config.raft))
return None
self._sdark_images, ccd_dict = extract_raft_unbiased_images(
self._sdark_file_dict, mask_dict=self._mask_file_dict)
self._sdark_arrays = extract_raft_imaging_data(self._sdark_images,
ccd_dict)
out_data = outlier_raft_dict(self._sdark_arrays, 0., 25.)
fp_dict = build_defect_dict(self._sdark_images,
fp_type='bright',
abs_thresh=50)
dtables = TableDict()
dtables.make_datatable('defects', fp_dict)
dtables.make_datatable('outliers', out_data)
return dtables
def get_data_columns_from_file(fname, tname, clist):
"""Get a column from a particular table
Parameters
----------
filetyple : `str`
Type of file in `FileFormatDict`
tname : `str`
Table name
clist : `list`
Column names
kwargs
Passed to the class file name formatting function
Returns
-------
odict : `dict`
Dictionary mapping column name to data
"""
dtables = TableDict(fname, [tname])
dtab = dtables[tname]
return {key:dtab[key] for key in clist}
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
# Get the slot, if needed
#slot = self.config.slot
tmpl_files = data['TMPL']
# Get the superbias frame, if needed
#mask_files = self.get_mask_files()
#superbias_frame = self.get_superbias_frame(mask_files)
self.log_info_slot_msg(self.config, "")
# This is a dictionary of dictionaries to store all the
# data you extract from the tmpl_files
data_dict = {}
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, tmpl_files))
for key, val in data_dict.items():
dtables.make_datatable(key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
output data tables
"""
self.safe_update(**kwargs)
slots = ALL_SLOTS
# This is a dictionary of dictionaries to store all the
# data you extract from the base_files
data_dict = dict(slot=np.ndarray((144)))
for islot, slot in enumerate(slots):
infile = data[slot]
try:
dtables = TableDict(infile, ['amplifier_results'])
except FileNotFoundError:
return None
table = dtables['amplifier_results']
data_dict['slot'][16 * islot:16 * (islot + 1)] = islot
for key in table.keys():
if key not in data_dict:
data_dict[key] = np.ndarray((144))
data_dict[key][16 * islot:16 * (islot + 1)] = table[key]
dtables = TableDict()
dtables.make_datatable('eo_results', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
self.log_info_slot_msg(self.config, "")
pd_calib_file = self.get_pd_calib_file()
gains = np.ones((17))
basename = data[0]
qe_data = QE_Data()
qe_data.read_fits(basename)
qe_data.incidentPower(pd_calib_file)
qe_data.calculate_QE(gains)
qe_curves_data_dict = qe_data.make_qe_curves_data_dict()
bands_data_dict = qe_data.make_bands_data_dict()
dtables = TableDict()
dtables.make_datatable('qe_curves', qe_curves_data_dict)
dtables.make_datatable('bands', bands_data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the gains and widths from the f355 clusters
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
self.log.warn("Ignoring butler")
use_all = self.config.use_all
data_dict = dict(kalpha_peak=[],
kalpha_sigma=[],
ncluster=[],
ngood=[],
gain=[],
gain_error=[],
fit_xmin=[],
fit_xmax=[],
fit_pars=[],
fit_nbins=[],
sigmax_median=[],
sigmay_median=[],
slot=[],
amp=[])
self.log_info_raft_msg(self.config, "")
for islot, slot in enumerate(ALL_SLOTS):
self.log_progress(" %s" % slot)
basename = data[slot]
dtables = TableDict(basename)
for amp in range(16):
table = dtables['amp%02i' % (amp + 1)]
if use_all:
mask = np.ones((len(table)), bool)
else:
mask = (np.fabs(table['XPOS'] - table['XPEAK']) < 1)*\
(np.fabs(table['YPOS'] - table['YPEAK']) < 1)
tablevals = table[mask]['DN']
gainfitter = Fe55GainFitter(tablevals)
try:
kalpha_peak, kalpha_sigma = gainfitter.fit(bins=100)
gain = gainfitter.gain
gain_error = gainfitter.gain_error
pars = gainfitter.pars
except Exception:
kalpha_peak, kalpha_sigma = (np.nan, np.nan)
gain = np.nan
gain_error = np.nan
pars = np.nan * np.ones((4))
data_dict['kalpha_peak'].append(kalpha_peak)
data_dict['kalpha_sigma'].append(kalpha_sigma)
data_dict['gain'].append(gain)
data_dict['gain_error'].append(gain_error)
xra = gainfitter.xrange
data_dict['ncluster'].append(mask.size)
data_dict['ngood'].append(mask.sum())
if xra is None:
data_dict['fit_xmin'].append(np.nan)
data_dict['fit_xmax'].append(np.nan)
else:
data_dict['fit_xmin'].append(xra[0])
data_dict['fit_xmax'].append(xra[1])
data_dict['fit_pars'].append(pars)
data_dict['fit_nbins'].append(100.)
data_dict['sigmax_median'].append(np.median(table['SIGMAX']))
data_dict['sigmay_median'].append(np.median(table['SIGMAY']))
data_dict['slot'].append(islot)
data_dict['amp'].append(amp)
self.log_progress("Done!")
outtables = TableDict()
outtables.make_datatable("fe55_gain_stats", data_dict)
return outtables
def extract(self, butler, data, **kwargs):
"""Extract the summary statistics about the fluctuations
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
self.log.warn("Ignoring bulter")
data_dict = dict(s_row_min_mean=[],
s_row_min_median=[],
s_row_min_std=[],
s_row_min_min=[],
s_row_min_max=[],
s_row_max_mean=[],
s_row_max_median=[],
s_row_max_std=[],
s_row_max_min=[],
s_row_max_max=[],
p_col_min_mean=[],
p_col_min_median=[],
p_col_min_std=[],
p_col_min_min=[],
p_col_min_max=[],
p_col_max_mean=[],
p_col_max_median=[],
p_col_max_std=[],
p_col_max_min=[],
p_col_max_max=[],
slot=[])
self.log_info_raft_msg(self.config, "")
for islot, slot in enumerate(ALL_SLOTS):
self.log_progress(" %s" % slot)
basename = data[slot]
datapath = basename.replace('_stats.fits', '.fits')
try:
dtables = TableDict(datapath, ['stack-row_s', 'stack-col_p'])
table_s = dtables['stack-row_s']
table_p = dtables['stack-col_p']
except FileNotFoundError:
self.log.error("Could not open %s\n" % datapath)
table_s = None
table_p = None
if table_s is not None:
tablevals_s_min = table_s['stack_mean'].min(0)
tablevals_s_max = table_s['stack_mean'].max(0)
else:
tablevals_s_min = [0., 0.]
tablevals_s_max = [0., 0.]
if table_p is not None:
tablevals_p_min = table_p['stack_mean'].min(0)
tablevals_p_max = table_p['stack_mean'].max(0)
else:
tablevals_p_min = [0., 0.]
tablevals_p_max = [0., 0.]
data_dict['s_row_min_mean'].append(np.mean(tablevals_s_min))
data_dict['s_row_min_median'].append(np.median(tablevals_s_min))
data_dict['s_row_min_std'].append(np.std(tablevals_s_min))
data_dict['s_row_min_min'].append(np.min(tablevals_s_min))
data_dict['s_row_min_max'].append(np.max(tablevals_s_min))
data_dict['s_row_max_mean'].append(np.mean(tablevals_s_max))
data_dict['s_row_max_median'].append(np.median(tablevals_s_max))
data_dict['s_row_max_std'].append(np.std(tablevals_s_max))
data_dict['s_row_max_min'].append(np.min(tablevals_s_max))
data_dict['s_row_max_max'].append(np.max(tablevals_s_max))
data_dict['p_col_min_mean'].append(np.mean(tablevals_p_min))
data_dict['p_col_min_median'].append(np.median(tablevals_p_min))
data_dict['p_col_min_std'].append(np.std(tablevals_p_min))
data_dict['p_col_min_min'].append(np.min(tablevals_p_min))
data_dict['p_col_min_max'].append(np.max(tablevals_p_min))
data_dict['p_col_max_mean'].append(np.mean(tablevals_p_max))
data_dict['p_col_max_median'].append(np.median(tablevals_p_max))
data_dict['p_col_max_std'].append(np.std(tablevals_p_max))
data_dict['p_col_max_min'].append(np.min(tablevals_p_max))
data_dict['p_col_max_max'].append(np.max(tablevals_p_max))
data_dict['slot'].append(islot)
self.log_progress("Done!")
outtables = TableDict()
outtables.make_datatable("biasosstack_stats", data_dict)
return outtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
if self.config.teststand == 'ts8':
flat1_files = data['FLAT1']
flat2_files = data['FLAT2']
elif self.config.teststand == 'bot':
flat1_files = data['FLAT0']
flat2_files = data['FLAT1']
bias_type = self.get_bias_algo()
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
nlc = self.get_nonlinearirty_correction()
#slot_idx = ALL_SLOTS.index(self.config.slot)
self.log_info_slot_msg(self.config, "%i %i files" % (len(flat1_files), len(flat2_files)))
# This is a dictionary of dictionaries to store all the
# data you extract from the flat_files
data_dict = dict(FLUX=[],
EXPTIME=[],
MONDIODE1=[],
MONDIODE2=[],
MONOCH_SLIT_B=[])
for i in range(1, 17):
data_dict['AMP%02i_RATIO' % i] = []
data_dict['AMP%02i_MEAN' % i] = []
data_dict['AMP%02i_CORRMEAN' % i] = []
data_dict['AMP%02i_VAR' % i] = []
data_dict['AMP%02i_SIGNAL' % i] = []
data_dict['AMP%02i_MEAN1' % i] = []
data_dict['AMP%02i_MEAN2' % i] = []
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
for ifile, (id_1, id_2) in enumerate(zip(flat1_files, flat2_files)):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
flat_1 = self.get_ccd(butler, id_1, mask_files)
flat_2 = self.get_ccd(butler, id_2, mask_files)
amps = get_amp_list(flat_1)
exp_time_1 = get_exposure_time(flat_1)
exp_time_2 = get_exposure_time(flat_2)
if exp_time_1 != exp_time_2:
self.log.warn("Exposure times do not match for:\n%s\n%s\n %0.3F %0.3F. Skipping Pair\n"
% (id_1, id_2, exp_time_1, exp_time_2))
continue
mondiode_1 = get_monodiode_val_from_data_id(id_1, exp_time_1,
self.config.teststand, butler)
mondiode_2 = get_monodiode_val_from_data_id(id_2, exp_time_2,
self.config.teststand, butler)
if mondiode_1 is None or mondiode_2 is None:
self.log.warn("No monitoring data for:\n%s\n%s\n Skipping Pair\n"
% (id_1, id_2))
continue
flux = (exp_time_1 * mondiode_1 + exp_time_2 * mondiode_2)/2.
data_dict['EXPTIME'].append(exp_time_1)
data_dict['MONDIODE1'].append(mondiode_1)
data_dict['MONDIODE2'].append(mondiode_2)
data_dict['FLUX'].append(flux)
try:
data_dict['MONOCH_SLIT_B'].append(get_mono_slit_b(flat_1))
except KeyError:
data_dict['MONOCH_SLIT_B'].append(0.)
ccd_1_ims = unbiased_ccd_image_dict(flat_1, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging', nonlinearity=nlc)
ccd_2_ims = unbiased_ccd_image_dict(flat_2, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging', nonlinearity=nlc)
for i, amp in enumerate(amps):
image_1 = ccd_1_ims[amp]
image_2 = ccd_2_ims[amp]
fstats = self.get_pair_stats(image_1, image_2)
signal = fstats[1]
#if gains is not None:
# signal *= gains[slot_idx][i]
data_dict['AMP%02i_RATIO' % (i+1)].append(fstats[0])
data_dict['AMP%02i_MEAN' % (i+1)].append(fstats[1])
data_dict['AMP%02i_CORRMEAN' % (i+1)].append(fstats[2])
data_dict['AMP%02i_VAR' % (i+1)].append(fstats[3])
data_dict['AMP%02i_SIGNAL' % (i+1)].append(signal)
data_dict['AMP%02i_MEAN1' % (i+1)].append(fstats[4])
data_dict['AMP%02i_MEAN2' % (i+1)].append(fstats[5])
self.log_progress("Done!")
primary_hdu = fits.PrimaryHDU()
primary_hdu.header['NAMPS'] = 16
dtables = TableDict(primary=primary_hdu)
dtables.make_datatable('files', make_file_dict(butler, flat1_files + flat2_files))
dtables.make_datatable('flat', data_dict)
return dtables
