def extract(self, butler, data, **kwargs):
"""Stack the overscan region from all the amps on a sensor
to look for coherent read noise
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
bias_files = data['BIAS']
bias_type = self.get_bias_algo()
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files=mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(bias_files))
stack_arrays = {}
nfiles = len(bias_files)
for ifile, bias_file in enumerate(bias_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, bias_file, mask_files)
if ifile == 0:
dim_array_dict = get_dimension_arrays_from_ccd(ccd)
for key, val in dim_array_dict.items():
stack_arrays[key] = np.zeros((nfiles, 16, len(val)))
stack_by_amps(stack_arrays,
ccd,
ifile=ifile,
bias_type=bias_type,
superbias_frame=superbias_frame)
self.log_progress("Done!")
stackdata_dict = convert_stack_arrays_to_dict(stack_arrays,
dim_array_dict, nfiles)
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
for key, val in stackdata_dict.items():
dtables.make_datatable('stack-%s' % key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Plot the row-wise and col-wise struture
in a series of bias frames
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
slot = self.config.slot
bias_files = data['BIAS']
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files=mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(bias_files))
biasstruct_data = {}
for ifile, bias_file in enumerate(bias_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, bias_file, mask_files)
if ifile == 0:
dim_array_dict = get_dimension_arrays_from_ccd(ccd)
for key, val in dim_array_dict.items():
biasstruct_data[key] = {key: val}
self.get_ccd_data(ccd,
biasstruct_data,
slot=slot,
ifile=ifile,
nfiles_used=len(bias_files),
superbias_frame=superbias_frame)
self.log_progress("Done!")
if not biasstruct_data:
return None
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
for key, val in biasstruct_data.items():
dtables.make_datatable('biasst-%s' % key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
gains = np.ones((17))
ppump_files = data['PPUMP']
print(ppump_files)
ppump_file = ppump_files[0]
mask_files = self.get_mask_files()
self.log_info_slot_msg(self.config, "")
# This is a dictionary of dictionaries to store all the
# data you extract from the ppump_files
data_dict = {}
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
ccd = self.get_ccd(butler, ppump_file, mask_files, masked_ccd=True)
my_traps = Traps(ccd,
gains,
cycles=self.config.cycles,
C2_thresh=self.config.C2_thresh,
C3_thresh=self.config.C3_thresh,
nx=self.config.bkg_nx,
ny=self.config.bkg_ny,
edge_rolloff=self.config.edge_rolloff)
data_dict = my_traps.make_data_dict()
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, ppump_files))
dtables.make_datatable('traps', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the row-wise and col-wise struture in a superbias frame
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
slot = self.config.slot
if butler is not None:
self.log.warn("Ignoring butler")
if data is not None:
self.log.warn("Ignoring data")
mask_files = self.get_mask_files()
superbias = self.get_superbias_frame(mask_files=mask_files)
if superbias is None:
return None
self.log_info_slot_msg(self.config, "")
biasstruct_data = {}
dim_array_dict = get_dimension_arrays_from_ccd(superbias)
for key, val in dim_array_dict.items():
biasstruct_data[key] = {key: val}
self.get_ccd_data(superbias,
biasstruct_data,
slot=slot,
bias_type=None,
std=self.config.std,
superbias_frame=None)
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(None, [slot]))
for key, val in biasstruct_data.items():
dtables.make_datatable('biasst-%s' % key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the bias as function of row
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
bias_files = data['BIAS']
if not bias_files:
return None
mask_files = self.get_mask_files()
self.log_info_slot_msg(self.config, "%i files" % len(bias_files))
biasval_data = {}
for ifile, bias_file in enumerate(bias_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, bias_file, mask_files)
if ifile == 0:
dims = get_dims_from_ccd(ccd)
xrow_s = np.linspace(0, dims['nrow_s'] - 1, dims['nrow_s'])
self.get_ccd_data(ccd,
biasval_data,
ifile=ifile,
nfiles=len(bias_files))
#Need to truncate the row array to match the data
a_row = biasval_data[sorted(biasval_data.keys())[0]]
biasval_data['row_s'] = xrow_s[0:len(a_row)]
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
dtables.make_datatable('biasval', biasval_data)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the correlations between the serial overscan for each amp on a raft
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
slots = ALL_SLOTS
overscans = []
for slot in slots:
bias_files = data[slot]['BIAS']
mask_files = self.get_mask_files(slot=slot)
superbias_frame = self.get_superbias_frame(mask_files, slot=slot)
ccd = self.get_ccd(butler, bias_files[0], [])
overscans += self.get_ccd_data(butler, ccd, superbias_frame=superbias_frame)
namps = len(overscans)
if self.config.covar:
data = np.array([np.cov(overscans[i[0]].ravel(),
overscans[i[1]].ravel())[0, 1]
for i in itertools.product(range(namps), range(namps))])
else:
data = np.array([np.corrcoef(overscans[i[0]].ravel(),
overscans[i[1]].ravel())[0, 1]
for i in itertools.product(range(namps), range(namps))])
data = data.reshape(namps, namps)
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
dtables.make_datatable('correl', dict(correl=data))
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
# Get the slot, if needed
#slot = self.config.slot
tmpl_files = data['TMPL']
# Get the superbias frame, if needed
#mask_files = self.get_mask_files()
#superbias_frame = self.get_superbias_frame(mask_files)
self.log_info_slot_msg(self.config, "")
# This is a dictionary of dictionaries to store all the
# data you extract from the tmpl_files
data_dict = {}
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, tmpl_files))
for key, val in data_dict.items():
dtables.make_datatable(key, val)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
if self.config.teststand == 'ts8':
flat1_files = data['FLAT1']
flat2_files = data['FLAT2']
elif self.config.teststand == 'bot':
flat1_files = data['FLAT0']
flat2_files = data['FLAT1']
bias_type = self.get_bias_algo()
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
nlc = self.get_nonlinearirty_correction()
#slot_idx = ALL_SLOTS.index(self.config.slot)
self.log_info_slot_msg(self.config, "%i %i files" % (len(flat1_files), len(flat2_files)))
# This is a dictionary of dictionaries to store all the
# data you extract from the flat_files
data_dict = dict(FLUX=[],
EXPTIME=[],
MONDIODE1=[],
MONDIODE2=[],
MONOCH_SLIT_B=[])
for i in range(1, 17):
data_dict['AMP%02i_RATIO' % i] = []
data_dict['AMP%02i_MEAN' % i] = []
data_dict['AMP%02i_CORRMEAN' % i] = []
data_dict['AMP%02i_VAR' % i] = []
data_dict['AMP%02i_SIGNAL' % i] = []
data_dict['AMP%02i_MEAN1' % i] = []
data_dict['AMP%02i_MEAN2' % i] = []
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
for ifile, (id_1, id_2) in enumerate(zip(flat1_files, flat2_files)):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
flat_1 = self.get_ccd(butler, id_1, mask_files)
flat_2 = self.get_ccd(butler, id_2, mask_files)
amps = get_amp_list(flat_1)
exp_time_1 = get_exposure_time(flat_1)
exp_time_2 = get_exposure_time(flat_2)
if exp_time_1 != exp_time_2:
self.log.warn("Exposure times do not match for:\n%s\n%s\n %0.3F %0.3F. Skipping Pair\n"
% (id_1, id_2, exp_time_1, exp_time_2))
continue
mondiode_1 = get_monodiode_val_from_data_id(id_1, exp_time_1,
self.config.teststand, butler)
mondiode_2 = get_monodiode_val_from_data_id(id_2, exp_time_2,
self.config.teststand, butler)
if mondiode_1 is None or mondiode_2 is None:
self.log.warn("No monitoring data for:\n%s\n%s\n Skipping Pair\n"
% (id_1, id_2))
continue
flux = (exp_time_1 * mondiode_1 + exp_time_2 * mondiode_2)/2.
data_dict['EXPTIME'].append(exp_time_1)
data_dict['MONDIODE1'].append(mondiode_1)
data_dict['MONDIODE2'].append(mondiode_2)
data_dict['FLUX'].append(flux)
try:
data_dict['MONOCH_SLIT_B'].append(get_mono_slit_b(flat_1))
except KeyError:
data_dict['MONOCH_SLIT_B'].append(0.)
ccd_1_ims = unbiased_ccd_image_dict(flat_1, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging', nonlinearity=nlc)
ccd_2_ims = unbiased_ccd_image_dict(flat_2, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging', nonlinearity=nlc)
for i, amp in enumerate(amps):
image_1 = ccd_1_ims[amp]
image_2 = ccd_2_ims[amp]
fstats = self.get_pair_stats(image_1, image_2)
signal = fstats[1]
#if gains is not None:
# signal *= gains[slot_idx][i]
data_dict['AMP%02i_RATIO' % (i+1)].append(fstats[0])
data_dict['AMP%02i_MEAN' % (i+1)].append(fstats[1])
data_dict['AMP%02i_CORRMEAN' % (i+1)].append(fstats[2])
data_dict['AMP%02i_VAR' % (i+1)].append(fstats[3])
data_dict['AMP%02i_SIGNAL' % (i+1)].append(signal)
data_dict['AMP%02i_MEAN1' % (i+1)].append(fstats[4])
data_dict['AMP%02i_MEAN2' % (i+1)].append(fstats[5])
self.log_progress("Done!")
primary_hdu = fits.PrimaryHDU()
primary_hdu.header['NAMPS'] = 16
dtables = TableDict(primary=primary_hdu)
dtables.make_datatable('files', make_file_dict(butler, flat1_files + flat2_files))
dtables.make_datatable('flat', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
slot = self.config.slot
bias_type = self.get_bias_algo()
flat1_files = data['FLAT1']
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(flat1_files))
sflat_table_file = self.get_filename_from_format(
RAFT_SFLAT_TABLE_FORMATTER, "sflat.fits")
sflat_tables = TableDict(sflat_table_file)
# dictionary of dictionaries of lists of bounding
# boxes, keyed by slot, amp
bbox_dict = construct_bbox_dict(sflat_tables['defects'])
slot_bbox_dict = bbox_dict[slot]
slot_idx_dict = dict(S00=0,
S01=1,
S02=2,
S10=3,
S11=4,
S12=5,
S20=6,
S21=7,
S22=8)
# This is a dictionary of dictionaries to store all the
# data you extract from the flat1_files
fp_dict = dict(slot=[],
amp=[],
x_corner=[],
y_corner=[],
x_size=[],
y_size=[],
med_flux_full=[],
exptime=[],
mondiode=[],
amp_median=[],
ratio_full=[])
for i in range(4):
fp_dict['ratio_%i' % i] = []
fp_dict['med_flux_%i' % i] = []
fp_dict['npix_%i' % i] = []
fp_dict['npix_0p2_%i' % i] = []
# Analysis goes here, you should fill fp_dict with data extracted
# by the analysis
#
for ifile, flat1_file in enumerate(flat1_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, flat1_file, mask_files)
# To be appended while looping over bounding boxes
exptime = get_exposure_time(ccd)
mondiode_val = get_mondiode_val(ccd)
islot = slot_idx_dict[slot]
unbiased_images = unbiased_ccd_image_dict(
ccd, bias=bias_type, superbias_frame=superbias_frame)
for iamp, (amp, image) in enumerate(unbiased_images.items()):
bbox_list = slot_bbox_dict[iamp]
amp_median = np.median(image.array)
#fill fp_dict
frac_thresh = kwargs.get('frac_thresh', 0.9)
thresh_float = frac_thresh * amp_median
thresh_0p2_float = (1. - (1. - frac_thresh) * 0.2) * amp_median
max_bbox = min(len(bbox_list), 10)
#print(i, amp, bbox_list[0:max_bbox])
#print(max_bbox)
for bbox in bbox_list[0:max_bbox]:
try:
cutout = image[bbox].array
except Exception:
print("cutout failed", slot, amp, bbox)
continue
#print(ifile, bbox, cutout)
ratio_full = np.mean(cutout) / amp_median
cutout_median_full = np.median(cutout)
peak_idx = cutout.argmax()
peak_x = bbox.getMinX() + int(peak_idx % cutout.shape[1])
peak_y = bbox.getMinY() + int(peak_idx / cutout.shape[1])
peak = afwGeom.Point2I(peak_x, peak_y)
extent = afwGeom.Extent2I(1, 1)
bbox_expand = afwGeom.Box2I(peak, extent)
#npix = np.array([1, 9, 25, 49])
npix_cumul = np.array([1, 8, 16, 24])
sums = np.zeros((4))
meds = np.zeros((4))
over_thresh = np.zeros((4), int)
over_0p2_thresh = np.zeros((4), int)
sums_cumul = np.zeros((4))
meds_cumul = np.zeros((4))
over_thresh_cumul = np.zeros((4), int)
over_0p2_thresh_cumul = np.zeros((4), int)
for j in range(4):
try:
cuttout_array = image[bbox_expand].array
except LengthError:
break
sums[j] = cuttout_array.sum()
meds[j] = np.median(cuttout_array)
over_thresh[j] = (cuttout_array > thresh_float).sum()
over_0p2_thresh[j] = (cuttout_array >
thresh_0p2_float).sum()
if j > 0:
sums_cumul[j] = sums[j] - sums[j - 1]
meds_cumul[j] = meds[j] - meds[j - 1]
over_thresh_cumul[
j] = over_thresh[j] - over_thresh[j - 1]
over_0p2_thresh_cumul[j] = over_0p2_thresh[
j] - over_0p2_thresh[j - 1]
else:
sums_cumul[j] = sums[j]
meds_cumul[j] = meds[j]
over_thresh_cumul[j] = over_thresh[j]
over_0p2_thresh_cumul[j] = over_0p2_thresh[j]
bbox_expand.grow(1)
means_cumul = sums_cumul / (npix_cumul * amp_median)
fp_dict['exptime'].append(exptime)
fp_dict['mondiode'].append(mondiode_val)
fp_dict['amp'].append(iamp)
fp_dict['slot'].append(islot)
fp_dict['amp_median'].append(amp_median)
fp_dict['x_corner'].append(bbox.getMinX())
fp_dict['y_corner'].append(bbox.getMinY())
fp_dict['x_size'].append(bbox.getWidth())
fp_dict['y_size'].append(bbox.getHeight())
fp_dict['med_flux_full'].append(cutout_median_full)
fp_dict['ratio_full'].append(ratio_full)
for i in range(4):
fp_dict['ratio_%i' % i].append(means_cumul[i])
fp_dict['med_flux_%i' % i].append(meds_cumul[i])
fp_dict['npix_%i' % i].append(over_thresh_cumul[i])
fp_dict['npix_0p2_%i' % i].append(
over_0p2_thresh_cumul[i])
sys.stdout.write("!\n")
sys.stdout.flush()
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, flat1_files))
dtables.make_datatable('footprints', fp_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Compare frames to superflat
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
sflat_l : `dict`
Dictionary keyed by amp of the low exposure superflats
sflat_h : `dict`
Dictionary keyed by amp of the high exposure superflats
ratio_images : `dict`
Dictionary keyed by amp of the low/high ratio images
"""
self.safe_update(**kwargs)
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
bias_type = self.get_bias_algo()
sflat_files = data['SFLAT']
if not sflat_files:
self.log_warn_slot_msg(self.config, "No superflat files")
return None
self.log_info_slot_msg(self.config, "%i files" % (len(sflat_files)))
# This is a dictionary of dictionaries to store all the
# data you extract from the flat_files
data_dict = dict(FLUX=[],
EXPTIME=[],
MONDIODE=[])
for i in range(1, 17):
data_dict['AMP%02i_FLAT_MEAN' % i] = []
data_dict['AMP%02i_FLAT_MEDIAN' % i] = []
data_dict['AMP%02i_FLAT_VAR' % i] = []
data_dict['AMP%02i_FLAT_ROWMEAN' % i] = []
data_dict['AMP%02i_FLAT_COLMEAN' % i] = []
for ifile, sflat_file in enumerate(sflat_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
sflat = self.get_ccd(butler, sflat_file, mask_files)
exp_time = get_exposure_time(sflat)
mondiode = get_monodiode_val_from_data_id(sflat_file, exp_time,
self.config.teststand, butler)
if mondiode is None:
continue
flux = exp_time * mondiode
data_dict['EXPTIME'].append(exp_time)
data_dict['MONDIODE'].append(mondiode)
data_dict['FLUX'].append(flux)
ccd_ims = unbiased_ccd_image_dict(sflat, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging')
amps = get_amp_list(sflat)
for amp in amps:
image = ccd_ims[amp]
fstats = self.get_stats(image)
data_dict['AMP%02i_FLAT_MEAN' % amp].append(fstats[0])
data_dict['AMP%02i_FLAT_MEDIAN' % amp].append(fstats[1])
data_dict['AMP%02i_FLAT_VAR' % amp].append(fstats[2])
data_dict['AMP%02i_FLAT_ROWMEAN' % amp].append(image.image.array.mean(0))
data_dict['AMP%02i_FLAT_COLMEAN' % amp].append(image.image.array.mean(1))
self.log_progress("Done!")
primary_hdu = fits.PrimaryHDU()
primary_hdu.header['NAMPS'] = 16
dtables = TableDict(primary=primary_hdu)
dtables.make_datatable('files', make_file_dict(butler, sflat_files))
dtables.make_datatable('stability', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
slot = self.config.slot
qe_files = data['LAMBDA']
lams = []
for file in qe_files:
lam = file.split('flat_')[1].split('_')[0]
lams.append(lam)
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
bias_type = self.get_bias_algo()
self.log_info_slot_msg(self.config, "%i files" % len(qe_files))
sflat_table_file = self.get_filename_from_format(RAFT_SFLAT_TABLE_FORMATTER, "sflat.fits")
sflat_tables = TableDict(sflat_table_file)
bbox_dict = construct_bbox_dict(sflat_tables['defects'])
slot_bbox_dict = bbox_dict[slot]
# This is a dictionary of dictionaries to store all the
# data you extract from the qe_files
data_dict = dict(SLOT=[], AMP=[], XCORNER=[], YCORNER=[], XSIZE=[], YSIZE=[])
for lam in lams:
data_dict['FLUX_' + lam] = []
data_dict['MED_' + lam] = []
temp_list = []
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
# For this slot, loop over qe files, loop over amplifiers, loop over dust spots in that amplifier,
# record the flux and median for each dust spot
# Then loop over dust spots CCDs, amplifiers, and QE files to accumulate the output dictionary
ccd = self.get_ccd(butler, qe_files[0], mask_files)
amps = get_amp_list(ccd)
spot = 0
for ifile, qe_file in enumerate(qe_files):
lam = qe_file.split('flat_')[1].split('_')[0]
for i, amp in enumerate(amps):
bbox_list = slot_bbox_dict[amp]
if len(bbox_list) > 100:
if i == 0:
self.log.warn("Skipping slot:amp %s:%i with %i defects" % (slot, amp, len(bbox_list)))
continue
ccd = self.get_ccd(butler, qe_file, mask_files)
regions = get_geom_regions(ccd, amp)
serial_oscan = regions['serial_overscan']
imaging = regions['imaging']
img = get_raw_image(ccd, amp)
if superbias_frame is not None:
if butler is not None:
superbias_im = get_raw_image(superbias_frame, amp)
else:
superbias_im = get_raw_image(superbias_frame, amp)
else:
superbias_im = None
image = unbias_amp(img, serial_oscan, bias_type=bias_type,
superbias_im=superbias_im, region=imaging)
for bbox in bbox_list:
if ifile == 0:
data_dict['SLOT'].append(slot)
#data_dict['LAM'].append(lam)
data_dict['AMP'].append(amp)
data_dict['XCORNER'].append(bbox.getBeginX())
data_dict['YCORNER'].append(bbox.getBeginY())
data_dict['XSIZE'].append(bbox.getWidth())
data_dict['YSIZE'].append(bbox.getHeight())
try:
cutout = image[bbox]
except Exception:
pass
# Here evaluate the 'flux' of the feature, relative to the median
# value of the amplifier image. May also want to assemble bounding
# box corners into a ds9 region file, CCD by CCD
med = np.median(image.array)
flux = np.sum(cutout.array) - bbox.getWidth()*bbox.getHeight()*med
spot += 1
#temp_dict[spot].append((lam, flux, med))
temp_list.append((lam, flux, med))
for i, tmp_data in enumerate(temp_list):
#data_dict['SLOT'].append(temp_dict['SLOT'][i])
#data_dict['AMP'].append(temp_dict['AMP'][i])
#lam = temp_dict['LAM'][i]
lam, flux, med = tmp_data
data_dict['FLUX_' + lam].append(flux) #temp_dict['FLUX'][i])
data_dict['MED_' + lam].append(med) #np.median(image.array))
sys.stdout.write("!\n")
sys.stdout.flush()
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, qe_files))
#dtables.make_datatable('dust_color', data_dict)
dtables.make_datatable('dust_color_hack', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the data from the overscan region
to estimate the deffered charge
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
raise ValueError(
"FlatOverscanTask not implemented for Butlerized data")
flat_files = data['FLAT1']
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(flat_files))
#fitter = OverscanFit(num_oscan_pixels=self.config.num_oscan_pixels,
# minflux=self.config.minflux, maxflux=self.config.maxflux)
gains = self.get_gains()
if gains is None:
gains = np.ones((17))
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
for ifile, flat_id in enumerate(flat_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
flat = self.get_ccd(butler,
flat_id,
mask_files,
bias_frame=superbias_frame)
fitter.process_image(flat, gains)
self.xmax = fitter.xmax_val
self.log_progress("Done!")
data_dict = fitter.build_output_dict()
primary = fitter.output[0]
primary.header['NAMPS'] = 16
dtables = TableDict(primary=primary)
for key, val in data_dict.items():
dtables.make_datatable(key.lower(), val)
dtables.make_datatable('files', make_file_dict(butler, flat_files))
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the statistics from the superbias frames
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
if butler is not None:
self.log.warn("Ignoring butler")
stats_data = {}
self.log_info_raft_msg(self.config, "")
slot_list = self.config.slots
if slot_list is None:
slot_list = ALL_SLOTS
for islot, slot in enumerate(slot_list):
superbias_file = data[slot]
if self.config.stat is not None:
superbias_file = superbias_file.replace('_superbias_', '_%s_' % self.config.stat)
if not os.path.exists(superbias_file):
self.log.warn("Skipping missing file for %s:%s" % (self.config.raft, slot))
continue
self.log_progress(" %s" % slot)
mask_files = self.get_mask_files(slot=slot)
try:
superbias_frame = self.get_ccd(None, superbias_file, mask_files)
except Exception:
self.log.warn("Skipping %s:%s:%s" % (self.config.run, self.config.raft, slot))
superbias_frame = None
self.get_superbias_stats(superbias_frame, stats_data, islot)
self.log_progress("Done!")
if not stats_data:
return None
stats_data['slot'] = np.arange(9)
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(None, slot_list))
dtables.make_datatable('stats', stats_data)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the correlations between the imaging section
and the overscan regions in a series of bias frames
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
bias_files = data['BIAS']
mask_files = self.get_mask_files()
self.log_info_slot_msg(self.config, "%i files" % len(bias_files))
ref_frames = {}
nfiles = len(bias_files)
s_correl = np.ndarray((16, nfiles - 1))
p_correl = np.ndarray((16, nfiles - 1))
for ifile, bias_file in enumerate(bias_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, bias_file, mask_files)
if ifile == 0:
dims = get_dims_from_ccd(ccd)
nrow_i = dims['nrow_i']
ncol_i = dims['ncol_i']
amps = get_amp_list(ccd)
for i, amp in enumerate(amps):
regions = get_geom_regions(ccd, amp)
image = get_raw_image(ccd, amp)
ref_frames[i] = get_image_frames_2d(image, regions)
continue
self.get_ccd_data(ccd,
ref_frames,
ifile=ifile,
s_correl=s_correl,
p_correl=p_correl,
nrow_i=nrow_i,
ncol_i=ncol_i)
self.log_progress("Done!")
data = {}
for i in range(16):
data['s_correl_a%02i' % i] = s_correl[i]
data['p_correl_a%02i' % i] = p_correl[i]
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
dtables.make_datatable("correl", data)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
qe_files = data['LAMBDA']
bias_type = self.get_bias_algo()
corrections = np.ones((17))
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(qe_files))
# This is a dictionary of dictionaries to store all the
# data you extract from the qe_files
data_dict = dict(WL=[], EXPTIME=[], MONDIODE=[])
for i in range(1, 17):
data_dict['AMP%02i_MEDIAN' % i] = []
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
for ifile, qe_file in enumerate(qe_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, qe_file, mask_files)
data_dict['WL'].append(get_mono_wl(ccd))
data_dict['EXPTIME'].append(get_exposure_time(ccd))
data_dict['MONDIODE'].append(get_mondiode_val(ccd))
unbiased_images = unbiased_ccd_image_dict(
ccd, bias=bias_type, superbias_frame=superbias_frame)
for i, (amp, image) in enumerate(unbiased_images.items()):
if corrections is not None:
image *= corrections[amp]
value = self.median(image)
data_dict['AMP%02i_MEDIAN' % (i + 1)].append(value)
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, qe_files))
dtables.make_datatable('qe_med', data_dict)
return dtables