def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
if self.config.teststand == 'ts8':
flat1_files = data['FLAT1']
flat2_files = data['FLAT2']
elif self.config.teststand == 'bot':
flat1_files = data['FLAT0']
flat2_files = data['FLAT1']
bias_type = self.get_bias_algo()
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
nlc = self.get_nonlinearirty_correction()
#slot_idx = ALL_SLOTS.index(self.config.slot)
self.log_info_slot_msg(self.config, "%i %i files" % (len(flat1_files), len(flat2_files)))
# This is a dictionary of dictionaries to store all the
# data you extract from the flat_files
data_dict = dict(FLUX=[],
EXPTIME=[],
MONDIODE1=[],
MONDIODE2=[],
MONOCH_SLIT_B=[])
for i in range(1, 17):
data_dict['AMP%02i_RATIO' % i] = []
data_dict['AMP%02i_MEAN' % i] = []
data_dict['AMP%02i_CORRMEAN' % i] = []
data_dict['AMP%02i_VAR' % i] = []
data_dict['AMP%02i_SIGNAL' % i] = []
data_dict['AMP%02i_MEAN1' % i] = []
data_dict['AMP%02i_MEAN2' % i] = []
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
for ifile, (id_1, id_2) in enumerate(zip(flat1_files, flat2_files)):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
flat_1 = self.get_ccd(butler, id_1, mask_files)
flat_2 = self.get_ccd(butler, id_2, mask_files)
amps = get_amp_list(flat_1)
exp_time_1 = get_exposure_time(flat_1)
exp_time_2 = get_exposure_time(flat_2)
if exp_time_1 != exp_time_2:
self.log.warn("Exposure times do not match for:\n%s\n%s\n %0.3F %0.3F. Skipping Pair\n"
% (id_1, id_2, exp_time_1, exp_time_2))
continue
mondiode_1 = get_monodiode_val_from_data_id(id_1, exp_time_1,
self.config.teststand, butler)
mondiode_2 = get_monodiode_val_from_data_id(id_2, exp_time_2,
self.config.teststand, butler)
if mondiode_1 is None or mondiode_2 is None:
self.log.warn("No monitoring data for:\n%s\n%s\n Skipping Pair\n"
% (id_1, id_2))
continue
flux = (exp_time_1 * mondiode_1 + exp_time_2 * mondiode_2)/2.
data_dict['EXPTIME'].append(exp_time_1)
data_dict['MONDIODE1'].append(mondiode_1)
data_dict['MONDIODE2'].append(mondiode_2)
data_dict['FLUX'].append(flux)
try:
data_dict['MONOCH_SLIT_B'].append(get_mono_slit_b(flat_1))
except KeyError:
data_dict['MONOCH_SLIT_B'].append(0.)
ccd_1_ims = unbiased_ccd_image_dict(flat_1, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging', nonlinearity=nlc)
ccd_2_ims = unbiased_ccd_image_dict(flat_2, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging', nonlinearity=nlc)
for i, amp in enumerate(amps):
image_1 = ccd_1_ims[amp]
image_2 = ccd_2_ims[amp]
fstats = self.get_pair_stats(image_1, image_2)
signal = fstats[1]
#if gains is not None:
# signal *= gains[slot_idx][i]
data_dict['AMP%02i_RATIO' % (i+1)].append(fstats[0])
data_dict['AMP%02i_MEAN' % (i+1)].append(fstats[1])
data_dict['AMP%02i_CORRMEAN' % (i+1)].append(fstats[2])
data_dict['AMP%02i_VAR' % (i+1)].append(fstats[3])
data_dict['AMP%02i_SIGNAL' % (i+1)].append(signal)
data_dict['AMP%02i_MEAN1' % (i+1)].append(fstats[4])
data_dict['AMP%02i_MEAN2' % (i+1)].append(fstats[5])
self.log_progress("Done!")
primary_hdu = fits.PrimaryHDU()
primary_hdu.header['NAMPS'] = 16
dtables = TableDict(primary=primary_hdu)
dtables.make_datatable('files', make_file_dict(butler, flat1_files + flat2_files))
dtables.make_datatable('flat', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
slot = self.config.slot
bias_type = self.get_bias_algo()
flat1_files = data['FLAT1']
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(flat1_files))
sflat_table_file = self.get_filename_from_format(
RAFT_SFLAT_TABLE_FORMATTER, "sflat.fits")
sflat_tables = TableDict(sflat_table_file)
# dictionary of dictionaries of lists of bounding
# boxes, keyed by slot, amp
bbox_dict = construct_bbox_dict(sflat_tables['defects'])
slot_bbox_dict = bbox_dict[slot]
slot_idx_dict = dict(S00=0,
S01=1,
S02=2,
S10=3,
S11=4,
S12=5,
S20=6,
S21=7,
S22=8)
# This is a dictionary of dictionaries to store all the
# data you extract from the flat1_files
fp_dict = dict(slot=[],
amp=[],
x_corner=[],
y_corner=[],
x_size=[],
y_size=[],
med_flux_full=[],
exptime=[],
mondiode=[],
amp_median=[],
ratio_full=[])
for i in range(4):
fp_dict['ratio_%i' % i] = []
fp_dict['med_flux_%i' % i] = []
fp_dict['npix_%i' % i] = []
fp_dict['npix_0p2_%i' % i] = []
# Analysis goes here, you should fill fp_dict with data extracted
# by the analysis
#
for ifile, flat1_file in enumerate(flat1_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, flat1_file, mask_files)
# To be appended while looping over bounding boxes
exptime = get_exposure_time(ccd)
mondiode_val = get_mondiode_val(ccd)
islot = slot_idx_dict[slot]
unbiased_images = unbiased_ccd_image_dict(
ccd, bias=bias_type, superbias_frame=superbias_frame)
for iamp, (amp, image) in enumerate(unbiased_images.items()):
bbox_list = slot_bbox_dict[iamp]
amp_median = np.median(image.array)
#fill fp_dict
frac_thresh = kwargs.get('frac_thresh', 0.9)
thresh_float = frac_thresh * amp_median
thresh_0p2_float = (1. - (1. - frac_thresh) * 0.2) * amp_median
max_bbox = min(len(bbox_list), 10)
#print(i, amp, bbox_list[0:max_bbox])
#print(max_bbox)
for bbox in bbox_list[0:max_bbox]:
try:
cutout = image[bbox].array
except Exception:
print("cutout failed", slot, amp, bbox)
continue
#print(ifile, bbox, cutout)
ratio_full = np.mean(cutout) / amp_median
cutout_median_full = np.median(cutout)
peak_idx = cutout.argmax()
peak_x = bbox.getMinX() + int(peak_idx % cutout.shape[1])
peak_y = bbox.getMinY() + int(peak_idx / cutout.shape[1])
peak = afwGeom.Point2I(peak_x, peak_y)
extent = afwGeom.Extent2I(1, 1)
bbox_expand = afwGeom.Box2I(peak, extent)
#npix = np.array([1, 9, 25, 49])
npix_cumul = np.array([1, 8, 16, 24])
sums = np.zeros((4))
meds = np.zeros((4))
over_thresh = np.zeros((4), int)
over_0p2_thresh = np.zeros((4), int)
sums_cumul = np.zeros((4))
meds_cumul = np.zeros((4))
over_thresh_cumul = np.zeros((4), int)
over_0p2_thresh_cumul = np.zeros((4), int)
for j in range(4):
try:
cuttout_array = image[bbox_expand].array
except LengthError:
break
sums[j] = cuttout_array.sum()
meds[j] = np.median(cuttout_array)
over_thresh[j] = (cuttout_array > thresh_float).sum()
over_0p2_thresh[j] = (cuttout_array >
thresh_0p2_float).sum()
if j > 0:
sums_cumul[j] = sums[j] - sums[j - 1]
meds_cumul[j] = meds[j] - meds[j - 1]
over_thresh_cumul[
j] = over_thresh[j] - over_thresh[j - 1]
over_0p2_thresh_cumul[j] = over_0p2_thresh[
j] - over_0p2_thresh[j - 1]
else:
sums_cumul[j] = sums[j]
meds_cumul[j] = meds[j]
over_thresh_cumul[j] = over_thresh[j]
over_0p2_thresh_cumul[j] = over_0p2_thresh[j]
bbox_expand.grow(1)
means_cumul = sums_cumul / (npix_cumul * amp_median)
fp_dict['exptime'].append(exptime)
fp_dict['mondiode'].append(mondiode_val)
fp_dict['amp'].append(iamp)
fp_dict['slot'].append(islot)
fp_dict['amp_median'].append(amp_median)
fp_dict['x_corner'].append(bbox.getMinX())
fp_dict['y_corner'].append(bbox.getMinY())
fp_dict['x_size'].append(bbox.getWidth())
fp_dict['y_size'].append(bbox.getHeight())
fp_dict['med_flux_full'].append(cutout_median_full)
fp_dict['ratio_full'].append(ratio_full)
for i in range(4):
fp_dict['ratio_%i' % i].append(means_cumul[i])
fp_dict['med_flux_%i' % i].append(meds_cumul[i])
fp_dict['npix_%i' % i].append(over_thresh_cumul[i])
fp_dict['npix_0p2_%i' % i].append(
over_0p2_thresh_cumul[i])
sys.stdout.write("!\n")
sys.stdout.flush()
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, flat1_files))
dtables.make_datatable('footprints', fp_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
output data tables
"""
self.safe_update(**kwargs)
slots = self.config.slots
if slots is None:
slots = ALL_SLOTS
if butler is not None:
self.log.warn("Ignoring butler")
dark_current_data = dict(median=[],
stdev=[],
mean=[],
fit_mean=[],
fit_width=[],
fit_dof=[],
fit_chi2=[],
exptime=[],
current=[],
slot=[],
amp=[])
self.log_info_raft_msg(self.config, "")
for islot, slot in enumerate(slots):
self.log_progress(" %s" % slot)
mask_files = self.get_mask_files(slot=slot)
superdark_file = data[slot]
if not os.path.exists(superdark_file):
self.log.warn(" %s does not exist, skipping" % superdark_file)
continue
superdark_frame = self.get_ccd(None, superdark_file, mask_files)
exptime = get_exposure_time(superdark_frame)
amps = get_amp_list(superdark_frame)
for iamp, amp in enumerate(amps):
regions = get_geom_regions(superdark_frame, amp)
imaging = regions['imaging']
superdark_im = get_raw_image(superdark_frame, amp).image
image_data = superdark_im[imaging].array
median = np.median(image_data)
stdev = np.std(image_data)
hist_bins = np.linspace(median - 5 * stdev, median + 5 * stdev, 101)
hist = np.histogram(image_data, bins=hist_bins)
fit_result = gauss_fit(hist)
fit_pars = fit_result[0]
dark_current_data['median'].append(median)
dark_current_data['stdev'].append(stdev)
dark_current_data['mean'].append(np.mean(image_data))
dark_current_data['fit_mean'].append(fit_pars[1])
dark_current_data['fit_width'].append(fit_pars[2])
dark_current_data['fit_dof'].append(0.)
dark_current_data['fit_chi2'].append(0.)
dark_current_data['exptime'].append(exptime)
dark_current_data['current'].append(median/exptime)
dark_current_data['slot'].append(islot)
dark_current_data['amp'].append(iamp)
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('dark_current', dark_current_data)
return dtables
def extract(self, butler, data, **kwargs):
"""Compare frames to superflat
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
sflat_l : `dict`
Dictionary keyed by amp of the low exposure superflats
sflat_h : `dict`
Dictionary keyed by amp of the high exposure superflats
ratio_images : `dict`
Dictionary keyed by amp of the low/high ratio images
"""
self.safe_update(**kwargs)
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
bias_type = self.get_bias_algo()
sflat_files = data['SFLAT']
if not sflat_files:
self.log_warn_slot_msg(self.config, "No superflat files")
return None
self.log_info_slot_msg(self.config, "%i files" % (len(sflat_files)))
# This is a dictionary of dictionaries to store all the
# data you extract from the flat_files
data_dict = dict(FLUX=[],
EXPTIME=[],
MONDIODE=[])
for i in range(1, 17):
data_dict['AMP%02i_FLAT_MEAN' % i] = []
data_dict['AMP%02i_FLAT_MEDIAN' % i] = []
data_dict['AMP%02i_FLAT_VAR' % i] = []
data_dict['AMP%02i_FLAT_ROWMEAN' % i] = []
data_dict['AMP%02i_FLAT_COLMEAN' % i] = []
for ifile, sflat_file in enumerate(sflat_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
sflat = self.get_ccd(butler, sflat_file, mask_files)
exp_time = get_exposure_time(sflat)
mondiode = get_monodiode_val_from_data_id(sflat_file, exp_time,
self.config.teststand, butler)
if mondiode is None:
continue
flux = exp_time * mondiode
data_dict['EXPTIME'].append(exp_time)
data_dict['MONDIODE'].append(mondiode)
data_dict['FLUX'].append(flux)
ccd_ims = unbiased_ccd_image_dict(sflat, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging')
amps = get_amp_list(sflat)
for amp in amps:
image = ccd_ims[amp]
fstats = self.get_stats(image)
data_dict['AMP%02i_FLAT_MEAN' % amp].append(fstats[0])
data_dict['AMP%02i_FLAT_MEDIAN' % amp].append(fstats[1])
data_dict['AMP%02i_FLAT_VAR' % amp].append(fstats[2])
data_dict['AMP%02i_FLAT_ROWMEAN' % amp].append(image.image.array.mean(0))
data_dict['AMP%02i_FLAT_COLMEAN' % amp].append(image.image.array.mean(1))
self.log_progress("Done!")
primary_hdu = fits.PrimaryHDU()
primary_hdu.header['NAMPS'] = 16
dtables = TableDict(primary=primary_hdu)
dtables.make_datatable('files', make_file_dict(butler, sflat_files))
dtables.make_datatable('stability', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
qe_files = data['LAMBDA']
bias_type = self.get_bias_algo()
corrections = np.ones((17))
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
self.log_info_slot_msg(self.config, "%i files" % len(qe_files))
# This is a dictionary of dictionaries to store all the
# data you extract from the qe_files
data_dict = dict(WL=[], EXPTIME=[], MONDIODE=[])
for i in range(1, 17):
data_dict['AMP%02i_MEDIAN' % i] = []
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
for ifile, qe_file in enumerate(qe_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, qe_file, mask_files)
data_dict['WL'].append(get_mono_wl(ccd))
data_dict['EXPTIME'].append(get_exposure_time(ccd))
data_dict['MONDIODE'].append(get_mondiode_val(ccd))
unbiased_images = unbiased_ccd_image_dict(
ccd, bias=bias_type, superbias_frame=superbias_frame)
for i, (amp, image) in enumerate(unbiased_images.items()):
if corrections is not None:
image *= corrections[amp]
value = self.median(image)
data_dict['AMP%02i_MEDIAN' % (i + 1)].append(value)
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, qe_files))
dtables.make_datatable('qe_med', data_dict)
return dtables