def get_ccd_data(self, ccd, data, **kwargs):
"""Get the bias values and update the data dictionary
Parameters
----------
ccd : `MaskedCCD`
The ccd we are getting data from
data : `dict`
The data we are updating
Keywords
--------
slot : `str`
The slot name
ifile : `int`
The file index
nfiles_used : `int`
Total number of files
"""
slot = self.config.slot
bias_type = self.get_bias_algo()
ifile = kwargs['ifile']
nfiles = kwargs['nfiles']
amps = get_amp_list(ccd)
for i, amp in enumerate(amps):
regions = get_geom_regions(ccd, amp)
serial_oscan = regions['serial_overscan']
img = get_raw_image(ccd, amp)
bimg = imutil.bias_image(img, serial_oscan, bias_method=bias_type)
bimg_row_mean = bimg[serial_oscan].getArray().mean(1)
key_str = "biasval_%s_a%02i" % (slot, i)
if key_str not in data:
data[key_str] = np.ndarray((len(bimg_row_mean), nfiles))
data[key_str][:, ifile] = bimg_row_mean
def get_ccd_data(self, ccd, ref_frames, **kwargs):
"""Get the bias values and update the data dictionary
Parameters
----------
ccd : `MaskedCCD`
The ccd we are getting data from
data : `dict`
The data we are updating
Keywords
--------
ifile : `int`
The file index
s_correl : `array`
Serial overscan correlations
p_correl : `array`
Parallel overscan correlations
"""
ifile = kwargs['ifile']
s_correl = kwargs['s_correl']
p_correl = kwargs['p_correl']
nrow_i = kwargs['nrow_i']
ncol_i = kwargs['ncol_i']
amps = get_amp_list(ccd)
for i, amp in enumerate(amps):
regions = get_geom_regions(ccd, amp)
image = get_raw_image(ccd, amp)
frames = get_image_frames_2d(image, regions)
del_i_array = frames['imaging'] - ref_frames[i]['imaging']
del_s_array = frames['serial_overscan'] - ref_frames[i][
'serial_overscan']
del_p_array = frames['parallel_overscan'] - ref_frames[i][
'parallel_overscan']
dd_s = del_s_array.mean(1)[0:nrow_i] - del_i_array.mean(1)
dd_p = del_p_array.mean(0)[0:ncol_i] - del_i_array.mean(0)
mask_s = np.fabs(dd_s) < self.clip_value
mask_p = np.fabs(dd_p) < self.clip_value
s_correl[i, ifile - 1] = np.corrcoef(
del_s_array.mean(1)[0:nrow_i][mask_s], dd_s[mask_s])[0, 1]
p_correl[i, ifile - 1] = np.corrcoef(
del_p_array.mean(0)[0:ncol_i][mask_p], dd_p[mask_p])[0, 1]
def get_ccd_data(self, butler, ccd, **kwargs):
"""Get the serial overscan data
Parameters
----------
butler : `Butler`
The data butler
ccd : `MaskedCCD`
The ccd we are getting data from
data : `dict`
The data we are updating
Keywords
--------
superbias_frame : `MaskedCCD`
The superbias frame to subtract away
boundry : `int`
Size of buffer around edge of overscan region
Returns
-------
overscans : `list`
The overscan data
"""
amps = get_amp_list(ccd)
superbias_frame = kwargs.get('superbias_frame', None)
overscans = []
for amp in amps:
superbias_im = self.get_superbias_amp_image(butler, superbias_frame, amp)
regions = get_geom_regions(ccd, amp)
serial_oscan = regions['serial_overscan']
img = get_raw_image(ccd, amp)
image = unbias_amp(img, serial_oscan, bias_type=None, superbias_im=superbias_im).image
oscan_copy = copy.deepcopy(serial_oscan)
oscan_copy.grow(-self.boundry)
oscan_data = image[oscan_copy]
step_x = regions['step_x']
step_y = regions['step_y']
overscans.append(oscan_data.getArray()[::step_x, ::step_y])
return overscans
def unbias(input_file,
output_file,
bias_method,
bias_method_col=None,
superbias_file=None):
# build output file by copying input
shutil.copyfile(input_file, output_file)
ccd = get_ccd_from_id(None, input_file, [])
hdulist = fits.open(input_file)
if superbias_file is not None:
superbias_ccd = get_ccd_from_id(None, superbias_file, [])
else:
superbias_ccd = None
amps = get_amp_list(ccd)
offset = get_amp_offset(ccd, superbias_ccd)
for i, amp in enumerate(amps):
regions = get_geom_regions(ccd, amp)
serial_oscan = regions['serial_overscan']
parallel_oscan = regions['parallel_overscan']
img = get_raw_image(ccd, amp)
if superbias_ccd is not None:
superbias_im = get_raw_image(superbias_frame, amp + offset)
else:
superbias_im = None
image = unbias_amp(img,
serial_oscan,
bias_type=bias_method,
superbias_im=superbias_im,
bias_type_col=bias_method_col,
parallel_oscan=parallel_oscan)
fits.update(output_file,
image.image.array,
amp,
header=hdulist[amp].header)
def get_superbias_stats(superbias, stats_data, islot):
"""Get the serial overscan data
Parameters
----------
butler : `Butler`
The data butler
superbias : `MaskedCCD`
The ccd we are getting data from
stats_data : `dict`
The data we are updating
Keywords
--------
islot : `int`
The slot index
"""
if 'mean' not in stats_data:
stats_data['mean'] = np.zeros((9, 16))
stats_data['median'] = np.zeros((9, 16))
stats_data['std'] = np.zeros((9, 16))
stats_data['min'] = np.zeros((9, 16))
stats_data['max'] = np.zeros((9, 16))
if superbias is None:
return
amps = get_amp_list(superbias)
for i, amp in enumerate(amps):
img = get_raw_image(superbias, amp).image
stats_data['mean'][islot, i] = img.array.mean()
stats_data['median'][islot, i] = np.median(img.array)
stats_data['std'][islot, i] = img.array.std()
stats_data['min'][islot, i] = img.array.min()
stats_data['max'][islot, i] = img.array.max()
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
if self.config.teststand == 'ts8':
flat1_files = data['FLAT1']
flat2_files = data['FLAT2']
elif self.config.teststand == 'bot':
flat1_files = data['FLAT0']
flat2_files = data['FLAT1']
bias_type = self.get_bias_algo()
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
nlc = self.get_nonlinearirty_correction()
#slot_idx = ALL_SLOTS.index(self.config.slot)
self.log_info_slot_msg(self.config, "%i %i files" % (len(flat1_files), len(flat2_files)))
# This is a dictionary of dictionaries to store all the
# data you extract from the flat_files
data_dict = dict(FLUX=[],
EXPTIME=[],
MONDIODE1=[],
MONDIODE2=[],
MONOCH_SLIT_B=[])
for i in range(1, 17):
data_dict['AMP%02i_RATIO' % i] = []
data_dict['AMP%02i_MEAN' % i] = []
data_dict['AMP%02i_CORRMEAN' % i] = []
data_dict['AMP%02i_VAR' % i] = []
data_dict['AMP%02i_SIGNAL' % i] = []
data_dict['AMP%02i_MEAN1' % i] = []
data_dict['AMP%02i_MEAN2' % i] = []
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
for ifile, (id_1, id_2) in enumerate(zip(flat1_files, flat2_files)):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
flat_1 = self.get_ccd(butler, id_1, mask_files)
flat_2 = self.get_ccd(butler, id_2, mask_files)
amps = get_amp_list(flat_1)
exp_time_1 = get_exposure_time(flat_1)
exp_time_2 = get_exposure_time(flat_2)
if exp_time_1 != exp_time_2:
self.log.warn("Exposure times do not match for:\n%s\n%s\n %0.3F %0.3F. Skipping Pair\n"
% (id_1, id_2, exp_time_1, exp_time_2))
continue
mondiode_1 = get_monodiode_val_from_data_id(id_1, exp_time_1,
self.config.teststand, butler)
mondiode_2 = get_monodiode_val_from_data_id(id_2, exp_time_2,
self.config.teststand, butler)
if mondiode_1 is None or mondiode_2 is None:
self.log.warn("No monitoring data for:\n%s\n%s\n Skipping Pair\n"
% (id_1, id_2))
continue
flux = (exp_time_1 * mondiode_1 + exp_time_2 * mondiode_2)/2.
data_dict['EXPTIME'].append(exp_time_1)
data_dict['MONDIODE1'].append(mondiode_1)
data_dict['MONDIODE2'].append(mondiode_2)
data_dict['FLUX'].append(flux)
try:
data_dict['MONOCH_SLIT_B'].append(get_mono_slit_b(flat_1))
except KeyError:
data_dict['MONOCH_SLIT_B'].append(0.)
ccd_1_ims = unbiased_ccd_image_dict(flat_1, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging', nonlinearity=nlc)
ccd_2_ims = unbiased_ccd_image_dict(flat_2, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging', nonlinearity=nlc)
for i, amp in enumerate(amps):
image_1 = ccd_1_ims[amp]
image_2 = ccd_2_ims[amp]
fstats = self.get_pair_stats(image_1, image_2)
signal = fstats[1]
#if gains is not None:
# signal *= gains[slot_idx][i]
data_dict['AMP%02i_RATIO' % (i+1)].append(fstats[0])
data_dict['AMP%02i_MEAN' % (i+1)].append(fstats[1])
data_dict['AMP%02i_CORRMEAN' % (i+1)].append(fstats[2])
data_dict['AMP%02i_VAR' % (i+1)].append(fstats[3])
data_dict['AMP%02i_SIGNAL' % (i+1)].append(signal)
data_dict['AMP%02i_MEAN1' % (i+1)].append(fstats[4])
data_dict['AMP%02i_MEAN2' % (i+1)].append(fstats[5])
self.log_progress("Done!")
primary_hdu = fits.PrimaryHDU()
primary_hdu.header['NAMPS'] = 16
dtables = TableDict(primary=primary_hdu)
dtables.make_datatable('files', make_file_dict(butler, flat1_files + flat2_files))
dtables.make_datatable('flat', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Compare frames to superflat
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
sflat_l : `dict`
Dictionary keyed by amp of the low exposure superflats
sflat_h : `dict`
Dictionary keyed by amp of the high exposure superflats
ratio_images : `dict`
Dictionary keyed by amp of the low/high ratio images
"""
self.safe_update(**kwargs)
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
bias_type = self.get_bias_algo()
sflat_files = data['SFLAT']
if not sflat_files:
self.log_warn_slot_msg(self.config, "No superflat files")
return None
self.log_info_slot_msg(self.config, "%i files" % (len(sflat_files)))
# This is a dictionary of dictionaries to store all the
# data you extract from the flat_files
data_dict = dict(FLUX=[],
EXPTIME=[],
MONDIODE=[])
for i in range(1, 17):
data_dict['AMP%02i_FLAT_MEAN' % i] = []
data_dict['AMP%02i_FLAT_MEDIAN' % i] = []
data_dict['AMP%02i_FLAT_VAR' % i] = []
data_dict['AMP%02i_FLAT_ROWMEAN' % i] = []
data_dict['AMP%02i_FLAT_COLMEAN' % i] = []
for ifile, sflat_file in enumerate(sflat_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
sflat = self.get_ccd(butler, sflat_file, mask_files)
exp_time = get_exposure_time(sflat)
mondiode = get_monodiode_val_from_data_id(sflat_file, exp_time,
self.config.teststand, butler)
if mondiode is None:
continue
flux = exp_time * mondiode
data_dict['EXPTIME'].append(exp_time)
data_dict['MONDIODE'].append(mondiode)
data_dict['FLUX'].append(flux)
ccd_ims = unbiased_ccd_image_dict(sflat, bias=bias_type,
superbias_frame=superbias_frame,
trim='imaging')
amps = get_amp_list(sflat)
for amp in amps:
image = ccd_ims[amp]
fstats = self.get_stats(image)
data_dict['AMP%02i_FLAT_MEAN' % amp].append(fstats[0])
data_dict['AMP%02i_FLAT_MEDIAN' % amp].append(fstats[1])
data_dict['AMP%02i_FLAT_VAR' % amp].append(fstats[2])
data_dict['AMP%02i_FLAT_ROWMEAN' % amp].append(image.image.array.mean(0))
data_dict['AMP%02i_FLAT_COLMEAN' % amp].append(image.image.array.mean(1))
self.log_progress("Done!")
primary_hdu = fits.PrimaryHDU()
primary_hdu.header['NAMPS'] = 16
dtables = TableDict(primary=primary_hdu)
dtables.make_datatable('files', make_file_dict(butler, sflat_files))
dtables.make_datatable('stability', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
Output data tables
"""
self.safe_update(**kwargs)
slot = self.config.slot
qe_files = data['LAMBDA']
lams = []
for file in qe_files:
lam = file.split('flat_')[1].split('_')[0]
lams.append(lam)
mask_files = self.get_mask_files()
superbias_frame = self.get_superbias_frame(mask_files)
bias_type = self.get_bias_algo()
self.log_info_slot_msg(self.config, "%i files" % len(qe_files))
sflat_table_file = self.get_filename_from_format(RAFT_SFLAT_TABLE_FORMATTER, "sflat.fits")
sflat_tables = TableDict(sflat_table_file)
bbox_dict = construct_bbox_dict(sflat_tables['defects'])
slot_bbox_dict = bbox_dict[slot]
# This is a dictionary of dictionaries to store all the
# data you extract from the qe_files
data_dict = dict(SLOT=[], AMP=[], XCORNER=[], YCORNER=[], XSIZE=[], YSIZE=[])
for lam in lams:
data_dict['FLUX_' + lam] = []
data_dict['MED_' + lam] = []
temp_list = []
# Analysis goes here, you should fill data_dict with data extracted
# by the analysis
#
# For this slot, loop over qe files, loop over amplifiers, loop over dust spots in that amplifier,
# record the flux and median for each dust spot
# Then loop over dust spots CCDs, amplifiers, and QE files to accumulate the output dictionary
ccd = self.get_ccd(butler, qe_files[0], mask_files)
amps = get_amp_list(ccd)
spot = 0
for ifile, qe_file in enumerate(qe_files):
lam = qe_file.split('flat_')[1].split('_')[0]
for i, amp in enumerate(amps):
bbox_list = slot_bbox_dict[amp]
if len(bbox_list) > 100:
if i == 0:
self.log.warn("Skipping slot:amp %s:%i with %i defects" % (slot, amp, len(bbox_list)))
continue
ccd = self.get_ccd(butler, qe_file, mask_files)
regions = get_geom_regions(ccd, amp)
serial_oscan = regions['serial_overscan']
imaging = regions['imaging']
img = get_raw_image(ccd, amp)
if superbias_frame is not None:
if butler is not None:
superbias_im = get_raw_image(superbias_frame, amp)
else:
superbias_im = get_raw_image(superbias_frame, amp)
else:
superbias_im = None
image = unbias_amp(img, serial_oscan, bias_type=bias_type,
superbias_im=superbias_im, region=imaging)
for bbox in bbox_list:
if ifile == 0:
data_dict['SLOT'].append(slot)
#data_dict['LAM'].append(lam)
data_dict['AMP'].append(amp)
data_dict['XCORNER'].append(bbox.getBeginX())
data_dict['YCORNER'].append(bbox.getBeginY())
data_dict['XSIZE'].append(bbox.getWidth())
data_dict['YSIZE'].append(bbox.getHeight())
try:
cutout = image[bbox]
except Exception:
pass
# Here evaluate the 'flux' of the feature, relative to the median
# value of the amplifier image. May also want to assemble bounding
# box corners into a ds9 region file, CCD by CCD
med = np.median(image.array)
flux = np.sum(cutout.array) - bbox.getWidth()*bbox.getHeight()*med
spot += 1
#temp_dict[spot].append((lam, flux, med))
temp_list.append((lam, flux, med))
for i, tmp_data in enumerate(temp_list):
#data_dict['SLOT'].append(temp_dict['SLOT'][i])
#data_dict['AMP'].append(temp_dict['AMP'][i])
#lam = temp_dict['LAM'][i]
lam, flux, med = tmp_data
data_dict['FLUX_' + lam].append(flux) #temp_dict['FLUX'][i])
data_dict['MED_' + lam].append(med) #np.median(image.array))
sys.stdout.write("!\n")
sys.stdout.flush()
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, qe_files))
#dtables.make_datatable('dust_color', data_dict)
dtables.make_datatable('dust_color_hack', data_dict)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract data
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
output data tables
"""
self.safe_update(**kwargs)
slots = self.config.slots
if slots is None:
slots = ALL_SLOTS
if butler is not None:
self.log.warn("Ignoring butler")
dark_current_data = dict(median=[],
stdev=[],
mean=[],
fit_mean=[],
fit_width=[],
fit_dof=[],
fit_chi2=[],
exptime=[],
current=[],
slot=[],
amp=[])
self.log_info_raft_msg(self.config, "")
for islot, slot in enumerate(slots):
self.log_progress(" %s" % slot)
mask_files = self.get_mask_files(slot=slot)
superdark_file = data[slot]
if not os.path.exists(superdark_file):
self.log.warn(" %s does not exist, skipping" % superdark_file)
continue
superdark_frame = self.get_ccd(None, superdark_file, mask_files)
exptime = get_exposure_time(superdark_frame)
amps = get_amp_list(superdark_frame)
for iamp, amp in enumerate(amps):
regions = get_geom_regions(superdark_frame, amp)
imaging = regions['imaging']
superdark_im = get_raw_image(superdark_frame, amp).image
image_data = superdark_im[imaging].array
median = np.median(image_data)
stdev = np.std(image_data)
hist_bins = np.linspace(median - 5 * stdev, median + 5 * stdev, 101)
hist = np.histogram(image_data, bins=hist_bins)
fit_result = gauss_fit(hist)
fit_pars = fit_result[0]
dark_current_data['median'].append(median)
dark_current_data['stdev'].append(stdev)
dark_current_data['mean'].append(np.mean(image_data))
dark_current_data['fit_mean'].append(fit_pars[1])
dark_current_data['fit_width'].append(fit_pars[2])
dark_current_data['fit_dof'].append(0.)
dark_current_data['fit_chi2'].append(0.)
dark_current_data['exptime'].append(exptime)
dark_current_data['current'].append(median/exptime)
dark_current_data['slot'].append(islot)
dark_current_data['amp'].append(iamp)
self.log_progress("Done!")
dtables = TableDict()
dtables.make_datatable('dark_current', dark_current_data)
return dtables
def extract(self, butler, data, **kwargs):
"""Extract the correlations between the imaging section
and the overscan regions in a series of bias frames
Parameters
----------
butler : `Butler`
The data butler
data : `dict`
Dictionary (or other structure) contain the input data
kwargs
Used to override default configuration
Returns
-------
dtables : `TableDict`
The resulting data
"""
self.safe_update(**kwargs)
bias_files = data['BIAS']
mask_files = self.get_mask_files()
self.log_info_slot_msg(self.config, "%i files" % len(bias_files))
ref_frames = {}
nfiles = len(bias_files)
s_correl = np.ndarray((16, nfiles - 1))
p_correl = np.ndarray((16, nfiles - 1))
for ifile, bias_file in enumerate(bias_files):
if ifile % 10 == 0:
self.log_progress(" %i" % ifile)
ccd = self.get_ccd(butler, bias_file, mask_files)
if ifile == 0:
dims = get_dims_from_ccd(ccd)
nrow_i = dims['nrow_i']
ncol_i = dims['ncol_i']
amps = get_amp_list(ccd)
for i, amp in enumerate(amps):
regions = get_geom_regions(ccd, amp)
image = get_raw_image(ccd, amp)
ref_frames[i] = get_image_frames_2d(image, regions)
continue
self.get_ccd_data(ccd,
ref_frames,
ifile=ifile,
s_correl=s_correl,
p_correl=p_correl,
nrow_i=nrow_i,
ncol_i=ncol_i)
self.log_progress("Done!")
data = {}
for i in range(16):
data['s_correl_a%02i' % i] = s_correl[i]
data['p_correl_a%02i' % i] = p_correl[i]
dtables = TableDict()
dtables.make_datatable('files', make_file_dict(butler, bias_files))
dtables.make_datatable("correl", data)
return dtables
def get_ccd_data(self, ccd, data, **kwargs):
"""Get the bias values and update the data dictionary
Parameters
----------
ccd : `MaskedCCD`
The ccd we are getting data from
data : `dict`
The data we are updating
Keywords
--------
slot : `str`
The slot name
ifile : `int`
The file index
nfiles_used : `int`
Total number of files
bias_type : `str`
Method to use to construct bias
std : `bool`
Used standard deviation instead of mean
superbias_frame : `MaskedCCD`
The superbias frame to subtract away
"""
nfiles_used = kwargs.get('nfiles_used', 1)
ifile = kwargs.get('ifile', 0)
slot = kwargs.get('slot')
superbias_frame = kwargs.get('superbias_frame', None)
offset = get_amp_offset(ccd, superbias_frame)
bias_type = self.get_bias_algo()
bias_type_col = self.get_bias_col_algo()
amps = get_amp_list(ccd)
for i, amp in enumerate(amps):
regions = get_geom_regions(ccd, amp)
serial_oscan = regions['serial_overscan']
parallel_oscan = regions['parallel_overscan']
img = get_raw_image(ccd, amp)
if superbias_frame is not None:
superbias_im = get_raw_image(superbias_frame, amp + offset)
else:
superbias_im = None
image = unbias_amp(img,
serial_oscan,
bias_type=bias_type,
bias_type_col=bias_type_col,
superbias_im=superbias_im,
parallel_oscan=parallel_oscan)
frames = get_image_frames_2d(image, regions)
for key, region in zip(REGION_KEYS, REGION_NAMES):
framekey_row = "row_%s" % key
framekey_col = "col_%s" % key
struct = array_struct(frames[region], do_std=self.config.std)
key_str = "biasst_%s_a%02i" % (slot, i)
if key_str not in data[framekey_row]:
data[framekey_row][key_str] = np.ndarray(
(len(struct['rows']), nfiles_used))
if key_str not in data[framekey_col]:
data[framekey_col][key_str] = np.ndarray(
(len(struct['cols']), nfiles_used))
data[framekey_row][key_str][:, ifile] = struct['rows']
data[framekey_col][key_str][:, ifile] = struct['cols']