def beam_recons(led_names,
ref_slot='11',
ref_amp=4,
ref_pix_x=1000,
ref_pix_y=256,
npix_for_avg=30,
npix_beam_image=300):
mean_bias_file = ref_slot + '_mean_bias_image_RTM-006.fits'
recons = {}
for led in led_names:
config = u.load_ccob_config(
'/home/combet/ccob-wb/ccob_config_RTM-006.yaml')
config['led_name'] = led
config['path'] = os.path.join(config['path'], config['led_name'])
flist = sorted(u.find_files(config, slot=ref_slot))
ccd_dict = sensorTest.MaskedCCD(flist[0],
bias_frame=os.path.join(
config['tmp_dir'], mean_bias_file))
eotest_results_file = os.path.join(
config['eo_data_path'],
'{}_eotest_results.fits'.format(ccd_dict.md('LSST_NUM')))
gains_dict = u.gains(eotest_results_file)
nodes = {}
nodes['xarr'] = []
nodes['yarr'] = []
nodes['val'] = []
for i, f in enumerate(sorted(flist)):
nodes['xarr'].append(
float(os.path.basename(f).split('_')[3].split('x')[1]))
nodes['yarr'].append(
float(os.path.basename(f).split('_')[4].split('y')[1]))
ccd_dict = sensorTest.MaskedCCD(f,
bias_frame=os.path.join(
config['tmp_dir'],
mean_bias_file))
image = ccd_dict.unbiased_and_trimmed_image(ref_amp)
image *= gains_dict[ref_amp]
arr = image.getArrays()
nodes['val'].append(
np.mean(
(arr[0][ref_pix_x - npix_for_avg / 2:ref_pix_x +
npix_for_avg / 2, ref_pix_y -
npix_for_avg / 2:ref_pix_y + npix_for_avg / 2])))
f_interp = interpolate.interp2d(np.unique(nodes['xarr']),
np.unique(nodes['yarr']),
nodes['val'],
kind='cubic')
xarr2 = np.linspace(min(nodes['xarr']), max(nodes['xarr']),
npix_beam_image)
yarr2 = np.linspace(min(nodes['yarr']), max(nodes['yarr']),
npix_beam_image)
tmp = f_inerp(xarr2, yarr2)
recons[led] = tmp / max(tmp.flatten())
return recons
def persistence_task(run, det_name, bias_files, superbias_frame, mask_files):
"""Single sensor execution of the persistence analysis."""
file_prefix = make_file_prefix(run, det_name)
data = defaultdict(list)
ccd = None
for bias_file in bias_files:
ccd = sensorTest.MaskedCCD(bias_file, mask_files=mask_files,
bias_frame=superbias_frame)
tseqnum = ccd.md.get('TSEQNUM')
for amp in ccd:
amp_image = ccd.unbiased_and_trimmed_image(amp)
stats = afwMath.makeStatistics(amp_image,
afwMath.MEAN | afwMath.STDEV,
ccd.stat_ctrl)
data['tseqnum'].append(tseqnum)
data['amp'].append(amp)
data['mean_signal'].append(stats.getValue(afwMath.MEAN))
data['stdev'].append(stats.getValue(afwMath.STDEV))
df = pd.DataFrame(data=data)
outfile = f'{file_prefix}_persistence_data.pickle'
df.to_pickle(outfile)
fig = plt.figure()
for amp in ccd:
my_df = df.query(f'amp == {amp}')
plt.scatter(my_df['tseqnum'], my_df['mean_signal'], s=2, label=f'{amp}')
xmax = 1.2*(np.max(df['tseqnum']) - np.min(df['tseqnum'])) \
+ np.min(df['tseqnum'])
axis = plt.axis()
plt.xlim(axis[0], xmax)
plt.legend(fontsize='x-small')
plt.xlabel('test sequence number')
plt.ylabel('mean residual signal (ADU)')
plt.title(f'{file_prefix} persistence test')
plt.savefig(f'{file_prefix}_persistence.png')
plt.close()
def make_image(config, slot_names, mean_frame_pattern='_mean_bias_image.fits'):
"""
Make the mosaic image of the entire raft, when illuminated by the CCOB
according to config. Returns:
- the raw image of the raft
- the corrected image where mean bias frame has been removed and gains applied
"""
file_list = sorted(find_files(config))
fits_files_dict = {slot_names[i] : file_list[i] for i in range(len(file_list))}
ccd_dict = {}
ccd_dict_wbias = {}
gains_dict = {}
for slot in slot_names:
mean_bias_file = slot + mean_frame_pattern
ccd_dict[slot] = sensorTest.MaskedCCD(fits_files_dict[slot])
outfile = os.path.join(config['tmp_dir'],'ccd' + slot + '.fits')
image={}
ccd_dict_wbias[slot]=sensorTest.MaskedCCD(fits_files_dict[slot],\
bias_frame=os.path.join(config['tmp_dir'],mean_bias_file))
outfile_wbias = os.path.join(config['tmp_dir'],'ccd' + slot + '_wbias.fits')
image_wbias={}
eotest_results_file = os.path.join(config['eo_data_path'],'{}_eotest_results.fits'.format(ccd_dict[slot].md('LSST_NUM')))
gains_dict[slot] = gains(eotest_results_file)
for amp in ccd_dict[slot]:
image[amp] = ccd_dict[slot].bias_subtracted_image(amp)
image[amp] *= gains_dict[slot][amp]
image_wbias[amp] = ccd_dict_wbias[slot].bias_subtracted_image(amp)
image_wbias[amp] *= gains_dict[slot][amp]
imutils.writeFits({amp: image_wbias[amp].getImage() for amp in ccd_dict_wbias[slot]},
outfile_wbias, fits_files_dict[slot])
imutils.writeFits({amp: image[amp].getImage() for amp in ccd_dict[slot]},
outfile, fits_files_dict[slot])
fits_files_dict_corr={slot : os.path.join(config['tmp_dir'],'ccd'+slot+'.fits') for slot in slot_names}
fits_files_dict_corr_wbias={slot : os.path.join(config['tmp_dir'],'ccd'+slot+'_wbias.fits') for slot in slot_names}
im_corr = raft.RaftMosaic(fits_files_dict_corr, bias_subtract=False)
im_corr_wbias = raft.RaftMosaic(fits_files_dict_corr_wbias, bias_subtract=False)
im_raw = raft.RaftMosaic(fits_files_dict, bias_subtract=False)
return im_raw, im_corr, im_corr_wbias
def __init__(self, mask_file):
"""
Parameters
----------
mask_file: str
Filename of a mask file generated with the eotest code.
"""
self.ccd = sensorTest.MaskedCCD(mask_file)
def __init__(self,
fits_files,
gains=None,
bias_subtract=True,
nx=12700,
ny=12700,
nx_segments=8,
ny_segments=2,
segment_processor=None):
"""
Constructor.
Parameters
----------
fits_files : dict
Dictionary of single sensor FITS files, keyed by raft slot
name. These files should conform to LCA-13501.
gains : dict, optional
Dictionary (keyed by slot name) of dictionaries (one per
FITS file) of system gain values for each amp. Default:
None (i.e., do not apply gain correction).
bias_subtract : bool, optional
Flag do to a bias subtraction based on the serial overscan.
Default: True
nx : int, optional
Number of pixels in the x (serial) direction. Default: 12700
ny : int, optional
Number of pixels in the y (parallel) direction. Default: 12700
nx_segments : int, optional
Number of segments in the x (serial) direction. Default: 8
ny_segments : int, optional
Number of pixels in the y (parallel) direction. Default: 2
segment_processor : function, optional
Function to apply to pixel data in each segment. If None (default),
then set do the standard bias subtraction and gain correction.
"""
self.fits_files = fits_files
self.raft_name = fits.open(
fits_files.values()[0])[0].header['RAFTNAME']
self.wl = fits.open(fits_files.values()[0])[0].header['MONOWL']
self.image_array = np.zeros((nx, ny), dtype=np.float32)
self.nx = nx
self.ny = ny
self.nx_segments = nx_segments
self.ny_segments = ny_segments
self.segment_processor = segment_processor
self._amp_coords = defaultdict(dict)
if gains is None:
# Assume unit gain for all amplifiers.
unit_gains = dict([(i, 1) for i in range(1, 17)])
gains = dict([(slot, unit_gains) for slot in fits_files])
for slot, filename in fits_files.items():
print("processing", os.path.basename(filename))
ccd = sensorTest.MaskedCCD(filename)
hdu_list = fits.open(filename)
for amp, hdu in zip(ccd, hdu_list[1:]):
self._set_segment(slot, ccd, amp, hdu, gains[slot][amp],
bias_subtract)
def test_EPERTask_run_parallel(self):
"Test the EPERTask.run method for the parallel direction."
ccd = sensorTest.MaskedCCD(self.fits_file)
task = sensorTest.EPERTask()
task.config.direction = 'p'
task.config.cti = True
task.config.verbose = self.verbose
cti, bias_ests = task.run(self.fits_file, 1, range(1, 17),
self.overscans)
nrows = ccd.amp_geom.imaging.getHeight()
cti_expected = (float(self.overscans * self.overscan_value) /
float(self.imaging_value) / float(nrows))
for amp in ccd:
self.assertEqual(cti[amp].value, cti_expected)
def test_EPERTask_run_serial(self):
"Test the EPERTask.run method for the serial direction."
ccd = sensorTest.MaskedCCD(self.fits_file)
task = sensorTest.EPERTask()
task.config.direction = 's'
task.config.cti = True
task.config.verbose = self.verbose
cti, bias_ests = task.run(self.fits_file, 1, range(1, 17),
self.overscans)
ncols = (ccd.amp_geom.prescan.getWidth() +
ccd.amp_geom.imaging.getWidth())
cti_expected = (float(self.overscans * self.overscan_value) /
float(self.imaging_value) / float(ncols))
for amp in ccd:
self.assertEqual(cti[amp].value, cti_expected)
def diff_image_arrays(flat1_file, flat2_file, bias_frame, buffer=10):
"""
Compute the difference images for each amp, using the Astier
weighting scheme to account for somewhat different exposure times,
and return a dict of the image arrays, keyed by amp.
"""
image_arrays = dict()
ccd1 = sensorTest.MaskedCCD(flat1_file, bias_frame=bias_frame)
ccd2 = sensorTest.MaskedCCD(flat2_file, bias_frame=bias_frame)
imaging_bbox = ccd1.amp_geom.imaging
imaging_bbox.grow(-buffer)
for amp in ccd1:
image1 = ccd1.unbiased_and_trimmed_image(amp, imaging=imaging_bbox)
image2 = ccd2.unbiased_and_trimmed_image(amp, imaging=imaging_bbox)
mean1 = afwMath.makeStatistics(image1, afwMath.MEAN, ccd1.stat_ctrl)\
.getValue()
mean2 = afwMath.makeStatistics(image2, afwMath.MEAN, ccd1.stat_ctrl)\
.getValue()
fmean = (mean1 + mean2) / 2.
image1 *= mean2 / fmean
image2 *= mean1 / fmean
image1 -= image2
image_arrays[amp] = copy.deepcopy(image1.getImage().getArray())
return image_arrays
def test_get_median_signal_levels(self):
"""
Test the get_median_signal_levels function on the flat and bias
frames.
"""
flat = sensorTest.MaskedCCD(self.flat_file)
overscan = flat.amp_geom.serial_overscan
imaging = flat.amp_geom.imaging
imaging_signals \
= aliveness_utils.get_median_signal_levels(flat, imaging)
oscan_signals \
= aliveness_utils.get_median_signal_levels(flat, overscan)
for amp in oscan_signals:
signal = imaging_signals[amp] - oscan_signals[amp]
self.assertGreater(signal, 0.9*self.nominal_signal)
self.assertLess(signal, 1.1*self.nominal_signal)
def make_fits(QE_map, amp_coord, outfile, template_file):
"""
Saves the mosaicked QE image into a fits file, using the default format
Parameters
----------
outfile: string
Name of the output fits file into which the data will be saved
template_file: string
Name of the file to be used as template in the writeFits function
"""
amp_dict = {}
for amp_pos in amp_coord.keys():
amp = amp_coord[amp_pos]['amp']
xmin = amp_coord[amp_pos]['xmin']
xmax = amp_coord[amp_pos]['xmax']
ymin = amp_coord[amp_pos]['ymin']
ymax = amp_coord[amp_pos]['ymax']
flipx = amp_coord[amp_pos]['flipx']
flipy = amp_coord[amp_pos]['flipy']
detsec = amp_coord[amp_pos]['detsec']
datasec = amp_coord[amp_pos]['datasec']
foo = QE_map[ymin:ymax, xmin:xmax]
if flipx:
amp_dict[amp] = foo[:, ::-1]
elif flipy:
amp_dict[amp] = foo[::-1, :]
else:
amp_dict[amp] = foo
ccd_dict = sensorTest.MaskedCCD(template_file)
shape = ccd_dict[1].getImage().getArray().shape
amp_dict_w_overscan = {}
for amp in amp_dict:
arr = np.zeros(shape)
arr[datasec['ymin'] - 1:datasec['ymax'],
datasec['xmin'] - 1:datasec['xmax']] = amp_dict[amp]
amp_dict_w_overscan[amp] = arr
u.writeFits_from_dict(amp_dict_w_overscan, outfile,
template_file) #,bitpix=-32)
def test_subimage_parallel(self):
"Test the SubImage class for the parallel direction."
ccd = sensorTest.MaskedCCD(self.fits_file)
task = sensorTest.EPERTask()
task.config.direction = 'p'
task.config.verbose = self.verbose
for amp in ccd:
subimage = SubImage(ccd, amp, self.overscans, task)
lastrow = ccd.amp_geom.imaging.getMaxY()
ncols = ccd.amp_geom.imaging.getWidth()
last_imaging_row = subimage(lastrow).getImage().getArray().flatten()
self.assertEqual(ncols, len(last_imaging_row))
self.assertEqual(ncols*self.imaging_value, sum(last_imaging_row))
for irow in range(1, self.overscans+1):
overscan_row = subimage(lastrow + irow)
self.assertEqual(ncols*self.overscan_value,
sum(overscan_row.getImage().getArray().flatten()))
def test_subimage_serial(self):
"Test the SubImage class for the serial direction."
ccd = sensorTest.MaskedCCD(self.fits_file)
task = sensorTest.EPERTask()
task.config.direction = 's'
task.config.verbose = self.verbose
for amp in ccd:
subimage = SubImage(ccd, amp, self.overscans, task)
lastcol = ccd.amp_geom.imaging.getMaxX()
nrows = ccd.amp_geom.imaging.getHeight()
last_imaging_col = subimage(lastcol).getImage().getArray().flatten()
self.assertEqual(nrows, len(last_imaging_col))
self.assertEqual(nrows*self.imaging_value, sum(last_imaging_col))
for icol in range(1, self.overscans+1):
overscan_col = subimage(lastcol + icol)
self.assertEqual(nrows*self.overscan_value,
sum(overscan_col.getImage().getArray().flatten()))
def make_mask(med_file,
gains,
outfile,
ethresh=0.1,
colthresh=20,
mask_plane='BAD'):
"""
Create bright pixel mask from a medianed single sensor dark frame.
Parameters
----------
med_file: str
Filename of the medianded dark frame.
gains: dict
Gains (e-/ADU) of the sixteen amplifiers in the CCD, keyed by amp.
outfile: str
Output filename of the mask file.
ethresh: float, optional
Threshold in e- per second per pixel for identifying a bright
pixel defect. Default: 0.1
colthresh: float, optional
Threshold in e- per second per column for identifying a bright
column. Default: 20
mask_plane: str, optional
Name of the mask plane. Default: 'BAD'
Returns
-------
IsMasked object
"""
ccd = sensorTest.MaskedCCD(med_file)
exptime = ccd.md.get('EXPTIME')
pixels, columns = {}, {}
for amp in ccd:
bright_pixels \
= sensorTest.BrightPixels(ccd, amp, exptime, gains[amp],
ethresh=ethresh, colthresh=colthresh)
pixels[amp], columns[amp] = bright_pixels.find()
sensorTest.generate_mask(med_file,
outfile,
mask_plane,
pixels=pixels,
columns=columns)
return IsMasked(outfile)
def bias_stability_task(run, det_name, bias_files, nsigma=10):
"""Compute amp-wise bias stability time histories and serial profiles."""
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
data = defaultdict(list)
fig = plt.figure(figsize=(16, 16))
xlabel_amps = (13, 14, 15, 16)
ylabel_amps = (1, 5, 9, 13)
ax = {amp: fig.add_subplot(4, 4, amp) for amp in range(1, 17)}
for bias_file in bias_files:
with fits.open(bias_file) as hdus:
temps = dict()
for i in range(1, 10):
key = f'TEMP{i}'
if key in hdus['REB_COND'].header:
temps[key] = hdus['REB_COND'].header[key]
ccd = sensorTest.MaskedCCD(bias_file)
for amp in ccd:
# Retrieve the per row overscan subtracted imaging section.
amp_image = ccd.unbiased_and_trimmed_image(amp)
# Plot the median of each column versus serial pixel number.
imarr = amp_image.getImage().array
ax[amp].plot(range(imarr.shape[1]), np.median(imarr, axis=0))
# Compute 10-sigma clipped mean and stdev
mean, stdev = image_stats(amp_image, nsigma=nsigma)
data['raft'].append(raft)
data['slot'].append(slot)
data['tseqnum'].append(ccd.md.get('TSEQNUM'))
for key, value in temps.items():
data[key].append(value)
data['MJD'].append(ccd.md.get('MJD-OBS'))
data['amp'].append(amp)
data['mean'].append(mean)
data['stdev'].append(stdev)
plt.suptitle(f'{det_name}, Run {run}\nmedian signal (ADU) vs column')
plt.tight_layout(rect=(0, 0, 1, 0.95))
for amp in ccd:
ax[amp].annotate(f'amp {amp}', (0.5, 0.95),
xycoords='axes fraction', ha='center')
plt.savefig(f'{file_prefix}_bias_serial_profiles.png')
df = pd.DataFrame(data=data)
df.to_pickle(f'{file_prefix}_bias_frame_stats.pickle')
def load_ccd(self, led_name='red'):
"""
Load and generate mosaic image from a given sensor illuminated
by the CCOB. The path to the data is provided in the self.config_file_data file.
Parameters
----------
led_name: choice of the CCOB LED. Either one of ['nm960','nm850','nm750,'red','blue,'uv']
"""
#config_file_data = '../ccob_config_RTM-006.yaml'
# config_data = u.load_ccob_config(self.config_file_data)
# config_beam = u.load_ccob_config(self.config_file_beam)
self.config_data['led_name'] = led_name
slot = self.beam.properties['ref_slot']
file_list=sorted(u.find_files(self.config_data, slot=slot))
mean_slot_file = slot+'_mean_ccob_image.fits'
imutils.fits_mean_file(file_list, os.path.join(self.config_data['tmp_dir'],mean_slot_file))
fits_file = os.path.join(self.config_data['tmp_dir'],mean_slot_file)
gains_dict={}
ccd_dict={}
#bias_frames = glob.glob(os.path.join(config['path'], slot+'_bias*'))
#mean_bias_file = slot+'_mean_bias_image_RTM-006_new.fits'
#imutils.fits_mean_file(bias_frames, os.path.join(config['tmp_dir'],mean_bias_file))
#ccd_dict = sensorTest.MaskedCCD(fits_file, bias_frame=os.path.join(self.config_data['tmp_dir'],mean_bias_file))
superbias_frame = make_superbias_frame(self.config_data, slot=slot)
ccd_dict = sensorTest.MaskedCCD(fits_file, bias_frame=os.path.join(self.config_data['tmp_dir'],superbias_frame))
eotest_results_file = os.path.join(self.config_data['eo_data_path'],\
'{}_eotest_results.fits'.format(ccd_dict.md('LSST_NUM')))
gains_dict = u.gains(eotest_results_file)
self.ccd = {}
self.ccd['mosaic'], self.ccd['amp_coord'] = u.make_ccd_2d_array(fits_file, gains=gains_dict)
self.ccd['xpos_ccob'] = self.config_data['xpos']
self.ccd['ypos_ccob'] = self.config_data['ypos']
return self.ccd
def recons(self,
ref_slot='11',
ref_amp=4,
ref_pix_x=1000,
ref_pix_y=256,
npix_for_avg=30):
self.properties["ref_slot"] = ref_slot
self.properties["ref_amp"] = ref_amp
self.properties["ref_pix_x"] = int(ref_pix_x)
self.properties["ref_pix_y"] = int(ref_pix_y)
self.properties["npix_for_avg"] = int(npix_for_avg)
recons = {}
led = self.config['led_name']
print(led)
flist = sorted(u.find_files(self.config, slot=ref_slot))
nodes = {}
nodes['xarr'] = []
nodes['yarr'] = []
nodes['val'] = []
for i, f in enumerate(sorted(flist)):
nodes['xarr'].append(
float(os.path.basename(f).split('_')[3].split('x')[1]))
nodes['yarr'].append(
float(os.path.basename(f).split('_')[4].split('y')[1]))
ccd_dict = sensorTest.MaskedCCD(f)
image = ccd_dict.unbiased_and_trimmed_image(ref_amp)
arr = image.getArrays()
nodes['val'].append(np.mean((arr[0][int(ref_pix_x-npix_for_avg/2):int(ref_pix_x+npix_for_avg/2),\
int(ref_pix_y-npix_for_avg/2):int(ref_pix_y+npix_for_avg/2)])))
f_interp = interpolate.interp2d(np.unique(nodes['xarr']),
np.unique(nodes['yarr']),
nodes['val'],
kind='cubic')
self.beam_image['nodes'] = nodes
self.beam_image['f_interp'] = f_interp
def test_generate_mask(self):
"Test the generated mask for expected masked and unmasked pixels."
sensorTest.generate_mask(self.template_file,
self.mask_file,
mask_plane='TRAPS',
pixels=self.pixels,
columns=self.columns,
temp_mask_image='my_temp_mask_file.fits')
ccd = sensorTest.MaskedCCD(self.mask_file)
for amp in self.pixels:
image = imutils.trim(ccd[amp].getImage(), ccd.amp_geom.imaging)
imarr = image.getArray()
for ix, iy in self.pixels[amp]:
self.assertNotEqual(0, imarr[iy][ix])
for xoffset, yoffset in ((-1, -1), (-1, 0), (-1, 1), (0, -1),
(0, 1), (1, -1), (1, 0), (1, 1)):
self.assertEqual(0, imarr[iy+yoffset][ix+xoffset])
for amp in self.columns:
image = imutils.trim(ccd[amp].getImage(), ccd.amp_geom.imaging)
imarr = image.getArray()
for ix in self.columns[amp]:
self.assertNotEqual(0, imarr[0][ix])
self.assertEqual(imarr[0][ix]*imarr.shape[0], sum(imarr[:, ix]))
ccs_setup_class=CcsRaftSetup)
# Perform noise analysis of last bias frame taken.
raft_id = siteUtils.getUnitId()
run_number = siteUtils.getRunNumber()
raft = camera_components.Raft.create_from_etrav(raft_id)
results_files = dict()
bias_files = dict()
bbox = None
for slot, sensor_id in raft.items():
# glob the bias files for each sensor. Use the last one in the sequence.
bias_files[slot] = sorted(
glob.glob('%s/%s_conn_bias_*.fits' % (slot, sensor_id)))[-2:]
ccd = sensorTest.MaskedCCD(bias_files[slot][-1])
if bbox is None:
bbox = ccd.amp_geom.serial_overscan
bbox.grow(-10)
outfile = '%s_eotest_results.fits' % sensor_id
results = sensorTest.EOTestResults(outfile)
for amp in ccd:
image = ccd[amp].getImage()
oscan = image.Factory(image, bbox)
read_noise \
= afw_math.makeStatistics(oscan, afw_math.STDEVCLIP).getValue()
results.add_seg_result(amp, 'READ_NOISE', read_noise)
results.write()
results_files[slot] = outfile
def raft_level_oscan_correlations(bias_files,
buffer=10,
title='',
vrange=None,
stretch=viz.LinearStretch,
figsize=(8, 8)):
"""
Compute the correlation coefficients between the overscan pixels
of the 144 amplifiers in raft.
Parameters
----------
bias_files: dict
Dictionary of bias image files, indexed by sensor slot id.
buffer: int [10]
Buffer region around perimeter of serial overscan region to
avoid when computing the correlation coefficients.
title: str ['']
Plot title.
vrange: (float, float) [None]
Minimum and maximum values for color scale range. If None, then
the range of the central 98th percentile of the absolute value
of the data is used.
stretch: astropy.visualization.BaseStretch [LinearStretch]
Stretch to use for the color scale.
Returns
-------
(matplotlib.figure.Figure, np.array): The figure containing the plot and
the numpy array containing the correlation coefficients.
"""
slots = 'S00 S01 S02 S10 S11 S12 S20 S21 S22'.split()
overscans = []
ccd0 = sensorTest.MaskedCCD(list(bias_files.values())[0])
bbox = ccd0.amp_geom.serial_overscan
bbox.grow(-buffer)
for slot in slots:
if slot not in bias_files:
for amp in ccd0:
overscans.append(np.zeros((bbox.getHeight(), bbox.getWidth())))
else:
ccd = sensorTest.MaskedCCD(bias_files[slot])
for amp in ccd:
image = ccd[amp].getImage()
overscans.append(image.Factory(image, bbox).getArray())
namps = len(overscans)
data = np.array([
np.corrcoef(overscans[i[0]].ravel(), overscans[i[1]].ravel())[0, 1]
for i in itertools.product(range(namps), range(namps))
])
data = data.reshape((namps, namps))
fig = plt.figure(figsize=figsize)
ax = fig.add_subplot(111)
ax.set_title(title, fontsize='medium')
interval = viz.PercentileInterval(98.)
if vrange is None:
vrange = interval.get_limits(np.abs(data.ravel()))
norm = ImageNormalize(vmin=vrange[0], vmax=vrange[1], stretch=stretch())
image = ax.imshow(data, interpolation='none', norm=norm)
plt.colorbar(image)
set_ticks(ax, slots, amps=16)
return fig, data
def normed_mean_response_vscol(sflat_file):
"""
For an input .fits file, calculates the normalized sigma clipped
mean flux vs. Col# for a group of Rows returns two arrays for
the top and bottom section of the CCD
"""
amc = sensorTest.MaskedCCD(sflat_file)
amps = imutils.allAmps(sflat_file)
ncol = amc.amp_geom.nx
sensor_type = amc.amp_geom.vendor.lower()
imaging = amc.amp_geom.imaging
# use 200 rows close to the amplifier
row_lo = 10
row_hi = 210
# top row
averow_top = np.zeros(ncol*8)
for i_amp in range(1, 8+1):
# Segments 10-17
anamp = imutils.trim(amc[i_amp], imaging=imaging)
anamp_im = anamp.getImage()
anamp_arr = anamp_im.getArray()
# use a robust mean
anamp_meanbyrow, _, _ \
= stats.sigma_clipped_stats(anamp_arr[row_lo:row_hi, :], axis=0)
# normalize
nmean_byrow = anamp_meanbyrow/np.median(anamp_meanbyrow)
lopix = 0 + (i_amp-1)*ncol
hipix = ncol + (i_amp-1)*ncol
averow_top[lopix:hipix] = np.flip(nmean_byrow)
# bot row
averow_bot = np.zeros((ncol*8))
for j_amp in range(16, 8, -1):
if j_amp not in amps:
continue
# Segments 00-07
# i_amp goes from 1 to 8, in order of increasing Yccs
i_amp = 17 - j_amp
anamp = imutils.trim(amc[j_amp], imaging=imaging)
anamp_im = anamp.getImage()
anamp_arr = anamp_im.getArray()
# use a robust mean
anamp_meanbyrow, _, _ \
= stats.sigma_clipped_stats(anamp_arr[row_lo:row_hi, :], axis=0)
# normalize
nmean_byrow = anamp_meanbyrow/np.median(anamp_meanbyrow)
lopix = 0 + (i_amp-1)*ncol
hipix = ncol + (i_amp-1)*ncol
if sensor_type == 'e2v':
averow_bot[lopix:hipix] = nmean_byrow
elif sensor_type == 'itl':
averow_bot[lopix:hipix] = np.flip(nmean_byrow)
# analyze the gaps between amplifiers for Divisidero Tearing, and
# find the max(abs) deviation in the +-2 columns at the boundaries
max_divisidero_tearing = [] # 14 entries per CCD
for k in range(1, 7+1):
collo = ncol*k - 2 # 2nd to last column in Amplifier
max_divisidero = np.max(np.abs(averow_top[collo:collo+4] - 1.0)) # +-2 columns
max_divisidero_tearing.append(max_divisidero)
for k in range(1, 7+1):
if k + 8 not in amps:
continue
collo = ncol*k - 2 # 2nd to last column in Amplifier
max_divisidero = np.max(np.abs(averow_bot[collo:collo+4] - 1.0)) # +-2 columns
max_divisidero_tearing.append(max_divisidero)
return averow_top, averow_bot, max_divisidero_tearing
def __init__(self,
fits_files,
gains=None,
bias_subtract=True,
nx=12700,
ny=12700,
nx_segments=8,
ny_segments=2,
segment_processor=None,
bias_frames=None,
dark_currents=None):
"""
Parameters
----------
fits_files : dict
Dictionary of single sensor FITS files, keyed by raft slot
name. These files should conform to LCA-13501.
gains : dict [None]
Dictionary (keyed by slot name) of dictionaries (one per
FITS file) of system gain values for each amp. If
None, then do not apply gain correction.
bias_subtract : bool [True]
Flag do to a bias subtraction based on the serial overscan
or provided bias frame.
nx : int [12700]
Number of pixels in the x (serial) direction.
ny : int, [12700]
Number of pixels in the y (parallel) direction.
nx_segments : int [8]
Number of segments in the x (serial) direction.
ny_segments : int [2]
Number of pixels in the y (parallel) direction.
segment_processor : function [None]
Function to apply to pixel data in each segment. If None,
then set do the standard bias subtraction and gain correction.
bias_frames : dict [None]
Dictionary of single sensor bias frames, keyed by raft slot.
If None, then just do the bias level subtraction using
overscan region.
dark_currents : dict [None]
Dictionary of dictionaries of dark current values per amp
in e-/s, keyed by raft slot and by amp number. If None, then
dark current subtraction is not applied.
"""
self.fits_files = fits_files
with fits.open(list(fits_files.values())[0]) as hdu_list:
self.raft_name = hdu_list[0].header['RAFTNAME']
try:
self.wl = hdu_list[0].header['MONOWL']
except KeyError:
wl = 0
self.image_array = np.zeros((nx, ny), dtype=np.float32)
self.nx = nx
self.ny = ny
self.nx_segments = nx_segments
self.ny_segments = ny_segments
self.segment_processor = segment_processor
self._amp_coords = defaultdict(dict)
if gains is None:
# Assume unit gain for all amplifiers.
unit_gains = dict([(i, 1) for i in range(1, 17)])
gains = dict([(slot, unit_gains) for slot in fits_files])
for slot, filename in list(fits_files.items()):
#print("processing", os.path.basename(filename))
bias_frame = bias_frames[slot] if bias_frames is not None else None
ccd = sensorTest.MaskedCCD(filename, bias_frame=bias_frame)
if dark_currents is not None:
try:
dark_time = ccd.md.get('DARKTIME')
except:
dark_time = ccd.md.get('EXPTIME')
with fits.open(filename) as hdu_list:
for amp, hdu in zip(ccd, hdu_list[1:]):
if dark_currents:
dark_correction = dark_time * dark_currents[slot][amp]
else:
dark_correction = 0
self._set_segment(slot, ccd, amp, hdu, gains[slot][amp],
bias_subtract, dark_correction)
def read_multibunch(self, config, dirlist=None, silent=False):
""" Reads the data from a bunch of reference pixels after a CCOB scan and fills in self.raw_data
and self.properties
Parameters
----------
config : dict
Contains all required information to reconstruct the beam
dirlist : list
List of directories containing the data
silent : boolean
If True, track the progress of the beam reconstruction
"""
self.properties["ref_raft"] = ref_raft = 'R22' if (
'ref_raft' not in config) else config['ref_raft']
self.properties["ref_slot"] = ref_slot = 'S11' if (
'ref_slot' not in config) else config['ref_slot']
self.properties["ref_amp"] = ref_amps = [5] if (
'ref_amps' not in config) else config['ref_amps']
self.properties["ref_pix_x"] = ref_pix_x = 1000 if (
'ref_pix_x' not in config) else int(config['ref_pix_x'])
self.properties["ref_pix_y"] = ref_pix_y = 256 if (
'ref_pix_y' not in config) else int(config['ref_pix_y'])
self.properties["npix_for_avg"] = npix_for_avg = 30 if (
'npix_for_avg' not in config) else int(config['npix_for_avg'])
biasfile = None if ('biasfile' not in config) else config['biasfile']
outdir = './' if ('tmpdir' not in config) else config['tmpdir']
recons = {}
led = self.config['led_name']
dirlist = sorted(dirlist)
if 'xarr' not in self.raw_data: self.raw_data['xarr'] = []
if 'yarr' not in self.raw_data: self.raw_data['yarr'] = []
if 'val' not in self.raw_data: self.raw_data['val'] = {}
if 'pd_value' not in self.raw_data: self.raw_data['pd_value'] = []
for i, d in enumerate(dirlist):
dd = os.path.basename(d)
xcurr = float(dd.split('_')[2])
ycurr = float(dd.split('_')[3])
l = list(zip(self.raw_data['xarr'], self.raw_data['yarr']))
# print(i)
if (xcurr, ycurr) in l:
# print('here')
continue
self.raw_data['xarr'].append(xcurr)
self.raw_data['yarr'].append(ycurr)
f = glob.glob(
os.path.join(d, "*" + ref_raft + "*" + ref_slot + '*'))
bb = fits.open(f[0])
xh = np.round(bb[0].header['BOTXCAM'], 1)
yh = np.round(bb[0].header['BOTYCAM'], 1)
# self.raw_data['xarr'].append(np.round(bb[0].header['BOTX'],1))
# self.raw_data['yarr'].append(np.round(bb[0].header['BOTY'],1))
self.raw_data['pd_value'].append(bb[0].header['CCOBADC'])
bb.close()
ccd_dict = sensorTest.MaskedCCD(f[0], bias_frame=biasfile)
for amp in ref_amps:
image = ccd_dict.unbiased_and_trimmed_image(amp)
arr = image.getArrays()
val = np.mean((arr[0][int(ref_pix_x-npix_for_avg/2):int(ref_pix_x+npix_for_avg/2),\
int(ref_pix_y-npix_for_avg/2):int(ref_pix_y+npix_for_avg/2)]))
if amp in self.raw_data['val']:
self.raw_data['val'][amp].append(val)
else:
self.raw_data['val'][amp] = [val]
if not silent:
#print(os.path.basename(f[0]))
print(amp, self.raw_data['xarr'][-1], xh,
self.raw_data['yarr'][-1], yh, val)
#print(self.raw_data['val'])
# Reordering the data
x = self.raw_data['xarr']
y = self.raw_data['yarr']
val = self.raw_data['val']
pd = self.raw_data['pd_value']
newx = [x for x, _, _, _ in sorted(zip(x, y, val[ref_amps[0]], pd))]
newy = [y for _, y, _, _ in sorted(zip(x, y, val[ref_amps[0]], pd))]
newpd = [pd for _, _, _, pd in sorted(zip(x, y, val[ref_amps[0]], pd))]
newval = {}
for amp in ref_amps:
newval[amp] = [
val for _, _, val, _ in sorted(zip(x, y, val[amp], pd))
]
self.raw_data['xarr'] = newx
self.raw_data['yarr'] = newy
self.raw_data['val'] = newval
self.raw_data['pd_value'] = newpd
if outdir is not None:
for amp in ref_amps:
outf = os.path.join(
outdir, 'beam_raw_' + led + '_' + ref_raft + '_' +
ref_slot + '_' + str(amp) + '_' + str(ref_pix_x) + '_' +
str(ref_pix_y) + '.txt')
np.savetxt(outf,
np.array([newx, newy, newval[amp], newpd]).T,
delimiter=' ')
dir='.',
suffix='.fits').name
imutils.fits_median_file(darks, med_file, bitpix=-32)
mask_file = tempfile.NamedTemporaryFile(prefix='tmp_mask_',
dir='.',
suffix='.fits').name
is_masked = make_mask(med_file, gains, mask_file)
fp_id, x0, y0, pixel_values = [], [], [], []
index = -1
exptime = 0
num_pix = 0
for dark in darks:
print "processing", dark
ccd = sensorTest.MaskedCCD(dark, mask_files=(mask_file, ))
exptime += ccd.md.get('EXPTIME')
for amp in ccd.keys():
image = ccd[amp]
image -= bg_image(ccd, amp)
image = image.Factory(image, ccd.amp_geom.imaging)
num_pix += np.prod(image.getImage().getArray().shape)
flags = afw_math.MEDIAN | afw_math.STDEVCLIP
stats = afw_math.makeStatistics(image, flags, ccd.stat_ctrl)
median = stats.getValue(afw_math.MEDIAN)
stdev = stats.getValue(afw_math.STDEVCLIP)
threshold = afw_detect.Threshold(median + args.nsig * stdev)
fp_set = afw_detect.FootprintSet(image, threshold)
for fp in fp_set.getFootprints():
def make_ccd_2d_array(infile, biasfile=None, gains=None):
'''
Generate a 2d array of a sensor image (trimmed, bias subtracted) from
an input fits file and a gain dictionary.
Function adapted from sensorTest.plot_flat()
'''
ccd = sensorTest.MaskedCCD(infile, bias_frame=biasfile)
foo = fits.open(infile)
datasec = parse_geom_kwd(foo[1].header['DATASEC'])
# Specialize to science sensor or wavefront sensor geometries.
nx_segments = 8
ny_segments = len(ccd) // nx_segments
nx = nx_segments * (datasec['xmax'] - datasec['xmin'] + 1)
ny = ny_segments * (datasec['ymax'] - datasec['ymin'] + 1)
mosaic = np.zeros((ny, nx), dtype=np.float)
amp_coords = {}
for ypos in range(ny_segments):
for xpos in range(nx_segments):
amp = ypos * nx_segments + xpos + 1
#
# Determine subarray boundaries in the mosaicked image array
# from DETSEC keywords for each segment.
detsec = parse_geom_kwd(foo[amp].header['DETSEC'])
datasec = parse_geom_kwd(foo[amp].header['DATASEC'])
xmin = nx - max(detsec['xmin'], detsec['xmax'])
xmax = nx - min(detsec['xmin'], detsec['xmax']) + 1
ymin = ny - max(detsec['ymin'], detsec['ymax'])
ymax = ny - min(detsec['ymin'], detsec['ymax']) + 1
#
#
# Extract the bias-subtracted masked image for this segment.
segment_image = ccd.unbiased_and_trimmed_image(amp)
subarr = segment_image.getImage().getArray()
#
# Determine flips in x- and y-directions in order to
# get the (1, 1) pixel in the lower right corner.
flipx = False
flipy = False
if detsec['xmax'] > detsec['xmin']: # flip in x-direction
subarr = subarr[:, ::-1]
flipx = True
if detsec['ymax'] > detsec['ymin']: # flip in y-direction
subarr = subarr[::-1, :]
flipy = True
#
# Convert from ADU to e-
if gains is not None:
subarr *= gains[amp]
#
# Save coordinates of segment for later use
amp_coords[(xpos, ypos)] = {
'amp': amp,
'segment': foo[amp].header['EXTNAME'],
'xmin': xmin,
'xmax': xmax,
'ymin': ymin,
'ymax': ymax,
'flipx': flipx,
'flipy': flipy,
'detsec': detsec,
'datasec': datasec
}
# Set the subarray in the mosaic.
mosaic[ymin:ymax, xmin:xmax] = subarr
return mosaic, amp_coords
def fit_superflats(datapath, sensor_id):
filepath = lambda level: \
os.path.join(datapath, '%s_superflat_%s.fits' % (sensor_id, level))
ccd_low = sensorTest.MaskedCCD(filepath('low'))
ccd_high = sensorTest.MaskedCCD(filepath('high'))
TrailedChargeResults = namedtuple('TrailedChargeResults',
['fit_results', 'plots'])
fit_results = FitResults(sensor_id)
my_plots = MultiPanelPlot(4, 4, figsize=(12, 12))
for amp in ccd_high:
tc_low = TrailedCharge(ccd_low, amp, lastskip=4)
tc_high = TrailedCharge(ccd_high, amp, lastskip=4)
# Fix bias levels to last column value
tc_low.bias_per_pixel = tc_low.oscan_values[-1] / tc_low.nrows
tc_high.bias_per_pixel = tc_high.oscan_values[-1] / tc_high.nrows
# Initial parameter estimates
cti_0 = tc_high.cti()
tau = 1. / np.log(
(tc_high.oscan_values[1] - tc_high.oscan_values[-1]) /
(tc_high.oscan_values[2] - tc_high.oscan_values[-1]))
q0 = ((tc_high.oscan_values[1] / tc_high.nrows -
tc_high.bias_per_pixel) / np.exp(-2 / tau) / tc_high.q_lastcol)
p0 = q0, tau, cti_0
bounds = ((0, None), (0, None), (0, None))
result_low = scipy.optimize.minimize(tc_low,
p0,
method='L-BFGS-B',
bounds=bounds)
fit_results.add_results('low', amp, result_low)
result_high = scipy.optimize.minimize(tc_high,
p0,
method='L-BFGS-B',
bounds=bounds)
fit_results.add_results('high', amp, result_high)
tc_joint = MultiObjectiveFunctions([tc_low, tc_high])
result = scipy.optimize.minimize(tc_joint,
p0,
method='L-BFGS-B',
bounds=bounds)
fit_results.add_results('joint', amp, result)
my_plots.add_subplot()
tc_low.plot_fit(result_low.x, label='low flux-only', linewidth=1)
tc_high.plot_fit(result_high.x,
color='red',
label='high flux-only',
linewidth=1)
tc_low.plot_model(result.x, color='green', marker='--', label='joint')
tc_high.plot_model(result.x, color='green', marker='--')
plt.annotate('Amp %i' % amp, (0.1, 0.9),
xycoords='axes fraction',
size='x-small',
horizontalalignment='left')
if tc_high.oscan_values[0] / tc_low.oscan_values[0] > 5:
plt.yscale('log')
axis_range = list(plt.axis())
axis_range[:2] = 0.5, axis_range[1] + 0.5
plt.axis(axis_range)
plt.legend(loc=0, fontsize='x-small')
my_plots.set_title(sensor_id)
my_plots.set_xlabel('overscan pixel')
my_plots.set_ylabel('ADU / pixel')
return TrailedChargeResults(fit_results, my_plots)
