def mean_overscans(imagefile, nexp=7, plots=None, verbose=False):
if nexp is None:
nexp = 7
result = [
] # decay constant, amplitude, flux of image, residue of first overscan, sensor number, segment, run number
for segment in range(1, 16 + 1):
header = getheader(imagefile)
info = [
header['LSST_NUM'], 'Segment {}'.format(segment),
header['RUNNUM'].split()[0]
]
image_untrimmed = ImageF(imagefile, segment)
amp = makeAmplifierGeometry(imagefile)
flat_mean = np.mean(
imutils.trim(image_untrimmed, imaging=amp.imaging).array[:, -1])
# mean value of the last column of the image
image = imutils.trim(image_untrimmed, imaging=amp.serial_overscan)
overscans_mean = [
np.mean([image.array[i, j] for i in np.arange(len(image.array))])
for j in range(len(image.array[0]))
]
bias = np.mean(overscans_mean[5:-2])
over_mean_subfit = overscans_mean[:nexp]
params, cov = curve_fit(exp_fit,
np.arange(nexp),
over_mean_subfit,
p0=(10, 10, 20000),
bounds=([.1, 0, 0], [20, 300, 50000]))
residue = params[1] / (flat_mean - bias)
result.append([params[0], params[1], flat_mean, residue, *info])
if verbose:
print('Segment {seg}:\n Decay : {p[0]:<10.03g} pixels\n Amplitude: {p[1]:<10.03g} '\
'ADU\n Offset : {p[2]:<10.03g} ADU'.format(seg=segment, p=params))
if plots is None:
continue
fig = plt.figure(figsize=(10, 10))
plt.plot(over_mean_subfit, ls='none', marker='.')
xfit = np.linspace(0, nexp - 1, 50)
plt.plot(xfit, [exp_fit(x, *params) for x in xfit])
plt.title(
'Superflat Mean Serial Overscan in {0} {1} Run {2}'.format(*info))
plt.figtext(0.5,0.5,('Decay constant: {p[0]:.03g} pixels \nAmplitude: {p[1]:.03g}'\
' ADU\nImage flux: {0:.00f} ADU\nResidue in first overscan pixel: {1:.03%}').format(flat_mean,residue,p=params))
plt.ylabel('ADU')
plt.xlabel('Pixel')
plt.legend(['Data', 'Fit'])
fig.patch.set_facecolor('white')
plt.savefig(('{0}/{1}_{2}_run{3}.png'.format(plots,
*info)).replace(" ", ""))
plt.close(fig)
return result
def pixel_counts(ccd_file, input_mask=None):
"""
Based on the sensor geometry and an optional input mask, compute
the total number of pixels in the imaging regions of the
amplifiers and the total number of masked pixels within the
imaging regions. If no input mask is given, the standard rolloff
mask for the vendor device is used.
@return <total number of imaging region pixels>, <number of masked pixels>
"""
if input_mask is not None:
mask_file = input_mask
else:
mask_file = tempfile.mkstemp(suffix='.fits', dir='.')[-1]
rolloff_mask(ccd_file, mask_file)
ccd = MaskedCCD(mask_file)
num_masked = 0
num_total = 0
imaging = ccd.amp_geom.imaging
for amp in ccd:
imarr = imutils.trim(ccd[amp].getImage(), imaging).getArray()
num_masked += len(np.where(imarr != 0)[0])
num_total += imarr.shape[0]*imarr.shape[1]
if input_mask is None:
try:
os.remove(mask_file)
except OSError:
pass
return num_total, num_masked
def unbiased_and_trimmed_image(self,
amp,
overscan=None,
imaging=None,
**kwargs):
"""
Return an offset-corrected image where the offset values generated using
either of the bias(), bias_row(), bias_func() or bias_spline() methods from
image_utils.py. The default bias method is set to bias_row(). Keyword arguments
can be passed depending on which bias method is used.
Keyword Arguments:
fit_order: The order of the polynomial. This only needs to be specified when
using the 'func' method. The default is: 1.
k: The degree of the spline fit. This only needs to be specified when using
the 'spline' method. The default is: 3.
s: The amount of smoothing to be applied to the fit. This only needs to be
specified when using the 'spline' method. The default is: 18000.
t: The number of knots. If None, finds the number of knots to use for a given
smoothing factor, s. This only needs to be specified when using the 'spline'
method. The default is: None.
"""
unbiased_image = self.bias_subtracted_image(amp, overscan, **kwargs)
if imaging is None:
imaging = self.amp_geom.imaging
mi = imutils.trim(unbiased_image, imaging)
if self._applyMasks:
self.applyInterpolateFromMask(mi)
return mi
def unbiased_and_trimmed_image(self,
amp,
overscan=None,
imaging=None,
fit_order=1):
unbiased_image = self.bias_subtracted_image(amp, overscan, fit_order)
if imaging is None:
imaging = self.amp_geom.imaging
mi = imutils.trim(unbiased_image, imaging)
if self._applyMasks:
self.applyInterpolateFromMask(mi)
return mi
def test_generate_mask(self):
"Test the generated mask for expected masked and unmasked pixels."
sensorTest.generate_mask(self.template_file,
self.mask_file,
mask_plane='TRAPS',
pixels=self.pixels,
columns=self.columns,
temp_mask_image='my_temp_mask_file.fits')
ccd = sensorTest.MaskedCCD(self.mask_file)
for amp in self.pixels:
image = imutils.trim(ccd[amp].getImage(), ccd.amp_geom.imaging)
imarr = image.getArray()
for ix, iy in self.pixels[amp]:
self.assertNotEqual(0, imarr[iy][ix])
for xoffset, yoffset in ((-1, -1), (-1, 0), (-1, 1), (0, -1),
(0, 1), (1, -1), (1, 0), (1, 1)):
self.assertEqual(0, imarr[iy+yoffset][ix+xoffset])
for amp in self.columns:
image = imutils.trim(ccd[amp].getImage(), ccd.amp_geom.imaging)
imarr = image.getArray()
for ix in self.columns[amp]:
self.assertNotEqual(0, imarr[0][ix])
self.assertEqual(imarr[0][ix]*imarr.shape[0], sum(imarr[:, ix]))
def __init__(self, ccd, amp, bg_reg=(10, 10)):
self.ccd = ccd
self.amp = amp
self.ccdtemp = ccd.md.get('CCDTEMP')
self.fe55_yield = Fe55Yield(self.ccdtemp)
raw_image = ccd[amp]
try:
self.imarr = raw_image.getArray()
except AttributeError:
self.imarr = raw_image.getImage().getArray()
self.image = imutils.trim(raw_image, imaging=ccd.amp_geom.imaging)
self.image -= self._bg_image(*bg_reg)
flags = afwMath.MEANCLIP | afwMath.STDEVCLIP
stats = afwMath.makeStatistics(self.image, flags, self.ccd.stat_ctrl)
self.mean = stats.getValue(afwMath.MEANCLIP)
self.stdev = stats.getValue(afwMath.STDEVCLIP)
self.footprint_signal = self._footprint_signal_spans
def unbias_amp(img,
serial_oscan,
bias_type=None,
superbias_im=None,
region=None,
bias_type_col=None,
parallel_oscan=None):
"""Unbias the data from a particular amp
Paramters
---------
img : `ImageF`
The image
serial_oscan : `Box2I`
Serial overscan bounding box
bias_type : `str` or `None`
Method of unbiasing to applly
superbias_im : `ImageF`
Optional superbias frame to subtract off
region : `Box2I`
Return to return data for
Returns
-------
iamge : `ImageF`
The unbiased image
"""
if bias_type is not None:
image = imutil.unbias_and_trim(img,
serial_oscan,
bias_method=bias_type,
bias_frame=superbias_im,
imaging=region,
bias_method_col=bias_type_col,
overscan_col=parallel_oscan)
else:
image = img
if superbias_im is not None:
image -= superbias_im
if region is not None:
image = imutil.trim(image, region)
return image
def flat_gain(image1,
image2,
count=1000,
dx=100,
dy=100,
binsize=1,
seed=None):
"""
Calculate the gain and noise of a CCD camera system by
examining two flat field images. The calculation is the standard
mean/variance thing.
"""
# If seed is None, the seed is generated from /dev/urandom.
random.seed(seed)
# Unbias using the mean bias of the two images.
bmean = (imutils.bias(image1) + imutils.bias(image2)) / 2.
image1 -= bmean
image2 -= bmean
# Trim prescan and overscan.
image1 = imutils.trim(image1)
image2 = imutils.trim(image2)
# Rebin into binsize x binsize pixels.
im1 = imutils.rebin(image1, binsize)
im2 = imutils.rebin(image2, binsize)
if dx > im1.getWidth():
dx = im1.getWidth()
if dy > im1.getHeight():
dy = im1.getHeight()
# Sample detector at size=count locations.
try:
xarr = random.randint(im1.getWidth() - dx - 1, size=count)
except ValueError:
# Rebinned image width smaller than requested dx, so just
# generate subregions using the full x extent.
xarr = np.zeros(count, dtype=np.int)
try:
yarr = random.randint(im1.getHeight() - dy - 1, size=count)
except ValueError:
# Rebinned image height smaller than requested dy, so just
# generate subregions using the full y extent.
yarr = np.zeros(count, dtype=np.int)
gains = []
ntrial = 0
exception_count = 0
for x, y in zip(xarr, yarr):
bbox = lsstGeom.Box2I(lsstGeom.Point2I(int(x), int(y)),
lsstGeom.Extent2I(dx, dy))
imarr1 = im1.Factory(im1, bbox).getArray()
imarr2 = im2.Factory(im2, bbox).getArray()
# Calculate flat ratio and correct subarray of image 2.
fratio = np.mean(imarr1 / imarr2)
imarr2 *= fratio
# Calculate the mean value of the flat field images.
fmean = (np.mean(imarr1) + np.mean(imarr2)) / 2.
# Calculate the variance of the flat difference image.
fvar = np.var(imarr1 - imarr2) / 2.
gains.append(fvar / fmean)
gain = 1. / np.median(gains) # gain in Ne/DN
return gain, im1, im2
def _bg_image(self, nx, ny):
bg_ctrl = afwMath.BackgroundControl(nx, ny, self.ccd.stat_ctrl)
bg = afwMath.makeBackground(self.ccd[self.amp], bg_ctrl)
image_region = self.ccd.amp_geom.imaging
return imutils.trim(bg.getImageF(), imaging=image_region)
def run(self, sensor_id, dark_files, mask_files, gains, bias_frame=None):
imutils.check_temperatures(dark_files,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
median_images = {}
md = imutils.Metadata(dark_files[0], 1)
for amp in imutils.allAmps(dark_files[0]):
median_images[amp] = imutils.fits_median(dark_files,
imutils.dm_hdu(amp))
medfile = os.path.join(self.config.output_dir,
'%s_median_dark_current.fits' % sensor_id)
imutils.writeFits(median_images, medfile, dark_files[0])
ccd = MaskedCCD(medfile, mask_files=mask_files, bias_frame=bias_frame)
dark95s = {}
exptime = md.get('EXPTIME')
if self.config.verbose:
self.log.info("Amp 95 percentile median")
dark_curr_pixels = []
dark_curr_pixels_per_amp = {}
for amp in ccd:
imaging_region = ccd.amp_geom.imaging
overscan = ccd.amp_geom.serial_overscan
image = imutils.unbias_and_trim(ccd[amp].getImage(), overscan,
imaging_region)
mask = imutils.trim(ccd[amp].getMask(), imaging_region)
imarr = image.getArray()
mskarr = mask.getArray()
pixels = imarr.reshape(1, imarr.shape[0] * imarr.shape[1])[0]
masked = mskarr.reshape(1, mskarr.shape[0] * mskarr.shape[1])[0]
unmasked = [
pixels[i] for i in range(len(pixels)) if masked[i] == 0
]
unmasked.sort()
unmasked = np.array(unmasked) * gains[amp] / exptime
dark_curr_pixels_per_amp[amp] = unmasked
dark_curr_pixels.extend(unmasked)
try:
dark95s[amp] = unmasked[int(len(unmasked) * 0.95)]
median = unmasked[len(unmasked) / 2]
except IndexError as eobj:
print str(eobj)
dark95s[amp] = -1.
median = -1.
if self.config.verbose:
self.log.info("%2i %.2e %.2e" %
(amp, dark95s[amp], median))
#
# Compute 95th percentile dark current for CCD as a whole.
#
dark_curr_pixels = sorted(dark_curr_pixels)
darkcurr95 = dark_curr_pixels[int(len(dark_curr_pixels) * 0.95)]
dark95mean = np.mean(dark95s.values())
if self.config.verbose:
#self.log.info("CCD: mean 95 percentile value = %s" % dark95mean)
self.log.info("CCD-wide 95 percentile value = %s" % darkcurr95)
#
# Update header of dark current median image file with dark
# files used and dark95 values, and write dark95 values to the
# eotest results file.
#
results_file = self.config.eotest_results_file
if results_file is None:
results_file = os.path.join(self.config.output_dir,
'%s_eotest_results.fits' % sensor_id)
results = EOTestResults(results_file, namps=len(ccd))
output = fits.open(medfile)
for i, dark in enumerate(dark_files):
output[0].header['DARK%02i' % i] = os.path.basename(dark)
# Write overall dark current 95th percentile
results.output['AMPLIFIER_RESULTS'].header['DARK95'] = darkcurr95
for amp in ccd:
output[0].header['DARK95%s' %
imutils.channelIds[amp]] = dark95s[amp]
results.add_seg_result(amp, 'DARK_CURRENT_95', dark95s[amp])
fitsWriteto(output, medfile, clobber=True, checksum=True)
results.write(clobber=True)
return dark_curr_pixels_per_amp, dark95s
def normed_mean_response_vscol(sflat_file):
"""
For an input .fits file, calculates the normalized sigma clipped
mean flux vs. Col# for a group of Rows returns two arrays for
the top and bottom section of the CCD
"""
amc = sensorTest.MaskedCCD(sflat_file)
amps = imutils.allAmps(sflat_file)
ncol = amc.amp_geom.nx
sensor_type = amc.amp_geom.vendor.lower()
imaging = amc.amp_geom.imaging
# use 200 rows close to the amplifier
row_lo = 10
row_hi = 210
# top row
averow_top = np.zeros(ncol*8)
for i_amp in range(1, 8+1):
# Segments 10-17
anamp = imutils.trim(amc[i_amp], imaging=imaging)
anamp_im = anamp.getImage()
anamp_arr = anamp_im.getArray()
# use a robust mean
anamp_meanbyrow, _, _ \
= stats.sigma_clipped_stats(anamp_arr[row_lo:row_hi, :], axis=0)
# normalize
nmean_byrow = anamp_meanbyrow/np.median(anamp_meanbyrow)
lopix = 0 + (i_amp-1)*ncol
hipix = ncol + (i_amp-1)*ncol
averow_top[lopix:hipix] = np.flip(nmean_byrow)
# bot row
averow_bot = np.zeros((ncol*8))
for j_amp in range(16, 8, -1):
if j_amp not in amps:
continue
# Segments 00-07
# i_amp goes from 1 to 8, in order of increasing Yccs
i_amp = 17 - j_amp
anamp = imutils.trim(amc[j_amp], imaging=imaging)
anamp_im = anamp.getImage()
anamp_arr = anamp_im.getArray()
# use a robust mean
anamp_meanbyrow, _, _ \
= stats.sigma_clipped_stats(anamp_arr[row_lo:row_hi, :], axis=0)
# normalize
nmean_byrow = anamp_meanbyrow/np.median(anamp_meanbyrow)
lopix = 0 + (i_amp-1)*ncol
hipix = ncol + (i_amp-1)*ncol
if sensor_type == 'e2v':
averow_bot[lopix:hipix] = nmean_byrow
elif sensor_type == 'itl':
averow_bot[lopix:hipix] = np.flip(nmean_byrow)
# analyze the gaps between amplifiers for Divisidero Tearing, and
# find the max(abs) deviation in the +-2 columns at the boundaries
max_divisidero_tearing = [] # 14 entries per CCD
for k in range(1, 7+1):
collo = ncol*k - 2 # 2nd to last column in Amplifier
max_divisidero = np.max(np.abs(averow_top[collo:collo+4] - 1.0)) # +-2 columns
max_divisidero_tearing.append(max_divisidero)
for k in range(1, 7+1):
if k + 8 not in amps:
continue
collo = ncol*k - 2 # 2nd to last column in Amplifier
max_divisidero = np.max(np.abs(averow_bot[collo:collo+4] - 1.0)) # +-2 columns
max_divisidero_tearing.append(max_divisidero)
return averow_top, averow_bot, max_divisidero_tearing