def run(self, sensor_id, qe_files, pd_ratio_file, mask_files, gains,
bias_frame=None, medians_file=None, vendor_data=False,
correction_image=None, mondiode_func=None):
imutils.check_temperatures(qe_files, self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
qe_data = QE.QE_Data(verbose=self.config.verbose, logger=self.log,
mondiode_func=mondiode_func)
if medians_file is None:
medians_file = os.path.join(self.config.output_dir,
'%s_QE_medians.txt' % sensor_id)
qe_data.calculate_medians(qe_files, medians_file,
mask_files=mask_files,
bias_frame=bias_frame,
overwrite=True,
correction_image=correction_image)
qe_data.read_medians(medians_file)
if vendor_data:
qe_data.incidentPower_e2v()
else:
qe_data.incidentPower(pd_ratio_file)
qe_data.calculate_QE(gains, amps=imutils.allAmps(qe_files[0]))
fits_outfile = os.path.join(self.config.output_dir,
'%s_QE.fits' % sensor_id)
qe_data.write_fits_tables(fits_outfile)
def run(self, sensor_id, dark_files, mask_files, gains, bias_frame=None):
imutils.check_temperatures(dark_files,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
median_images = {}
for amp in imutils.allAmps(dark_files[0]):
median_images[amp] = imutils.fits_median(dark_files,
imutils.dm_hdu(amp))
medfile = os.path.join(self.config.output_dir,
'%s_median_dark_bp.fits' % sensor_id)
imutils.writeFits(median_images, medfile, dark_files[0])
ccd = MaskedCCD(medfile, mask_files=mask_files, bias_frame=bias_frame)
md = imutils.Metadata(dark_files[0], 1)
exptime = ccd.md.get('EXPTIME')
total_bright_pixels = 0
total_bright_columns = 0
if self.config.verbose:
self.log.info("Amp # bright pixels # bright columns")
#
# Write bright pixel and column counts to results file.
#
results_file = self.config.eotest_results_file
if results_file is None:
results_file = os.path.join(self.config.output_dir,
'%s_eotest_results.fits' % sensor_id)
results = EOTestResults(results_file, namps=len(ccd))
pixels = {}
columns = {}
for amp in ccd:
bright_pixels = BrightPixels(ccd, amp, exptime, gains[amp])
pixels[amp], columns[amp] = bright_pixels.find()
pix_count = len(pixels[amp])
col_count = len(columns[amp])
total_bright_pixels += pix_count
total_bright_columns += col_count
results.add_seg_result(amp, 'NUM_BRIGHT_PIXELS', pix_count)
results.add_seg_result(amp, 'NUM_BRIGHT_COLUMNS', col_count)
self.log.info("%2i %i %i" %
(amp, pix_count, col_count))
if self.config.verbose:
self.log.info("Total bright pixels: %i" % total_bright_pixels)
self.log.info("Total bright columns: %i" % total_bright_columns)
results.write(clobber=True)
# Generate the mask file based on the pixel and columns.
mask_file = os.path.join(self.config.output_dir,
'%s_bright_pixel_mask.fits' % sensor_id)
if os.path.isfile(mask_file):
os.remove(mask_file)
generate_mask(medfile,
mask_file,
self.config.mask_plane,
pixels=pixels,
columns=columns)
def eotest_check_input_data(rootdir='.', use_baselined=True):
"""
Check electro-optical datasets according to relevant specs in
LCA-128-E, LCA-10103-A, and LCA-10140-A?. Not all specs are tested.
rootdir is assumed to point to a directory that ends in the sensor
ID, i.e., it is of the form "<...>/NNN-MM". Checks are performed
only for files and directories below rootdir.
"""
errors = {'dirs': [], 'temperature': [], 'lambda': []}
full_path = lambda x: os.path.join(rootdir, x)
if use_baselined: # Use the baselined version in docushare
print "Assuming baselined version of LCA-10140-A (from docushare)"
test_types = ('dark', 'fe55', 'flat', 'lambda', 'spot',
'superflat_500', 'trap')
file_pattern = os.path.join('*', '*.fits')
else: # Use the version in the most recent revision (2014-04-11)
print "Assuming 2014-04-11 revision of LCA-10140-A"
test_types = ('dark', 'fe55', 'flat', 'lambda', 'spot', 'sflat_500',
'trap')
file_pattern = os.path.join('*', '*', '*.fits')
files = {}
#
# Check basic directory structure and glob for filenames as
# specified in LCA-10140-A.
#
for test in test_types:
target = full_path(test)
if not os.path.isdir(target):
what = "Expected test type subdir %(target)s not found" % locals()
errors['dirs'].append(what)
glob_target = os.path.join(rootdir, test, file_pattern)
files[test] = glob.glob(glob_target)
if len(files[test]) == 0:
what = "No files found for test type %(test)s in %(glob_target)s" \
% locals()
errors['dirs'].append(what)
#
# Check temperature ranges of datasets need for read noise (fe55),
# crosstalk (spot), dark current (dark), and QE (lambda) as
# specified in LCA-128-E.
#
for test in ('fe55', 'spot', 'dark', 'lambda'):
for infile in files[test]:
try:
imutils.check_temperatures([infile], 1., setpoint=-95)
except RuntimeError, eObj:
errors['temperature'].append(eObj.message)
def run(self, sensor_id, xtalk_files, mask_files, system_xtalk_file=None):
imutils.check_temperatures(xtalk_files,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
#
# Test if we have a single (and therefore multi-aggressor) file.
#
if len(xtalk_files) == 1:
xtalk_files = xtalk_files[0]
xtalk = make_crosstalk_matrix(xtalk_files, mask_files=mask_files)
if system_xtalk_file is not None:
system_xtalk_matrix = CrosstalkMatrix(system_xtalk_file)
xtalk = xtalk - system_xtalk_matrix
xtalk.write_fits(
os.path.join(self.config.output_dir,
'%s_xtalk_matrix.fits' % sensor_id))
def run(self, sensor_id, pre_flat_darks, flat, post_flat_darks, mask_files,
gains):
darks = list(pre_flat_darks) + list(post_flat_darks)
imutils.check_temperatures(darks,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
# Check that pre-flat dark frames all have the same exposure time
md = imutils.Metadata(pre_flat_darks[0], 1)
exptime = md.get('EXPTIME')
for item in pre_flat_darks[1:]:
md = imutils.Metadata(item, 1)
if exptime != md.get('EXPTIME'):
raise RuntimeError("Exposure times of pre-flat darks differ.")
# Make a median image of the preflat darks
median_images = {}
for amp in imutils.allAmps(darks[0]):
median_images[amp] = imutils.fits_median(pre_flat_darks,
imutils.dm_hdu(amp))
medfile = os.path.join(self.config.output_dir,
'%s_persistence_dark_median.fits' % sensor_id)
imutils.writeFits(median_images, medfile, darks[0])
ccd = MaskedCCD(medfile, mask_files=mask_files)
# Define the sub-region for assessing the deferred charge.
# This is the same bounding box for all segments, so use amp=1.
image = ccd.unbiased_and_trimmed_image(1)
xllc = ((image.getWidth() - self.config.region_size) / 2. -
self.config.region_x_offset)
yllc = ((image.getHeight() - self.config.region_size) / 2. -
self.config.region_y_offset)
imaging_reg = afwGeom.Box2I(
afwGeom.Point2I(int(xllc), int(yllc)),
afwGeom.Extent2I(self.config.region_size, self.config.region_size))
overscan = ccd.amp_geom.serial_overscan
# Compute reference dark current for each segment.
dc_ref = {}
for amp in ccd:
mi = imutils.unbias_and_trim(ccd[amp], overscan, imaging_reg)
dc_ref[amp] = afwMath.makeStatistics(mi, afwMath.MEDIAN,
ccd.stat_ctrl).getValue()
dc_ref[amp] *= gains[amp] / exptime
# Extract reference time for computing the time dependence
# of the deferred charge as the observation time + exposure time
# from the saturated flat.
tref = readout_time(flat)
# Loop over post-flat darks, compute median e-/pixel in
# subregion, subtract dc_ref*exptime, persist, and report the
# deferred charge vs time (using MJD-OBS + EXPTIME) for each amp.
deferred_charges = []
times = []
for dark in post_flat_darks:
ccd = MaskedCCD(dark, mask_files=mask_files)
dt = readout_time(dark) - tref
times.append(dt.sec)
exptime = ccd.md.get('EXPTIME')
charge = {}
for amp in ccd:
mi = imutils.unbias_and_trim(ccd[amp], overscan, imaging_reg)
estimators = afwMath.MEDIAN | afwMath.STDEV
stats = afwMath.makeStatistics(mi, estimators, ccd.stat_ctrl)
value = (stats.getValue(afwMath.MEDIAN) * gains[amp] -
dc_ref[amp] * exptime)
stdev = (stats.getValue(afwMath.STDEV) * gains[amp] -
dc_ref[amp] * exptime)
charge[amp] = (value, stdev)
deferred_charges.append(charge)
if self.config.verbose:
for amp in ccd:
self.log.info("amp: %i" % amp)
for i, time in enumerate(times):
self.log.info("%.1f %e %e" %
(time, deferred_charges[i][amp][0],
deferred_charges[i][amp][1]))
outfile = os.path.join(self.config.output_dir,
'%s_persistence.fits' % sensor_id)
self.write(times, deferred_charges, outfile, clobber=True)
def run(self,
sensor_id,
bias_files,
gains,
system_noise_files=None,
system_noise=None,
mask_files=(),
use_overscan=False):
all_amps = imutils.allAmps(bias_files[0])
imutils.check_temperatures(bias_files,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
outfiles = []
Ntot = NoiseDistributions(amps=all_amps)
Nsys = NoiseDistributions(amps=all_amps)
if system_noise_files is None:
system_noise_files = [None] * len(bias_files)
for i, bias, sysnoise in zip(range(len(bias_files)), bias_files,
system_noise_files):
outfile = "%s_read_noise_%03i.fits" % (sensor_id, i)
outfile = os.path.join(self.config.output_dir, outfile)
outfiles.append(outfile)
if self.config.verbose:
self.log.info("Processing %s %s -> %s" %
(bias, sysnoise, outfile))
# Determine the nominal imaging region from the bias file.
ccd = MaskedCCD(bias)
if use_overscan:
imaging = ccd.amp_geom.serial_overscan
dx = imaging.getWidth() / 2
dy = self.config.dy
nsamp = self.config.nsamp
else:
imaging = ccd.amp_geom.imaging
dx = self.config.dx
dy = self.config.dy
nsamp = self.config.nsamp
#
# Create a single sub-region sampler so that the same
# sub-regions will be used for both the bias and system
# noise frames.
#
sampler = imutils.SubRegionSampler(dx, dy, nsamp, imaging=imaging)
Ntot_amp = noise_dists(bias, gains, sampler, mask_files=mask_files)
Nsys_amp = noise_dists(sysnoise,
gains,
sampler,
mask_files=mask_files)
_write_read_noise_dists(outfile, Ntot_amp, Nsys_amp, gains, bias,
sysnoise)
#
# Accumulate noise distributions for final median calculation
#
Ntot.append(Ntot_amp)
Nsys.append(Nsys_amp)
results_file = self.config.eotest_results_file
if results_file is None:
results_file = os.path.join(self.config.output_dir,
'%s_eotest_results.fits' % sensor_id)
results = EOTestResults(results_file, namps=len(ccd))
if self.config.verbose:
self.log.info("Amp read noise total noise system noise")
for amp in ccd:
Ntot_med = imutils.median(Ntot[amp])
if system_noise is not None:
Nsys_med = float(system_noise[amp])
else:
Nsys_med = imutils.median(Nsys[amp])
var = Ntot_med**2 - Nsys_med**2
if var >= 0:
Nread = np.sqrt(var)
else:
Nread = -1
line = "%2s %7.4f %7.4f %7.4f" % (
amp, Nread, Ntot_med, Nsys_med)
if self.config.verbose:
self.log.info(line)
results.add_seg_result(amp, 'READ_NOISE', Nread)
results.add_seg_result(amp, 'TOTAL_NOISE', Ntot_med)
results.add_seg_result(amp, 'SYSTEM_NOISE', Nsys_med)
results.write(clobber=True)
def run(self, sensor_id, dark_files, mask_files, gains, bias_frame=None):
imutils.check_temperatures(dark_files,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
median_images = {}
md = imutils.Metadata(dark_files[0], 1)
for amp in imutils.allAmps(dark_files[0]):
median_images[amp] = imutils.fits_median(dark_files,
imutils.dm_hdu(amp))
medfile = os.path.join(self.config.output_dir,
'%s_median_dark_current.fits' % sensor_id)
imutils.writeFits(median_images, medfile, dark_files[0])
ccd = MaskedCCD(medfile, mask_files=mask_files, bias_frame=bias_frame)
dark95s = {}
exptime = md.get('EXPTIME')
if self.config.verbose:
self.log.info("Amp 95 percentile median")
dark_curr_pixels = []
dark_curr_pixels_per_amp = {}
for amp in ccd:
imaging_region = ccd.amp_geom.imaging
overscan = ccd.amp_geom.serial_overscan
image = imutils.unbias_and_trim(ccd[amp].getImage(), overscan,
imaging_region)
mask = imutils.trim(ccd[amp].getMask(), imaging_region)
imarr = image.getArray()
mskarr = mask.getArray()
pixels = imarr.reshape(1, imarr.shape[0] * imarr.shape[1])[0]
masked = mskarr.reshape(1, mskarr.shape[0] * mskarr.shape[1])[0]
unmasked = [
pixels[i] for i in range(len(pixels)) if masked[i] == 0
]
unmasked.sort()
unmasked = np.array(unmasked) * gains[amp] / exptime
dark_curr_pixels_per_amp[amp] = unmasked
dark_curr_pixels.extend(unmasked)
try:
dark95s[amp] = unmasked[int(len(unmasked) * 0.95)]
median = unmasked[len(unmasked) / 2]
except IndexError as eobj:
print str(eobj)
dark95s[amp] = -1.
median = -1.
if self.config.verbose:
self.log.info("%2i %.2e %.2e" %
(amp, dark95s[amp], median))
#
# Compute 95th percentile dark current for CCD as a whole.
#
dark_curr_pixels = sorted(dark_curr_pixels)
darkcurr95 = dark_curr_pixels[int(len(dark_curr_pixels) * 0.95)]
dark95mean = np.mean(dark95s.values())
if self.config.verbose:
#self.log.info("CCD: mean 95 percentile value = %s" % dark95mean)
self.log.info("CCD-wide 95 percentile value = %s" % darkcurr95)
#
# Update header of dark current median image file with dark
# files used and dark95 values, and write dark95 values to the
# eotest results file.
#
results_file = self.config.eotest_results_file
if results_file is None:
results_file = os.path.join(self.config.output_dir,
'%s_eotest_results.fits' % sensor_id)
results = EOTestResults(results_file, namps=len(ccd))
output = fits.open(medfile)
for i, dark in enumerate(dark_files):
output[0].header['DARK%02i' % i] = os.path.basename(dark)
# Write overall dark current 95th percentile
results.output['AMPLIFIER_RESULTS'].header['DARK95'] = darkcurr95
for amp in ccd:
output[0].header['DARK95%s' %
imutils.channelIds[amp]] = dark95s[amp]
results.add_seg_result(amp, 'DARK_CURRENT_95', dark95s[amp])
fitsWriteto(output, medfile, clobber=True, checksum=True)
results.write(clobber=True)
return dark_curr_pixels_per_amp, dark95s
def eotest_check_input_data(rootdir='.', use_baselined=True):
"""
Check electro-optical datasets according to relevant specs in
LCA-128-E, LCA-10103-A, and LCA-10140-A?. Not all specs are tested.
rootdir is assumed to point to a directory that ends in the sensor
ID, i.e., it is of the form "<...>/NNN-MM". Checks are performed
only for files and directories below rootdir.
"""
errors = {'dirs': [], 'temperature': [], 'lambda': []}
def full_path(x):
return os.path.join(rootdir, x)
if use_baselined: # Use the baselined version in docushare
print("Assuming baselined version of LCA-10140-A (from docushare)")
test_types = ('dark', 'fe55', 'flat', 'lambda', 'spot',
'superflat_500', 'trap')
file_pattern = os.path.join('*', '*.fits')
else: # Use the version in the most recent revision (2014-04-11)
print("Assuming 2014-04-11 revision of LCA-10140-A")
test_types = ('dark', 'fe55', 'flat', 'lambda', 'spot', 'sflat_500',
'trap')
file_pattern = os.path.join('*', '*', '*.fits')
files = {}
#
# Check basic directory structure and glob for filenames as
# specified in LCA-10140-A.
#
for test in test_types:
target = full_path(test)
if not os.path.isdir(target):
what = "Expected test type subdir %(target)s not found" % locals()
errors['dirs'].append(what)
glob_target = os.path.join(rootdir, test, file_pattern)
files[test] = glob.glob(glob_target)
if len(files[test]) == 0:
what = "No files found for test type %(test)s in %(glob_target)s" \
% locals()
errors['dirs'].append(what)
#
# Check temperature ranges of datasets need for read noise (fe55),
# crosstalk (spot), dark current (dark), and QE (lambda) as
# specified in LCA-128-E.
#
for test in ('fe55', 'spot', 'dark', 'lambda'):
for infile in files[test]:
try:
imutils.check_temperatures([infile], 1., setpoint=-95)
except RuntimeError as eObj:
errors['temperature'].append(eObj.message)
#
# Check for required wavelengths for QE and PRNU as specified in
# LCA-128-E.
#
required_wls = (330, 350, 370, 450, 500, 620, 750, 870, 1000)
acquired_wls = []
for item in files['lambda']:
wl = int(np.round(fits.open(item)[0].header['MONOWL']))
acquired_wls.append(wl)
for target_wl in required_wls:
if target_wl not in acquired_wls:
what = 'Flat at %(target_wl)s nm missing from lambda dataset' \
% locals()
errors['lambda'].append(what)
#
# Print summary of failures.
#
for test in errors:
if errors[test]:
print("%i error(s) of type %s:" % (len(errors[test]), test))
for message in errors[test]:
print(" ", message)
print()
def run(self,
sensor_id,
infiles,
mask_files,
bias_frame=None,
fe55_catalog=None,
minClustersPerAmp=None,
chiprob_min=0.1,
accuracy_req=0,
hist_nsig=10,
linearity_correction=None):
imutils.check_temperatures(infiles,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
if self.config.verbose and fe55_catalog is None:
self.log.info("Input files:")
for item in infiles:
self.log.info(" %s" % item)
#
# Detect and fit 2D Gaussian to Fe55 charge clusters,
# accumulating the results by amplifier.
#
fitter = PsfGaussFit(nsig=self.config.nsig, fit_xy=self.config.fit_xy)
gains, gain_errors, sigma_modes = {}, {}, {}
if fe55_catalog is None:
for infile in infiles:
if self.config.verbose:
self.log.info("processing %s" % infile)
ccd = MaskedCCD(infile,
mask_files=mask_files,
bias_frame=bias_frame,
linearity_correction=linearity_correction)
for amp in ccd:
if self.config.verbose:
self.log.info(" amp %i" % amp)
if amp in gain_errors:
gain_accuracy = np.abs(gain_errors[amp] / gains[amp])
if self.config.verbose:
message = " Relative gain accuracy, dgain/gain " \
+ "= %.2e" % gain_accuracy
self.log.info(message)
if gain_accuracy < accuracy_req:
# Requested accuracy already obtained, so
# skip cluster fitting.
continue
fitter.process_image(ccd, amp, logger=self.log)
gains, gain_errors, sigma_modes = \
self.fit_gains(fitter, gains, gain_errors, sigma_modes,
amps=[amp], hist_nsig=hist_nsig)
if self.config.output_file is None:
psf_results = os.path.join(
self.config.output_dir, '%s_psf_results_nsig%i.fits' %
(sensor_id, self.config.nsig))
else:
psf_results = self.config.output_file
if self.config.verbose:
self.log.info("Writing psf results file to %s" % psf_results)
fitter.write_results(outfile=psf_results)
namps = len(ccd)
else:
fitter.read_fe55_catalog(fe55_catalog)
namps = fits.open(fe55_catalog)[0].header['NAMPS']
results_file = self.config.eotest_results_file
if results_file is None:
results_file = os.path.join(self.config.output_dir,
'%s_eotest_results.fits' % sensor_id)
if self.config.verbose:
self.log.info("Writing gain and psf sigma results to %s" %
results_file)
results = EOTestResults(results_file, namps=namps)
for amp in gains:
results.add_seg_result(amp, 'GAIN', gains[amp])
results.add_seg_result(amp, 'GAIN_ERROR', gain_errors[amp])
results.add_seg_result(amp, 'PSF_SIGMA', sigma_modes[amp])
results.write(clobber=True)
return gains