def validate_dark_defects(results, det_names):
"""Validate and persist dark defects results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
mask_file = '%s_dark_pixel_mask.fits' % file_prefix
if not os.path.isfile(mask_file):
missing_det_names.append(det_name)
continue
eotestUtils.addHeaderData(mask_file,
TESTTYPE='SFLAT_500',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(mask_file))
superflat = '%s_median_sflat.fits' % file_prefix
eotestUtils.addHeaderData(superflat,
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(superflat))
eotest_results = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(eotest_results)
amps = data['AMP']
npixels = data['NUM_DARK_PIXELS']
ncolumns = data['NUM_DARK_COLUMNS']
for amp, npix, ncol in zip(amps, npixels, ncolumns):
results.append(
lcatr.schema.valid(lcatr.schema.get('dark_defects_BOT'),
amp=amp,
dark_pixels=npix,
dark_columns=ncol,
slot=slot,
raft=raft))
# Persist the png files.
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='SFLAT_500',
TEST_CATEGORY='EO')
filename = '%s_superflat_dark_defects.png' % file_prefix
results.extend(
siteUtils.persist_png_files(filename,
file_prefix,
metadata=metadata))
report_missing_data("validate_dark_defects", missing_det_names)
return results
def validate_traps(results, det_names):
"""Validate and persist trap results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
trap_file = '%s_traps.fits' % file_prefix
if not os.path.isfile(trap_file):
missing_det_names.append(det_name)
continue
eotestUtils.addHeaderData(trap_file,
TESTTYPE='TRAP',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(trap_file))
mask_file = '%s_traps_mask.fits' % file_prefix
results.append(siteUtils.make_fileref(mask_file))
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
num_traps = data['NUM_TRAPS']
for amp, ntrap in zip(amps, num_traps):
results.append(
lcatr.schema.valid(lcatr.schema.get('traps_BOT'),
amp=amp,
num_traps=ntrap,
slot=slot,
raft=raft))
report_missing_data("validate_traps", missing_det_names)
return results
def persist_fe55_gains():
"""Persist only the Fe55 gains from the results file."""
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
# Save eotest results file with nominal gains.
ccd_vendor = sensor_id.split('-')[0].upper()
results_file = '%s_eotest_results.fits' % sensor_id
eotestUtils.addHeaderData(results_file,
LSST_NUM=sensor_id,
TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
results.append(lcatr.schema.fileref.make(results_file))
# Persist nominal values to eT results database.
amps = imutils.allAmps()
gain_data = np.ones(len(amps))
gain_errors = np.zeros(len(amps))
sigmas = np.zeros(len(amps))
for amp, gain_value, gain_error, sigma in zip(amps, gain_data,
gain_errors, sigmas):
if not np.isfinite(gain_error):
gain_error = -1
results.append(
lcatr.schema.valid(lcatr.schema.get('fe55_raft_analysis'),
amp=amp,
gain=gain_value,
gain_error=gain_error,
psf_sigma=sigma,
slot=slot,
sensor_id=sensor_id))
return results
def validate_bias_frame(results, det_names):
"""Validate and persist medianed bias frames."""
run = siteUtils.getRunNumber()
missing_det_names = set()
for det_name in det_names:
file_prefix = make_file_prefix(run, det_name)
bias_frame = f'{file_prefix}_median_bias.fits'
rolloff_mask = f'{file_prefix}_edge_rolloff_mask.fits'
pca_bias_file = f'{file_prefix}_pca_bias.fits'
pca_superbias = f'{file_prefix}_pca_superbias.fits'
# Add/update the metadata to the primary HDU of these files.
for fitsfile in (bias_frame, rolloff_mask, pca_bias_file,
pca_superbias):
if os.path.isfile(fitsfile):
eotestUtils.addHeaderData(fitsfile,
TESTTYPE='BIAS',
DATE=eotestUtils.utc_now_isoformat())
results.append(lcatr.schema.fileref.make(fitsfile))
else:
missing_det_names.add(det_name)
# Persist the PCA bias model file.
pca_bias_model = f'{file_prefix}_pca_bias.pickle'
if os.path.isfile(pca_bias_model):
results.append(lcatr.schema.fileref.make(pca_bias_model))
else:
missing_det_names.add(det_name)
report_missing_data('validate_bias_frames', missing_det_names)
return results
def persist_traps_analysis():
"""Persist the results from the traps analysis."""
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
ccd_vendor = sensor_id.split('-')[0].upper()
trap_file = '%s_traps.fits' % sensor_id
eotestUtils.addHeaderData(trap_file,
LSST_NUM=sensor_id,
TESTTYPE='TRAP',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
results.append(siteUtils.make_fileref(trap_file, folder=slot))
mask_file = '%s_traps_mask.fits' % sensor_id
results.append(siteUtils.make_fileref(mask_file, folder=slot))
results_file = '%s_eotest_results.fits' % sensor_id
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
num_traps = data['NUM_TRAPS']
for amp, ntrap in zip(amps, num_traps):
results.append(
lcatr.schema.valid(lcatr.schema.get('traps_raft'),
amp=amp,
num_traps=ntrap,
slot=slot,
sensor_id=sensor_id))
return results
def validate_qe(results, det_names):
"""Validate the QE results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
qe_results_file = '%s_QE.fits' % file_prefix
if not os.path.isfile(qe_results_file):
missing_det_names.append(det_name)
continue
with fits.open(qe_results_file) as qe_results:
qe_data = qe_results['QE_BANDS'].data
QE = OrderedDict((band, []) for band in qe_data.field('BAND'))
for amp in range(1, 17):
values = qe_data.field('AMP%02i' % amp)
for band, value in zip(QE, values):
QE[band].append(value)
for band in QE:
for amp in range(1, 17):
results.append(
lcatr.schema.valid(lcatr.schema.get('qe_BOT_analysis'),
band=band,
QE=QE[band][amp - 1],
amp=amp,
slot=slot,
raft=raft))
qe_files = glob.glob('%s_*QE*.fits' % file_prefix)
for item in qe_files:
eotestUtils.addHeaderData(item,
TESTTYPE='LAMBDA',
DATE=eotestUtils.utc_now_isoformat())
results.extend([siteUtils.make_fileref(item) for item in qe_files])
# Persist the png files.
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='LAMBDA',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('%s*qe.png' % file_prefix,
file_prefix,
metadata=metadata))
results.extend(
siteUtils.persist_png_files('%s*flat.png' % file_prefix,
file_prefix,
metadata=metadata))
report_missing_data("validate_qe", missing_det_names)
return results
def validate_brighter_fatter(results, det_names):
"""Validate the brighter-fatter results."""
run = siteUtils.getRunNumber()
missing_det_names = set()
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
bf_results = '%s_bf.fits' % file_prefix
if not os.path.isfile(bf_results):
missing_det_names.add(det_name)
continue
eotestUtils.addHeaderData(bf_results,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(bf_results))
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
columns = (data['AMP'], data['BF_XCORR'], data['BF_XCORR_ERR'],
data['BF_YCORR'], data['BF_YCORR_ERR'], data['BF_SLOPEX'],
data['BF_SLOPEX_ERR'], data['BF_SLOPEY'],
data['BF_SLOPEY_ERR'], data['BF_MEAN'])
for amp, bf_xcorr, bf_xcorr_err, bf_ycorr, bf_ycorr_err, \
bf_slopex, bf_slopex_err, bf_slopey, bf_slopey_err, bf_mean \
in zip(*columns):
results.append(
lcatr.schema.valid(lcatr.schema.get('brighter_fatter_BOT'),
amp=amp,
bf_xcorr=bf_xcorr,
bf_xcorr_err=bf_xcorr_err,
bf_ycorr=bf_ycorr,
bf_ycorr_err=bf_ycorr_err,
bf_slopex=bf_slopex,
bf_slopex_err=bf_slopex_err,
bf_slopey=bf_slopey,
bf_slopey_err=bf_slopey_err,
bf_mean=bf_mean,
slot=slot,
raft=raft))
# Persist the png files.
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='FLAT',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('%s*brighter-fatter.png' % file_prefix,
file_prefix,
metadata=metadata))
return results
def validate_read_noise(results, det_names):
"""Validate and persist read noise results."""
run = siteUtils.getRunNumber()
missing_det_names = set()
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
read_noise_file = '%s_eotest_results.fits' % file_prefix
if not os.path.isfile(read_noise_file):
# No data for this detector, so note that and continue
# with the others.
missing_det_names.add(det_name)
continue
data = sensorTest.EOTestResults(read_noise_file)
amps = data['AMP']
read_noise_data = data['READ_NOISE']
system_noise_data = data['SYSTEM_NOISE']
total_noise_data = data['TOTAL_NOISE']
for amp, read_noise, system_noise, total_noise \
in zip(amps, read_noise_data, system_noise_data, total_noise_data):
results.append(lcatr.schema.valid(
lcatr.schema.get('read_noise_BOT'),
amp=amp, read_noise=read_noise, system_noise=system_noise,
total_noise=total_noise, slot=slot, raft=raft))
files = glob.glob('%s_read_noise?*.fits' % file_prefix)
for fitsfile in files:
eotestUtils.addHeaderData(fitsfile, TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat())
data_products = [siteUtils.make_fileref(item) for item in files]
results.extend(data_products)
# Persist the png files.
metadata = dict(DETECTOR=det_name, TESTTYPE='FE55', TEST_CATEGORY='EO',
RUN=run)
filename = '%s_correlated_noise.png' % file_prefix
results.extend(siteUtils.persist_png_files(filename, file_prefix,
metadata=metadata))
# Persist the raft-level overscan correlation plots.
for raft in camera_info.get_installed_raft_names():
metadata = dict(TESTTYPE='FE55', TEST_CATEGORY='EO', RAFT=raft, RUN=run)
file_prefix = make_file_prefix(run, raft)
filename = '%s_overscan_correlations.png' % file_prefix
results.extend(siteUtils.persist_png_files(filename, file_prefix,
metadata=metadata))
report_missing_data("validate_read_noise", missing_det_names)
return results
def validate_ptc(results, det_names):
"""Validate the PTC results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
ptc_results = '%s_ptc.fits' % file_prefix
if not os.path.isfile(ptc_results):
missing_det_names.append(det_name)
continue
eotestUtils.addHeaderData(ptc_results,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(ptc_results))
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
columns = (data['AMP'], data['PTC_GAIN'], data['PTC_GAIN_ERROR'],
data['PTC_A00'], data['PTC_A00_ERROR'], data['PTC_NOISE'],
data['PTC_NOISE_ERROR'], data['PTC_TURNOFF'])
for amp, gain, gain_error, a00, a00_error,\
noise, noise_error, turnoff in zip(*columns):
results.append(
lcatr.schema.valid(lcatr.schema.get('ptc_BOT'),
amp=amp,
ptc_gain=gain,
ptc_gain_error=gain_error,
ptc_a00=a00,
ptc_a00_error=a00_error,
ptc_noise=noise,
ptc_noise_error=noise_error,
ptc_turnoff=turnoff,
slot=slot,
raft=raft))
# Persist the png files.
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='FLAT',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('%s*ptcs.png' % file_prefix,
file_prefix,
metadata=metadata))
report_missing_data("validate_ptc", missing_det_names)
return results
def validate_flat_pairs(results, det_names):
"""Validate the flat pair analysis results."""
run = siteUtils.getRunNumber()
missing_det_names = set()
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
det_resp_data = '%s_det_response.fits' % file_prefix
if not os.path.isfile(det_resp_data):
missing_det_names.add(det_name)
continue
eotestUtils.addHeaderData(det_resp_data, DETECTOR=det_name,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(det_resp_data))
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
max_observed_signal_data = data['MAX_OBSERVED_SIGNAL']
max_frac_dev_data = data['MAX_FRAC_DEV']
row_mean_var_slope_data = data['ROW_MEAN_VAR_SLOPE']
linearity_turnoff_data = data['LINEARITY_TURNOFF']
for amp, max_observed_signal, max_frac_dev, row_mean_var_slope, \
linearity_turnoff in zip(amps, max_observed_signal_data,
max_frac_dev_data,
row_mean_var_slope_data,
linearity_turnoff_data):
results.append(lcatr.schema.valid(
lcatr.schema.get('flat_pairs_BOT'),
amp=amp, max_observed_signal=max_observed_signal,
max_frac_dev=max_frac_dev,
row_mean_var_slope=row_mean_var_slope,
linearity_turnoff=linearity_turnoff, slot=slot, raft=raft))
# Persist the png files.
metadata = dict(DETECTOR=det_name, RUN=run,
TESTTYPE='FLAT', TEST_CATEGORY='EO')
results.extend(siteUtils.persist_png_files(('%s_linearity*.png'
% file_prefix),
file_prefix,
metadata=metadata))
results.extend(siteUtils.persist_png_files(('%s_row_means_variance.png'
% file_prefix),
file_prefix,
metadata=metadata))
report_missing_data("validate_flat_pairs", missing_det_names)
return results
def validate_flat_pairs(results, det_names):
"""Validate the flat pair analysis results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
det_resp_data = '%s_det_response.fits' % file_prefix
if not os.path.isfile(det_resp_data):
missing_det_names.append(det_name)
continue
eotestUtils.addHeaderData(det_resp_data,
DETECTOR=det_name,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(det_resp_data))
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
full_well_data = data['FULL_WELL']
max_frac_dev_data = data['MAX_FRAC_DEV']
for amp, full_well, max_frac_dev in zip(amps, full_well_data,
max_frac_dev_data):
results.append(
lcatr.schema.valid(lcatr.schema.get('flat_pairs_BOT'),
amp=amp,
full_well=full_well,
max_frac_dev=max_frac_dev,
slot=slot,
raft=raft))
# Persist the png files.
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='FLAT',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files(('%s_linearity*.png' % file_prefix),
file_prefix,
metadata=metadata))
report_missing_data("validate_flat_pairs", missing_det_names)
return results
def validate_raft_results(results, raft_names):
"""Validate the raft level results."""
run = siteUtils.getRunNumber()
slot_names = camera_info.get_slot_names()
md = siteUtils.DataCatalogMetadata(ORIGIN=siteUtils.getSiteName(),
TEST_CATEGORY='EO',
DATA_PRODUCT='EOTEST_RESULTS')
missing_raft_names = []
for raft_name in raft_names:
for slot_name in slot_names:
det_name = make_file_prefix(raft_name, slot_name)
file_prefix = make_file_prefix(run, det_name)
results_file = '{}_eotest_results.fits'.format(file_prefix)
if not os.path.isfile(results_file):
if raft_name not in missing_raft_names:
missing_raft_names.append(raft_name)
continue
eotestUtils.addHeaderData(results_file,
DETECTOR=det_name,
DATE=eotestUtils.utc_now_isoformat(),
RUNNUM=run)
results.append(
lcatr.schema.fileref.make(results_file,
metadata=md(SLOT=slot_name,
RAFT=raft_name)))
# Persist the png files.
png_file_list = '{}_raft_results_task_png_files.txt'.format(raft_name)
if not os.path.isfile(png_file_list):
continue
with open(png_file_list, 'r') as input_:
png_files = [x.strip() for x in input_]
metadata = dict(TEST_CATEGORY='EO', DETECTOR=det_name, RUN=run)
results.extend(
siteUtils.persist_png_files('',
file_prefix,
png_files=png_files,
metadata=metadata))
report_missing_data("validate_raft_results",
missing_raft_names,
components='rafts',
total=21)
return results
def validate_cte(results, det_names):
"""Validate the CTE task results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
superflats \
= sorted(glob.glob('{}_superflat_*.fits'.format(file_prefix)))
if not superflats:
missing_det_names.append(det_name)
continue
for item in superflats:
eotestUtils.addHeaderData(item,
FILENAME=item,
DATE=eotestUtils.utc_now_isoformat())
results.extend([siteUtils.make_fileref(x) for x in superflats])
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
cti_high_serial = data['CTI_HIGH_SERIAL']
cti_high_serial_error = data['CTI_HIGH_SERIAL_ERROR']
cti_high_parallel = data['CTI_HIGH_PARALLEL']
cti_high_parallel_error = data['CTI_HIGH_PARALLEL_ERROR']
cti_low_serial = data['CTI_LOW_SERIAL']
cti_low_serial_error = data['CTI_LOW_SERIAL_ERROR']
cti_low_parallel = data['CTI_LOW_PARALLEL']
cti_low_parallel_error = data['CTI_LOW_PARALLEL_ERROR']
for values in zip(amps, cti_high_serial, cti_high_serial_error,
cti_high_parallel, cti_high_parallel_error,
cti_low_serial, cti_low_serial_error,
cti_low_parallel, cti_low_parallel_error):
results.append(
lcatr.schema.valid(lcatr.schema.get('cte_BOT'),
amp=values[0],
cti_high_serial=values[1],
cti_high_serial_error=values[2],
cti_high_parallel=values[3],
cti_high_parallel_error=values[4],
cti_low_serial=values[5],
cti_low_serial_error=values[6],
cti_low_parallel=values[7],
cti_low_parallel_error=values[8],
slot=slot,
raft=raft))
# Persist the png files.
png_file_list = '{}_cte_task_png_files.txt'.format(det_name)
with open(png_file_list, 'r') as input_:
png_files = [x.strip() for x in input_]
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='SFLAT_500',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('',
file_prefix,
png_files=png_files,
metadata=metadata))
report_missing_data("validate_cte", missing_det_names)
return results
#!/usr/bin/env python
import lsst.eotest.sensor as sensorTest
import lcatr.schema
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
mask_file = '%s_dark_pixel_mask.fits' % sensor_id
eotestUtils.addHeaderData(mask_file,
LSST_NUM=sensor_id,
TESTTYPE='SFLAT_500',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results = [lcatr.schema.fileref.make(mask_file)]
superflat = '%s_median_sflat.fits' % sensor_id
eotestUtils.addHeaderData(superflat, DATE=eotestUtils.utc_now_isoformat())
results.append(lcatr.schema.fileref.make(superflat))
eotest_results = '%s_eotest_results.fits' % sensor_id
data = sensorTest.EOTestResults(eotest_results)
amps = data['AMP']
npixels = data['NUM_DARK_PIXELS']
ncolumns = data['NUM_DARK_COLUMNS']
for amp, npix, ncol in zip(amps, npixels, ncolumns):
results.append(
lcatr.schema.valid(lcatr.schema.get('dark_defects'),
amp=amp,
dark_pixels=npix,
dark_columns=ncol))
def persist_fe55_analysis_results():
"""Persist the results from the full analysis."""
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
ccd_vendor = sensor_id.split('-')[0].upper()
# The output files from producer script.
gain_file = '%(sensor_id)s_eotest_results.fits' % locals()
psf_results = glob.glob('%(sensor_id)s_psf_results*.fits' %
locals())[0]
rolloff_mask = '%(sensor_id)s_rolloff_defects_mask.fits' % locals()
output_files = gain_file, psf_results, rolloff_mask
# Add/update the metadata to the primary HDU of these files.
for fitsfile in output_files:
eotestUtils.addHeaderData(fitsfile,
LSST_NUM=sensor_id,
TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
#
# Persist the median bias FITS file.
#
bias_median_file = glob.glob(f'{sensor_id}_*_median_bias.fits')[0]
results.append(siteUtils.make_fileref(bias_median_file, folder=slot))
# Persist the png files.
metadata = dict(CCD_MANU=ccd_vendor,
LSST_NUM=sensor_id,
TESTTYPE='FE55',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('%s*.png' % sensor_id,
sensor_id,
folder=slot,
metadata=metadata))
data = sensorTest.EOTestResults(gain_file)
amps = data['AMP']
gain_data = data['GAIN']
gain_errors = data['GAIN_ERROR']
sigmas = data['PSF_SIGMA']
for amp, gain_value, gain_error, sigma in zip(amps, gain_data,
gain_errors, sigmas):
if not np.isfinite(gain_error):
gain_error = -1
results.append(
lcatr.schema.valid(lcatr.schema.get('fe55_raft_analysis'),
amp=amp,
gain=gain_value,
gain_error=gain_error,
psf_sigma=sigma,
slot=slot,
sensor_id=sensor_id))
results.extend([lcatr.schema.fileref.make(x) for x in output_files])
return results
#!/usr/bin/env python
import lsst.eotest.sensor as sensorTest
import lcatr.schema
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
mask_file = '%s_dark_pixel_mask.fits' % sensor_id
eotestUtils.addHeaderData(mask_file, LSST_NUM=sensor_id, TESTTYPE='SFLAT_500',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results = [lcatr.schema.fileref.make(mask_file)]
superflat = '%s_median_sflat.fits' % sensor_id
eotestUtils.addHeaderData(superflat, DATE=eotestUtils.utc_now_isoformat())
results.append(lcatr.schema.fileref.make(superflat))
eotest_results = '%s_eotest_results.fits' % sensor_id
data = sensorTest.EOTestResults(eotest_results)
amps = data['AMP']
npixels = data['NUM_DARK_PIXELS']
ncolumns = data['NUM_DARK_COLUMNS']
for amp, npix, ncol in zip(amps, npixels, ncolumns):
results.append(lcatr.schema.valid(lcatr.schema.get('dark_defects'),
amp=amp,
dark_pixels=npix,
dark_columns=ncol))
results.extend(siteUtils.jobInfo())
import glob
import lsst.eotest.sensor as sensorTest
import lcatr.schema
import siteUtils
import eotestUtils
import camera_components
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
ccd_vendor = sensor_id.split('-')[0].upper()
mask_file = '%s_bright_pixel_mask.fits' % sensor_id
eotestUtils.addHeaderData(mask_file, LSST_NUM=sensor_id, TESTTYPE='DARK',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
results.append(siteUtils.make_fileref(mask_file, folder=slot))
bias_frame = glob.glob(f'{sensor_id}_*_median_bias.fits')[0]
eotestUtils.addHeaderData(bias_frame, DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(bias_frame, folder=slot))
medianed_dark = '%s_median_dark_bp.fits' % sensor_id
eotestUtils.addHeaderData(medianed_dark,
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(medianed_dark, folder=slot))
eotest_results = '%s_eotest_results.fits' % sensor_id
data = sensorTest.EOTestResults(eotest_results)
amps = data['AMP']
def validate_flat_pairs(results, det_names):
"""Validate the flat pair analysis results."""
run = siteUtils.getRunNumber()
missing_det_names = set()
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
det_resp_data = '%s_det_response.fits' % file_prefix
if not os.path.isfile(det_resp_data):
missing_det_names.add(det_name)
continue
eotestUtils.addHeaderData(det_resp_data,
DETECTOR=det_name,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(det_resp_data))
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
max_observed_signal_data = data['MAX_OBSERVED_SIGNAL']
max_frac_dev_data = data['MAX_FRAC_DEV']
row_mean_var_slope_data = data['ROW_MEAN_VAR_SLOPE']
linearity_turnoff_data = data['LINEARITY_TURNOFF']
for amp, max_observed_signal, max_frac_dev, row_mean_var_slope, \
linearity_turnoff in zip(amps, max_observed_signal_data,
max_frac_dev_data,
row_mean_var_slope_data,
linearity_turnoff_data):
results.append(
lcatr.schema.valid(lcatr.schema.get('flat_pairs_BOT'),
amp=amp,
max_observed_signal=max_observed_signal,
max_frac_dev=max_frac_dev,
row_mean_var_slope=row_mean_var_slope,
linearity_turnoff=linearity_turnoff,
slot=slot,
raft=raft))
# Persist the png files.
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='FLAT',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files(('%s_linearity*.png' % file_prefix),
file_prefix,
metadata=metadata))
results.extend(
siteUtils.persist_png_files(
('%s_row_means_variance.png' % file_prefix),
file_prefix,
metadata=metadata))
# Persist the raft-level imaging region correlation plots.
missing_raft_names = set()
for raft in camera_info.get_installed_raft_names():
metadata = dict(TESTTYPE='FLAT',
TEST_CATEGORY='EO',
RAFT=raft,
RUN=run)
file_prefix = make_file_prefix(run, raft)
filename = f'{file_prefix}_imaging_region_correlations.png'
if not os.path.isfile(filename):
missing_raft_names.add(raft)
continue
results.extend(
siteUtils.persist_png_files(filename,
file_prefix,
metadata=metadata))
# Persist any pd correction file specified in the lcatr.cfg file.
pd_corrections_file_env = 'LCATR_PD_CORRECTIONS_FILE'
if pd_corrections_file_env in os.environ:
pd_corrections_file = os.environ[pd_corrections_file_env]
shutil.copy(pd_corrections_file, '.')
fileref = siteUtils.make_fileref(os.path.basename(pd_corrections_file))
results.append(fileref)
report_missing_data("validate_flat_pairs", missing_det_names)
report_missing_data("validate_flat_pairs",
sorted(list(missing_raft_names)),
components='rafts',
total=21)
return results
def validate_fe55(results, det_names):
"""Validate and persist fe55 gain and psf results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
# The output files from producer script.
gain_file = '%(file_prefix)s_eotest_results.fits' % locals()
psf_results_files \
= glob.glob('%(file_prefix)s_psf_results*.fits' % locals())
if not os.path.isfile(gain_file) or not psf_results_files:
# Results for this detector are not available so note
# that and continue with the others.
missing_det_names.append(det_name)
continue
psf_results = psf_results_files[0]
rolloff_mask = '%(file_prefix)s_edge_rolloff_mask.fits' % locals()
output_files = psf_results, rolloff_mask
# Add/update the metadata to the primary HDU of these files.
for fitsfile in output_files:
eotestUtils.addHeaderData(fitsfile,
TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat())
results.extend([lcatr.schema.fileref.make(x) for x in output_files])
# Persist the median bias FITS file.
bias_frame \
= glob.glob('%(file_prefix)s_median_bias.fits' % locals())[0]
results.append(siteUtils.make_fileref(bias_frame))
# Persist the png files.
png_file_list = '{}_fe55_task_png_files.txt'.format(det_name)
with open(png_file_list, 'r') as input_:
png_files = [x.strip() for x in input_]
metadata = dict(TESTTYPE='FE55',
TEST_CATEGORY='EO',
DETECTOR=det_name,
RUN=run)
results.extend(
siteUtils.persist_png_files('',
file_prefix,
png_files=png_files,
metadata=metadata))
data = sensorTest.EOTestResults(gain_file)
amps = data['AMP']
gain_data = data['GAIN']
gain_errors = data['GAIN_ERROR']
sigmas = data['PSF_SIGMA']
for amp, gain_value, gain_error, sigma in zip(amps, gain_data,
gain_errors, sigmas):
if not np.isfinite(gain_error):
gain_error = -1
results.append(
lcatr.schema.valid(lcatr.schema.get('fe55_BOT_analysis'),
amp=amp,
gain=gain_value,
gain_error=gain_error,
psf_sigma=sigma,
slot=slot,
raft=raft))
report_missing_data('validate_fe55', missing_det_names)
return results
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
print("Processing:", slot, sensor_id)
if 'ccd2' in slot :
continue
wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id]
ccd_vendor = sensor_id.split('-')[0].upper()
# superflats = glob.glob('%(sensor_id)s_superflat_*.fits' % locals())
superflats = glob.glob('%s_superflat_*.fits' % wgSlotName)
for item in superflats:
eotestUtils.addHeaderData(item, FILENAME=item,
DATE=eotestUtils.utc_now_isoformat())
results.extend([siteUtils.make_fileref(x, folder=slot) for x in superflats])
results_file = '%s_eotest_results.fits' % wgSlotName
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
cti_high_serial = data['CTI_HIGH_SERIAL']
cti_high_serial_error = data['CTI_HIGH_SERIAL_ERROR']
cti_high_parallel = data['CTI_HIGH_PARALLEL']
cti_high_parallel_error = data['CTI_HIGH_PARALLEL_ERROR']
cti_low_serial = data['CTI_LOW_SERIAL']
cti_low_serial_error = data['CTI_LOW_SERIAL_ERROR']
cti_low_parallel = data['CTI_LOW_PARALLEL']
cti_low_parallel_error = data['CTI_LOW_PARALLEL_ERROR']
#!/usr/bin/env python
import glob
import pyfits
import numpy as np
import lsst.eotest.sensor as sensorTest
import lcatr.schema
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
# Save the output file from producer script.
persistence_file = '%(sensor_id)s_persistence.fits' % locals()
eotestUtils.addHeaderData(persistence_file, LSST_NUM=sensor_id,
TESTTYPE='PERSISTENCE',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results = [lcatr.schema.fileref.make(persistence_file)]
# Save the deferred charge for each amplifier from the first post-flat
# dark frame as the image persistence metric.
#
# Read the results from the FITS file.
persistence = pyfits.open(persistence_file)
times = persistence[1].data.field('TIME')
# Times of dark frames are relative to the MJD-OBS of the flat.
post_flat_times = times[np.where(times > 0)]
index = np.where(times == min(post_flat_times))[0][0]
#!/usr/bin/env python
import lsst.eotest.sensor as sensorTest
import lcatr.schema
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
mask_file = '%s_bright_pixel_mask.fits' % sensor_id
eotestUtils.addHeaderData(mask_file, LSST_NUM=sensor_id, TESTTYPE='DARK',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results = [lcatr.schema.fileref.make(mask_file)]
medianed_dark = '%s_median_dark_bp.fits' % sensor_id
eotestUtils.addHeaderData(medianed_dark, DATE=eotestUtils.utc_now_isoformat())
results.append(lcatr.schema.fileref.make(medianed_dark))
eotest_results = '%s_eotest_results.fits' % sensor_id
data = sensorTest.EOTestResults(eotest_results)
amps = data['AMP']
npixels = data['NUM_BRIGHT_PIXELS']
ncolumns = data['NUM_BRIGHT_COLUMNS']
for amp, npix, ncol in zip(amps, npixels, ncolumns):
results.append(lcatr.schema.valid(lcatr.schema.get('bright_defects'),
amp=amp,
bright_pixels=npix,
bright_columns=ncol))
results.extend(siteUtils.jobInfo())
