def persist_traps_analysis():
"""Persist the results from the traps analysis."""
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
ccd_vendor = sensor_id.split('-')[0].upper()
trap_file = '%s_traps.fits' % sensor_id
eotestUtils.addHeaderData(trap_file,
LSST_NUM=sensor_id,
TESTTYPE='TRAP',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
results.append(siteUtils.make_fileref(trap_file, folder=slot))
mask_file = '%s_traps_mask.fits' % sensor_id
results.append(siteUtils.make_fileref(mask_file, folder=slot))
results_file = '%s_eotest_results.fits' % sensor_id
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
num_traps = data['NUM_TRAPS']
for amp, ntrap in zip(amps, num_traps):
results.append(
lcatr.schema.valid(lcatr.schema.get('traps_raft'),
amp=amp,
num_traps=ntrap,
slot=slot,
sensor_id=sensor_id))
return results
def validate_flat_gain_stability(results, det_names):
"""Valdiate the output files from the flat_gain_stability analysis"""
if 'gainstability' not in get_analysis_types():
return results
run = siteUtils.getRunNumber()
missing_det_names = set()
for det_name in det_names:
file_prefix = make_file_prefix(run, det_name)
results_file = f'{file_prefix}_flat_signal_sequence.pickle'
if not os.path.isfile(results_file):
missing_det_names.add(det_name)
else:
md = dict(DATA_PRODUCT='flat_gain_stability_results')
results.append(siteUtils.make_fileref(results_file, metadata=md))
report_missing_data('validate_flat_gain_stability', missing_det_names)
unit_id = siteUtils.getUnitId()
png_files = glob.glob('*flat_gain_stability.png')
for png_file in png_files:
md = dict(DATA_PRODUCT='flat_gain_stability_plot')
if unit_id in png_file:
md['LsstId'] = unit_id
results.append(siteUtils.make_fileref(png_file, metadata=md))
return results
def validate_traps(results, det_names):
"""Validate and persist trap results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
trap_file = '%s_traps.fits' % file_prefix
if not os.path.isfile(trap_file):
missing_det_names.append(det_name)
continue
eotestUtils.addHeaderData(trap_file,
TESTTYPE='TRAP',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(trap_file))
mask_file = '%s_traps_mask.fits' % file_prefix
results.append(siteUtils.make_fileref(mask_file))
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
num_traps = data['NUM_TRAPS']
for amp, ntrap in zip(amps, num_traps):
results.append(
lcatr.schema.valid(lcatr.schema.get('traps_BOT'),
amp=amp,
num_traps=ntrap,
slot=slot,
raft=raft))
report_missing_data("validate_traps", missing_det_names)
return results
def validate_dark_defects(results, det_names):
"""Validate and persist dark defects results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
mask_file = '%s_dark_pixel_mask.fits' % file_prefix
if not os.path.isfile(mask_file):
missing_det_names.append(det_name)
continue
eotestUtils.addHeaderData(mask_file,
TESTTYPE='SFLAT_500',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(mask_file))
superflat = '%s_median_sflat.fits' % file_prefix
eotestUtils.addHeaderData(superflat,
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(superflat))
eotest_results = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(eotest_results)
amps = data['AMP']
npixels = data['NUM_DARK_PIXELS']
ncolumns = data['NUM_DARK_COLUMNS']
for amp, npix, ncol in zip(amps, npixels, ncolumns):
results.append(
lcatr.schema.valid(lcatr.schema.get('dark_defects_BOT'),
amp=amp,
dark_pixels=npix,
dark_columns=ncol,
slot=slot,
raft=raft))
# Persist the png files.
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='SFLAT_500',
TEST_CATEGORY='EO')
filename = '%s_superflat_dark_defects.png' % file_prefix
results.extend(
siteUtils.persist_png_files(filename,
file_prefix,
metadata=metadata))
report_missing_data("validate_dark_defects", missing_det_names)
return results
def validate_overscan(results, det_names):
"""Validate the overscan analysis results."""
run = siteUtils.getRunNumber()
results = []
missing_det_names = set()
for det_name in det_names:
file_prefix = make_file_prefix(run, det_name)
results_file = f'{file_prefix}_overscan_results.fits'
if not os.path.isfile(results_file):
missing_det_names.add(det_name)
else:
md = dict(DATA_PRODUCT='overscan_task_results',
RUN=run,
DETECTOR=det_name)
results.append(siteUtils.make_fileref(results_file, metadata=md))
png_files = (glob.glob(f'{file_prefix}_*_eper_*.png') +
glob.glob(f'{file_prefix}_*_overscan_*.png') +
glob.glob(f'{file_prefix}_*_cti.png'))
md = dict(TEST_CATEGORY='EO', DETECTOR=det_name, RUN=run)
results.extend(
siteUtils.persist_png_files('',
file_prefix,
png_files=png_files,
metadata=md))
report_missing_data('validate_overscan', missing_det_names)
return results
def persist_tearing_png_files(png_files, folder=None, metadata=None):
"""
Create the lcatr.schema.filerefs for persisting the png files
with the DataCatalog.
Parameters
----------
png_files: list
A list of png files for the tearing profile plots.
folder: str [None]
Folder under which to persist the png file. For raft-level
analysis, this would be the slot number of the CCD.
metadata: dict [None]
Any additional metadata to persist with the png files.
Returns
-------
list: This is a list of lcatr.schema.filerefs.
"""
if metadata is None:
metadata = dict()
md = siteUtils.DataCatalogMetadata(**metadata)
png_filerefs = []
for png_file in png_files:
dp = 'tearing_profiles'
lsst_id = os.path.basename(png_file).split('_')[0]
png_filerefs.append(
siteUtils.make_fileref(png_file,
folder=folder,
metadata=md(DATA_PRODUCT=dp,
LsstId=lsst_id)))
return png_filerefs
def validate_scan(results, det_names):
"""Validate scan mode analysis results."""
run = siteUtils.getRunNumber()
rafts = set()
for det_name in det_names:
raft, slot = det_name.split('_')
rafts.add(raft)
file_prefix = make_file_prefix(run, det_name)
disp_files = glob.glob('{}_*_dispersion.png'.format(file_prefix))
for item in disp_files:
md = dict(raft=raft, slot=slot, run=run)
results.append(siteUtils.make_fileref(item, metadata=md))
for raft in rafts:
multiscope_files = glob.glob('{}_{}_*multiscope.png'.format(raft, run))
for item in multiscope_files:
md = dict(raft=raft, run=run)
results.append(siteUtils.make_fileref(item, metadata=md))
return results
def validate_qe(results, det_names):
"""Validate the QE results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
qe_results_file = '%s_QE.fits' % file_prefix
if not os.path.isfile(qe_results_file):
missing_det_names.append(det_name)
continue
with fits.open(qe_results_file) as qe_results:
qe_data = qe_results['QE_BANDS'].data
QE = OrderedDict((band, []) for band in qe_data.field('BAND'))
for amp in range(1, 17):
values = qe_data.field('AMP%02i' % amp)
for band, value in zip(QE, values):
QE[band].append(value)
for band in QE:
for amp in range(1, 17):
results.append(
lcatr.schema.valid(lcatr.schema.get('qe_BOT_analysis'),
band=band,
QE=QE[band][amp - 1],
amp=amp,
slot=slot,
raft=raft))
qe_files = glob.glob('%s_*QE*.fits' % file_prefix)
for item in qe_files:
eotestUtils.addHeaderData(item,
TESTTYPE='LAMBDA',
DATE=eotestUtils.utc_now_isoformat())
results.extend([siteUtils.make_fileref(item) for item in qe_files])
# Persist the png files.
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='LAMBDA',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('%s*qe.png' % file_prefix,
file_prefix,
metadata=metadata))
results.extend(
siteUtils.persist_png_files('%s*flat.png' % file_prefix,
file_prefix,
metadata=metadata))
report_missing_data("validate_qe", missing_det_names)
return results
def validate_bias_stability(results, det_names):
"""Validate bias stability results."""
run = siteUtils.getRunNumber()
rafts = set()
for det_name in det_names:
raft, slot = det_name.split('_')
rafts.add(raft)
file_prefix = make_file_prefix(run, det_name)
profile_plots = f'{file_prefix}_bias_serial_profiles.png'
if not os.path.isfile(profile_plots):
continue
md = dict(raft=raft, slot=slot, run=run)
results.append(siteUtils.make_fileref(profile_plots, metadata=md))
for raft in rafts:
file_prefix = make_file_prefix(run, raft)
stats_file = f'{file_prefix}_bias_frame_stats.pickle'
if not os.path.isfile(stats_file):
continue
md = dict(raft=raft, run=run)
results.append(siteUtils.make_fileref(stats_file, metadata=md))
return results
def validate_brighter_fatter(results, det_names):
"""Validate the brighter-fatter results."""
run = siteUtils.getRunNumber()
missing_det_names = set()
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
bf_results = '%s_bf.fits' % file_prefix
if not os.path.isfile(bf_results):
missing_det_names.add(det_name)
continue
eotestUtils.addHeaderData(bf_results,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(bf_results))
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
columns = (data['AMP'], data['BF_XCORR'], data['BF_XCORR_ERR'],
data['BF_YCORR'], data['BF_YCORR_ERR'], data['BF_SLOPEX'],
data['BF_SLOPEX_ERR'], data['BF_SLOPEY'],
data['BF_SLOPEY_ERR'], data['BF_MEAN'])
for amp, bf_xcorr, bf_xcorr_err, bf_ycorr, bf_ycorr_err, \
bf_slopex, bf_slopex_err, bf_slopey, bf_slopey_err, bf_mean \
in zip(*columns):
results.append(
lcatr.schema.valid(lcatr.schema.get('brighter_fatter_BOT'),
amp=amp,
bf_xcorr=bf_xcorr,
bf_xcorr_err=bf_xcorr_err,
bf_ycorr=bf_ycorr,
bf_ycorr_err=bf_ycorr_err,
bf_slopex=bf_slopex,
bf_slopex_err=bf_slopex_err,
bf_slopey=bf_slopey,
bf_slopey_err=bf_slopey_err,
bf_mean=bf_mean,
slot=slot,
raft=raft))
# Persist the png files.
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='FLAT',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('%s*brighter-fatter.png' % file_prefix,
file_prefix,
metadata=metadata))
return results
def validate_read_noise(results, det_names):
"""Validate and persist read noise results."""
run = siteUtils.getRunNumber()
missing_det_names = set()
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
read_noise_file = '%s_eotest_results.fits' % file_prefix
if not os.path.isfile(read_noise_file):
# No data for this detector, so note that and continue
# with the others.
missing_det_names.add(det_name)
continue
data = sensorTest.EOTestResults(read_noise_file)
amps = data['AMP']
read_noise_data = data['READ_NOISE']
system_noise_data = data['SYSTEM_NOISE']
total_noise_data = data['TOTAL_NOISE']
for amp, read_noise, system_noise, total_noise \
in zip(amps, read_noise_data, system_noise_data, total_noise_data):
results.append(lcatr.schema.valid(
lcatr.schema.get('read_noise_BOT'),
amp=amp, read_noise=read_noise, system_noise=system_noise,
total_noise=total_noise, slot=slot, raft=raft))
files = glob.glob('%s_read_noise?*.fits' % file_prefix)
for fitsfile in files:
eotestUtils.addHeaderData(fitsfile, TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat())
data_products = [siteUtils.make_fileref(item) for item in files]
results.extend(data_products)
# Persist the png files.
metadata = dict(DETECTOR=det_name, TESTTYPE='FE55', TEST_CATEGORY='EO',
RUN=run)
filename = '%s_correlated_noise.png' % file_prefix
results.extend(siteUtils.persist_png_files(filename, file_prefix,
metadata=metadata))
# Persist the raft-level overscan correlation plots.
for raft in camera_info.get_installed_raft_names():
metadata = dict(TESTTYPE='FE55', TEST_CATEGORY='EO', RAFT=raft, RUN=run)
file_prefix = make_file_prefix(run, raft)
filename = '%s_overscan_correlations.png' % file_prefix
results.extend(siteUtils.persist_png_files(filename, file_prefix,
metadata=metadata))
report_missing_data("validate_read_noise", missing_det_names)
return results
def validate_flat_pairs(results, det_names):
"""Validate the flat pair analysis results."""
run = siteUtils.getRunNumber()
missing_det_names = set()
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
det_resp_data = '%s_det_response.fits' % file_prefix
if not os.path.isfile(det_resp_data):
missing_det_names.add(det_name)
continue
eotestUtils.addHeaderData(det_resp_data, DETECTOR=det_name,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(det_resp_data))
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
max_observed_signal_data = data['MAX_OBSERVED_SIGNAL']
max_frac_dev_data = data['MAX_FRAC_DEV']
row_mean_var_slope_data = data['ROW_MEAN_VAR_SLOPE']
linearity_turnoff_data = data['LINEARITY_TURNOFF']
for amp, max_observed_signal, max_frac_dev, row_mean_var_slope, \
linearity_turnoff in zip(amps, max_observed_signal_data,
max_frac_dev_data,
row_mean_var_slope_data,
linearity_turnoff_data):
results.append(lcatr.schema.valid(
lcatr.schema.get('flat_pairs_BOT'),
amp=amp, max_observed_signal=max_observed_signal,
max_frac_dev=max_frac_dev,
row_mean_var_slope=row_mean_var_slope,
linearity_turnoff=linearity_turnoff, slot=slot, raft=raft))
# Persist the png files.
metadata = dict(DETECTOR=det_name, RUN=run,
TESTTYPE='FLAT', TEST_CATEGORY='EO')
results.extend(siteUtils.persist_png_files(('%s_linearity*.png'
% file_prefix),
file_prefix,
metadata=metadata))
results.extend(siteUtils.persist_png_files(('%s_row_means_variance.png'
% file_prefix),
file_prefix,
metadata=metadata))
report_missing_data("validate_flat_pairs", missing_det_names)
return results
def validate_ptc(results, det_names):
"""Validate the PTC results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
ptc_results = '%s_ptc.fits' % file_prefix
if not os.path.isfile(ptc_results):
missing_det_names.append(det_name)
continue
eotestUtils.addHeaderData(ptc_results,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(ptc_results))
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
columns = (data['AMP'], data['PTC_GAIN'], data['PTC_GAIN_ERROR'],
data['PTC_A00'], data['PTC_A00_ERROR'], data['PTC_NOISE'],
data['PTC_NOISE_ERROR'], data['PTC_TURNOFF'])
for amp, gain, gain_error, a00, a00_error,\
noise, noise_error, turnoff in zip(*columns):
results.append(
lcatr.schema.valid(lcatr.schema.get('ptc_BOT'),
amp=amp,
ptc_gain=gain,
ptc_gain_error=gain_error,
ptc_a00=a00,
ptc_a00_error=a00_error,
ptc_noise=noise,
ptc_noise_error=noise_error,
ptc_turnoff=turnoff,
slot=slot,
raft=raft))
# Persist the png files.
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='FLAT',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('%s*ptcs.png' % file_prefix,
file_prefix,
metadata=metadata))
report_missing_data("validate_ptc", missing_det_names)
return results
def validate_nonlinearity(results, det_names):
"""Validate the nonlinearity analysis results."""
run = siteUtils.getRunNumber()
results = []
missing_det_names = set()
for det_name in det_names:
nlc_file = f'{make_file_prefix(run, det_name)}_nlc.fits'
if not os.path.isfile(nlc_file):
missing_det_names.add(det_name)
continue
md = dict(DATA_PRODUCT='nonlinearity_correction')
results.append(siteUtils.make_fileref(nlc_file, metadata=md))
report_missing_data('validate_nonlinearity', missing_det_names)
return results
def validate_flat_pairs(results, det_names):
"""Validate the flat pair analysis results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
det_resp_data = '%s_det_response.fits' % file_prefix
if not os.path.isfile(det_resp_data):
missing_det_names.append(det_name)
continue
eotestUtils.addHeaderData(det_resp_data,
DETECTOR=det_name,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(det_resp_data))
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
full_well_data = data['FULL_WELL']
max_frac_dev_data = data['MAX_FRAC_DEV']
for amp, full_well, max_frac_dev in zip(amps, full_well_data,
max_frac_dev_data):
results.append(
lcatr.schema.valid(lcatr.schema.get('flat_pairs_BOT'),
amp=amp,
full_well=full_well,
max_frac_dev=max_frac_dev,
slot=slot,
raft=raft))
# Persist the png files.
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='FLAT',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files(('%s_linearity*.png' % file_prefix),
file_prefix,
metadata=metadata))
report_missing_data("validate_flat_pairs", missing_det_names)
return results
def validate_persistence(results, det_names):
"""Validate the persistence analysis results."""
run = siteUtils.getRunNumber()
results = []
missing_det_names = set()
for det_name in det_names:
file_prefix = make_file_prefix(run, det_name)
data_file = f'{file_prefix}_persistence_data.pickle'
if not os.path.isfile(data_file):
missing_det_names.add(det_name)
continue
md = dict(DATA_PRODUCT='persistence_task_results', RUN=run,
DETECTOR=det_name)
results.append(siteUtils.make_fileref(data_file, metadata=md))
png_files = [f'{file_prefix}_persistence.png']
md = dict(TEST_CATEGORY='EO', DETECTOR=det_name, RUN=run)
results.extend(siteUtils.persist_png_files('', file_prefix,
png_files=png_files,
metadata=md))
report_missing_data('validate_persistence', missing_det_names)
return results
results = []
for slot, sensor_id in raft.items():
if 'ccd2' in slot:
continue
wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id]
ccd_vendor = sensor_id.split('-')[0].upper()
mask_file = '%s_dark_pixel_mask.fits' % wgSlotName
eotestUtils.addHeaderData(mask_file,
LSST_NUM=sensor_id,
TESTTYPE='SFLAT_500',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
results.append(siteUtils.make_fileref(mask_file))
superflat = '%s_median_sflat.fits' % wgSlotName
eotestUtils.addHeaderData(superflat, DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(superflat))
eotest_results = '%s_eotest_results.fits' % wgSlotName
data = sensorTest.EOTestResults(eotest_results)
amps = data['AMP']
npixels = data['NUM_DARK_PIXELS']
ncolumns = data['NUM_DARK_COLUMNS']
for amp, npix, ncol in zip(amps, npixels, ncolumns):
results.append(
lcatr.schema.valid(lcatr.schema.get('dark_defects_raft'),
amp=amp,
dark_pixels=npix,
output_files = gain_file, psf_results, rolloff_mask
# Add/update the metadata to the primary HDU of these files.
for fitsfile in output_files:
eotestUtils.addHeaderData(fitsfile,
LSST_NUM=sensor_id,
TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
#
# Persist the mean bias FITS file.
#
# bias_mean_file = glob.glob('%(sensor_id)s_mean_bias_*.fits' % locals())[0]
bias_mean_file = glob.glob('%s_mean_bias_*.fits' % wgSlotName)[0]
results.append(siteUtils.make_fileref(bias_mean_file, folder=slot))
# Persist the png files.
metadata = dict(CCD_MANU=ccd_vendor,
LSST_NUM=sensor_id,
TESTTYPE='FE55',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('%s*.png' % wgSlotName,
sensor_id,
folder=slot,
metadata=metadata))
data = sensorTest.EOTestResults(gain_file)
amps = data['AMP']
gain_data = data['GAIN']
import eotestUtils
import camera_components
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
ccd_vendor = sensor_id.split('-')[0].upper()
det_resp_data = '%s_det_response.fits' % sensor_id
eotestUtils.addHeaderData(det_resp_data, LSST_NUM=sensor_id,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
results.append(siteUtils.make_fileref(det_resp_data, folder=slot))
results_file = '%s_eotest_results.fits' % sensor_id
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
full_well_data = data['FULL_WELL']
max_frac_dev_data = data['MAX_FRAC_DEV']
for amp, full_well, max_frac_dev in zip(amps, full_well_data,
max_frac_dev_data):
results.append(lcatr.schema.valid(lcatr.schema.get('flat_pairs_raft'),
amp=amp, full_well=full_well,
max_frac_dev=max_frac_dev,
slot=slot,
sensor_id=sensor_id))
# Persist the png files.
lcatr.schema.valid(lcatr.schema.get('qe_raft_analysis'),
band=band,
QE=np.mean(QE[band]),
slot=slot,
sensor_id=sensor_id))
qe_files = glob.glob('%s_*QE*.*' % sensor_id)
for item in qe_files:
if item.endswith('.fits'):
eotestUtils.addHeaderData(item,
LSST_NUM=sensor_id,
TESTTYPE='LAMBDA',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
results.extend(
[siteUtils.make_fileref(item, folder=slot) for item in qe_files])
# Persist the png files.
metadata = dict(CCD_MANU=ccd_vendor,
LSST_NUM=sensor_id,
TESTTYPE='LAMBDA',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('%s*.png' % sensor_id,
sensor_id,
folder=slot,
metadata=metadata))
sensor_id = raft.sensor_names[0]
qe_acq_job_id = siteUtils.get_prerequisite_job_id(
('S*/%s_lambda_flat_*.fits' % sensor_id),
jobname=siteUtils.getProcessName('qe_raft_acq'))
results = []
for slot, sensor_id in raft.items():
if 'ccd2' in slot:
continue
wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id]
ccd_vendor = sensor_id.split('-')[0].upper()
trap_file = '%s_traps.fits' % wgSlotName
eotestUtils.addHeaderData(trap_file,
LSST_NUM=sensor_id,
TESTTYPE='TRAP',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
results.append(siteUtils.make_fileref(trap_file, folder=slot))
mask_file = '%s_traps_mask.fits' % wgSlotName
results.append(siteUtils.make_fileref(mask_file, folder=slot))
results_file = '%s_eotest_results.fits' % wgSlotName
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
num_traps = data['NUM_TRAPS']
for amp, ntrap in zip(amps, num_traps):
results.append(
lcatr.schema.valid(lcatr.schema.get('traps_raft'),
amp=amp,
num_traps=ntrap,
slot=slot,
import lcatr.schema
import siteUtils
import eotestUtils
import camera_components
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
ccd_vendor = sensor_id.split('-')[0].upper()
mask_file = '%s_bright_pixel_mask.fits' % sensor_id
eotestUtils.addHeaderData(mask_file, LSST_NUM=sensor_id, TESTTYPE='DARK',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
results.append(siteUtils.make_fileref(mask_file, folder=slot))
bias_frame = glob.glob(f'{sensor_id}_*_median_bias.fits')[0]
eotestUtils.addHeaderData(bias_frame, DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(bias_frame, folder=slot))
medianed_dark = '%s_median_dark_bp.fits' % sensor_id
eotestUtils.addHeaderData(medianed_dark,
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(medianed_dark, folder=slot))
eotest_results = '%s_eotest_results.fits' % sensor_id
data = sensorTest.EOTestResults(eotest_results)
amps = data['AMP']
npixels = data['NUM_BRIGHT_PIXELS']
ncolumns = data['NUM_BRIGHT_COLUMNS']
def persist_fe55_analysis_results():
"""Persist the results from the full analysis."""
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
ccd_vendor = sensor_id.split('-')[0].upper()
# The output files from producer script.
gain_file = '%(sensor_id)s_eotest_results.fits' % locals()
psf_results = glob.glob('%(sensor_id)s_psf_results*.fits' %
locals())[0]
rolloff_mask = '%(sensor_id)s_rolloff_defects_mask.fits' % locals()
output_files = gain_file, psf_results, rolloff_mask
# Add/update the metadata to the primary HDU of these files.
for fitsfile in output_files:
eotestUtils.addHeaderData(fitsfile,
LSST_NUM=sensor_id,
TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
#
# Persist the median bias FITS file.
#
bias_median_file = glob.glob(f'{sensor_id}_*_median_bias.fits')[0]
results.append(siteUtils.make_fileref(bias_median_file, folder=slot))
# Persist the png files.
metadata = dict(CCD_MANU=ccd_vendor,
LSST_NUM=sensor_id,
TESTTYPE='FE55',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('%s*.png' % sensor_id,
sensor_id,
folder=slot,
metadata=metadata))
data = sensorTest.EOTestResults(gain_file)
amps = data['AMP']
gain_data = data['GAIN']
gain_errors = data['GAIN_ERROR']
sigmas = data['PSF_SIGMA']
for amp, gain_value, gain_error, sigma in zip(amps, gain_data,
gain_errors, sigmas):
if not np.isfinite(gain_error):
gain_error = -1
results.append(
lcatr.schema.valid(lcatr.schema.get('fe55_raft_analysis'),
amp=amp,
gain=gain_value,
gain_error=gain_error,
psf_sigma=sigma,
slot=slot,
sensor_id=sensor_id))
results.extend([lcatr.schema.fileref.make(x) for x in output_files])
return results
def validate_tearing(results, det_names):
"""Validate the tearing analysis results."""
run = siteUtils.getRunNumber()
schema = lcatr.schema.get('tearing_detection_BOT')
amps_schema = lcatr.schema.get('tearing_stats_BOT')
missing_det_names = set()
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
tearing_results_file = '%s_tearing_stats.pickle' % file_prefix
if not os.path.isfile(tearing_results_file):
missing_det_names.add(det_name)
continue
results.append(siteUtils.make_fileref(tearing_results_file))
with open(tearing_results_file, 'rb') as input_:
tearing_stats, amp_counts = pickle.load(input_)
for values in tearing_stats:
stats = dict(
zip(('job_name', 'subset', 'sensor_id', 'detections', 'slot',
'raft'),
list(values) + [slot, raft]))
results.append(lcatr.schema.valid(schema, **stats))
for amp, detections in amp_counts.items():
results.append(
lcatr.schema.valid(amps_schema,
amp=amp,
slot=slot,
raft=raft,
tearing_detections=detections))
png_files = sorted(glob.glob('*_tearing.png'))
results.extend(persist_tearing_png_files(png_files))
missing_raft_names = set()
for raft_name in camera_info.get_installed_raft_names():
try:
divisidero_plot = glob.glob(f'{raft_name}_*_divisidero.png')[0]
except IndexError:
missing_raft_names.add(raft_name)
continue
md = dict(DATA_PRODUCT='divisidero_tearing_plot', LsstId=raft_name)
results.append(siteUtils.make_fileref(divisidero_plot, metadata=md))
try:
divisidero_json_file \
= glob.glob(f'{raft_name}*max_divisidero.json')[0]
except IndexError:
missing_raft_names.add(raft_name)
continue
with open(divisidero_json_file, 'r') as fd:
max_devs = json.load(fd)
results.append(siteUtils.make_fileref(divisidero_json_file))
bot_schema = lcatr.schema.get('divisadero_tearing_BOT')
for slot, values in max_devs.items():
# Weed out nans and infinities.
max_dev_values = []
for value in values:
if np.isfinite(value):
max_dev_values.append(value)
else:
max_dev_values.append(0)
# Top half of CCD.
my_devs = max_dev_values[:7]
for amp, devs in enumerate(zip([0] + my_devs, my_devs + [0]), 1):
results.append(
lcatr.schema.valid(bot_schema,
amp=amp,
slot=slot,
raft=raft_name,
divisadero_max_dev=max(devs)))
if len(max_dev_values) == 7:
# This is a WF sensor.
continue
# Bottom half of CCD.
my_devs = max_dev_values[7:]
my_devs.reverse()
for amp, devs in enumerate(zip([0] + my_devs, my_devs + [0]), 9):
results.append(
lcatr.schema.valid(bot_schema,
amp=amp,
slot=slot,
raft=raft_name,
divisadero_max_dev=max(devs)))
report_missing_data("validate_tearing", missing_det_names)
report_missing_data("validate_tearing",
sorted(list(missing_raft_names)),
components='rafts',
total=25)
return results
results = []
for slot, sensor_id in raft.items():
ccd_vendor = sensor_id.split('-')[0].upper()
if 'ccd2' in slot:
continue
wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id]
ptc_results = '%s_ptc.fits' % wgSlotName
eotestUtils.addHeaderData(ptc_results,
LSST_NUM=sensor_id,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
results.append(siteUtils.make_fileref(ptc_results, folder=slot))
results_file = '%s_eotest_results.fits' % wgSlotName
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
ptc_gains = data['PTC_GAIN']
ptc_gain_errors = data['PTC_GAIN_ERROR']
for amp, gain, gain_error in zip(amps, ptc_gains, ptc_gain_errors):
results.append(
lcatr.schema.valid(lcatr.schema.get('ptc_raft'),
amp=amp,
ptc_gain=gain,
ptc_gain_error=gain_error,
slot=slot,
sensor_id=wgSlotName))
# Persist the png files.
slot=slot,
sensor_id=wgSlotName))
# sensor_id=sensor_id))
# fe55_acq_job_id = siteUtils.get_prerequisite_job_id('S*/%s_fe55_fe55_*.fits' % sensor_id,
fe55_acq_job_id = siteUtils.get_prerequisite_job_id('S*/%s_fe55_fe55_*.fits' % wgSlotName,
jobname=siteUtils.getProcessName('fe55_raft_acq'))
# files = glob.glob('%s_read_noise?*.fits' % sensor_id)
files = glob.glob('%s_read_noise?*.fits' % wgSlotName)
for fitsfile in files:
eotestUtils.addHeaderData(fitsfile, LSST_NUM=sensor_id, TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
data_products = [siteUtils.make_fileref(item, folder=slot)
for item in files]
results.extend(data_products)
# Persist the png files.
metadata = dict(CCD_MANU=ccd_vendor, LSST_NUM=sensor_id,
TESTTYPE='FE55', TEST_CATEGORY='EO')
results.extend(siteUtils.persist_png_files('%s*.png' % sensor_id,
sensor_id, folder=slot,
metadata=metadata))
# Persist the raft-level overscan correlation plot.
metadata = dict(LSST_NUM=raft_id, TESTTYPE='FE55', TEST_CATEGORY='EO')
results.extend(siteUtils.persist_png_files('%s*.png' % raft_id, raft_id,
metadata=metadata))
def validate_cte(results, det_names):
"""Validate the CTE task results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
superflats \
= sorted(glob.glob('{}_superflat_*.fits'.format(file_prefix)))
if not superflats:
missing_det_names.append(det_name)
continue
for item in superflats:
eotestUtils.addHeaderData(item,
FILENAME=item,
DATE=eotestUtils.utc_now_isoformat())
results.extend([siteUtils.make_fileref(x) for x in superflats])
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
cti_high_serial = data['CTI_HIGH_SERIAL']
cti_high_serial_error = data['CTI_HIGH_SERIAL_ERROR']
cti_high_parallel = data['CTI_HIGH_PARALLEL']
cti_high_parallel_error = data['CTI_HIGH_PARALLEL_ERROR']
cti_low_serial = data['CTI_LOW_SERIAL']
cti_low_serial_error = data['CTI_LOW_SERIAL_ERROR']
cti_low_parallel = data['CTI_LOW_PARALLEL']
cti_low_parallel_error = data['CTI_LOW_PARALLEL_ERROR']
for values in zip(amps, cti_high_serial, cti_high_serial_error,
cti_high_parallel, cti_high_parallel_error,
cti_low_serial, cti_low_serial_error,
cti_low_parallel, cti_low_parallel_error):
results.append(
lcatr.schema.valid(lcatr.schema.get('cte_BOT'),
amp=values[0],
cti_high_serial=values[1],
cti_high_serial_error=values[2],
cti_high_parallel=values[3],
cti_high_parallel_error=values[4],
cti_low_serial=values[5],
cti_low_serial_error=values[6],
cti_low_parallel=values[7],
cti_low_parallel_error=values[8],
slot=slot,
raft=raft))
# Persist the png files.
png_file_list = '{}_cte_task_png_files.txt'.format(det_name)
with open(png_file_list, 'r') as input_:
png_files = [x.strip() for x in input_]
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='SFLAT_500',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('',
file_prefix,
png_files=png_files,
metadata=metadata))
report_missing_data("validate_cte", missing_det_names)
return results
def validate_fe55(results, det_names):
"""Validate and persist fe55 gain and psf results."""
run = siteUtils.getRunNumber()
missing_det_names = []
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
# The output files from producer script.
gain_file = '%(file_prefix)s_eotest_results.fits' % locals()
psf_results_files \
= glob.glob('%(file_prefix)s_psf_results*.fits' % locals())
if not os.path.isfile(gain_file) or not psf_results_files:
# Results for this detector are not available so note
# that and continue with the others.
missing_det_names.append(det_name)
continue
psf_results = psf_results_files[0]
rolloff_mask = '%(file_prefix)s_edge_rolloff_mask.fits' % locals()
output_files = psf_results, rolloff_mask
# Add/update the metadata to the primary HDU of these files.
for fitsfile in output_files:
eotestUtils.addHeaderData(fitsfile,
TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat())
results.extend([lcatr.schema.fileref.make(x) for x in output_files])
# Persist the median bias FITS file.
bias_frame \
= glob.glob('%(file_prefix)s_median_bias.fits' % locals())[0]
results.append(siteUtils.make_fileref(bias_frame))
# Persist the png files.
png_file_list = '{}_fe55_task_png_files.txt'.format(det_name)
with open(png_file_list, 'r') as input_:
png_files = [x.strip() for x in input_]
metadata = dict(TESTTYPE='FE55',
TEST_CATEGORY='EO',
DETECTOR=det_name,
RUN=run)
results.extend(
siteUtils.persist_png_files('',
file_prefix,
png_files=png_files,
metadata=metadata))
data = sensorTest.EOTestResults(gain_file)
amps = data['AMP']
gain_data = data['GAIN']
gain_errors = data['GAIN_ERROR']
sigmas = data['PSF_SIGMA']
for amp, gain_value, gain_error, sigma in zip(amps, gain_data,
gain_errors, sigmas):
if not np.isfinite(gain_error):
gain_error = -1
results.append(
lcatr.schema.valid(lcatr.schema.get('fe55_BOT_analysis'),
amp=amp,
gain=gain_value,
gain_error=gain_error,
psf_sigma=sigma,
slot=slot,
raft=raft))
report_missing_data('validate_fe55', missing_det_names)
return results
results = []
for slot, sensor_id in raft.items():
print("Processing:", slot, sensor_id)
if 'ccd2' in slot :
continue
wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id]
ccd_vendor = sensor_id.split('-')[0].upper()
# superflats = glob.glob('%(sensor_id)s_superflat_*.fits' % locals())
superflats = glob.glob('%s_superflat_*.fits' % wgSlotName)
for item in superflats:
eotestUtils.addHeaderData(item, FILENAME=item,
DATE=eotestUtils.utc_now_isoformat())
results.extend([siteUtils.make_fileref(x, folder=slot) for x in superflats])
results_file = '%s_eotest_results.fits' % wgSlotName
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
cti_high_serial = data['CTI_HIGH_SERIAL']
cti_high_serial_error = data['CTI_HIGH_SERIAL_ERROR']
cti_high_parallel = data['CTI_HIGH_PARALLEL']
cti_high_parallel_error = data['CTI_HIGH_PARALLEL_ERROR']
cti_low_serial = data['CTI_LOW_SERIAL']
cti_low_serial_error = data['CTI_LOW_SERIAL_ERROR']
cti_low_parallel = data['CTI_LOW_PARALLEL']
cti_low_parallel_error = data['CTI_LOW_PARALLEL_ERROR']
def validate_flat_pairs(results, det_names):
"""Validate the flat pair analysis results."""
run = siteUtils.getRunNumber()
missing_det_names = set()
for det_name in det_names:
raft, slot = det_name.split('_')
file_prefix = make_file_prefix(run, det_name)
det_resp_data = '%s_det_response.fits' % file_prefix
if not os.path.isfile(det_resp_data):
missing_det_names.add(det_name)
continue
eotestUtils.addHeaderData(det_resp_data,
DETECTOR=det_name,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat())
results.append(siteUtils.make_fileref(det_resp_data))
results_file = '%s_eotest_results.fits' % file_prefix
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
max_observed_signal_data = data['MAX_OBSERVED_SIGNAL']
max_frac_dev_data = data['MAX_FRAC_DEV']
row_mean_var_slope_data = data['ROW_MEAN_VAR_SLOPE']
linearity_turnoff_data = data['LINEARITY_TURNOFF']
for amp, max_observed_signal, max_frac_dev, row_mean_var_slope, \
linearity_turnoff in zip(amps, max_observed_signal_data,
max_frac_dev_data,
row_mean_var_slope_data,
linearity_turnoff_data):
results.append(
lcatr.schema.valid(lcatr.schema.get('flat_pairs_BOT'),
amp=amp,
max_observed_signal=max_observed_signal,
max_frac_dev=max_frac_dev,
row_mean_var_slope=row_mean_var_slope,
linearity_turnoff=linearity_turnoff,
slot=slot,
raft=raft))
# Persist the png files.
metadata = dict(DETECTOR=det_name,
RUN=run,
TESTTYPE='FLAT',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files(('%s_linearity*.png' % file_prefix),
file_prefix,
metadata=metadata))
results.extend(
siteUtils.persist_png_files(
('%s_row_means_variance.png' % file_prefix),
file_prefix,
metadata=metadata))
# Persist the raft-level imaging region correlation plots.
missing_raft_names = set()
for raft in camera_info.get_installed_raft_names():
metadata = dict(TESTTYPE='FLAT',
TEST_CATEGORY='EO',
RAFT=raft,
RUN=run)
file_prefix = make_file_prefix(run, raft)
filename = f'{file_prefix}_imaging_region_correlations.png'
if not os.path.isfile(filename):
missing_raft_names.add(raft)
continue
results.extend(
siteUtils.persist_png_files(filename,
file_prefix,
metadata=metadata))
# Persist any pd correction file specified in the lcatr.cfg file.
pd_corrections_file_env = 'LCATR_PD_CORRECTIONS_FILE'
if pd_corrections_file_env in os.environ:
pd_corrections_file = os.environ[pd_corrections_file_env]
shutil.copy(pd_corrections_file, '.')
fileref = siteUtils.make_fileref(os.path.basename(pd_corrections_file))
results.append(fileref)
report_missing_data("validate_flat_pairs", missing_det_names)
report_missing_data("validate_flat_pairs",
sorted(list(missing_raft_names)),
components='rafts',
total=21)
return results
