bias_files = defaultdict(dict)
for slot, sensor_id in raft.items():
my_files = \
siteUtils.dependency_glob('S*/%s_fe55_bias_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('fe55_raft_acq'),
description='Bias files for noise correlations:')
for item in my_files:
timestamp = item.split('_')[-1].split('.')[0]
bias_files[timestamp][slot] = item
for timestamp in bias_files:
if len(bias_files[timestamp]) == 9:
return bias_files[timestamp]
raise RuntimeError("Could not find bias files for all nine CCDs.")
if __name__ == '__main__':
import matplotlib.pyplot as plt
from correlated_noise import raft_level_oscan_correlations
from multiprocessor_execution import sensor_analyses
processes = 9 # Reserve 1 process per CCD.
sensor_analyses(run_read_noise_task, processes=processes)
raft_id = os.environ['LCATR_UNIT_ID']
run = siteUtils.getRunNumber()
bias_files = get_bias_files(raft_id)
title = 'Overscan correlations, {}, Run {}'.format(raft_id, run)
plt.rcParams['figure.figsize'] = (8, 8)
raft_level_oscan_correlations(bias_files, title=title)
plt.savefig('{}_{}_overscan_correlations.png'.format(raft_id, run))
task = sensorTest.FlatPairTask()
task.run(sensor_id,
flat_files,
mask_files,
gains,
linearity_spec_range=(1e4, 9e4),
use_exptime=use_exptime)
results_file = '%s_eotest_results.fits' % sensor_id
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
Ne_bounds = (1e4, 9e4)
detresp_file = '%s_det_response.fits' % sensor_id
siteUtils.make_png_file(plots.linearity,
'%s_linearity.png' % file_prefix,
detresp_file=detresp_file,
max_dev=0.03,
use_exptime=use_exptime,
Ne_bounds=Ne_bounds)
siteUtils.make_png_file(plots.linearity_resids,
'%s_linearity_resids.png' % file_prefix,
detresp_file=detresp_file,
max_dev=0.03,
Ne_bounds=Ne_bounds,
use_exptime=use_exptime)
if __name__ == '__main__':
sensor_analyses(run_flat_pair_task)
import eotestUtils
acq_jobname = siteUtils.getProcessName('sflat_raft_acq')
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
sflat_files = siteUtils.dependency_glob('S*/%s_sflat_500_flat_H*.fits' % sensor_id,
jobname=acq_jobname,
description='Superflat files:')
bias_files = siteUtils.dependency_glob('S*/%s_sflat_bias*.fits' % sensor_id,
jobname=acq_jobname,
description='Bias files:')
bias_frame = eotestUtils.make_median_bias_frame(bias_files, sensor_id,
'sflat_raft_acq')
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
task = sensorTest.DarkPixelsTask()
task.run(sensor_id, sflat_files, mask_files, bias_frame=bias_frame)
siteUtils.make_png_file(sensorTest.plot_flat,
'%s_superflat_dark_defects.png' % file_prefix,
'%s_median_sflat.fits' % sensor_id,
title='%s, superflat for dark defects analysis' % sensor_id,
annotation='ADU/pixel', flatten=True, binsize=4)
if __name__ == '__main__':
from multiprocessor_execution import sensor_analyses
processes = 9 # Reserve 1 process per CCD.
sensor_analyses(run_dark_pixels_task, processes=processes)
sensorTest.plot_flat,
sflat_file.replace('.fits',
'.png').replace(sensor_id, file_prefix),
sflat_file,
title=('%s, CTE superflat, %s flux ' % (sensor_id, flux_level)),
annotation='ADU/pixel',
flatten=True,
binsize=4)
siteUtils.make_png_file(plots.cte_profiles,
('%s_serial_oscan_%s.png' %
(file_prefix, flux_level)),
flux_level,
sflat_file,
mask_files,
serial=True)
siteUtils.make_png_file(plots.cte_profiles,
('%s_parallel_oscan_%s.png' %
(file_prefix, flux_level)),
flux_level,
sflat_file,
mask_files,
serial=False)
if __name__ == '__main__':
from multiprocessor_execution import sensor_analyses
processes = 9 # Reserve 1 process per CCD.
sensor_analyses(run_cte_task, processes=processes)
import siteUtils
import eotestUtils
from multiprocessor_execution import sensor_analyses
def run_trap_task(sensor_id):
trap_file = siteUtils.dependency_glob(
'S*/%s_trap_ppump_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('ppump_raft_acq'),
description='Trap file:')[0]
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
# Omit rolloff defects mask since a trap in the rolloff edge region can
# affect the entire column.
mask_files = [
item for item in mask_files if item.find('rolloff_defects') == -1
]
print("Using mask files:")
for mask_file in mask_files:
print(" " + mask_file)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.TrapTask()
task.run(sensor_id, trap_file, mask_files, gains)
if __name__ == '__main__':
sensor_analyses(run_trap_task)
jobname=acq_jobname,
description='Bias files:')
bias_frame = eotestUtils.make_median_bias_frame(bias_files, sensor_id,
'dark_raft_acq')
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.BrightPixelsTask()
task.config.temp_set_point = -100.
task.run(sensor_id, dark_files, mask_files, gains, bias_frame=bias_frame)
siteUtils.make_png_file(
sensorTest.plot_flat,
'%s_medianed_dark.png' % file_prefix,
'%s_median_dark_bp.fits' % sensor_id,
title='%s, medianed dark for bright defects analysis' % sensor_id,
annotation='e-/pixel, gain-corrected, bias-subtracted',
bias_frame=bias_frame,
gains=gains,
binsize=4)
if __name__ == '__main__':
from multiprocessor_execution import sensor_analyses
processes = 9 # Reserve 1 process per CCD.
sensor_analyses(run_bright_pixels_task, processes=processes)
sensor_id=sensor_id)
task = sensorTest.DarkCurrentTask()
task.config.temp_set_point = -100.
dark_curr_pixels, dark95s \
= task.run(sensor_id, dark_files, mask_files, gains)
results_file \
= siteUtils.dependency_glob('%s_eotest_results.fits' % sensor_id,
jobname='read_noise_raft')[0]
eo_results = sensorTest.EOTestResults(results_file)
read_noise = dict(
pair for pair in zip(eo_results['AMP'], eo_results['TOTAL_NOISE']))
siteUtils.make_png_file(sensorTest.total_noise_histograms,
'%s_total_noise_hists.png' % file_prefix,
dark_curr_pixels,
read_noise,
dark95s,
exptime=16,
title=sensor_id)
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
siteUtils.make_png_file(plots.total_noise,
'%s_noise.png' % file_prefix,
dark95s=dark95s)
if __name__ == '__main__':
sensor_analyses(run_dark_current_task)
'%s_fe55_dists.png' % file_prefix,
fe55_file=fe55_file,
xrange_scale=3,
dn_range=dn_range)
siteUtils.make_png_file(plots.psf_dists,
'%s_psf_dists.png' % file_prefix,
fe55_file=fe55_file)
def write_nominal_gains(sensor_id, gain=1):
"""Write nominal gains in lieu of doing the Fe55 analysis."""
import lsst.eotest.sensor as sensorTest
results_file = '%s_eotest_results.fits' % sensor_id
results = sensorTest.EOTestResults(results_file)
for amp in range(1, 17):
results.add_seg_result(amp, 'GAIN', gain)
results.write(clobber=True)
if __name__ == '__main__':
import os
from multiprocessor_execution import sensor_analyses
processes = 9 # Reserve 1 process per CCD.
if os.environ.get("LCATR_SKIP_FE55_ANALYSIS", "False") == "True":
sensor_analyses(write_nominal_gains, processes=processes)
else:
sensor_analyses(run_fe55_task, processes=processes)
description='Dark files:')
bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' %
sensor_id,
description='Super bias frame:')[0]
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.DarkCurrentTask()
task.config.temp_set_point = -100.
dark_curr_pixels, dark95s \
= task.run(sensor_id, dark_files, mask_files, gains,
bias_frame=bias_frame)
results_file \
= siteUtils.dependency_glob('%s_eotest_results.fits' % sensor_id,
jobname='read_noise_raft')[0]
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
siteUtils.make_png_file(plots.total_noise,
'%s_noise.png' % file_prefix,
dark95s=dark95s)
if __name__ == '__main__':
from multiprocessor_execution import sensor_analyses
processes = 9 # Reserve 1 process per CCD.
sensor_analyses(run_dark_current_task, processes=processes)
import eotestUtils
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
flat_files = siteUtils.dependency_glob(
'S*/%s_flat*flat?_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('flat_pair_raft_acq'),
description='Flat files:')
bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' %
sensor_id,
description='Super bias frame:')[0]
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.PtcTask()
task.run(sensor_id, flat_files, mask_files, gains, bias_frame=bias_frame)
results_file = '%s_eotest_results.fits' % sensor_id
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
siteUtils.make_png_file(plots.ptcs,
'%s_ptcs.png' % file_prefix,
ptc_file='%s_ptc.fits' % sensor_id)
if __name__ == '__main__':
from multiprocessor_execution import sensor_analyses
processes = 9 # Reserve 1 process per CCD.
sensor_analyses(run_ptc_task, processes=processes)
import eotestUtils
from multiprocessor_execution import sensor_analyses
def run_bright_pixels_task(sensor_id):
"Single sensor execution of the bright pixels task."
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
dark_files = siteUtils.dependency_glob(
'S*/%s_dark_dark_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('dark_raft_acq'),
description='Dark files:')
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.BrightPixelsTask()
task.config.temp_set_point = -100.
task.run(sensor_id, dark_files, mask_files, gains)
siteUtils.make_png_file(
sensorTest.plot_flat,
'%s_medianed_dark.png' % file_prefix,
'%s_median_dark_bp.fits' % sensor_id,
title='%s, medianed dark for bright defects analysis' % sensor_id,
annotation='e-/pixel, gain-corrected, bias-subtracted')
if __name__ == '__main__':
sensor_analyses(run_bright_pixels_task)
# Since slit-width can be set individually for each exposure
# on TS-8 at IR-2 (LSSTTD-1231), we need to use the MONDIODE
# keyword for computing the integrated incident flux.
use_exptime = False
task = sensorTest.FlatPairTask()
task.run(sensor_id, flat_files, mask_files, gains,
linearity_spec_range=(1e4, 9e4), use_exptime=use_exptime,
bias_frame=bias_frame)
results_file = '%s_eotest_results.fits' % sensor_id
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
Ne_bounds = (1e4, 9e4)
detresp_file = '%s_det_response.fits' % sensor_id
siteUtils.make_png_file(plots.linearity,
'%s_linearity.png' % file_prefix,
detresp_file=detresp_file, max_dev=0.03,
use_exptime=use_exptime, Ne_bounds=Ne_bounds)
siteUtils.make_png_file(plots.linearity_resids,
'%s_linearity_resids.png' % file_prefix,
detresp_file=detresp_file, max_dev=0.03,
Ne_bounds=Ne_bounds, use_exptime=use_exptime)
if __name__ == '__main__':
from multiprocessor_execution import sensor_analyses
processes = 9 # Reserve 1 process per CCD.
sensor_analyses(run_flat_pair_task, processes=processes)
"""
Producer script for raft-level PTC analysis.
"""
from __future__ import print_function
import lsst.cr_eotest.sensor as sensorTest
import siteUtils
import eotestUtils
from multiprocessor_execution import sensor_analyses
def run_ptc_task(sensor_id):
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
flat_files = siteUtils.dependency_glob('S*/%s_flat*flat?_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('flat_pair_raft_acq'),
description='Flat files:')
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.PtcTask()
task.run(sensor_id, flat_files, mask_files, gains)
results_file = '%s_eotest_results.fits' % sensor_id
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
siteUtils.make_png_file(plots.ptcs,
'%s_ptcs.png' % file_prefix,
ptc_file='%s_ptc.fits' % sensor_id)
if __name__ == '__main__':
sensor_analyses(run_ptc_task)
for sflat_file in superflat_files:
flux_level = 'low'
if sflat_file.find('high') != -1:
flux_level = 'high'
siteUtils.make_png_file(
sensorTest.plot_flat,
sflat_file.replace('.fits',
'.png').replace(sensor_id, file_prefix),
sflat_file,
title=('%s, CTE supeflat, %s flux ' % (sensor_id, flux_level)),
annotation='ADU/pixel')
siteUtils.make_png_file(plots.cte_profiles,
('%s_serial_oscan_%s.png' %
(file_prefix, flux_level)),
flux_level,
sflat_file,
mask_files,
serial=True)
siteUtils.make_png_file(plots.cte_profiles,
('%s_parallel_oscan_%s.png' %
(file_prefix, flux_level)),
flux_level,
sflat_file,
mask_files,
serial=False)
if __name__ == '__main__':
sensor_analyses(run_cte_task)
from __future__ import print_function
import pickle
import siteUtils
from multiprocessor_execution import sensor_analyses
from tearing_detection import tearing_detection
acq_jobs = {('flat_pair_raft_acq', 'N/A'): 'S*/%s_flat*flat?_*.fits',
('qe_raft_acq', 'N/A'): 'S*/%s_lambda_flat_*.fits',
('sflat_raft_acq', 'low_flux'): 'S*/%s_sflat_500_flat_L*.fits',
('sflat_raft_acq', 'high_flux'): 'S*/%s_sflat_500_flat_H*.fits'}
def run_tearing_detection(sensor_id):
"""
Loop over the acquisition jobs and perform tearing analysis on each.
"""
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
tearing_stats = []
for job_key, pattern in acq_jobs.items():
job_name, subset = job_key
flats = siteUtils.dependency_glob(pattern % sensor_id,
jobname=siteUtils.getProcessName(job_name),
description='Flat files:')
tearing_found, _ = tearing_detection(flats)
tearing_stats.append((job_name, subset, sensor_id, len(tearing_found)))
with open('%s_tearing_stats.pkl' % file_prefix, 'wb') as output:
pickle.dump(tearing_stats, output)
if __name__ == '__main__':
sensor_analyses(run_tearing_detection)
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.QeTask()
task.config.temp_set_point = -100.
task.run(sensor_id, lambda_files, pd_ratio_file, mask_files, gains,
correction_image=correction_image)
results_file \
= siteUtils.dependency_glob('%s_eotest_results.fits' % sensor_id,
jobname='fe55_raft_analysis',
description='Fe55 results file')[0]
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
siteUtils.make_png_file(plots.qe,
'%s_qe.png' % file_prefix,
qe_file='%s_QE.fits' % sensor_id)
try:
plots.flat_fields(os.path.dirname(lambda_files[0]),
annotation='e-/pixel, gain-corrected, bias-subtracted')
except Exception as eobj:
print("Exception raised while creating flat fields:")
print(str(eobj))
if __name__ == '__main__':
sensor_analyses(run_qe_task)
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
# Omit rolloff defects mask since a trap in the rolloff edge region can
# affect the entire column.
bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' %
sensor_id,
description='Super bias frame:')[0]
mask_files = [
item for item in mask_files if item.find('rolloff_defects') == -1
]
print("Using mask files:")
for mask_file in mask_files:
print(" " + mask_file)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.TrapTask()
task.run(sensor_id, trap_file, mask_files, gains, bias_frame=bias_frame)
if __name__ == '__main__':
import os
from multiprocessor_execution import sensor_analyses
processes = 9 # Reserve 1 process per CCD.
if os.environ.get('LCATR_SKIP_TRAPS_ANALYSIS', 'False') == 'True':
pass
else:
sensor_analyses(run_trap_task, processes=processes)
with open('%s_tearing_stats.pkl' % file_prefix, 'wb') as output:
pickle.dump(tearing_stats, output)
if __name__ == '__main__':
import os
import json
import matplotlib.pyplot as plt
import siteUtils
import camera_components
import lsst.eotest.raft as raftTest
from multiprocessor_execution import sensor_analyses
processes = 9 # Reserve 1 process per CCD.
sensor_analyses(run_tearing_detection, processes=processes)
# Divisidero tearing analysis.
run = siteUtils.getRunNumber()
raft_unit_id = siteUtils.getUnitId()
files = siteUtils.dependency_glob('*median_sflat.fits',
jobname='dark_defects_raft',
description='Superflat files:')
raft = camera_components.Raft.create_from_etrav(raft_unit_id,
db_name='Prod')
det_map = dict([(sensor_id, slot) for slot, sensor_id in raft.items()])
sflat_files = dict()
for item in files:
slot = det_map[os.path.basename(item).split('_')[0]]
sflat_files[slot] = item
# raft = camera_components.Raft.create_from_etrav(raft_id)
# wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id];
"Single sensor execution of the dark pixels task."
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
sflat_files = siteUtils.dependency_glob(
'S*/%s_sflat_500_flat_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('sflat_raft_acq'),
description='Superflat files:')
print("sflat query: ", 'S*/%s_sflat_500_flat_H*.fits' % sensor_id)
print("sflat_files = ", sflat_files)
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
task = sensorTest.DarkPixelsTask()
task.run(sensor_id, sflat_files, mask_files)
siteUtils.make_png_file(sensorTest.plot_flat,
'%s_superflat_dark_defects.png' % file_prefix,
'%s_median_sflat.fits' % sensor_id,
title='%s, superflat for dark defects analysis' %
sensor_id,
annotation='ADU/pixel')
if __name__ == '__main__':
sensor_analyses(run_dark_pixels_task)
def get_bias_files(raft_id=None):
"""Get the bias files from the Fe55 acquisition."""
if raft_id is None:
raft_id = os.environ['LCATR_UNIT_ID']
raft = camera_components.Raft.create_from_etrav(raft_id)
bias_files = dict()
for slot, sensor_id in raft.items():
wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id]
print('bias_files slot = ', slot)
bias_files[slot] \
= siteUtils.dependency_glob('S*/%s_fe55_bias_*.fits' % wgSlotName,
jobname=siteUtils.getProcessName('fe55_raft_acq'),
description='Bias files for noise correlations:')[0]
# = siteUtils.dependency_glob('S*/%s_fe55_bias_*.fits' % sensor_id,
return bias_files
if __name__ == '__main__':
sensor_analyses(run_read_noise_task)
raft_id = os.environ['LCATR_UNIT_ID']
run = siteUtils.getRunNumber()
bias_files = get_bias_files(raft_id)
title = 'Overscan correlations, {}, Run {}'.format(raft_id, run)
plt.rcParams['figure.figsize'] = (8, 8)
# raft_level_oscan_correlations(bias_files, title=title)
# plt.savefig('{}_{}_overscan_correlations.png'.format(raft_id, run))
sensorTest.rolloff_mask(fe55_files[0], rolloff_mask_file)
task = sensorTest.Fe55Task()
task.config.temp_set_point = -100.
task.run(sensor_id, fe55_files, (rolloff_mask_file,), accuracy_req=0.01)
# Fe55 gain and psf analysis results plots for the test report.
results_file = '%s_eotest_results.fits' % sensor_id
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
siteUtils.make_png_file(plots.gains,
'%s_gains.png' % file_prefix)
siteUtils.make_png_file(sensorTest.plot_flat,
'%s_mean_bias.png' % file_prefix,
mean_bias_file,
title='%s, mean bias frame' % sensor_id,
annotation='ADU/pixel, overscan-subtracted')
fe55_file = glob.glob('%s_psf_results*.fits' % sensor_id)[0]
siteUtils.make_png_file(plots.fe55_dists,
'%s_fe55_dists.png' % file_prefix,
fe55_file=fe55_file)
siteUtils.make_png_file(plots.psf_dists,
'%s_psf_dists.png' % file_prefix,
fe55_file=fe55_file)
if __name__ == '__main__':
sensor_analyses(run_fe55_task)