def read_noise_jh_task(det_name):
"""JH version of the single sensor read noise task."""
import os
import glob
import logging
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, read_noise_task, get_mask_files
logger = logging.getLogger('read_noise_jh_task')
logger.setLevel(logging.INFO)
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
nbias = os.environ.get('LCATR_NUM_BIAS_FRAMES', 10)
bias_files \
= siteUtils.dependency_glob(glob_pattern('read_noise', det_name),
acq_jobname=acq_jobname)[:nbias]
if not bias_files:
logger.info(
"read_noise_task: Needed data files are missing "
"for detector %s", det_name)
return None
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
mask_files = get_mask_files(det_name)
return read_noise_task(run,
det_name,
bias_files,
gains,
mask_files=mask_files)
def run_read_noise_task(sensor_id):
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
bias_files = siteUtils.dependency_glob(
'S*/%s_fe55_fe55_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('fe55_raft_acq'),
description='Fe55 files for read noise:')
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
system_noise = None
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
task = sensorTest.ReadNoiseTask()
task.config.temp_set_point = -100.
task.run(sensor_id,
bias_files,
gains,
system_noise=system_noise,
mask_files=mask_files,
use_overscan=True)
# Compute amp-amp correlated noise.
_, corr_fig, _ = correlated_noise(bias_files,
target=0,
make_plots=True,
title=sensor_id)
plt.figure(corr_fig.number)
plt.savefig('%s_correlated_noise.png' % file_prefix)
def bias_frame_jh_task(det_name):
"""JH version of the bias_frame_task."""
import os
import siteUtils
import json
from bot_eo_analyses import glob_pattern, bias_frame_task, \
bias_stability_task
run = siteUtils.getRunNumber()
acq_jobname = siteUtils.getProcessName('BOT_acq')
bias_files \
= siteUtils.dependency_glob(glob_pattern('bias_frame', det_name),
acq_jobname=acq_jobname,
description='Bias frames:')
if not bias_files:
print("bias_frame_task: Needed data files are missing for detector",
det_name)
return None
bias_stability_files \
= siteUtils.dependency_glob(glob_pattern('bias_stability', det_name),
acq_jobname=acq_jobname,
description='Bias stability frames:')
if not bias_stability_files:
print("bias_stability_task: Needed data files are missing for detector",
det_name)
return None
bias_stability_files = sorted(bias_stability_files)
bias_stability_task(run, det_name, bias_stability_files)
return bias_frame_task(run, det_name, bias_files[1:])
def run_bf_task(sensor_id):
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
flat_files = siteUtils.dependency_glob(
'S*/%s_flat*flat1*.fits' % sensor_id,
jobname=siteUtils.getProcessName('flat_pair_raft_acq'),
description='Flat files:')
bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' %
sensor_id,
description='Superbias files:')[0]
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
task = sensorTest.BFTask()
task.run(sensor_id,
flat_files,
mask_files=mask_files,
bias_frame=bias_frame)
results_file = '%s_eotest_results.fits' % sensor_id
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
siteUtils.make_png_file(plots.bf_curves,
'%s_brighter-fatter.png' % file_prefix,
bf_file='%s_bf.fits' % sensor_id)
def dark_current_jh_task(det_name):
"""JH version of single sensor execution of the dark current task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, dark_current_task,\
plot_ccd_total_noise, get_mask_files
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
dark_files \
= siteUtils.dependency_glob(glob_pattern('dark_current', det_name),
acq_jobname=acq_jobname,
description="Dark current frames:")
if not dark_files:
print("dark_current_task: No dark files found for detector", det_name)
return None
mask_files = get_mask_files(det_name)
eotest_results_file \
= siteUtils.dependency_glob('{}_eotest_results.fits'.format(file_prefix),
jobname='read_noise_BOT')[0]
gains = get_amplifier_gains('{}_eotest_results.fits'.format(file_prefix))
bias_frame = bias_filename(run, det_name)
dark_curr_pixels, dark95s \
= dark_current_task(run, det_name, dark_files, gains,
mask_files=mask_files, bias_frame=bias_frame)
plot_ccd_total_noise(run, det_name, dark_curr_pixels, dark95s,
eotest_results_file)
return dark_curr_pixels, dark95s
def run_dark_pixels_task(sensor_id):
print("run_dark_pixels_task: sensor_id = ", sensor_id)
# raft_id = os.environ['LCATR_UNIT_ID']
# raft = camera_components.Raft.create_from_etrav(raft_id)
# wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id];
"Single sensor execution of the dark pixels task."
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
sflat_files = siteUtils.dependency_glob(
'S*/%s_sflat_500_flat_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('sflat_raft_acq'),
description='Superflat files:')
print("sflat query: ", 'S*/%s_sflat_500_flat_H*.fits' % sensor_id)
print("sflat_files = ", sflat_files)
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
task = sensorTest.DarkPixelsTask()
task.run(sensor_id, sflat_files, mask_files)
siteUtils.make_png_file(sensorTest.plot_flat,
'%s_superflat_dark_defects.png' % file_prefix,
'%s_median_sflat.fits' % sensor_id,
title='%s, superflat for dark defects analysis' %
sensor_id,
annotation='ADU/pixel')
def run_dark_pixels_task(sensor_id):
"Single sensor execution of the dark pixels task."
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
acq_jobname = siteUtils.getProcessName('sflat_raft_acq')
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
sflat_files = siteUtils.dependency_glob('S*/%s_sflat_500_flat_H*.fits' % sensor_id,
jobname=acq_jobname,
description='Superflat files:')
bias_files = siteUtils.dependency_glob('S*/%s_sflat_bias*.fits' % sensor_id,
jobname=acq_jobname,
description='Bias files:')
bias_frame = eotestUtils.make_median_bias_frame(bias_files, sensor_id,
'sflat_raft_acq')
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
task = sensorTest.DarkPixelsTask()
task.run(sensor_id, sflat_files, mask_files, bias_frame=bias_frame)
siteUtils.make_png_file(sensorTest.plot_flat,
'%s_superflat_dark_defects.png' % file_prefix,
'%s_median_sflat.fits' % sensor_id,
title='%s, superflat for dark defects analysis' % sensor_id,
annotation='ADU/pixel', flatten=True, binsize=4)
def dark_defects_jh_task(det_name):
"""JH version of single sensor execution of the dark defects task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, dark_defects_task, get_mask_files
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
sflat_files \
= siteUtils.dependency_glob(glob_pattern('dark_defects', det_name),
acq_jobname=acq_jobname)
if not sflat_files:
print("dark_defects_task: No high flux superflat files found for",
det_name)
return None
mask_files = sorted(glob.glob(f'{file_prefix}*mask*.fits'))
bias_frame = bias_filename(run, det_name)
return dark_defects_task(run,
det_name,
sflat_files,
mask_files=mask_files,
bias_frame=bias_frame)
def run_trap_task(sensor_id):
"""Run the traps analysis for a single sensor."""
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
trap_file = siteUtils.dependency_glob(
'S*/%s_trap_ppump_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('ppump_raft_acq'),
description='Trap file:')[0]
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
# Omit rolloff defects mask since a trap in the rolloff edge region can
# affect the entire column.
bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' %
sensor_id,
description='Super bias frame:')[0]
mask_files = [
item for item in mask_files if item.find('rolloff_defects') == -1
]
print("Using mask files:")
for mask_file in mask_files:
print(" " + mask_file)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.TrapTask()
task.run(sensor_id, trap_file, mask_files, gains, bias_frame=bias_frame)
def ptc_jh_task(det_name):
"""JH version of single sensor execution of the PTC task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, ptc_task, get_mask_files
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
flat_files = siteUtils.dependency_glob(glob_pattern('ptc', det_name),
acq_jobname=acq_jobname)
if not flat_files:
print("ptc_task: Flat pairs files not found for detector", det_name)
return None
mask_files = get_mask_files(det_name)
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
bias_frame = bias_filename(run, det_name)
return ptc_task(run,
det_name,
flat_files,
gains,
mask_files=mask_files,
bias_frame=bias_frame)
def bf_jh_task(det_name):
"""JH version of single sensor execution of the brighter-fatter task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
bias_filename, bf_task, find_flat2_bot, get_mask_files,\
get_amplifier_gains
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
flat_files \
= siteUtils.dependency_glob(glob_pattern('brighter_fatter', det_name),
acq_jobname=acq_jobname)
if not flat_files:
print("bf_jh_task: Flat pairs files not found for detector", det_name)
return None
flat_files = [_ for _ in flat_files if 'flat1' in _]
mask_files = get_mask_files(det_name)
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
bias_frame = bias_filename(run, det_name)
return bf_task(run,
det_name,
flat_files,
gains,
mask_files=mask_files,
flat2_finder=find_flat2_bot,
bias_frame=bias_frame)
def qe_jh_task(det_name):
"""JH version of single sensor execution of the QE task."""
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
lambda_files = siteUtils.dependency_glob(glob_pattern('qe', det_name),
acq_jobname=acq_jobname)
if not lambda_files:
print("qe_task: QE scan files not found for detector", det_name)
return None
pd_ratio_file = eotestUtils.getPhotodiodeRatioFile()
if pd_ratio_file is None:
message = ("The BOT photodiode ratio file is " +
"not given in config/%s/eotest_calibrations.cfg."
% siteUtils.getSiteName())
raise RuntimeError(message)
# correction_image = eotestUtils.getIlluminationNonUniformityImage()
# if correction_image is None:
# print()
# print("WARNING: The correction image file is not given in")
# print("config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName())
# print("No correction for non-uniform illumination will be applied.")
# print()
# sys.stdout.flush()
mask_files = get_mask_files(det_name)
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
bias_frame = bias_filename(run, det_name)
return qe_task(run, det_name, lambda_files, pd_ratio_file, gains,
mask_files=mask_files, bias_frame=bias_frame,
mondiode_func=mondiode_value)
def persistence_jh_task(det_name):
"""JH version of the persistence_task."""
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern, \
bias_frame_task, get_mask_files, get_bot_eo_config, persistence_task
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
bias_files \
= siteUtils.dependency_glob(glob_pattern('persistence_bias', det_name),
acq_jobname=acq_jobname,
description='Persistence bias frames:')
dark_files \
= siteUtils.dependency_glob(glob_pattern('persistence_dark', det_name),
acq_jobname=acq_jobname,
description='Persistence dark frames:')
if not bias_files or not dark_files:
print("persistence_task: Needed data files are missing for detector",
det_name)
return None
# Sort by test sequence number, i.e., by filenames.
bias_files = sorted(bias_files)
dark_files = sorted(dark_files)
# Make a superbias frame using the pre-exposure persistence bias
# files, skipping the first exposure.
superbias_frame = f'{file_prefix}_persistence_superbias.fits'
bias_frame_task(run, det_name, bias_files, bias_frame=superbias_frame)
return persistence_task(run, det_name, dark_files, superbias_frame,
get_mask_files(det_name))
def bright_defects_jh_task(det_name):
"""JH version of single sensor bright pixels task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, bright_defects_task, get_mask_files
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
dark_files \
= siteUtils.dependency_glob(glob_pattern('bright_defects', det_name),
acq_jobname=acq_jobname)
if not dark_files:
print("bright_defects_task: Needed data files missing for detector",
det_name)
return None
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
mask_files = sorted(glob.glob(f'{file_prefix}*mask*.fits'))
bias_frame = bias_filename(run, det_name)
return bright_defects_task(run,
det_name,
dark_files,
gains,
mask_files=mask_files,
bias_frame=bias_frame)
def run_ptc_task(sensor_id):
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
flat_files = siteUtils.dependency_glob(
'S*/%s_flat*flat?_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('flat_pair_raft_acq'),
description='Flat files:')
bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' %
sensor_id,
description='Super bias frame:')[0]
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.PtcTask()
task.run(sensor_id, flat_files, mask_files, gains, bias_frame=bias_frame)
results_file = '%s_eotest_results.fits' % sensor_id
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
siteUtils.make_png_file(plots.ptcs,
'%s_ptcs.png' % file_prefix,
ptc_file='%s_ptc.fits' % sensor_id)
def flat_gain_stability_jh_task(det_name):
"""JH version of single sensor execution of the flat pairs task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
bias_filename, flat_gain_stability_task,\
get_mask_files, medianed_dark_frame
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
flat_files = siteUtils.dependency_glob(glob_pattern('tearing', det_name),
acq_jobname=acq_jobname)
if not flat_files:
print("flat_gain_stability_task: Flat pairs files not found for",
det_name)
return None
mask_files = get_mask_files(det_name)
bias_frame = bias_filename(run, det_name)
dark_frame = medianed_dark_frame(det_name)
return flat_gain_stability_task(run,
det_name,
flat_files,
mask_files=mask_files,
bias_frame=bias_frame,
dark_frame=dark_frame)
def run_dark_current_task(sensor_id):
"Single sensor execution of dark current analysis."
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
dark_files = siteUtils.dependency_glob(
'S*/%s_dark_dark_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('dark_raft_acq'),
description='Dark files:')
bias_frame = siteUtils.dependency_glob('%s_sflat*median_bias.fits' %
sensor_id,
description='Super bias frame:')[0]
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.DarkCurrentTask()
task.config.temp_set_point = -100.
dark_curr_pixels, dark95s \
= task.run(sensor_id, dark_files, mask_files, gains,
bias_frame=bias_frame)
results_file \
= siteUtils.dependency_glob('%s_eotest_results.fits' % sensor_id,
jobname='read_noise_raft')[0]
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
siteUtils.make_png_file(plots.total_noise,
'%s_noise.png' % file_prefix,
dark95s=dark95s)
def run_qe_task(sensor_id):
try:
if hw_objects[sensor_id] :
return
except:
hw_objects[sensor_id] = True
print("hw_objects = ",hw_objects)
"Single sensor execution of the QE task."
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
lambda_files = siteUtils.dependency_glob('S*/%s_lambda_flat_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('qe_raft_acq'),
description='Lambda files:')
pd_ratio_file = eotestUtils.getPhotodiodeRatioFile()
if pd_ratio_file is None:
message = ("The test-stand specific photodiode ratio file is " +
"not given in config/%s/eotest_calibrations.cfg."
% siteUtils.getSiteName())
raise RuntimeError(message)
correction_image = eotestUtils.getIlluminationNonUniformityImage()
if correction_image is None:
print()
print("WARNING: The correction image file is not given in")
print("config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName())
print("No correction for non-uniform illumination will be applied.")
print()
sys.stdout.flush()
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.QeTask()
task.config.temp_set_point = -100.
task.run(sensor_id, lambda_files, pd_ratio_file, mask_files, gains,
correction_image=correction_image)
results_file \
= siteUtils.dependency_glob('%s_eotest_results.fits' % sensor_id,
jobname='fe55_raft_analysis',
description='Fe55 results file')[0]
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
siteUtils.make_png_file(plots.qe,
'%s_qe.png' % file_prefix,
qe_file='%s_QE.fits' % sensor_id)
try:
plots.flat_fields(os.path.dirname(lambda_files[0]),
annotation='e-/pixel, gain-corrected, bias-subtracted')
except Exception as eobj:
print("Exception raised while creating flat fields:")
print(str(eobj))
def run_tearing_detection(sensor_id):
"""
Loop over the acquisition jobs and perform tearing analysis on each.
"""
import pickle
import lsst.eotest.image_utils as imutils
import lsst.eotest.sensor as sensorTest
import siteUtils
from tearing_detection import tearing_detection
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
tearing_stats = []
# Create a super bias frame.
bias_files = siteUtils.dependency_glob(
'S*/%s_flat_bias*.fits' % sensor_id,
jobname=siteUtils.getProcessName('flat_pair_raft_acq'),
description='Bias files:')
if bias_files:
bias_frame = '%s_superbias.fits' % sensor_id
amp_geom = sensorTest.makeAmplifierGeometry(bias_files[0])
imutils.superbias_file(bias_files[:10], amp_geom.serial_overscan,
bias_frame)
else:
bias_frame = None
acq_jobs = {
('flat_pair_raft_acq', 'N/A'): 'S*/%s_flat*flat?_*.fits',
('qe_raft_acq', 'N/A'): 'S*/%s_lambda_flat_*.fits',
('sflat_raft_acq', 'low_flux'): 'S*/%s_sflat_500_flat_L*.fits',
('sflat_raft_acq', 'high_flux'): 'S*/%s_sflat_500_flat_H*.fits'
}
for job_key, pattern in acq_jobs.items():
job_name, subset = job_key
flats = siteUtils.dependency_glob(
pattern % sensor_id,
jobname=siteUtils.getProcessName(job_name),
description='Flat files:')
tearing_found, _ = tearing_detection(flats, bias_frame=bias_frame)
tearing_stats.append((job_name, subset, sensor_id, len(tearing_found)))
with open('%s_tearing_stats.pkl' % file_prefix, 'wb') as output:
pickle.dump(tearing_stats, output)
def dirname_dependencyGlob(sensor_id, **kwds):
"""
Return the directory path with the FITS output files for the
specified jobname.
"""
if kwds.has_key('jobname'):
kwds['jobname'] = siteUtils.getProcessName(kwds['jobname'])
file0 = dependency_glob('%(sensor_id)s*.fits' % locals(), **kwds)[0]
dirname = os.path.split(file0)[0]
if dirname == '':
dirname = '.'
return dirname
def get_scan_mode_files(raft_name):
"""Get the scan mode filenames for a given raft, organized by
scan mode folder and slot.
"""
acq_jobname = siteUtils.getProcessName('BOT_acq')
pattern = glob_pattern('scan_mode', '{}_*'.format(raft_name))
files = siteUtils.dependency_glob(pattern, acq_jobname=acq_jobname)
scan_mode_files = defaultdict(dict)
for item in files:
scan_dir = os.path.basename(os.path.dirname(item))
slot = os.path.basename(item).split('.')[0].split('_')[-1]
scan_mode_files[scan_dir][slot] = item
return scan_mode_files
def get_files(data_keys, device_name):
"""
Get the files needed by the current job for the specified device,
e.g., 'R22_S11' or 'R22', for CCDs or rafts, respectively.
"""
acq_jobname = siteUtils.getProcessName('BOT_acq')
files = set()
for data_key in data_keys:
pattern = glob_pattern(data_key, f'{device_name}*')
files = files.union(
siteUtils.dependency_glob(pattern,
acq_jobname=acq_jobname,
verbose=False))
return files
def dirname_dependencyGlob(ccd_vendor, **kwds):
"""
Return the directory path with the FITS output files for the
specified jobname.
"""
if 'jobname' in kwds:
kwds['jobname'] = siteUtils.getProcessName(kwds['jobname'])
file0 = dependency_glob('S*/%(ccd_vendor)s*.fits' % locals(), **kwds)[0]
# Apply os.path.dirname twice to omit both slot folder and file
# basename.
dirname = os.path.dirname(os.path.dirname(file0))
if dirname == '':
dirname = '.'
return os.path.join(dirname, 'S*')
def raft_divisidero_tearing(raft_name):
"""JH version of divisidero tearing analysis of BOT data."""
import os
from collections import defaultdict
import json
import matplotlib.pyplot as plt
import siteUtils
from lsst.eotest.sensor.cteTask import superflat
import lsst.eotest.raft as raftTest
from bot_eo_analyses import glob_pattern, bias_filename, get_mask_files
run = siteUtils.getRunNumber()
pattern = glob_pattern('divisadero_tearing', f'{raft_name}_*')
acq_jobname = siteUtils.getProcessName('BOT_acq')
bot_data = siteUtils.dependency_glob(pattern, acq_jobname=acq_jobname)
if not bot_data:
return
sflat_files = defaultdict(list)
for item in bot_data:
slot = item.split('.')[0].split('_')[-1]
sflat_files[slot].append(item)
median_sflats = dict()
for slot, files in sflat_files.items():
det_name = '_'.join((raft_name, slot))
outfile = f'{det_name}_{run}_median_sflat.fits'
bias_frame = bias_filename(run, det_name)
median_sflats[slot] = superflat(files,
outfile=outfile,
bias_frame=bias_frame)
mask_files = dict()
for slot in sflat_files:
det_name = '_'.join((raft_name, slot))
mask_files[slot] = get_mask_files(det_name)
title = f'Run {run} {raft_name}'
acq_run = os.environ.get('LCATR_ACQ_RUN', None)
if acq_run is not None:
title += f' (acq {acq_run})'
max_divisidero_tearing \
= raftTest.ana_divisidero_tearing(median_sflats, mask_files,
title=title)
plt.savefig(f'{raft_name}_{run}_divisidero.png')
with open(f'{raft_name}_{run}_max_divisidero.json', 'w') as fd:
json.dump(max_divisidero_tearing, fd)
def bias_frame_jh_task(det_name):
"""JH version of the bias_frame_task."""
import os
import siteUtils
import json
from bot_eo_analyses import glob_pattern, bias_frame_task, \
bias_stability_task, pca_corrected_superbias
run = siteUtils.getRunNumber()
acq_jobname = siteUtils.getProcessName('BOT_acq')
bias_files \
= siteUtils.dependency_glob(glob_pattern('bias_frame', det_name),
acq_jobname=acq_jobname,
description='Bias frames:')
# Skip the first 5 bias files to avoid transient features in the
# imaging array from after just starting the run.
bias_files = sorted(bias_files, key=os.path.basename)[5:]
if not bias_files:
print("bias_frame_task: Needed data files are missing for detector",
det_name)
return None
bias_stability_files \
= siteUtils.dependency_glob(glob_pattern('bias_stability', det_name),
acq_jobname=acq_jobname,
description='Bias stability frames:')
if not bias_stability_files:
print("bias_stability_task: Needed data files are missing for detector",
det_name)
return None
superbias_file, bias_model_components \
= bias_frame_task(run, det_name, bias_files)
bias_stability_files = sorted(bias_stability_files)
if not os.environ.get('LCATR_USE_PCA_BIAS_FIT', "True") == 'True':
bias_model_components = None
if siteUtils.get_analysis_run('bias') == 'rowcol':
bias_model_components = 'rowcol', superbias_file
print("bias_model_components:", bias_model_components)
bias_stability_task(run, det_name, bias_stability_files,
bias_model_components=bias_model_components)
if (bias_model_components is not None and
bias_model_components[0] != 'rowcol'):
pca_corrected_superbias(run, det_name, bias_files,
bias_model_components)
return superbias_file
def gain_stability_jh_task(det_name):
"JH version of the gain stability analysis task."
import os
import siteUtils
from bot_eo_analyses import glob_pattern, gain_stability_task
run = siteUtils.getRunNumber()
acq_jobname = siteUtils.getProcessName('BOT_acq')
fe55_files = siteUtils.dependency_glob(glob_pattern('fe55', det_name),
acq_jobname=acq_jobname)
if not fe55_files:
print("fe55_task: Needed data files missing for detector", det_name)
return None
return gain_stability_task(run, det_name, fe55_files)
def tearing_jh_task(det_name):
"""JH version of single sensor execution of the tearing task."""
import siteUtils
from bot_eo_analyses import glob_pattern, bias_filename, tearing_task
run = siteUtils.getRunNumber()
acq_jobname = siteUtils.getProcessName('BOT_acq')
flat_files = siteUtils.dependency_glob(glob_pattern('tearing', det_name),
acq_jobname=acq_jobname)
if not flat_files:
print("tearing_task: Flat files not found for detector", det_name)
return None
bias_frame = bias_filename(run, det_name)
return tearing_task(run, det_name, flat_files, bias_frame=bias_frame)
def flat_pairs_jh_task(det_name):
"""JH version of single sensor execution of the flat pairs task."""
import os
import glob
import siteUtils
import json
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, flat_pairs_task, mondiode_value,\
get_mask_files, medianed_dark_frame
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
flat_files \
= siteUtils.dependency_glob(glob_pattern('flat_pairs', det_name),
acq_jobname=acq_jobname)
if not flat_files:
print("flat_pairs_task: Flat pairs files not found for detector",
det_name)
return None
mask_files = get_mask_files(det_name)
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
bias_frame = bias_filename(run, det_name)
dark_frame = medianed_dark_frame(det_name)
if 'LCATR_PD_CORRECTIONS_FILE' in os.environ:
filter_corr_file = os.environ['LCATR_PD_CORRECTIONS_FILE']
else:
filter_corr_file = os.path.join(os.environ['EOANALYSISJOBSDIR'],
'data', 'pd_filter_corrections.json')
print('Using pd_filter_corrections file:', filter_corr_file, flush=True)
with open(filter_corr_file) as fd:
filter_corrections = json.load(fd)
return flat_pairs_task(run,
det_name,
flat_files,
gains,
mask_files=mask_files,
bias_frame=bias_frame,
mondiode_func=mondiode_value,
dark_frame=dark_frame,
filter_corrections=filter_corrections)
def run_tearing_detection(sensor_id):
"""
Loop over the acquisition jobs and perform tearing analysis on each.
"""
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
tearing_stats = []
for job_key, pattern in acq_jobs.items():
job_name, subset = job_key
flats = siteUtils.dependency_glob(pattern % sensor_id,
jobname=siteUtils.getProcessName(job_name),
description='Flat files:')
tearing_found, _ = tearing_detection(flats)
tearing_stats.append((job_name, subset, sensor_id, len(tearing_found)))
with open('%s_tearing_stats.pkl' % file_prefix, 'wb') as output:
pickle.dump(tearing_stats, output)
def run_flat_pair_task(sensor_id):
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
flat_files = siteUtils.dependency_glob(
'S*/%s_flat*flat?_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('flat_pair_raft_acq'),
description='Flat files:')
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
use_exptime = True
if siteUtils.getSiteName() == 'SLAC':
# Since slit-width can be set individually for each exposure
# on TS-8 at IR-2 (LSSTTD-1231), we need to use the MONDIODE
# keyword for computing the integrated incident flux.
use_exptime = False
task = sensorTest.FlatPairTask()
task.run(sensor_id,
flat_files,
mask_files,
gains,
linearity_spec_range=(1e4, 9e4),
use_exptime=use_exptime)
results_file = '%s_eotest_results.fits' % sensor_id
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
Ne_bounds = (1e4, 9e4)
detresp_file = '%s_det_response.fits' % sensor_id
siteUtils.make_png_file(plots.linearity,
'%s_linearity.png' % file_prefix,
detresp_file=detresp_file,
max_dev=0.03,
use_exptime=use_exptime,
Ne_bounds=Ne_bounds)
siteUtils.make_png_file(plots.linearity_resids,
'%s_linearity_resids.png' % file_prefix,
detresp_file=detresp_file,
max_dev=0.03,
Ne_bounds=Ne_bounds,
use_exptime=use_exptime)
#!/usr/bin/env python
import os
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
siteUtils.aggregate_job_ids()
sensor_id = siteUtils.getUnitId()
dark_files = siteUtils.dependency_glob('*_persistence_dark_*.fits',
jobname=siteUtils.getProcessName('persist_acq'),
description='Dark files:',
sort=True)
flat = siteUtils.dependency_glob('*_persistence_flat_*.fits',
jobname=siteUtils.getProcessName('persist_acq'),
description='Flat file:')[0]
pre_flat_darks = dark_files[:3]
post_flat_darks = dark_files[3:]
mask_files = eotestUtils.glob_mask_files()
gains = eotestUtils.getSensorGains()
task = sensorTest.PersistenceTask()
task.run(sensor_id, pre_flat_darks, flat, post_flat_darks, mask_files, gains)
#!/usr/bin/env python
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
siteUtils.aggregate_job_ids()
sensor_id = siteUtils.getUnitId()
trap_file = siteUtils.dependency_glob('*_trap_ppump_*.fits',
jobname=siteUtils.getProcessName('ppump_acq'),
description='Trap file:')[0]
mask_files = eotestUtils.glob_mask_files()
# Omit rolloff defects mask since a trap in the rolloff edge region can
# affect the entire column.
mask_files = [item for item in mask_files if item.find('rolloff_defects') == -1]
print("Using mask files:")
for mask_file in mask_files:
print(" " + mask_file)
gains = eotestUtils.getSensorGains()
task = sensorTest.TrapTask()
task.run(sensor_id, trap_file, mask_files, gains)
#!/usr/bin/env python
import siteUtils
import metUtils
from flatnessTask import flatnessTask
sensor_id = siteUtils.getUnitId()
# Find the metrology scan data
met_file = siteUtils.dependency_glob('*.dat',jobname=siteUtils.getProcessName('Room_Temp_Measurement_After_Thermal_Cycle'),description='')
met_file = met_file[0] # siteUtils returns met_file as a list with one member;
# here take the first (and only) member
# The dtype below indicates the source of the data, which is always OGP
# for sensors measured at BNL
#flatnessTask(sensor_id, met_file, dtype='OGP', pickle_file='flatness.pickle')
results.append(eotestUtils.eotestCalibrations())
sensor_id = siteUtils.getUnitId()
qe_data = pyfits.open('%s_QE.fits' % sensor_id)['QE_BANDS'].data
QE = OrderedDict((band, []) for band in qe_data.field('BAND'))
for amp in range(1, 17):
values = qe_data.field('AMP%02i' % amp)
for band, value in zip(QE, values):
QE[band].append(value)
for band in QE:
results.append(lcatr.schema.valid(lcatr.schema.get('qe_analysis'),
band=band, QE=np.mean(QE[band])))
qe_acq_job_id = siteUtils.get_prerequisite_job_id('*_lambda_flat_*.fits',
jobname=siteUtils.getProcessName('qe_acq'))
md = dict(photodiode_ratio_file=dict(JOB_ID=qe_acq_job_id),
illumination_non_uniformity_file=dict(JOB_ID=qe_acq_job_id))
results.extend(eotestUtils.eotestCalibsPersist('photodiode_ratio_file',
'illumination_non_uniformity_file',
metadata=md))
qe_files = glob.glob('*QE*.*')
for item in qe_files:
if item.endswith('.fits'):
eotestUtils.addHeaderData(item, LSST_NUM=sensor_id, TESTTYPE='LAMBDA',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results.extend([lcatr.schema.fileref.make(item) for item in qe_files])
lcatr.schema.write_file(results)
lcatr.schema.validate_file()
#!/usr/bin/env python
import siteUtils
import metUtils
from flatnessTask import flatnessTask
sensor_id = siteUtils.getUnitId()
# Find the metrology scan data
met_file = siteUtils.dependency_glob('*.dat',jobname=siteUtils.getProcessName('Pump_and_Room_Temp_Measurement'),description='')
met_file = met_file[0] # siteUtils returns met_file as a list with one member;
# here take the first (and only) member
# The dtype below indicates the source of the data, which is always OGP
# for sensors measured at BNL
#flatnessTask(sensor_id, met_file, dtype='ETU', pickle_file='flatness.pickle')
#!/usr/bin/env python
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
siteUtils.aggregate_job_ids()
sensor_id = siteUtils.getUnitId()
dark_files = siteUtils.dependency_glob('*_dark_dark_*.fits',
jobname=siteUtils.getProcessName('dark_acq'),
description='Dark files:')
mask_files = eotestUtils.glob_mask_files()
gains = eotestUtils.getSensorGains()
task = sensorTest.BrightPixelsTask()
task.run(sensor_id, dark_files, mask_files, gains)
system_noise_data = data['SYSTEM_NOISE']
total_noise_data = data['TOTAL_NOISE']
for amp, read_noise, system_noise, total_noise in zip(amps, read_noise_data,
system_noise_data,
total_noise_data):
results.append(lcatr.schema.valid(lcatr.schema.get('read_noise'),
amp=amp, read_noise=read_noise,
system_noise=system_noise,
total_noise=total_noise))
results.extend(siteUtils.jobInfo())
results.append(eotestUtils.eotestCalibrations())
fe55_acq_job_id = siteUtils.get_prerequisite_job_id('*_fe55_fe55_*.fits',
jobname=siteUtils.getProcessName('fe55_acq'))
md = dict(system_noise_file=dict(JOB_ID=fe55_acq_job_id))
results.extend(eotestUtils.eotestCalibsPersist('system_noise_file',
metadata=md))
files = glob.glob('*read_noise?*.fits')
for fitsfile in files:
eotestUtils.addHeaderData(fitsfile, LSST_NUM=sensor_id, TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
data_products = [lcatr.schema.fileref.make(item) for item in files]
results.extend(data_products)
lcatr.schema.write_file(results)
lcatr.schema.validate_file()
#!/usr/bin/env python
import sys
import os
import siteUtils
import metUtils
from flatnessTask import flatnessTask
raft_id = siteUtils.getUnitId()
# Find the TS5 metrology scan data by constructing the name of the data-taking step
acqjobname = siteUtils.getJobName().replace('_Analysis', '')
if ("Room_Temp_Measurement" in acqjobname and not "After" in acqjobname):
acqjobname = "Pump_and_" + acqjobname
print('flatness_ts5:')
print(acqjobname)
print(siteUtils.getProcessName(acqjobname))
print('----')
# siteUtils returns a list with one member;
# here take the first (and only) member
infile = siteUtils.dependency_glob('*.csv',
jobname=siteUtils.getProcessName(acqjobname),
description='')[0]
print "infile = %s" % infile
flatnessTask(raft_id, infile, dtype='TS5', pickle_file='flatness_ts5.pickle')
#!/usr/bin/env python
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
siteUtils.aggregate_job_ids()
sensor_id = siteUtils.getUnitId()
flat_files = siteUtils.dependency_glob('*_flat*flat?_*.fits',
jobname=siteUtils.getProcessName('flat_acq'),
description='Flat files:')
mask_files = eotestUtils.glob_mask_files()
gains = eotestUtils.getSensorGains()
task = sensorTest.FlatPairTask()
task.run(sensor_id, flat_files, mask_files, gains, use_exptime=True)
def processName_dependencyGlob(*args, **kwds):
if kwds.has_key('jobname'):
kwds['jobname'] = siteUtils.getProcessName(kwds['jobname'])
return dependency_glob(*args, **kwds)
#!/usr/bin/env python
import siteUtils
import metUtils
from flatnessTask import flatnessTask
sensor_id = siteUtils.getUnitId()
# Find the metrology scan data
met_file = siteUtils.dependency_glob('*.dat',jobname=siteUtils.getProcessName('Cooling_Measurement-1'),description='')
met_file = met_file[0] # siteUtils returns met_file as a list with one member;
# here take the first (and only) member
# The dtype below indicates the source of the data, which is always OGP
# for sensors measured at BNL
#flatnessTask(sensor_id, met_file, dtype='OGP', pickle_file='flatness.pickle')
#!/usr/bin/env python
import sys
import os
import siteUtils
import metUtils
from flatnessTask import flatnessTask
#os.system("rsync -r [email protected]:jobHarness/jh_stage/e2v-CCD/An-e2v-CCD/\* /home/LSSTuser/jobHarness/jh_stage/e2v-CCD/")
sensor_id = siteUtils.getUnitId()
ccd_vendor = siteUtils.getCcdVendor()
# Find the OGP metrology scan data
#flat_file = siteUtils.dependency_glob('*/*/*_AbsHeight*.DAT',
direcs = siteUtils.dependency_glob('*',
jobname=siteUtils.getProcessName('OGP_Dimensional_Metrology_Upload'),description='OGP Absolute Height scan:')
print "direcs:"
print direcs
print "------------------------------"
dimdir = ""
for line in direcs :
if "DimensionalMetrology" in line :
dimdir = line
print "dimensional metrology directory = %s" % dimdir
flat_file = ""
for root, dirs, flat_files in os.walk(dimdir):
#!/usr/bin/env python
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
# Use Fe55 exposures and the overscan region instead of the bias
# frames since the vendor data are not guaranteed to have the same
# gains for the bias frames.
bias_files = siteUtils.datacatalog_glob('*_fe55_fe55_*.fits',
testtype="FE55",
imgtype="FE55",
description='Bias files (using overscan):')
gains = eotestUtils.getSensorGains(jobname='fe55_offline')
system_noise_files = siteUtils.dependency_glob('noise_*.fits',
jobname=siteUtils.getProcessName('system_noise'))
if not system_noise_files:
system_noise_files = None
mask_files = eotestUtils.glob_mask_files()
task = sensorTest.ReadNoiseTask()
task.run(sensor_id, bias_files, gains,
system_noise_files=system_noise_files, mask_files=mask_files,
use_overscan=True)
#!/usr/bin/env python
import sys
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
spot_files = siteUtils.dependency_glob('*_spot_*.fits',
jobname=siteUtils.getProcessName('xtalk_acq'),
description='Spot files:')
mask_files = eotestUtils.glob_mask_files()
system_xtalk_file = eotestUtils.getSystemCrosstalkFile()
if system_xtalk_file is None:
print
print "WARNING: The system crosstalk file is not given in"
print "config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName()
print "No correction for system crosstalk will be performed."
print
sys.stdout.flush()
task = sensorTest.CrosstalkTask()
task.run(sensor_id, spot_files, mask_files, system_xtalk_file=system_xtalk_file)