def bright_defects_jh_task(det_name):
"""JH version of single sensor bright pixels task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, bright_defects_task, get_mask_files
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
dark_files \
= siteUtils.dependency_glob(glob_pattern('bright_defects', det_name),
acq_jobname=acq_jobname)
if not dark_files:
print("bright_defects_task: Needed data files missing for detector",
det_name)
return None
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
mask_files = sorted(glob.glob(f'{file_prefix}*mask*.fits'))
bias_frame = bias_filename(run, det_name)
return bright_defects_task(run,
det_name,
dark_files,
gains,
mask_files=mask_files,
bias_frame=bias_frame)
def flat_gain_stability_jh_task(det_name):
"""JH version of single sensor execution of the flat pairs task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
bias_filename, flat_gain_stability_task,\
get_mask_files, medianed_dark_frame
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
flat_files = siteUtils.dependency_glob(glob_pattern('tearing', det_name),
acq_jobname=acq_jobname)
if not flat_files:
print("flat_gain_stability_task: Flat pairs files not found for",
det_name)
return None
mask_files = get_mask_files(det_name)
bias_frame = bias_filename(run, det_name)
dark_frame = medianed_dark_frame(det_name)
return flat_gain_stability_task(run,
det_name,
flat_files,
mask_files=mask_files,
bias_frame=bias_frame,
dark_frame=dark_frame)
def bf_jh_task(det_name):
"""JH version of single sensor execution of the brighter-fatter task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
bias_filename, bf_task, find_flat2_bot, get_mask_files,\
get_amplifier_gains
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
flat_files \
= siteUtils.dependency_glob(glob_pattern('brighter_fatter', det_name),
acq_jobname=acq_jobname)
if not flat_files:
print("bf_jh_task: Flat pairs files not found for detector", det_name)
return None
flat_files = [_ for _ in flat_files if 'flat1' in _]
mask_files = get_mask_files(det_name)
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
bias_frame = bias_filename(run, det_name)
return bf_task(run,
det_name,
flat_files,
gains,
mask_files=mask_files,
flat2_finder=find_flat2_bot,
bias_frame=bias_frame)
def ptc_jh_task(det_name):
"""JH version of single sensor execution of the PTC task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, ptc_task, get_mask_files
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
flat_files = siteUtils.dependency_glob(glob_pattern('ptc', det_name),
acq_jobname=acq_jobname)
if not flat_files:
print("ptc_task: Flat pairs files not found for detector", det_name)
return None
mask_files = get_mask_files(det_name)
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
bias_frame = bias_filename(run, det_name)
return ptc_task(run,
det_name,
flat_files,
gains,
mask_files=mask_files,
bias_frame=bias_frame)
def dark_defects_jh_task(det_name):
"""JH version of single sensor execution of the dark defects task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, dark_defects_task, get_mask_files
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
sflat_files \
= siteUtils.dependency_glob(glob_pattern('dark_defects', det_name),
acq_jobname=acq_jobname)
if not sflat_files:
print("dark_defects_task: No high flux superflat files found for",
det_name)
return None
mask_files = sorted(glob.glob(f'{file_prefix}*mask*.fits'))
bias_frame = bias_filename(run, det_name)
return dark_defects_task(run,
det_name,
sflat_files,
mask_files=mask_files,
bias_frame=bias_frame)
def dark_current_jh_task(det_name):
"""JH version of single sensor execution of the dark current task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, dark_current_task,\
plot_ccd_total_noise, get_mask_files
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
dark_files \
= siteUtils.dependency_glob(glob_pattern('dark_current', det_name),
acq_jobname=acq_jobname,
description="Dark current frames:")
if not dark_files:
print("dark_current_task: No dark files found for detector", det_name)
return None
mask_files = get_mask_files(det_name)
eotest_results_file \
= siteUtils.dependency_glob('{}_eotest_results.fits'.format(file_prefix),
jobname='read_noise_BOT')[0]
gains = get_amplifier_gains('{}_eotest_results.fits'.format(file_prefix))
bias_frame = bias_filename(run, det_name)
dark_curr_pixels, dark95s \
= dark_current_task(run, det_name, dark_files, gains,
mask_files=mask_files, bias_frame=bias_frame)
plot_ccd_total_noise(run, det_name, dark_curr_pixels, dark95s,
eotest_results_file)
return dark_curr_pixels, dark95s
def get_isr_files(det_name, run):
"""Get bias, dark, and mask files."""
files = set()
try:
files.add(bias_filename(run, det_name))
except IndexError:
pass
try:
files.add(medianed_dark_frame(det_name))
except IndexError:
pass
files = files.union(get_mask_files(det_name))
return files
def raft_divisidero_tearing(raft_name):
"""JH version of divisidero tearing analysis of BOT data."""
import os
from collections import defaultdict
import json
import matplotlib.pyplot as plt
import siteUtils
from lsst.eotest.sensor.cteTask import superflat
import lsst.eotest.raft as raftTest
from bot_eo_analyses import glob_pattern, bias_filename, get_mask_files
run = siteUtils.getRunNumber()
pattern = glob_pattern('divisadero_tearing', f'{raft_name}_*')
acq_jobname = siteUtils.getProcessName('BOT_acq')
bot_data = siteUtils.dependency_glob(pattern, acq_jobname=acq_jobname)
if not bot_data:
return
sflat_files = defaultdict(list)
for item in bot_data:
slot = item.split('.')[0].split('_')[-1]
sflat_files[slot].append(item)
median_sflats = dict()
for slot, files in sflat_files.items():
det_name = '_'.join((raft_name, slot))
outfile = f'{det_name}_{run}_median_sflat.fits'
bias_frame = bias_filename(run, det_name)
median_sflats[slot] = superflat(files,
outfile=outfile,
bias_frame=bias_frame)
mask_files = dict()
for slot in sflat_files:
det_name = '_'.join((raft_name, slot))
mask_files[slot] = get_mask_files(det_name)
title = f'Run {run} {raft_name}'
acq_run = os.environ.get('LCATR_ACQ_RUN', None)
if acq_run is not None:
title += f' (acq {acq_run})'
max_divisidero_tearing \
= raftTest.ana_divisidero_tearing(median_sflats, mask_files,
title=title)
plt.savefig(f'{raft_name}_{run}_divisidero.png')
with open(f'{raft_name}_{run}_max_divisidero.json', 'w') as fd:
json.dump(max_divisidero_tearing, fd)
def tearing_jh_task(det_name):
"""JH version of single sensor execution of the tearing task."""
import siteUtils
from bot_eo_analyses import glob_pattern, bias_filename, tearing_task
run = siteUtils.getRunNumber()
acq_jobname = siteUtils.getProcessName('BOT_acq')
flat_files = siteUtils.dependency_glob(glob_pattern('tearing', det_name),
acq_jobname=acq_jobname)
if not flat_files:
print("tearing_task: Flat files not found for detector", det_name)
return None
bias_frame = bias_filename(run, det_name)
return tearing_task(run, det_name, flat_files, bias_frame=bias_frame)
def flat_pairs_jh_task(det_name):
"""JH version of single sensor execution of the flat pairs task."""
import os
import glob
import siteUtils
import json
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, flat_pairs_task, mondiode_value,\
get_mask_files, medianed_dark_frame
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
flat_files \
= siteUtils.dependency_glob(glob_pattern('flat_pairs', det_name),
acq_jobname=acq_jobname)
if not flat_files:
print("flat_pairs_task: Flat pairs files not found for detector",
det_name)
return None
mask_files = get_mask_files(det_name)
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
bias_frame = bias_filename(run, det_name)
dark_frame = medianed_dark_frame(det_name)
if 'LCATR_PD_CORRECTIONS_FILE' in os.environ:
filter_corr_file = os.environ['LCATR_PD_CORRECTIONS_FILE']
else:
filter_corr_file = os.path.join(os.environ['EOANALYSISJOBSDIR'],
'data', 'pd_filter_corrections.json')
print('Using pd_filter_corrections file:', filter_corr_file, flush=True)
with open(filter_corr_file) as fd:
filter_corrections = json.load(fd)
return flat_pairs_task(run,
det_name,
flat_files,
gains,
mask_files=mask_files,
bias_frame=bias_frame,
mondiode_func=mondiode_value,
dark_frame=dark_frame,
filter_corrections=filter_corrections)
def dark_current_jh_task(det_name):
"""JH version of single sensor execution of the dark current task."""
from collections import defaultdict
from astropy.io import fits
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, dark_current_task,\
plot_ccd_total_noise, get_mask_files
from bot_data_handling import most_common_dark_files
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
dark_files \
= siteUtils.dependency_glob(glob_pattern('dark_current', det_name),
acq_jobname=acq_jobname,
description="Dark current frames:")
if not dark_files:
print("dark_current_task: No dark files found for detector", det_name)
return None
dark_files_linear_fit = list(dark_files)
dark_files = most_common_dark_files(dark_files)
if len(dark_files_linear_fit) == len(dark_files):
# These data only have one integration time, so skip linear
# fit of dark current signal vs integration time.
dark_files_linear_fit = None
mask_files = get_mask_files(det_name)
eotest_results_file \
= siteUtils.dependency_glob('{}_eotest_results.fits'.format(file_prefix),
jobname='read_noise_BOT')[0]
gains = get_amplifier_gains('{}_eotest_results.fits'.format(file_prefix))
bias_frame = bias_filename(run, det_name)
dark_curr_pixels, dark95s \
= dark_current_task(run, det_name, dark_files, gains,
mask_files=mask_files, bias_frame=bias_frame,
dark_files_linear_fit=dark_files_linear_fit)
plot_ccd_total_noise(run, det_name, dark_curr_pixels, dark95s,
eotest_results_file)
return dark_curr_pixels, dark95s
def get_isr_files(det_name, run):
"""Get bias, dark, and mask files."""
files = set()
try:
bias_fn = bias_filename(run, det_name)
if isinstance(bias_fn, str):
if os.path.isfile(bias_fn):
files.add(bias_fn)
elif isinstance(bias_fn, (tuple, list)) and bias_fn[0] == 'rowcol':
if bias_fn[1] is not None:
files.add(bias_fn[1])
else:
files = files.union(bias_fn)
except IndexError:
pass
try:
files.add(medianed_dark_frame(det_name))
except IndexError:
pass
files = files.union(get_mask_files(det_name))
return files
def persistence_jh_task(det_name):
"""JH version of the persistence_task."""
import os
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern, \
bias_frame_task, get_mask_files, get_bot_eo_config, persistence_task, \
bias_filename
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
bias_files \
= siteUtils.dependency_glob(glob_pattern('persistence_bias', det_name),
acq_jobname=acq_jobname,
description='Persistence bias frames:')
dark_files \
= siteUtils.dependency_glob(glob_pattern('persistence_dark', det_name),
acq_jobname=acq_jobname,
description='Persistence dark frames:')
if not bias_files or not dark_files:
print("persistence_task: Needed data files are missing for detector",
det_name)
return None
# Sort by test sequence number, i.e., by filenames.
bias_files = sorted(bias_files)
dark_files = sorted(dark_files)
use_pca_bias = os.environ.get('LCATR_USE_PCA_BIAS_FIT', "True") == 'True'
if use_pca_bias:
superbias_frame = bias_filename(run, det_name)
else:
# Make a superbias frame using the pre-exposure persistence bias
# files, skipping the first exposure.
superbias_frame = f'{file_prefix}_persistence_superbias.fits'
bias_frame_task(run, det_name, bias_files, bias_frame=superbias_frame)
return persistence_task(run, det_name, dark_files, superbias_frame,
get_mask_files(det_name))
def traps_jh_task(det_name):
"""JH version of single sensor execution of the traps analysis task."""
import glob
import siteUtils
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, traps_task, get_mask_files
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
trap_files = siteUtils.dependency_glob(glob_pattern('traps', det_name),
acq_jobname=acq_jobname)
if not trap_files:
print("traps_task: No pocket pumping file found for detector",
det_name)
return None
trap_file = trap_files[0]
mask_files = get_mask_files(det_name)
# Omit rolloff defects mask since a trap in the rolloff edge region can
# affect the entire column.
mask_files \
= [item for item in mask_files if item.find('edge_rolloff') == -1]
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
bias_frame = bias_filename(run, det_name)
return traps_task(run,
det_name,
trap_file,
gains,
mask_files=mask_files,
bias_frame=bias_frame)
def raft_jh_signal_correlations(raft_name):
"""JH version of raft-level signal-correlation analysis."""
import os
import logging
import numpy as np
import matplotlib.pyplot as plt
import siteUtils
from bot_eo_analyses import bias_filename, make_file_prefix, \
append_acq_run, glob_pattern
from signal_correlations import raft_level_signal_correlations
logger = logging.getLogger('raft_jh_noise_correlations')
logger.setLevel(logging.INFO)
# Use the high signal superflat files.
pattern = glob_pattern('raft_signal_correlations', f'{raft_name}_S??')
acq_jobname = siteUtils.getProcessName('BOT_acq')
sflat_files = siteUtils.dependency_glob(pattern, acq_jobname=acq_jobname)
folders = sorted(
list(set([os.path.basename(os.path.dirname(_)) for _ in sflat_files])))
logger.info(f'folders: {folders}')
if not folders:
logger.info('No data found for this raft, so skip the '
'signal correlation analysis.')
return
flat1_files = dict()
flat2_files = dict()
for item in sflat_files:
folder = os.path.basename(os.path.dirname(item))
if folder not in folders[:2]:
continue
logger.info(f'item: {item}')
logger.info(f'folder: {folder}')
basename = os.path.basename(item)
logger.info(f'basename: {basename}')
slot = basename.split('_')[-1][:-len('.fits')]
if folder == folders[0]:
flat1_files[slot] = item
elif folder == folders[1]:
flat2_files[slot] = item
logger.info('flat pair files:')
for slot in flat1_files:
logger.info(' ' + flat1_files[slot])
logger.info(' ' + flat2_files[slot])
# Find the median bias files for the target raft.
run = siteUtils.getRunNumber()
bias_files = dict()
for slot in flat1_files.keys():
det_name = '_'.join((raft_name, slot))
bias_files[slot] = bias_filename(run, det_name)
file_prefix = make_file_prefix(run, raft_name)
title = append_acq_run("Imaging region correlations, "
f"Run {run}, {raft_name}")
raft_level_signal_correlations(flat1_files,
flat2_files,
bias_files,
title=title)
plt.savefig('{}_imaging_region_correlations.png'.format(file_prefix))
def cte_jh_task(det_name):
"""JH version of single sensor execution of the CTE task."""
import os
import glob
import shutil
import siteUtils
import lsst.eotest.sensor as sensorTest
from bot_eo_analyses import make_file_prefix, glob_pattern,\
get_amplifier_gains, bias_filename, cte_task, plot_cte_results,\
get_mask_files
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
sflat_high_files \
= siteUtils.dependency_glob(glob_pattern('cte_high', det_name),
acq_jobname=acq_jobname)
sflat_low_files \
= siteUtils.dependency_glob(glob_pattern('cte_low', det_name),
acq_jobname=acq_jobname)
if not sflat_high_files and not sflat_low_files:
print("cte_task: Superflat files not found for detector", det_name)
return None
mask_files = get_mask_files(det_name)
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
# Write gains to local eotest_results_file, which cte_task will update.
namps = 16 if 'SW' not in det_name else 8
results = sensorTest.EOTestResults(eotest_results_file, namps=namps)
for amp, gain in gains.items():
results.add_seg_result(amp, 'GAIN', gain)
results.write()
bias_frame = bias_filename(run, det_name)
# Omit rolloff defects mask since it would mask some of the edges used
# in the eper method.
mask_files \
= [item for item in mask_files if item.find('edge_rolloff') == -1]
png_files = []
for flux_level, sflat_files in zip(('high', 'low'),
(sflat_high_files, sflat_low_files)):
superflat_file = cte_task(run, det_name, sflat_files, gains,
mask_files=mask_files, flux_level=flux_level,
bias_frame=bias_frame)
png_files.extend(plot_cte_results(run, det_name, superflat_file,
eotest_results_file,
mask_files=mask_files))
png_file_list = '{}_cte_task_png_files.txt'.format(det_name)
with open(png_file_list, 'w') as output:
for item in png_files:
if os.path.isfile(item):
output.write('{}\n'.format(item))
return None