def flat_gain_stability_jh_task(det_name): """JH version of single sensor execution of the flat pairs task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ bias_filename, flat_gain_stability_task,\ get_mask_files, medianed_dark_frame run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files = siteUtils.dependency_glob(glob_pattern('tearing', det_name), acq_jobname=acq_jobname) if not flat_files: print("flat_gain_stability_task: Flat pairs files not found for", det_name) return None mask_files = get_mask_files(det_name) bias_frame = bias_filename(run, det_name) dark_frame = medianed_dark_frame(det_name) return flat_gain_stability_task(run, det_name, flat_files, mask_files=mask_files, bias_frame=bias_frame, dark_frame=dark_frame)
def bf_jh_task(det_name): """JH version of single sensor execution of the brighter-fatter task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ bias_filename, bf_task, find_flat2_bot, get_mask_files,\ get_amplifier_gains run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files \ = siteUtils.dependency_glob(glob_pattern('brighter_fatter', det_name), acq_jobname=acq_jobname) if not flat_files: print("bf_jh_task: Flat pairs files not found for detector", det_name) return None flat_files = [_ for _ in flat_files if 'flat1' in _] mask_files = get_mask_files(det_name) eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(run, det_name) return bf_task(run, det_name, flat_files, gains, mask_files=mask_files, flat2_finder=find_flat2_bot, bias_frame=bias_frame)
def ptc_jh_task(det_name): """JH version of single sensor execution of the PTC task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, ptc_task, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files = siteUtils.dependency_glob(glob_pattern('ptc', det_name), acq_jobname=acq_jobname) if not flat_files: print("ptc_task: Flat pairs files not found for detector", det_name) return None mask_files = get_mask_files(det_name) eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(run, det_name) return ptc_task(run, det_name, flat_files, gains, mask_files=mask_files, bias_frame=bias_frame)
def persistence_jh_task(det_name): """JH version of the persistence_task.""" import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern, \ bias_frame_task, get_mask_files, get_bot_eo_config, persistence_task run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') bias_files \ = siteUtils.dependency_glob(glob_pattern('persistence_bias', det_name), acq_jobname=acq_jobname, description='Persistence bias frames:') dark_files \ = siteUtils.dependency_glob(glob_pattern('persistence_dark', det_name), acq_jobname=acq_jobname, description='Persistence dark frames:') if not bias_files or not dark_files: print("persistence_task: Needed data files are missing for detector", det_name) return None # Sort by test sequence number, i.e., by filenames. bias_files = sorted(bias_files) dark_files = sorted(dark_files) # Make a superbias frame using the pre-exposure persistence bias # files, skipping the first exposure. superbias_frame = f'{file_prefix}_persistence_superbias.fits' bias_frame_task(run, det_name, bias_files, bias_frame=superbias_frame) return persistence_task(run, det_name, dark_files, superbias_frame, get_mask_files(det_name))
def read_noise_jh_task(det_name): """JH version of the single sensor read noise task.""" import os import glob import logging import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, read_noise_task, get_mask_files logger = logging.getLogger('read_noise_jh_task') logger.setLevel(logging.INFO) run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') nbias = os.environ.get('LCATR_NUM_BIAS_FRAMES', 10) bias_files \ = siteUtils.dependency_glob(glob_pattern('read_noise', det_name), acq_jobname=acq_jobname)[:nbias] if not bias_files: logger.info( "read_noise_task: Needed data files are missing " "for detector %s", det_name) return None eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) mask_files = get_mask_files(det_name) return read_noise_task(run, det_name, bias_files, gains, mask_files=mask_files)
def dark_current_jh_task(det_name): """JH version of single sensor execution of the dark current task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, dark_current_task,\ plot_ccd_total_noise, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') dark_files \ = siteUtils.dependency_glob(glob_pattern('dark_current', det_name), acq_jobname=acq_jobname, description="Dark current frames:") if not dark_files: print("dark_current_task: No dark files found for detector", det_name) return None mask_files = get_mask_files(det_name) eotest_results_file \ = siteUtils.dependency_glob('{}_eotest_results.fits'.format(file_prefix), jobname='read_noise_BOT')[0] gains = get_amplifier_gains('{}_eotest_results.fits'.format(file_prefix)) bias_frame = bias_filename(run, det_name) dark_curr_pixels, dark95s \ = dark_current_task(run, det_name, dark_files, gains, mask_files=mask_files, bias_frame=bias_frame) plot_ccd_total_noise(run, det_name, dark_curr_pixels, dark95s, eotest_results_file) return dark_curr_pixels, dark95s
def get_isr_files(det_name, run): """Get bias, dark, and mask files.""" files = set() try: files.add(bias_filename(run, det_name)) except IndexError: pass try: files.add(medianed_dark_frame(det_name)) except IndexError: pass files = files.union(get_mask_files(det_name)) return files
def raft_divisidero_tearing(raft_name): """JH version of divisidero tearing analysis of BOT data.""" import os from collections import defaultdict import json import matplotlib.pyplot as plt import siteUtils from lsst.eotest.sensor.cteTask import superflat import lsst.eotest.raft as raftTest from bot_eo_analyses import glob_pattern, bias_filename, get_mask_files run = siteUtils.getRunNumber() pattern = glob_pattern('divisadero_tearing', f'{raft_name}_*') acq_jobname = siteUtils.getProcessName('BOT_acq') bot_data = siteUtils.dependency_glob(pattern, acq_jobname=acq_jobname) if not bot_data: return sflat_files = defaultdict(list) for item in bot_data: slot = item.split('.')[0].split('_')[-1] sflat_files[slot].append(item) median_sflats = dict() for slot, files in sflat_files.items(): det_name = '_'.join((raft_name, slot)) outfile = f'{det_name}_{run}_median_sflat.fits' bias_frame = bias_filename(run, det_name) median_sflats[slot] = superflat(files, outfile=outfile, bias_frame=bias_frame) mask_files = dict() for slot in sflat_files: det_name = '_'.join((raft_name, slot)) mask_files[slot] = get_mask_files(det_name) title = f'Run {run} {raft_name}' acq_run = os.environ.get('LCATR_ACQ_RUN', None) if acq_run is not None: title += f' (acq {acq_run})' max_divisidero_tearing \ = raftTest.ana_divisidero_tearing(median_sflats, mask_files, title=title) plt.savefig(f'{raft_name}_{run}_divisidero.png') with open(f'{raft_name}_{run}_max_divisidero.json', 'w') as fd: json.dump(max_divisidero_tearing, fd)
def flat_pairs_jh_task(det_name): """JH version of single sensor execution of the flat pairs task.""" import os import glob import siteUtils import json from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, flat_pairs_task, mondiode_value,\ get_mask_files, medianed_dark_frame run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files \ = siteUtils.dependency_glob(glob_pattern('flat_pairs', det_name), acq_jobname=acq_jobname) if not flat_files: print("flat_pairs_task: Flat pairs files not found for detector", det_name) return None mask_files = get_mask_files(det_name) eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(run, det_name) dark_frame = medianed_dark_frame(det_name) if 'LCATR_PD_CORRECTIONS_FILE' in os.environ: filter_corr_file = os.environ['LCATR_PD_CORRECTIONS_FILE'] else: filter_corr_file = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'data', 'pd_filter_corrections.json') print('Using pd_filter_corrections file:', filter_corr_file, flush=True) with open(filter_corr_file) as fd: filter_corrections = json.load(fd) return flat_pairs_task(run, det_name, flat_files, gains, mask_files=mask_files, bias_frame=bias_frame, mondiode_func=mondiode_value, dark_frame=dark_frame, filter_corrections=filter_corrections)
def dark_current_jh_task(det_name): """JH version of single sensor execution of the dark current task.""" from collections import defaultdict from astropy.io import fits import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, dark_current_task,\ plot_ccd_total_noise, get_mask_files from bot_data_handling import most_common_dark_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') dark_files \ = siteUtils.dependency_glob(glob_pattern('dark_current', det_name), acq_jobname=acq_jobname, description="Dark current frames:") if not dark_files: print("dark_current_task: No dark files found for detector", det_name) return None dark_files_linear_fit = list(dark_files) dark_files = most_common_dark_files(dark_files) if len(dark_files_linear_fit) == len(dark_files): # These data only have one integration time, so skip linear # fit of dark current signal vs integration time. dark_files_linear_fit = None mask_files = get_mask_files(det_name) eotest_results_file \ = siteUtils.dependency_glob('{}_eotest_results.fits'.format(file_prefix), jobname='read_noise_BOT')[0] gains = get_amplifier_gains('{}_eotest_results.fits'.format(file_prefix)) bias_frame = bias_filename(run, det_name) dark_curr_pixels, dark95s \ = dark_current_task(run, det_name, dark_files, gains, mask_files=mask_files, bias_frame=bias_frame, dark_files_linear_fit=dark_files_linear_fit) plot_ccd_total_noise(run, det_name, dark_curr_pixels, dark95s, eotest_results_file) return dark_curr_pixels, dark95s
def get_isr_files(det_name, run): """Get bias, dark, and mask files.""" files = set() try: bias_fn = bias_filename(run, det_name) if isinstance(bias_fn, str): if os.path.isfile(bias_fn): files.add(bias_fn) elif isinstance(bias_fn, (tuple, list)) and bias_fn[0] == 'rowcol': if bias_fn[1] is not None: files.add(bias_fn[1]) else: files = files.union(bias_fn) except IndexError: pass try: files.add(medianed_dark_frame(det_name)) except IndexError: pass files = files.union(get_mask_files(det_name)) return files
def traps_jh_task(det_name): """JH version of single sensor execution of the traps analysis task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, traps_task, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') trap_files = siteUtils.dependency_glob(glob_pattern('traps', det_name), acq_jobname=acq_jobname) if not trap_files: print("traps_task: No pocket pumping file found for detector", det_name) return None trap_file = trap_files[0] mask_files = get_mask_files(det_name) # Omit rolloff defects mask since a trap in the rolloff edge region can # affect the entire column. mask_files \ = [item for item in mask_files if item.find('edge_rolloff') == -1] eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(run, det_name) return traps_task(run, det_name, trap_file, gains, mask_files=mask_files, bias_frame=bias_frame)
def cte_jh_task(det_name): """JH version of single sensor execution of the CTE task.""" import os import glob import shutil import siteUtils import lsst.eotest.sensor as sensorTest from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, cte_task, plot_cte_results,\ get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') sflat_high_files \ = siteUtils.dependency_glob(glob_pattern('cte_high', det_name), acq_jobname=acq_jobname) sflat_low_files \ = siteUtils.dependency_glob(glob_pattern('cte_low', det_name), acq_jobname=acq_jobname) if not sflat_high_files and not sflat_low_files: print("cte_task: Superflat files not found for detector", det_name) return None mask_files = get_mask_files(det_name) eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) # Write gains to local eotest_results_file, which cte_task will update. namps = 16 if 'SW' not in det_name else 8 results = sensorTest.EOTestResults(eotest_results_file, namps=namps) for amp, gain in gains.items(): results.add_seg_result(amp, 'GAIN', gain) results.write() bias_frame = bias_filename(run, det_name) # Omit rolloff defects mask since it would mask some of the edges used # in the eper method. mask_files \ = [item for item in mask_files if item.find('edge_rolloff') == -1] png_files = [] for flux_level, sflat_files in zip(('high', 'low'), (sflat_high_files, sflat_low_files)): superflat_file = cte_task(run, det_name, sflat_files, gains, mask_files=mask_files, flux_level=flux_level, bias_frame=bias_frame) png_files.extend(plot_cte_results(run, det_name, superflat_file, eotest_results_file, mask_files=mask_files)) png_file_list = '{}_cte_task_png_files.txt'.format(det_name) with open(png_file_list, 'w') as output: for item in png_files: if os.path.isfile(item): output.write('{}\n'.format(item)) return None