def bright_defects_jh_task(det_name): """JH version of single sensor bright pixels task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, bright_defects_task, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') dark_files \ = siteUtils.dependency_glob(glob_pattern('bright_defects', det_name), acq_jobname=acq_jobname) if not dark_files: print("bright_defects_task: Needed data files missing for detector", det_name) return None eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) mask_files = sorted(glob.glob(f'{file_prefix}*mask*.fits')) bias_frame = bias_filename(run, det_name) return bright_defects_task(run, det_name, dark_files, gains, mask_files=mask_files, bias_frame=bias_frame)
def flat_gain_stability_jh_task(det_name): """JH version of single sensor execution of the flat pairs task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ bias_filename, flat_gain_stability_task,\ get_mask_files, medianed_dark_frame run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files = siteUtils.dependency_glob(glob_pattern('tearing', det_name), acq_jobname=acq_jobname) if not flat_files: print("flat_gain_stability_task: Flat pairs files not found for", det_name) return None mask_files = get_mask_files(det_name) bias_frame = bias_filename(run, det_name) dark_frame = medianed_dark_frame(det_name) return flat_gain_stability_task(run, det_name, flat_files, mask_files=mask_files, bias_frame=bias_frame, dark_frame=dark_frame)
def bf_jh_task(det_name): """JH version of single sensor execution of the brighter-fatter task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ bias_filename, bf_task, find_flat2_bot, get_mask_files,\ get_amplifier_gains run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files \ = siteUtils.dependency_glob(glob_pattern('brighter_fatter', det_name), acq_jobname=acq_jobname) if not flat_files: print("bf_jh_task: Flat pairs files not found for detector", det_name) return None flat_files = [_ for _ in flat_files if 'flat1' in _] mask_files = get_mask_files(det_name) eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(run, det_name) return bf_task(run, det_name, flat_files, gains, mask_files=mask_files, flat2_finder=find_flat2_bot, bias_frame=bias_frame)
def ptc_jh_task(det_name): """JH version of single sensor execution of the PTC task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, ptc_task, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files = siteUtils.dependency_glob(glob_pattern('ptc', det_name), acq_jobname=acq_jobname) if not flat_files: print("ptc_task: Flat pairs files not found for detector", det_name) return None mask_files = get_mask_files(det_name) eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(run, det_name) return ptc_task(run, det_name, flat_files, gains, mask_files=mask_files, bias_frame=bias_frame)
def dark_defects_jh_task(det_name): """JH version of single sensor execution of the dark defects task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, dark_defects_task, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') sflat_files \ = siteUtils.dependency_glob(glob_pattern('dark_defects', det_name), acq_jobname=acq_jobname) if not sflat_files: print("dark_defects_task: No high flux superflat files found for", det_name) return None mask_files = sorted(glob.glob(f'{file_prefix}*mask*.fits')) bias_frame = bias_filename(run, det_name) return dark_defects_task(run, det_name, sflat_files, mask_files=mask_files, bias_frame=bias_frame)
def dark_current_jh_task(det_name): """JH version of single sensor execution of the dark current task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, dark_current_task,\ plot_ccd_total_noise, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') dark_files \ = siteUtils.dependency_glob(glob_pattern('dark_current', det_name), acq_jobname=acq_jobname, description="Dark current frames:") if not dark_files: print("dark_current_task: No dark files found for detector", det_name) return None mask_files = get_mask_files(det_name) eotest_results_file \ = siteUtils.dependency_glob('{}_eotest_results.fits'.format(file_prefix), jobname='read_noise_BOT')[0] gains = get_amplifier_gains('{}_eotest_results.fits'.format(file_prefix)) bias_frame = bias_filename(run, det_name) dark_curr_pixels, dark95s \ = dark_current_task(run, det_name, dark_files, gains, mask_files=mask_files, bias_frame=bias_frame) plot_ccd_total_noise(run, det_name, dark_curr_pixels, dark95s, eotest_results_file) return dark_curr_pixels, dark95s
def get_isr_files(det_name, run): """Get bias, dark, and mask files.""" files = set() try: files.add(bias_filename(run, det_name)) except IndexError: pass try: files.add(medianed_dark_frame(det_name)) except IndexError: pass files = files.union(get_mask_files(det_name)) return files
def raft_divisidero_tearing(raft_name): """JH version of divisidero tearing analysis of BOT data.""" import os from collections import defaultdict import json import matplotlib.pyplot as plt import siteUtils from lsst.eotest.sensor.cteTask import superflat import lsst.eotest.raft as raftTest from bot_eo_analyses import glob_pattern, bias_filename, get_mask_files run = siteUtils.getRunNumber() pattern = glob_pattern('divisadero_tearing', f'{raft_name}_*') acq_jobname = siteUtils.getProcessName('BOT_acq') bot_data = siteUtils.dependency_glob(pattern, acq_jobname=acq_jobname) if not bot_data: return sflat_files = defaultdict(list) for item in bot_data: slot = item.split('.')[0].split('_')[-1] sflat_files[slot].append(item) median_sflats = dict() for slot, files in sflat_files.items(): det_name = '_'.join((raft_name, slot)) outfile = f'{det_name}_{run}_median_sflat.fits' bias_frame = bias_filename(run, det_name) median_sflats[slot] = superflat(files, outfile=outfile, bias_frame=bias_frame) mask_files = dict() for slot in sflat_files: det_name = '_'.join((raft_name, slot)) mask_files[slot] = get_mask_files(det_name) title = f'Run {run} {raft_name}' acq_run = os.environ.get('LCATR_ACQ_RUN', None) if acq_run is not None: title += f' (acq {acq_run})' max_divisidero_tearing \ = raftTest.ana_divisidero_tearing(median_sflats, mask_files, title=title) plt.savefig(f'{raft_name}_{run}_divisidero.png') with open(f'{raft_name}_{run}_max_divisidero.json', 'w') as fd: json.dump(max_divisidero_tearing, fd)
def tearing_jh_task(det_name): """JH version of single sensor execution of the tearing task.""" import siteUtils from bot_eo_analyses import glob_pattern, bias_filename, tearing_task run = siteUtils.getRunNumber() acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files = siteUtils.dependency_glob(glob_pattern('tearing', det_name), acq_jobname=acq_jobname) if not flat_files: print("tearing_task: Flat files not found for detector", det_name) return None bias_frame = bias_filename(run, det_name) return tearing_task(run, det_name, flat_files, bias_frame=bias_frame)
def flat_pairs_jh_task(det_name): """JH version of single sensor execution of the flat pairs task.""" import os import glob import siteUtils import json from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, flat_pairs_task, mondiode_value,\ get_mask_files, medianed_dark_frame run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files \ = siteUtils.dependency_glob(glob_pattern('flat_pairs', det_name), acq_jobname=acq_jobname) if not flat_files: print("flat_pairs_task: Flat pairs files not found for detector", det_name) return None mask_files = get_mask_files(det_name) eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(run, det_name) dark_frame = medianed_dark_frame(det_name) if 'LCATR_PD_CORRECTIONS_FILE' in os.environ: filter_corr_file = os.environ['LCATR_PD_CORRECTIONS_FILE'] else: filter_corr_file = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'data', 'pd_filter_corrections.json') print('Using pd_filter_corrections file:', filter_corr_file, flush=True) with open(filter_corr_file) as fd: filter_corrections = json.load(fd) return flat_pairs_task(run, det_name, flat_files, gains, mask_files=mask_files, bias_frame=bias_frame, mondiode_func=mondiode_value, dark_frame=dark_frame, filter_corrections=filter_corrections)
def dark_current_jh_task(det_name): """JH version of single sensor execution of the dark current task.""" from collections import defaultdict from astropy.io import fits import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, dark_current_task,\ plot_ccd_total_noise, get_mask_files from bot_data_handling import most_common_dark_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') dark_files \ = siteUtils.dependency_glob(glob_pattern('dark_current', det_name), acq_jobname=acq_jobname, description="Dark current frames:") if not dark_files: print("dark_current_task: No dark files found for detector", det_name) return None dark_files_linear_fit = list(dark_files) dark_files = most_common_dark_files(dark_files) if len(dark_files_linear_fit) == len(dark_files): # These data only have one integration time, so skip linear # fit of dark current signal vs integration time. dark_files_linear_fit = None mask_files = get_mask_files(det_name) eotest_results_file \ = siteUtils.dependency_glob('{}_eotest_results.fits'.format(file_prefix), jobname='read_noise_BOT')[0] gains = get_amplifier_gains('{}_eotest_results.fits'.format(file_prefix)) bias_frame = bias_filename(run, det_name) dark_curr_pixels, dark95s \ = dark_current_task(run, det_name, dark_files, gains, mask_files=mask_files, bias_frame=bias_frame, dark_files_linear_fit=dark_files_linear_fit) plot_ccd_total_noise(run, det_name, dark_curr_pixels, dark95s, eotest_results_file) return dark_curr_pixels, dark95s
def get_isr_files(det_name, run): """Get bias, dark, and mask files.""" files = set() try: bias_fn = bias_filename(run, det_name) if isinstance(bias_fn, str): if os.path.isfile(bias_fn): files.add(bias_fn) elif isinstance(bias_fn, (tuple, list)) and bias_fn[0] == 'rowcol': if bias_fn[1] is not None: files.add(bias_fn[1]) else: files = files.union(bias_fn) except IndexError: pass try: files.add(medianed_dark_frame(det_name)) except IndexError: pass files = files.union(get_mask_files(det_name)) return files
def persistence_jh_task(det_name): """JH version of the persistence_task.""" import os import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern, \ bias_frame_task, get_mask_files, get_bot_eo_config, persistence_task, \ bias_filename run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') bias_files \ = siteUtils.dependency_glob(glob_pattern('persistence_bias', det_name), acq_jobname=acq_jobname, description='Persistence bias frames:') dark_files \ = siteUtils.dependency_glob(glob_pattern('persistence_dark', det_name), acq_jobname=acq_jobname, description='Persistence dark frames:') if not bias_files or not dark_files: print("persistence_task: Needed data files are missing for detector", det_name) return None # Sort by test sequence number, i.e., by filenames. bias_files = sorted(bias_files) dark_files = sorted(dark_files) use_pca_bias = os.environ.get('LCATR_USE_PCA_BIAS_FIT', "True") == 'True' if use_pca_bias: superbias_frame = bias_filename(run, det_name) else: # Make a superbias frame using the pre-exposure persistence bias # files, skipping the first exposure. superbias_frame = f'{file_prefix}_persistence_superbias.fits' bias_frame_task(run, det_name, bias_files, bias_frame=superbias_frame) return persistence_task(run, det_name, dark_files, superbias_frame, get_mask_files(det_name))
def traps_jh_task(det_name): """JH version of single sensor execution of the traps analysis task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, traps_task, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') trap_files = siteUtils.dependency_glob(glob_pattern('traps', det_name), acq_jobname=acq_jobname) if not trap_files: print("traps_task: No pocket pumping file found for detector", det_name) return None trap_file = trap_files[0] mask_files = get_mask_files(det_name) # Omit rolloff defects mask since a trap in the rolloff edge region can # affect the entire column. mask_files \ = [item for item in mask_files if item.find('edge_rolloff') == -1] eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(run, det_name) return traps_task(run, det_name, trap_file, gains, mask_files=mask_files, bias_frame=bias_frame)
def raft_jh_signal_correlations(raft_name): """JH version of raft-level signal-correlation analysis.""" import os import logging import numpy as np import matplotlib.pyplot as plt import siteUtils from bot_eo_analyses import bias_filename, make_file_prefix, \ append_acq_run, glob_pattern from signal_correlations import raft_level_signal_correlations logger = logging.getLogger('raft_jh_noise_correlations') logger.setLevel(logging.INFO) # Use the high signal superflat files. pattern = glob_pattern('raft_signal_correlations', f'{raft_name}_S??') acq_jobname = siteUtils.getProcessName('BOT_acq') sflat_files = siteUtils.dependency_glob(pattern, acq_jobname=acq_jobname) folders = sorted( list(set([os.path.basename(os.path.dirname(_)) for _ in sflat_files]))) logger.info(f'folders: {folders}') if not folders: logger.info('No data found for this raft, so skip the ' 'signal correlation analysis.') return flat1_files = dict() flat2_files = dict() for item in sflat_files: folder = os.path.basename(os.path.dirname(item)) if folder not in folders[:2]: continue logger.info(f'item: {item}') logger.info(f'folder: {folder}') basename = os.path.basename(item) logger.info(f'basename: {basename}') slot = basename.split('_')[-1][:-len('.fits')] if folder == folders[0]: flat1_files[slot] = item elif folder == folders[1]: flat2_files[slot] = item logger.info('flat pair files:') for slot in flat1_files: logger.info(' ' + flat1_files[slot]) logger.info(' ' + flat2_files[slot]) # Find the median bias files for the target raft. run = siteUtils.getRunNumber() bias_files = dict() for slot in flat1_files.keys(): det_name = '_'.join((raft_name, slot)) bias_files[slot] = bias_filename(run, det_name) file_prefix = make_file_prefix(run, raft_name) title = append_acq_run("Imaging region correlations, " f"Run {run}, {raft_name}") raft_level_signal_correlations(flat1_files, flat2_files, bias_files, title=title) plt.savefig('{}_imaging_region_correlations.png'.format(file_prefix))
def cte_jh_task(det_name): """JH version of single sensor execution of the CTE task.""" import os import glob import shutil import siteUtils import lsst.eotest.sensor as sensorTest from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, cte_task, plot_cte_results,\ get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') sflat_high_files \ = siteUtils.dependency_glob(glob_pattern('cte_high', det_name), acq_jobname=acq_jobname) sflat_low_files \ = siteUtils.dependency_glob(glob_pattern('cte_low', det_name), acq_jobname=acq_jobname) if not sflat_high_files and not sflat_low_files: print("cte_task: Superflat files not found for detector", det_name) return None mask_files = get_mask_files(det_name) eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) # Write gains to local eotest_results_file, which cte_task will update. namps = 16 if 'SW' not in det_name else 8 results = sensorTest.EOTestResults(eotest_results_file, namps=namps) for amp, gain in gains.items(): results.add_seg_result(amp, 'GAIN', gain) results.write() bias_frame = bias_filename(run, det_name) # Omit rolloff defects mask since it would mask some of the edges used # in the eper method. mask_files \ = [item for item in mask_files if item.find('edge_rolloff') == -1] png_files = [] for flux_level, sflat_files in zip(('high', 'low'), (sflat_high_files, sflat_low_files)): superflat_file = cte_task(run, det_name, sflat_files, gains, mask_files=mask_files, flux_level=flux_level, bias_frame=bias_frame) png_files.extend(plot_cte_results(run, det_name, superflat_file, eotest_results_file, mask_files=mask_files)) png_file_list = '{}_cte_task_png_files.txt'.format(det_name) with open(png_file_list, 'w') as output: for item in png_files: if os.path.isfile(item): output.write('{}\n'.format(item)) return None