def persistence_jh_task(det_name): """JH version of the persistence_task.""" import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern, \ bias_frame_task, get_mask_files, get_bot_eo_config, persistence_task run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') bias_files \ = siteUtils.dependency_glob(glob_pattern('persistence_bias', det_name), acq_jobname=acq_jobname, description='Persistence bias frames:') dark_files \ = siteUtils.dependency_glob(glob_pattern('persistence_dark', det_name), acq_jobname=acq_jobname, description='Persistence dark frames:') if not bias_files or not dark_files: print("persistence_task: Needed data files are missing for detector", det_name) return None # Sort by test sequence number, i.e., by filenames. bias_files = sorted(bias_files) dark_files = sorted(dark_files) # Make a superbias frame using the pre-exposure persistence bias # files, skipping the first exposure. superbias_frame = f'{file_prefix}_persistence_superbias.fits' bias_frame_task(run, det_name, bias_files, bias_frame=superbias_frame) return persistence_task(run, det_name, dark_files, superbias_frame, get_mask_files(det_name))
def bias_frame_jh_task(det_name): """JH version of the bias_frame_task.""" import os import siteUtils import json from bot_eo_analyses import glob_pattern, bias_frame_task, \ bias_stability_task run = siteUtils.getRunNumber() acq_jobname = siteUtils.getProcessName('BOT_acq') bias_files \ = siteUtils.dependency_glob(glob_pattern('bias_frame', det_name), acq_jobname=acq_jobname, description='Bias frames:') if not bias_files: print("bias_frame_task: Needed data files are missing for detector", det_name) return None bias_stability_files \ = siteUtils.dependency_glob(glob_pattern('bias_stability', det_name), acq_jobname=acq_jobname, description='Bias stability frames:') if not bias_stability_files: print("bias_stability_task: Needed data files are missing for detector", det_name) return None bias_stability_files = sorted(bias_stability_files) bias_stability_task(run, det_name, bias_stability_files) return bias_frame_task(run, det_name, bias_files[1:])
def bias_frame_jh_task(det_name): """JH version of the bias_frame_task.""" import os import siteUtils import json from bot_eo_analyses import glob_pattern, bias_frame_task, \ bias_stability_task, pca_corrected_superbias run = siteUtils.getRunNumber() acq_jobname = siteUtils.getProcessName('BOT_acq') bias_files \ = siteUtils.dependency_glob(glob_pattern('bias_frame', det_name), acq_jobname=acq_jobname, description='Bias frames:') # Skip the first 5 bias files to avoid transient features in the # imaging array from after just starting the run. bias_files = sorted(bias_files, key=os.path.basename)[5:] if not bias_files: print("bias_frame_task: Needed data files are missing for detector", det_name) return None bias_stability_files \ = siteUtils.dependency_glob(glob_pattern('bias_stability', det_name), acq_jobname=acq_jobname, description='Bias stability frames:') if not bias_stability_files: print("bias_stability_task: Needed data files are missing for detector", det_name) return None superbias_file, bias_model_components \ = bias_frame_task(run, det_name, bias_files) bias_stability_files = sorted(bias_stability_files) if not os.environ.get('LCATR_USE_PCA_BIAS_FIT', "True") == 'True': bias_model_components = None if siteUtils.get_analysis_run('bias') == 'rowcol': bias_model_components = 'rowcol', superbias_file print("bias_model_components:", bias_model_components) bias_stability_task(run, det_name, bias_stability_files, bias_model_components=bias_model_components) if (bias_model_components is not None and bias_model_components[0] != 'rowcol'): pca_corrected_superbias(run, det_name, bias_files, bias_model_components) return superbias_file
def dark_defects_jh_task(det_name): """JH version of single sensor execution of the dark defects task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, dark_defects_task, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') sflat_files \ = siteUtils.dependency_glob(glob_pattern('dark_defects', det_name), acq_jobname=acq_jobname) if not sflat_files: print("dark_defects_task: No high flux superflat files found for", det_name) return None mask_files = sorted(glob.glob(f'{file_prefix}*mask*.fits')) bias_frame = bias_filename(run, det_name) return dark_defects_task(run, det_name, sflat_files, mask_files=mask_files, bias_frame=bias_frame)
def dark_current_jh_task(det_name): """JH version of single sensor execution of the dark current task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, dark_current_task,\ plot_ccd_total_noise, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') dark_files \ = siteUtils.dependency_glob(glob_pattern('dark_current', det_name), acq_jobname=acq_jobname, description="Dark current frames:") if not dark_files: print("dark_current_task: No dark files found for detector", det_name) return None mask_files = get_mask_files(det_name) eotest_results_file \ = siteUtils.dependency_glob('{}_eotest_results.fits'.format(file_prefix), jobname='read_noise_BOT')[0] gains = get_amplifier_gains('{}_eotest_results.fits'.format(file_prefix)) bias_frame = bias_filename(run, det_name) dark_curr_pixels, dark95s \ = dark_current_task(run, det_name, dark_files, gains, mask_files=mask_files, bias_frame=bias_frame) plot_ccd_total_noise(run, det_name, dark_curr_pixels, dark95s, eotest_results_file) return dark_curr_pixels, dark95s
def bf_jh_task(det_name): """JH version of single sensor execution of the brighter-fatter task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ bias_filename, bf_task, find_flat2_bot, get_mask_files,\ get_amplifier_gains run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files \ = siteUtils.dependency_glob(glob_pattern('brighter_fatter', det_name), acq_jobname=acq_jobname) if not flat_files: print("bf_jh_task: Flat pairs files not found for detector", det_name) return None flat_files = [_ for _ in flat_files if 'flat1' in _] mask_files = get_mask_files(det_name) eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(run, det_name) return bf_task(run, det_name, flat_files, gains, mask_files=mask_files, flat2_finder=find_flat2_bot, bias_frame=bias_frame)
def ptc_jh_task(det_name): """JH version of single sensor execution of the PTC task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, ptc_task, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files = siteUtils.dependency_glob(glob_pattern('ptc', det_name), acq_jobname=acq_jobname) if not flat_files: print("ptc_task: Flat pairs files not found for detector", det_name) return None mask_files = get_mask_files(det_name) eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(run, det_name) return ptc_task(run, det_name, flat_files, gains, mask_files=mask_files, bias_frame=bias_frame)
def flat_gain_stability_jh_task(det_name): """JH version of single sensor execution of the flat pairs task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ bias_filename, flat_gain_stability_task,\ get_mask_files, medianed_dark_frame run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files = siteUtils.dependency_glob(glob_pattern('tearing', det_name), acq_jobname=acq_jobname) if not flat_files: print("flat_gain_stability_task: Flat pairs files not found for", det_name) return None mask_files = get_mask_files(det_name) bias_frame = bias_filename(run, det_name) dark_frame = medianed_dark_frame(det_name) return flat_gain_stability_task(run, det_name, flat_files, mask_files=mask_files, bias_frame=bias_frame, dark_frame=dark_frame)
def read_noise_jh_task(det_name): """JH version of the single sensor read noise task.""" import os import glob import logging import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, read_noise_task, get_mask_files logger = logging.getLogger('read_noise_jh_task') logger.setLevel(logging.INFO) run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') nbias = os.environ.get('LCATR_NUM_BIAS_FRAMES', 10) bias_files \ = siteUtils.dependency_glob(glob_pattern('read_noise', det_name), acq_jobname=acq_jobname)[:nbias] if not bias_files: logger.info( "read_noise_task: Needed data files are missing " "for detector %s", det_name) return None eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) mask_files = get_mask_files(det_name) return read_noise_task(run, det_name, bias_files, gains, mask_files=mask_files)
def bright_defects_jh_task(det_name): """JH version of single sensor bright pixels task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, bright_defects_task, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') dark_files \ = siteUtils.dependency_glob(glob_pattern('bright_defects', det_name), acq_jobname=acq_jobname) if not dark_files: print("bright_defects_task: Needed data files missing for detector", det_name) return None eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) mask_files = sorted(glob.glob(f'{file_prefix}*mask*.fits')) bias_frame = bias_filename(run, det_name) return bright_defects_task(run, det_name, dark_files, gains, mask_files=mask_files, bias_frame=bias_frame)
def raft_divisidero_tearing(raft_name): """JH version of divisidero tearing analysis of BOT data.""" import os from collections import defaultdict import json import matplotlib.pyplot as plt import siteUtils from lsst.eotest.sensor.cteTask import superflat import lsst.eotest.raft as raftTest from bot_eo_analyses import glob_pattern, bias_filename, get_mask_files run = siteUtils.getRunNumber() pattern = glob_pattern('divisadero_tearing', f'{raft_name}_*') acq_jobname = siteUtils.getProcessName('BOT_acq') bot_data = siteUtils.dependency_glob(pattern, acq_jobname=acq_jobname) if not bot_data: return sflat_files = defaultdict(list) for item in bot_data: slot = item.split('.')[0].split('_')[-1] sflat_files[slot].append(item) median_sflats = dict() for slot, files in sflat_files.items(): det_name = '_'.join((raft_name, slot)) outfile = f'{det_name}_{run}_median_sflat.fits' bias_frame = bias_filename(run, det_name) median_sflats[slot] = superflat(files, outfile=outfile, bias_frame=bias_frame) mask_files = dict() for slot in sflat_files: det_name = '_'.join((raft_name, slot)) mask_files[slot] = get_mask_files(det_name) title = f'Run {run} {raft_name}' acq_run = os.environ.get('LCATR_ACQ_RUN', None) if acq_run is not None: title += f' (acq {acq_run})' max_divisidero_tearing \ = raftTest.ana_divisidero_tearing(median_sflats, mask_files, title=title) plt.savefig(f'{raft_name}_{run}_divisidero.png') with open(f'{raft_name}_{run}_max_divisidero.json', 'w') as fd: json.dump(max_divisidero_tearing, fd)
def get_files(data_keys, device_name): """ Get the files needed by the current job for the specified device, e.g., 'R22_S11' or 'R22', for CCDs or rafts, respectively. """ acq_jobname = siteUtils.getProcessName('BOT_acq') files = set() for data_key in data_keys: pattern = glob_pattern(data_key, f'{device_name}*') files = files.union( siteUtils.dependency_glob(pattern, acq_jobname=acq_jobname, verbose=False)) return files
def tearing_jh_task(det_name): """JH version of single sensor execution of the tearing task.""" import siteUtils from bot_eo_analyses import glob_pattern, bias_filename, tearing_task run = siteUtils.getRunNumber() acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files = siteUtils.dependency_glob(glob_pattern('tearing', det_name), acq_jobname=acq_jobname) if not flat_files: print("tearing_task: Flat files not found for detector", det_name) return None bias_frame = bias_filename(run, det_name) return tearing_task(run, det_name, flat_files, bias_frame=bias_frame)
def gain_stability_jh_task(det_name): "JH version of the gain stability analysis task." import os import siteUtils from bot_eo_analyses import glob_pattern, gain_stability_task run = siteUtils.getRunNumber() acq_jobname = siteUtils.getProcessName('BOT_acq') fe55_files = siteUtils.dependency_glob(glob_pattern('fe55', det_name), acq_jobname=acq_jobname) if not fe55_files: print("fe55_task: Needed data files missing for detector", det_name) return None return gain_stability_task(run, det_name, fe55_files)
def flat_pairs_jh_task(det_name): """JH version of single sensor execution of the flat pairs task.""" import os import glob import siteUtils import json from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, flat_pairs_task, mondiode_value,\ get_mask_files, medianed_dark_frame run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') flat_files \ = siteUtils.dependency_glob(glob_pattern('flat_pairs', det_name), acq_jobname=acq_jobname) if not flat_files: print("flat_pairs_task: Flat pairs files not found for detector", det_name) return None mask_files = get_mask_files(det_name) eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(run, det_name) dark_frame = medianed_dark_frame(det_name) if 'LCATR_PD_CORRECTIONS_FILE' in os.environ: filter_corr_file = os.environ['LCATR_PD_CORRECTIONS_FILE'] else: filter_corr_file = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'data', 'pd_filter_corrections.json') print('Using pd_filter_corrections file:', filter_corr_file, flush=True) with open(filter_corr_file) as fd: filter_corrections = json.load(fd) return flat_pairs_task(run, det_name, flat_files, gains, mask_files=mask_files, bias_frame=bias_frame, mondiode_func=mondiode_value, dark_frame=dark_frame, filter_corrections=filter_corrections)
def raft_jh_noise_correlations(raft_name): """JH version of raft-level noise-correlation analysis.""" import os from collections import defaultdict import logging import siteUtils from camera_components import camera_info from bot_eo_analyses import raft_noise_correlations, glob_pattern logger = logging.getLogger('raft_jh_noise_correlations') logger.setLevel(logging.INFO) run = siteUtils.getRunNumber() acq_jobname = siteUtils.getProcessName('BOT_acq') bias_files \ = siteUtils.dependency_glob(glob_pattern('raft_noise_correlations', '{}_S??'.format(raft_name)), acq_jobname=acq_jobname) if not bias_files: logging.info("raft_noise_correlations: Missing bias files for raft %s", raft_name) return None bias_frame_dict = defaultdict(dict) for item in bias_files: # The exposure is identified by test type, image type, and # seqno in the name of the folder containing the FITS files. seqno = os.path.dirname(item).split('_')[-1] # slot_name is the last '_'-delimited field before '.fits'. slot_name = item.split('_')[-1].split('.')[0] bias_frame_dict[seqno][slot_name] = item bias_frame_files = dict() for seqno in bias_frame_dict: if len(bias_frame_dict[seqno]) == 9: bias_frame_files = bias_frame_dict[seqno] break else: if len(bias_frame_dict[seqno]) > len(bias_frame_files): bias_frame_files = bias_frame_dict[seqno] logger.info("bias frame files for raft_noise_correlations:") for key, value in bias_frame_files.items(): logger.info(" %s", value) return raft_noise_correlations(run, raft_name, bias_frame_files)
def dark_current_jh_task(det_name): """JH version of single sensor execution of the dark current task.""" from collections import defaultdict from astropy.io import fits import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, dark_current_task,\ plot_ccd_total_noise, get_mask_files from bot_data_handling import most_common_dark_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') dark_files \ = siteUtils.dependency_glob(glob_pattern('dark_current', det_name), acq_jobname=acq_jobname, description="Dark current frames:") if not dark_files: print("dark_current_task: No dark files found for detector", det_name) return None dark_files_linear_fit = list(dark_files) dark_files = most_common_dark_files(dark_files) if len(dark_files_linear_fit) == len(dark_files): # These data only have one integration time, so skip linear # fit of dark current signal vs integration time. dark_files_linear_fit = None mask_files = get_mask_files(det_name) eotest_results_file \ = siteUtils.dependency_glob('{}_eotest_results.fits'.format(file_prefix), jobname='read_noise_BOT')[0] gains = get_amplifier_gains('{}_eotest_results.fits'.format(file_prefix)) bias_frame = bias_filename(run, det_name) dark_curr_pixels, dark95s \ = dark_current_task(run, det_name, dark_files, gains, mask_files=mask_files, bias_frame=bias_frame, dark_files_linear_fit=dark_files_linear_fit) plot_ccd_total_noise(run, det_name, dark_curr_pixels, dark95s, eotest_results_file) return dark_curr_pixels, dark95s
def get_files(data_keys, device_name): """ Get the files needed by the current job for the specified device, e.g., 'R22_S11' or 'R22', for CCDs or rafts, respectively. """ acq_jobname = siteUtils.getProcessName('BOT_acq') files = set() for data_key in data_keys: pattern = glob_pattern(data_key, f'{device_name}*') files = files.union( siteUtils.dependency_glob(pattern, acq_jobname=acq_jobname, verbose=True)) # Remove any known bad exposures. bad_exposure_checker = siteUtils.BadExposureChecker() if not bad_exposure_checker.bad_exposures: return files good_files = set() for item in files: if bad_exposure_checker.is_bad(item): continue good_files.add(item) return good_files
def traps_jh_task(det_name): """JH version of single sensor execution of the traps analysis task.""" import glob import siteUtils from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, traps_task, get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') trap_files = siteUtils.dependency_glob(glob_pattern('traps', det_name), acq_jobname=acq_jobname) if not trap_files: print("traps_task: No pocket pumping file found for detector", det_name) return None trap_file = trap_files[0] mask_files = get_mask_files(det_name) # Omit rolloff defects mask since a trap in the rolloff edge region can # affect the entire column. mask_files \ = [item for item in mask_files if item.find('edge_rolloff') == -1] eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) bias_frame = bias_filename(run, det_name) return traps_task(run, det_name, trap_file, gains, mask_files=mask_files, bias_frame=bias_frame)
def raft_jh_signal_correlations(raft_name): """JH version of raft-level signal-correlation analysis.""" import os import logging import numpy as np import matplotlib.pyplot as plt import siteUtils from bot_eo_analyses import bias_filename, make_file_prefix, \ append_acq_run, glob_pattern from signal_correlations import raft_level_signal_correlations logger = logging.getLogger('raft_jh_noise_correlations') logger.setLevel(logging.INFO) # Use the high signal superflat files. pattern = glob_pattern('raft_signal_correlations', f'{raft_name}_S??') acq_jobname = siteUtils.getProcessName('BOT_acq') sflat_files = siteUtils.dependency_glob(pattern, acq_jobname=acq_jobname) folders = sorted( list(set([os.path.basename(os.path.dirname(_)) for _ in sflat_files]))) logger.info(f'folders: {folders}') if not folders: logger.info('No data found for this raft, so skip the ' 'signal correlation analysis.') return flat1_files = dict() flat2_files = dict() for item in sflat_files: folder = os.path.basename(os.path.dirname(item)) if folder not in folders[:2]: continue logger.info(f'item: {item}') logger.info(f'folder: {folder}') basename = os.path.basename(item) logger.info(f'basename: {basename}') slot = basename.split('_')[-1][:-len('.fits')] if folder == folders[0]: flat1_files[slot] = item elif folder == folders[1]: flat2_files[slot] = item logger.info('flat pair files:') for slot in flat1_files: logger.info(' ' + flat1_files[slot]) logger.info(' ' + flat2_files[slot]) # Find the median bias files for the target raft. run = siteUtils.getRunNumber() bias_files = dict() for slot in flat1_files.keys(): det_name = '_'.join((raft_name, slot)) bias_files[slot] = bias_filename(run, det_name) file_prefix = make_file_prefix(run, raft_name) title = append_acq_run("Imaging region correlations, " f"Run {run}, {raft_name}") raft_level_signal_correlations(flat1_files, flat2_files, bias_files, title=title) plt.savefig('{}_imaging_region_correlations.png'.format(file_prefix))
def cte_jh_task(det_name): """JH version of single sensor execution of the CTE task.""" import os import glob import shutil import siteUtils import lsst.eotest.sensor as sensorTest from bot_eo_analyses import make_file_prefix, glob_pattern,\ get_amplifier_gains, bias_filename, cte_task, plot_cte_results,\ get_mask_files run = siteUtils.getRunNumber() file_prefix = make_file_prefix(run, det_name) acq_jobname = siteUtils.getProcessName('BOT_acq') sflat_high_files \ = siteUtils.dependency_glob(glob_pattern('cte_high', det_name), acq_jobname=acq_jobname) sflat_low_files \ = siteUtils.dependency_glob(glob_pattern('cte_low', det_name), acq_jobname=acq_jobname) if not sflat_high_files and not sflat_low_files: print("cte_task: Superflat files not found for detector", det_name) return None mask_files = get_mask_files(det_name) eotest_results_file = '{}_eotest_results.fits'.format(file_prefix) gains = get_amplifier_gains(eotest_results_file) # Write gains to local eotest_results_file, which cte_task will update. namps = 16 if 'SW' not in det_name else 8 results = sensorTest.EOTestResults(eotest_results_file, namps=namps) for amp, gain in gains.items(): results.add_seg_result(amp, 'GAIN', gain) results.write() bias_frame = bias_filename(run, det_name) # Omit rolloff defects mask since it would mask some of the edges used # in the eper method. mask_files \ = [item for item in mask_files if item.find('edge_rolloff') == -1] png_files = [] for flux_level, sflat_files in zip(('high', 'low'), (sflat_high_files, sflat_low_files)): superflat_file = cte_task(run, det_name, sflat_files, gains, mask_files=mask_files, flux_level=flux_level, bias_frame=bias_frame) png_files.extend(plot_cte_results(run, det_name, superflat_file, eotest_results_file, mask_files=mask_files)) png_file_list = '{}_cte_task_png_files.txt'.format(det_name) with open(png_file_list, 'w') as output: for item in png_files: if os.path.isfile(item): output.write('{}\n'.format(item)) return None