def validate_read_noise(results, det_names): """Validate and persist read noise results.""" run = siteUtils.getRunNumber() missing_det_names = set() for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) read_noise_file = '%s_eotest_results.fits' % file_prefix if not os.path.isfile(read_noise_file): # No data for this detector, so note that and continue # with the others. missing_det_names.add(det_name) continue data = sensorTest.EOTestResults(read_noise_file) amps = data['AMP'] read_noise_data = data['READ_NOISE'] system_noise_data = data['SYSTEM_NOISE'] total_noise_data = data['TOTAL_NOISE'] for amp, read_noise, system_noise, total_noise \ in zip(amps, read_noise_data, system_noise_data, total_noise_data): results.append(lcatr.schema.valid( lcatr.schema.get('read_noise_BOT'), amp=amp, read_noise=read_noise, system_noise=system_noise, total_noise=total_noise, slot=slot, raft=raft)) files = glob.glob('%s_read_noise?*.fits' % file_prefix) for fitsfile in files: eotestUtils.addHeaderData(fitsfile, TESTTYPE='FE55', DATE=eotestUtils.utc_now_isoformat()) data_products = [siteUtils.make_fileref(item) for item in files] results.extend(data_products) # Persist the png files. metadata = dict(DETECTOR=det_name, TESTTYPE='FE55', TEST_CATEGORY='EO', RUN=run) filename = '%s_correlated_noise.png' % file_prefix results.extend(siteUtils.persist_png_files(filename, file_prefix, metadata=metadata)) # Persist the raft-level overscan correlation plots. for raft in camera_info.get_installed_raft_names(): metadata = dict(TESTTYPE='FE55', TEST_CATEGORY='EO', RAFT=raft, RUN=run) file_prefix = make_file_prefix(run, raft) filename = '%s_overscan_correlations.png' % file_prefix results.extend(siteUtils.persist_png_files(filename, file_prefix, metadata=metadata)) report_missing_data("validate_read_noise", missing_det_names) return results
#!/usr/bin/env ipython """ Producer script for BOT scan mode analysis. """ import os from camera_components import camera_info from scan_mode_analysis_jh_task import scan_mode_analysis_jh_task from bot_eo_analyses import get_analysis_types, run_python_task_or_cl_script if 'scan' in get_analysis_types(): scan_mode_script \ = os.path.join(os.environ['EOANALYSISJOBSDIR'], 'harnessed_jobs', 'scan_mode_analysis_BOT', 'v0', 'scan_mode_analysis_jh_task.py') installed_rafts = camera_info.get_installed_raft_names() run_python_task_or_cl_script(scan_mode_analysis_jh_task, scan_mode_script, device_names=installed_rafts)
def run_jh_tasks(*jh_tasks, device_names=None, processes=None, walltime=3600): """ Run functions to execute tasks under the job harness in parallel. These functions should take a device name as its only argument, and the parallelization will take place over device_names. Parameters ---------- jh_tasks: list-like container of functions These functions are serialized and dispatched to workers on remote nodes, so all dependencies should be imported in the bodies of the functions. device_names: list-like container of device names [None] List of sensors or rafts on which to operate. If None, then the installed sensors in the focal plane is used. processes: int [None] Number of processes to run in parallel. If None, then all available processes can be potentially used. walltime: float [3600] Walltime in seconds for python app execution. If the python app does not return within walltime, a parsl.app.errors.AppTimeout exception will be thrown. Raises ------ parsl.app.errors.AppTimeout Notes ----- Because the number of jh_task functions can vary, the keyword arguments should reference the keywords explicitly, i.e., one cannot rely on keyword position to pass those values. """ if device_names is None: device_names = camera_info.get_det_names() # Restrict to installed rafts or sensors. This function call # also writes the camera_info cache file for the eT db query. installed_rafts = camera_info.get_installed_raft_names() # Check if rafts are over-ridden in the lcatr.cfg file. override_rafts = os.environ.get('LCATR_RAFTS', None) if override_rafts is not None: installed_rafts = override_rafts.split('_') device_names = [_ for _ in device_names if _[:3] in installed_rafts] cwd = os.path.abspath('.') # Query eT database for file paths from a previous run, if # specified, and store in a pickle file. hj_fp_server = siteUtils.HarnessedJobFilePaths() # Query for file paths for other analysis runs, if specified in # the bot_eo_config_file. for analysis_type in ('badpixel', 'bias', 'dark', 'linearity', 'nonlinearity'): hj_fp_server.query_file_paths( siteUtils.get_analysis_run(analysis_type)) hj_fp_server_file = 'hj_fp_server.pkl' with open(hj_fp_server_file, 'wb') as output: pickle.dump(hj_fp_server, output) # Create a GetAmplifierGains object in order to query the eT # database for gain results from previous runs and write a pickle # file that can be loaded locally from disk by the various jh # tasks being run in parallel to avoid eT db access contention. GetAmplifierGains() for jh_task in jh_tasks: # Add 30 second sleep before launching jh_task processes in # parallel to allow for parsl process_pool_workers from the # previous set of jh_task processes to finish. time.sleep(30) run_device_analysis_pool(jh_task, device_names, processes=processes, cwd=cwd, walltime=walltime)
def validate_tearing(results, det_names): """Validate the tearing analysis results.""" run = siteUtils.getRunNumber() schema = lcatr.schema.get('tearing_detection_BOT') amps_schema = lcatr.schema.get('tearing_stats_BOT') missing_det_names = set() for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) tearing_results_file = '%s_tearing_stats.pickle' % file_prefix if not os.path.isfile(tearing_results_file): missing_det_names.add(det_name) continue results.append(siteUtils.make_fileref(tearing_results_file)) with open(tearing_results_file, 'rb') as input_: tearing_stats, amp_counts = pickle.load(input_) for values in tearing_stats: stats = dict( zip(('job_name', 'subset', 'sensor_id', 'detections', 'slot', 'raft'), list(values) + [slot, raft])) results.append(lcatr.schema.valid(schema, **stats)) for amp, detections in amp_counts.items(): results.append( lcatr.schema.valid(amps_schema, amp=amp, slot=slot, raft=raft, tearing_detections=detections)) png_files = sorted(glob.glob('*_tearing.png')) results.extend(persist_tearing_png_files(png_files)) missing_raft_names = set() for raft_name in camera_info.get_installed_raft_names(): try: divisidero_plot = glob.glob(f'{raft_name}_*_divisidero.png')[0] except IndexError: missing_raft_names.add(raft_name) continue md = dict(DATA_PRODUCT='divisidero_tearing_plot', LsstId=raft_name) results.append(siteUtils.make_fileref(divisidero_plot, metadata=md)) try: divisidero_json_file \ = glob.glob(f'{raft_name}*max_divisidero.json')[0] except IndexError: missing_raft_names.add(raft_name) continue with open(divisidero_json_file, 'r') as fd: max_devs = json.load(fd) results.append(siteUtils.make_fileref(divisidero_json_file)) bot_schema = lcatr.schema.get('divisadero_tearing_BOT') for slot, values in max_devs.items(): # Weed out nans and infinities. max_dev_values = [] for value in values: if np.isfinite(value): max_dev_values.append(value) else: max_dev_values.append(0) # Top half of CCD. my_devs = max_dev_values[:7] for amp, devs in enumerate(zip([0] + my_devs, my_devs + [0]), 1): results.append( lcatr.schema.valid(bot_schema, amp=amp, slot=slot, raft=raft_name, divisadero_max_dev=max(devs))) if len(max_dev_values) == 7: # This is a WF sensor. continue # Bottom half of CCD. my_devs = max_dev_values[7:] my_devs.reverse() for amp, devs in enumerate(zip([0] + my_devs, my_devs + [0]), 9): results.append( lcatr.schema.valid(bot_schema, amp=amp, slot=slot, raft=raft_name, divisadero_max_dev=max(devs))) report_missing_data("validate_tearing", missing_det_names) report_missing_data("validate_tearing", sorted(list(missing_raft_names)), components='rafts', total=25) return results
def validate_flat_pairs(results, det_names): """Validate the flat pair analysis results.""" run = siteUtils.getRunNumber() missing_det_names = set() for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) det_resp_data = '%s_det_response.fits' % file_prefix if not os.path.isfile(det_resp_data): missing_det_names.add(det_name) continue eotestUtils.addHeaderData(det_resp_data, DETECTOR=det_name, TESTTYPE='FLAT', DATE=eotestUtils.utc_now_isoformat()) results.append(siteUtils.make_fileref(det_resp_data)) results_file = '%s_eotest_results.fits' % file_prefix data = sensorTest.EOTestResults(results_file) amps = data['AMP'] max_observed_signal_data = data['MAX_OBSERVED_SIGNAL'] max_frac_dev_data = data['MAX_FRAC_DEV'] row_mean_var_slope_data = data['ROW_MEAN_VAR_SLOPE'] linearity_turnoff_data = data['LINEARITY_TURNOFF'] for amp, max_observed_signal, max_frac_dev, row_mean_var_slope, \ linearity_turnoff in zip(amps, max_observed_signal_data, max_frac_dev_data, row_mean_var_slope_data, linearity_turnoff_data): results.append( lcatr.schema.valid(lcatr.schema.get('flat_pairs_BOT'), amp=amp, max_observed_signal=max_observed_signal, max_frac_dev=max_frac_dev, row_mean_var_slope=row_mean_var_slope, linearity_turnoff=linearity_turnoff, slot=slot, raft=raft)) # Persist the png files. metadata = dict(DETECTOR=det_name, RUN=run, TESTTYPE='FLAT', TEST_CATEGORY='EO') results.extend( siteUtils.persist_png_files(('%s_linearity*.png' % file_prefix), file_prefix, metadata=metadata)) results.extend( siteUtils.persist_png_files( ('%s_row_means_variance.png' % file_prefix), file_prefix, metadata=metadata)) # Persist the raft-level imaging region correlation plots. missing_raft_names = set() for raft in camera_info.get_installed_raft_names(): metadata = dict(TESTTYPE='FLAT', TEST_CATEGORY='EO', RAFT=raft, RUN=run) file_prefix = make_file_prefix(run, raft) filename = f'{file_prefix}_imaging_region_correlations.png' if not os.path.isfile(filename): missing_raft_names.add(raft) continue results.extend( siteUtils.persist_png_files(filename, file_prefix, metadata=metadata)) # Persist any pd correction file specified in the lcatr.cfg file. pd_corrections_file_env = 'LCATR_PD_CORRECTIONS_FILE' if pd_corrections_file_env in os.environ: pd_corrections_file = os.environ[pd_corrections_file_env] shutil.copy(pd_corrections_file, '.') fileref = siteUtils.make_fileref(os.path.basename(pd_corrections_file)) results.append(fileref) report_missing_data("validate_flat_pairs", missing_det_names) report_missing_data("validate_flat_pairs", sorted(list(missing_raft_names)), components='rafts', total=21) return results
#!/usr/bin/env ipython """ Validator script for BOT raft-level results summaries. """ import glob import lcatr.schema import siteUtils from camera_components import camera_info from bot_eo_validators import validate_raft_results results = [] results = validate_raft_results(results, camera_info.get_installed_raft_names()) # # Validate focal plane heat map plots # unit_id = siteUtils.getUnitId() run = siteUtils.getRunNumber() png_files = glob.glob('{}_{}*.png'.format(unit_id, run)) results.extend(siteUtils.persist_png_files('', unit_id, png_files=png_files)) results.extend(siteUtils.jobInfo()) lcatr.schema.write_file(results) lcatr.schema.validate_file()