def persist_traps_analysis(): """Persist the results from the traps analysis.""" raft_id = siteUtils.getUnitId() raft = camera_components.Raft.create_from_etrav(raft_id) results = [] for slot, sensor_id in raft.items(): ccd_vendor = sensor_id.split('-')[0].upper() trap_file = '%s_traps.fits' % sensor_id eotestUtils.addHeaderData(trap_file, LSST_NUM=sensor_id, TESTTYPE='TRAP', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=ccd_vendor) results.append(siteUtils.make_fileref(trap_file, folder=slot)) mask_file = '%s_traps_mask.fits' % sensor_id results.append(siteUtils.make_fileref(mask_file, folder=slot)) results_file = '%s_eotest_results.fits' % sensor_id data = sensorTest.EOTestResults(results_file) amps = data['AMP'] num_traps = data['NUM_TRAPS'] for amp, ntrap in zip(amps, num_traps): results.append( lcatr.schema.valid(lcatr.schema.get('traps_raft'), amp=amp, num_traps=ntrap, slot=slot, sensor_id=sensor_id)) return results
def validate_flat_gain_stability(results, det_names): """Valdiate the output files from the flat_gain_stability analysis""" if 'gainstability' not in get_analysis_types(): return results run = siteUtils.getRunNumber() missing_det_names = set() for det_name in det_names: file_prefix = make_file_prefix(run, det_name) results_file = f'{file_prefix}_flat_signal_sequence.pickle' if not os.path.isfile(results_file): missing_det_names.add(det_name) else: md = dict(DATA_PRODUCT='flat_gain_stability_results') results.append(siteUtils.make_fileref(results_file, metadata=md)) report_missing_data('validate_flat_gain_stability', missing_det_names) unit_id = siteUtils.getUnitId() png_files = glob.glob('*flat_gain_stability.png') for png_file in png_files: md = dict(DATA_PRODUCT='flat_gain_stability_plot') if unit_id in png_file: md['LsstId'] = unit_id results.append(siteUtils.make_fileref(png_file, metadata=md)) return results
def validate_traps(results, det_names): """Validate and persist trap results.""" run = siteUtils.getRunNumber() missing_det_names = [] for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) trap_file = '%s_traps.fits' % file_prefix if not os.path.isfile(trap_file): missing_det_names.append(det_name) continue eotestUtils.addHeaderData(trap_file, TESTTYPE='TRAP', DATE=eotestUtils.utc_now_isoformat()) results.append(siteUtils.make_fileref(trap_file)) mask_file = '%s_traps_mask.fits' % file_prefix results.append(siteUtils.make_fileref(mask_file)) results_file = '%s_eotest_results.fits' % file_prefix data = sensorTest.EOTestResults(results_file) amps = data['AMP'] num_traps = data['NUM_TRAPS'] for amp, ntrap in zip(amps, num_traps): results.append( lcatr.schema.valid(lcatr.schema.get('traps_BOT'), amp=amp, num_traps=ntrap, slot=slot, raft=raft)) report_missing_data("validate_traps", missing_det_names) return results
def validate_dark_defects(results, det_names): """Validate and persist dark defects results.""" run = siteUtils.getRunNumber() missing_det_names = [] for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) mask_file = '%s_dark_pixel_mask.fits' % file_prefix if not os.path.isfile(mask_file): missing_det_names.append(det_name) continue eotestUtils.addHeaderData(mask_file, TESTTYPE='SFLAT_500', DATE=eotestUtils.utc_now_isoformat()) results.append(siteUtils.make_fileref(mask_file)) superflat = '%s_median_sflat.fits' % file_prefix eotestUtils.addHeaderData(superflat, DATE=eotestUtils.utc_now_isoformat()) results.append(siteUtils.make_fileref(superflat)) eotest_results = '%s_eotest_results.fits' % file_prefix data = sensorTest.EOTestResults(eotest_results) amps = data['AMP'] npixels = data['NUM_DARK_PIXELS'] ncolumns = data['NUM_DARK_COLUMNS'] for amp, npix, ncol in zip(amps, npixels, ncolumns): results.append( lcatr.schema.valid(lcatr.schema.get('dark_defects_BOT'), amp=amp, dark_pixels=npix, dark_columns=ncol, slot=slot, raft=raft)) # Persist the png files. metadata = dict(DETECTOR=det_name, RUN=run, TESTTYPE='SFLAT_500', TEST_CATEGORY='EO') filename = '%s_superflat_dark_defects.png' % file_prefix results.extend( siteUtils.persist_png_files(filename, file_prefix, metadata=metadata)) report_missing_data("validate_dark_defects", missing_det_names) return results
def validate_overscan(results, det_names): """Validate the overscan analysis results.""" run = siteUtils.getRunNumber() results = [] missing_det_names = set() for det_name in det_names: file_prefix = make_file_prefix(run, det_name) results_file = f'{file_prefix}_overscan_results.fits' if not os.path.isfile(results_file): missing_det_names.add(det_name) else: md = dict(DATA_PRODUCT='overscan_task_results', RUN=run, DETECTOR=det_name) results.append(siteUtils.make_fileref(results_file, metadata=md)) png_files = (glob.glob(f'{file_prefix}_*_eper_*.png') + glob.glob(f'{file_prefix}_*_overscan_*.png') + glob.glob(f'{file_prefix}_*_cti.png')) md = dict(TEST_CATEGORY='EO', DETECTOR=det_name, RUN=run) results.extend( siteUtils.persist_png_files('', file_prefix, png_files=png_files, metadata=md)) report_missing_data('validate_overscan', missing_det_names) return results
def persist_tearing_png_files(png_files, folder=None, metadata=None): """ Create the lcatr.schema.filerefs for persisting the png files with the DataCatalog. Parameters ---------- png_files: list A list of png files for the tearing profile plots. folder: str [None] Folder under which to persist the png file. For raft-level analysis, this would be the slot number of the CCD. metadata: dict [None] Any additional metadata to persist with the png files. Returns ------- list: This is a list of lcatr.schema.filerefs. """ if metadata is None: metadata = dict() md = siteUtils.DataCatalogMetadata(**metadata) png_filerefs = [] for png_file in png_files: dp = 'tearing_profiles' lsst_id = os.path.basename(png_file).split('_')[0] png_filerefs.append( siteUtils.make_fileref(png_file, folder=folder, metadata=md(DATA_PRODUCT=dp, LsstId=lsst_id))) return png_filerefs
def validate_scan(results, det_names): """Validate scan mode analysis results.""" run = siteUtils.getRunNumber() rafts = set() for det_name in det_names: raft, slot = det_name.split('_') rafts.add(raft) file_prefix = make_file_prefix(run, det_name) disp_files = glob.glob('{}_*_dispersion.png'.format(file_prefix)) for item in disp_files: md = dict(raft=raft, slot=slot, run=run) results.append(siteUtils.make_fileref(item, metadata=md)) for raft in rafts: multiscope_files = glob.glob('{}_{}_*multiscope.png'.format(raft, run)) for item in multiscope_files: md = dict(raft=raft, run=run) results.append(siteUtils.make_fileref(item, metadata=md)) return results
def validate_qe(results, det_names): """Validate the QE results.""" run = siteUtils.getRunNumber() missing_det_names = [] for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) qe_results_file = '%s_QE.fits' % file_prefix if not os.path.isfile(qe_results_file): missing_det_names.append(det_name) continue with fits.open(qe_results_file) as qe_results: qe_data = qe_results['QE_BANDS'].data QE = OrderedDict((band, []) for band in qe_data.field('BAND')) for amp in range(1, 17): values = qe_data.field('AMP%02i' % amp) for band, value in zip(QE, values): QE[band].append(value) for band in QE: for amp in range(1, 17): results.append( lcatr.schema.valid(lcatr.schema.get('qe_BOT_analysis'), band=band, QE=QE[band][amp - 1], amp=amp, slot=slot, raft=raft)) qe_files = glob.glob('%s_*QE*.fits' % file_prefix) for item in qe_files: eotestUtils.addHeaderData(item, TESTTYPE='LAMBDA', DATE=eotestUtils.utc_now_isoformat()) results.extend([siteUtils.make_fileref(item) for item in qe_files]) # Persist the png files. metadata = dict(DETECTOR=det_name, RUN=run, TESTTYPE='LAMBDA', TEST_CATEGORY='EO') results.extend( siteUtils.persist_png_files('%s*qe.png' % file_prefix, file_prefix, metadata=metadata)) results.extend( siteUtils.persist_png_files('%s*flat.png' % file_prefix, file_prefix, metadata=metadata)) report_missing_data("validate_qe", missing_det_names) return results
def validate_bias_stability(results, det_names): """Validate bias stability results.""" run = siteUtils.getRunNumber() rafts = set() for det_name in det_names: raft, slot = det_name.split('_') rafts.add(raft) file_prefix = make_file_prefix(run, det_name) profile_plots = f'{file_prefix}_bias_serial_profiles.png' if not os.path.isfile(profile_plots): continue md = dict(raft=raft, slot=slot, run=run) results.append(siteUtils.make_fileref(profile_plots, metadata=md)) for raft in rafts: file_prefix = make_file_prefix(run, raft) stats_file = f'{file_prefix}_bias_frame_stats.pickle' if not os.path.isfile(stats_file): continue md = dict(raft=raft, run=run) results.append(siteUtils.make_fileref(stats_file, metadata=md)) return results
def validate_brighter_fatter(results, det_names): """Validate the brighter-fatter results.""" run = siteUtils.getRunNumber() missing_det_names = set() for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) bf_results = '%s_bf.fits' % file_prefix if not os.path.isfile(bf_results): missing_det_names.add(det_name) continue eotestUtils.addHeaderData(bf_results, TESTTYPE='FLAT', DATE=eotestUtils.utc_now_isoformat()) results.append(siteUtils.make_fileref(bf_results)) results_file = '%s_eotest_results.fits' % file_prefix data = sensorTest.EOTestResults(results_file) columns = (data['AMP'], data['BF_XCORR'], data['BF_XCORR_ERR'], data['BF_YCORR'], data['BF_YCORR_ERR'], data['BF_SLOPEX'], data['BF_SLOPEX_ERR'], data['BF_SLOPEY'], data['BF_SLOPEY_ERR'], data['BF_MEAN']) for amp, bf_xcorr, bf_xcorr_err, bf_ycorr, bf_ycorr_err, \ bf_slopex, bf_slopex_err, bf_slopey, bf_slopey_err, bf_mean \ in zip(*columns): results.append( lcatr.schema.valid(lcatr.schema.get('brighter_fatter_BOT'), amp=amp, bf_xcorr=bf_xcorr, bf_xcorr_err=bf_xcorr_err, bf_ycorr=bf_ycorr, bf_ycorr_err=bf_ycorr_err, bf_slopex=bf_slopex, bf_slopex_err=bf_slopex_err, bf_slopey=bf_slopey, bf_slopey_err=bf_slopey_err, bf_mean=bf_mean, slot=slot, raft=raft)) # Persist the png files. metadata = dict(DETECTOR=det_name, RUN=run, TESTTYPE='FLAT', TEST_CATEGORY='EO') results.extend( siteUtils.persist_png_files('%s*brighter-fatter.png' % file_prefix, file_prefix, metadata=metadata)) return results
def validate_read_noise(results, det_names): """Validate and persist read noise results.""" run = siteUtils.getRunNumber() missing_det_names = set() for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) read_noise_file = '%s_eotest_results.fits' % file_prefix if not os.path.isfile(read_noise_file): # No data for this detector, so note that and continue # with the others. missing_det_names.add(det_name) continue data = sensorTest.EOTestResults(read_noise_file) amps = data['AMP'] read_noise_data = data['READ_NOISE'] system_noise_data = data['SYSTEM_NOISE'] total_noise_data = data['TOTAL_NOISE'] for amp, read_noise, system_noise, total_noise \ in zip(amps, read_noise_data, system_noise_data, total_noise_data): results.append(lcatr.schema.valid( lcatr.schema.get('read_noise_BOT'), amp=amp, read_noise=read_noise, system_noise=system_noise, total_noise=total_noise, slot=slot, raft=raft)) files = glob.glob('%s_read_noise?*.fits' % file_prefix) for fitsfile in files: eotestUtils.addHeaderData(fitsfile, TESTTYPE='FE55', DATE=eotestUtils.utc_now_isoformat()) data_products = [siteUtils.make_fileref(item) for item in files] results.extend(data_products) # Persist the png files. metadata = dict(DETECTOR=det_name, TESTTYPE='FE55', TEST_CATEGORY='EO', RUN=run) filename = '%s_correlated_noise.png' % file_prefix results.extend(siteUtils.persist_png_files(filename, file_prefix, metadata=metadata)) # Persist the raft-level overscan correlation plots. for raft in camera_info.get_installed_raft_names(): metadata = dict(TESTTYPE='FE55', TEST_CATEGORY='EO', RAFT=raft, RUN=run) file_prefix = make_file_prefix(run, raft) filename = '%s_overscan_correlations.png' % file_prefix results.extend(siteUtils.persist_png_files(filename, file_prefix, metadata=metadata)) report_missing_data("validate_read_noise", missing_det_names) return results
def validate_flat_pairs(results, det_names): """Validate the flat pair analysis results.""" run = siteUtils.getRunNumber() missing_det_names = set() for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) det_resp_data = '%s_det_response.fits' % file_prefix if not os.path.isfile(det_resp_data): missing_det_names.add(det_name) continue eotestUtils.addHeaderData(det_resp_data, DETECTOR=det_name, TESTTYPE='FLAT', DATE=eotestUtils.utc_now_isoformat()) results.append(siteUtils.make_fileref(det_resp_data)) results_file = '%s_eotest_results.fits' % file_prefix data = sensorTest.EOTestResults(results_file) amps = data['AMP'] max_observed_signal_data = data['MAX_OBSERVED_SIGNAL'] max_frac_dev_data = data['MAX_FRAC_DEV'] row_mean_var_slope_data = data['ROW_MEAN_VAR_SLOPE'] linearity_turnoff_data = data['LINEARITY_TURNOFF'] for amp, max_observed_signal, max_frac_dev, row_mean_var_slope, \ linearity_turnoff in zip(amps, max_observed_signal_data, max_frac_dev_data, row_mean_var_slope_data, linearity_turnoff_data): results.append(lcatr.schema.valid( lcatr.schema.get('flat_pairs_BOT'), amp=amp, max_observed_signal=max_observed_signal, max_frac_dev=max_frac_dev, row_mean_var_slope=row_mean_var_slope, linearity_turnoff=linearity_turnoff, slot=slot, raft=raft)) # Persist the png files. metadata = dict(DETECTOR=det_name, RUN=run, TESTTYPE='FLAT', TEST_CATEGORY='EO') results.extend(siteUtils.persist_png_files(('%s_linearity*.png' % file_prefix), file_prefix, metadata=metadata)) results.extend(siteUtils.persist_png_files(('%s_row_means_variance.png' % file_prefix), file_prefix, metadata=metadata)) report_missing_data("validate_flat_pairs", missing_det_names) return results
def validate_ptc(results, det_names): """Validate the PTC results.""" run = siteUtils.getRunNumber() missing_det_names = [] for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) ptc_results = '%s_ptc.fits' % file_prefix if not os.path.isfile(ptc_results): missing_det_names.append(det_name) continue eotestUtils.addHeaderData(ptc_results, TESTTYPE='FLAT', DATE=eotestUtils.utc_now_isoformat()) results.append(siteUtils.make_fileref(ptc_results)) results_file = '%s_eotest_results.fits' % file_prefix data = sensorTest.EOTestResults(results_file) columns = (data['AMP'], data['PTC_GAIN'], data['PTC_GAIN_ERROR'], data['PTC_A00'], data['PTC_A00_ERROR'], data['PTC_NOISE'], data['PTC_NOISE_ERROR'], data['PTC_TURNOFF']) for amp, gain, gain_error, a00, a00_error,\ noise, noise_error, turnoff in zip(*columns): results.append( lcatr.schema.valid(lcatr.schema.get('ptc_BOT'), amp=amp, ptc_gain=gain, ptc_gain_error=gain_error, ptc_a00=a00, ptc_a00_error=a00_error, ptc_noise=noise, ptc_noise_error=noise_error, ptc_turnoff=turnoff, slot=slot, raft=raft)) # Persist the png files. metadata = dict(DETECTOR=det_name, RUN=run, TESTTYPE='FLAT', TEST_CATEGORY='EO') results.extend( siteUtils.persist_png_files('%s*ptcs.png' % file_prefix, file_prefix, metadata=metadata)) report_missing_data("validate_ptc", missing_det_names) return results
def validate_nonlinearity(results, det_names): """Validate the nonlinearity analysis results.""" run = siteUtils.getRunNumber() results = [] missing_det_names = set() for det_name in det_names: nlc_file = f'{make_file_prefix(run, det_name)}_nlc.fits' if not os.path.isfile(nlc_file): missing_det_names.add(det_name) continue md = dict(DATA_PRODUCT='nonlinearity_correction') results.append(siteUtils.make_fileref(nlc_file, metadata=md)) report_missing_data('validate_nonlinearity', missing_det_names) return results
def validate_flat_pairs(results, det_names): """Validate the flat pair analysis results.""" run = siteUtils.getRunNumber() missing_det_names = [] for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) det_resp_data = '%s_det_response.fits' % file_prefix if not os.path.isfile(det_resp_data): missing_det_names.append(det_name) continue eotestUtils.addHeaderData(det_resp_data, DETECTOR=det_name, TESTTYPE='FLAT', DATE=eotestUtils.utc_now_isoformat()) results.append(siteUtils.make_fileref(det_resp_data)) results_file = '%s_eotest_results.fits' % file_prefix data = sensorTest.EOTestResults(results_file) amps = data['AMP'] full_well_data = data['FULL_WELL'] max_frac_dev_data = data['MAX_FRAC_DEV'] for amp, full_well, max_frac_dev in zip(amps, full_well_data, max_frac_dev_data): results.append( lcatr.schema.valid(lcatr.schema.get('flat_pairs_BOT'), amp=amp, full_well=full_well, max_frac_dev=max_frac_dev, slot=slot, raft=raft)) # Persist the png files. metadata = dict(DETECTOR=det_name, RUN=run, TESTTYPE='FLAT', TEST_CATEGORY='EO') results.extend( siteUtils.persist_png_files(('%s_linearity*.png' % file_prefix), file_prefix, metadata=metadata)) report_missing_data("validate_flat_pairs", missing_det_names) return results
def validate_persistence(results, det_names): """Validate the persistence analysis results.""" run = siteUtils.getRunNumber() results = [] missing_det_names = set() for det_name in det_names: file_prefix = make_file_prefix(run, det_name) data_file = f'{file_prefix}_persistence_data.pickle' if not os.path.isfile(data_file): missing_det_names.add(det_name) continue md = dict(DATA_PRODUCT='persistence_task_results', RUN=run, DETECTOR=det_name) results.append(siteUtils.make_fileref(data_file, metadata=md)) png_files = [f'{file_prefix}_persistence.png'] md = dict(TEST_CATEGORY='EO', DETECTOR=det_name, RUN=run) results.extend(siteUtils.persist_png_files('', file_prefix, png_files=png_files, metadata=md)) report_missing_data('validate_persistence', missing_det_names) return results
results = [] for slot, sensor_id in raft.items(): if 'ccd2' in slot: continue wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id] ccd_vendor = sensor_id.split('-')[0].upper() mask_file = '%s_dark_pixel_mask.fits' % wgSlotName eotestUtils.addHeaderData(mask_file, LSST_NUM=sensor_id, TESTTYPE='SFLAT_500', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=ccd_vendor) results.append(siteUtils.make_fileref(mask_file)) superflat = '%s_median_sflat.fits' % wgSlotName eotestUtils.addHeaderData(superflat, DATE=eotestUtils.utc_now_isoformat()) results.append(siteUtils.make_fileref(superflat)) eotest_results = '%s_eotest_results.fits' % wgSlotName data = sensorTest.EOTestResults(eotest_results) amps = data['AMP'] npixels = data['NUM_DARK_PIXELS'] ncolumns = data['NUM_DARK_COLUMNS'] for amp, npix, ncol in zip(amps, npixels, ncolumns): results.append( lcatr.schema.valid(lcatr.schema.get('dark_defects_raft'), amp=amp, dark_pixels=npix,
output_files = gain_file, psf_results, rolloff_mask # Add/update the metadata to the primary HDU of these files. for fitsfile in output_files: eotestUtils.addHeaderData(fitsfile, LSST_NUM=sensor_id, TESTTYPE='FE55', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=ccd_vendor) # # Persist the mean bias FITS file. # # bias_mean_file = glob.glob('%(sensor_id)s_mean_bias_*.fits' % locals())[0] bias_mean_file = glob.glob('%s_mean_bias_*.fits' % wgSlotName)[0] results.append(siteUtils.make_fileref(bias_mean_file, folder=slot)) # Persist the png files. metadata = dict(CCD_MANU=ccd_vendor, LSST_NUM=sensor_id, TESTTYPE='FE55', TEST_CATEGORY='EO') results.extend( siteUtils.persist_png_files('%s*.png' % wgSlotName, sensor_id, folder=slot, metadata=metadata)) data = sensorTest.EOTestResults(gain_file) amps = data['AMP'] gain_data = data['GAIN']
import eotestUtils import camera_components raft_id = siteUtils.getUnitId() raft = camera_components.Raft.create_from_etrav(raft_id) results = [] for slot, sensor_id in raft.items(): ccd_vendor = sensor_id.split('-')[0].upper() det_resp_data = '%s_det_response.fits' % sensor_id eotestUtils.addHeaderData(det_resp_data, LSST_NUM=sensor_id, TESTTYPE='FLAT', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=ccd_vendor) results.append(siteUtils.make_fileref(det_resp_data, folder=slot)) results_file = '%s_eotest_results.fits' % sensor_id data = sensorTest.EOTestResults(results_file) amps = data['AMP'] full_well_data = data['FULL_WELL'] max_frac_dev_data = data['MAX_FRAC_DEV'] for amp, full_well, max_frac_dev in zip(amps, full_well_data, max_frac_dev_data): results.append(lcatr.schema.valid(lcatr.schema.get('flat_pairs_raft'), amp=amp, full_well=full_well, max_frac_dev=max_frac_dev, slot=slot, sensor_id=sensor_id)) # Persist the png files.
lcatr.schema.valid(lcatr.schema.get('qe_raft_analysis'), band=band, QE=np.mean(QE[band]), slot=slot, sensor_id=sensor_id)) qe_files = glob.glob('%s_*QE*.*' % sensor_id) for item in qe_files: if item.endswith('.fits'): eotestUtils.addHeaderData(item, LSST_NUM=sensor_id, TESTTYPE='LAMBDA', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=ccd_vendor) results.extend( [siteUtils.make_fileref(item, folder=slot) for item in qe_files]) # Persist the png files. metadata = dict(CCD_MANU=ccd_vendor, LSST_NUM=sensor_id, TESTTYPE='LAMBDA', TEST_CATEGORY='EO') results.extend( siteUtils.persist_png_files('%s*.png' % sensor_id, sensor_id, folder=slot, metadata=metadata)) sensor_id = raft.sensor_names[0] qe_acq_job_id = siteUtils.get_prerequisite_job_id( ('S*/%s_lambda_flat_*.fits' % sensor_id), jobname=siteUtils.getProcessName('qe_raft_acq'))
results = [] for slot, sensor_id in raft.items(): if 'ccd2' in slot: continue wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id] ccd_vendor = sensor_id.split('-')[0].upper() trap_file = '%s_traps.fits' % wgSlotName eotestUtils.addHeaderData(trap_file, LSST_NUM=sensor_id, TESTTYPE='TRAP', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=ccd_vendor) results.append(siteUtils.make_fileref(trap_file, folder=slot)) mask_file = '%s_traps_mask.fits' % wgSlotName results.append(siteUtils.make_fileref(mask_file, folder=slot)) results_file = '%s_eotest_results.fits' % wgSlotName data = sensorTest.EOTestResults(results_file) amps = data['AMP'] num_traps = data['NUM_TRAPS'] for amp, ntrap in zip(amps, num_traps): results.append( lcatr.schema.valid(lcatr.schema.get('traps_raft'), amp=amp, num_traps=ntrap, slot=slot,
import lcatr.schema import siteUtils import eotestUtils import camera_components raft_id = siteUtils.getUnitId() raft = camera_components.Raft.create_from_etrav(raft_id) results = [] for slot, sensor_id in raft.items(): ccd_vendor = sensor_id.split('-')[0].upper() mask_file = '%s_bright_pixel_mask.fits' % sensor_id eotestUtils.addHeaderData(mask_file, LSST_NUM=sensor_id, TESTTYPE='DARK', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=ccd_vendor) results.append(siteUtils.make_fileref(mask_file, folder=slot)) bias_frame = glob.glob(f'{sensor_id}_*_median_bias.fits')[0] eotestUtils.addHeaderData(bias_frame, DATE=eotestUtils.utc_now_isoformat()) results.append(siteUtils.make_fileref(bias_frame, folder=slot)) medianed_dark = '%s_median_dark_bp.fits' % sensor_id eotestUtils.addHeaderData(medianed_dark, DATE=eotestUtils.utc_now_isoformat()) results.append(siteUtils.make_fileref(medianed_dark, folder=slot)) eotest_results = '%s_eotest_results.fits' % sensor_id data = sensorTest.EOTestResults(eotest_results) amps = data['AMP'] npixels = data['NUM_BRIGHT_PIXELS'] ncolumns = data['NUM_BRIGHT_COLUMNS']
def persist_fe55_analysis_results(): """Persist the results from the full analysis.""" raft_id = siteUtils.getUnitId() raft = camera_components.Raft.create_from_etrav(raft_id) results = [] for slot, sensor_id in raft.items(): ccd_vendor = sensor_id.split('-')[0].upper() # The output files from producer script. gain_file = '%(sensor_id)s_eotest_results.fits' % locals() psf_results = glob.glob('%(sensor_id)s_psf_results*.fits' % locals())[0] rolloff_mask = '%(sensor_id)s_rolloff_defects_mask.fits' % locals() output_files = gain_file, psf_results, rolloff_mask # Add/update the metadata to the primary HDU of these files. for fitsfile in output_files: eotestUtils.addHeaderData(fitsfile, LSST_NUM=sensor_id, TESTTYPE='FE55', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=ccd_vendor) # # Persist the median bias FITS file. # bias_median_file = glob.glob(f'{sensor_id}_*_median_bias.fits')[0] results.append(siteUtils.make_fileref(bias_median_file, folder=slot)) # Persist the png files. metadata = dict(CCD_MANU=ccd_vendor, LSST_NUM=sensor_id, TESTTYPE='FE55', TEST_CATEGORY='EO') results.extend( siteUtils.persist_png_files('%s*.png' % sensor_id, sensor_id, folder=slot, metadata=metadata)) data = sensorTest.EOTestResults(gain_file) amps = data['AMP'] gain_data = data['GAIN'] gain_errors = data['GAIN_ERROR'] sigmas = data['PSF_SIGMA'] for amp, gain_value, gain_error, sigma in zip(amps, gain_data, gain_errors, sigmas): if not np.isfinite(gain_error): gain_error = -1 results.append( lcatr.schema.valid(lcatr.schema.get('fe55_raft_analysis'), amp=amp, gain=gain_value, gain_error=gain_error, psf_sigma=sigma, slot=slot, sensor_id=sensor_id)) results.extend([lcatr.schema.fileref.make(x) for x in output_files]) return results
def validate_tearing(results, det_names): """Validate the tearing analysis results.""" run = siteUtils.getRunNumber() schema = lcatr.schema.get('tearing_detection_BOT') amps_schema = lcatr.schema.get('tearing_stats_BOT') missing_det_names = set() for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) tearing_results_file = '%s_tearing_stats.pickle' % file_prefix if not os.path.isfile(tearing_results_file): missing_det_names.add(det_name) continue results.append(siteUtils.make_fileref(tearing_results_file)) with open(tearing_results_file, 'rb') as input_: tearing_stats, amp_counts = pickle.load(input_) for values in tearing_stats: stats = dict( zip(('job_name', 'subset', 'sensor_id', 'detections', 'slot', 'raft'), list(values) + [slot, raft])) results.append(lcatr.schema.valid(schema, **stats)) for amp, detections in amp_counts.items(): results.append( lcatr.schema.valid(amps_schema, amp=amp, slot=slot, raft=raft, tearing_detections=detections)) png_files = sorted(glob.glob('*_tearing.png')) results.extend(persist_tearing_png_files(png_files)) missing_raft_names = set() for raft_name in camera_info.get_installed_raft_names(): try: divisidero_plot = glob.glob(f'{raft_name}_*_divisidero.png')[0] except IndexError: missing_raft_names.add(raft_name) continue md = dict(DATA_PRODUCT='divisidero_tearing_plot', LsstId=raft_name) results.append(siteUtils.make_fileref(divisidero_plot, metadata=md)) try: divisidero_json_file \ = glob.glob(f'{raft_name}*max_divisidero.json')[0] except IndexError: missing_raft_names.add(raft_name) continue with open(divisidero_json_file, 'r') as fd: max_devs = json.load(fd) results.append(siteUtils.make_fileref(divisidero_json_file)) bot_schema = lcatr.schema.get('divisadero_tearing_BOT') for slot, values in max_devs.items(): # Weed out nans and infinities. max_dev_values = [] for value in values: if np.isfinite(value): max_dev_values.append(value) else: max_dev_values.append(0) # Top half of CCD. my_devs = max_dev_values[:7] for amp, devs in enumerate(zip([0] + my_devs, my_devs + [0]), 1): results.append( lcatr.schema.valid(bot_schema, amp=amp, slot=slot, raft=raft_name, divisadero_max_dev=max(devs))) if len(max_dev_values) == 7: # This is a WF sensor. continue # Bottom half of CCD. my_devs = max_dev_values[7:] my_devs.reverse() for amp, devs in enumerate(zip([0] + my_devs, my_devs + [0]), 9): results.append( lcatr.schema.valid(bot_schema, amp=amp, slot=slot, raft=raft_name, divisadero_max_dev=max(devs))) report_missing_data("validate_tearing", missing_det_names) report_missing_data("validate_tearing", sorted(list(missing_raft_names)), components='rafts', total=25) return results
results = [] for slot, sensor_id in raft.items(): ccd_vendor = sensor_id.split('-')[0].upper() if 'ccd2' in slot: continue wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id] ptc_results = '%s_ptc.fits' % wgSlotName eotestUtils.addHeaderData(ptc_results, LSST_NUM=sensor_id, TESTTYPE='FLAT', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=ccd_vendor) results.append(siteUtils.make_fileref(ptc_results, folder=slot)) results_file = '%s_eotest_results.fits' % wgSlotName data = sensorTest.EOTestResults(results_file) amps = data['AMP'] ptc_gains = data['PTC_GAIN'] ptc_gain_errors = data['PTC_GAIN_ERROR'] for amp, gain, gain_error in zip(amps, ptc_gains, ptc_gain_errors): results.append( lcatr.schema.valid(lcatr.schema.get('ptc_raft'), amp=amp, ptc_gain=gain, ptc_gain_error=gain_error, slot=slot, sensor_id=wgSlotName)) # Persist the png files.
slot=slot, sensor_id=wgSlotName)) # sensor_id=sensor_id)) # fe55_acq_job_id = siteUtils.get_prerequisite_job_id('S*/%s_fe55_fe55_*.fits' % sensor_id, fe55_acq_job_id = siteUtils.get_prerequisite_job_id('S*/%s_fe55_fe55_*.fits' % wgSlotName, jobname=siteUtils.getProcessName('fe55_raft_acq')) # files = glob.glob('%s_read_noise?*.fits' % sensor_id) files = glob.glob('%s_read_noise?*.fits' % wgSlotName) for fitsfile in files: eotestUtils.addHeaderData(fitsfile, LSST_NUM=sensor_id, TESTTYPE='FE55', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=ccd_vendor) data_products = [siteUtils.make_fileref(item, folder=slot) for item in files] results.extend(data_products) # Persist the png files. metadata = dict(CCD_MANU=ccd_vendor, LSST_NUM=sensor_id, TESTTYPE='FE55', TEST_CATEGORY='EO') results.extend(siteUtils.persist_png_files('%s*.png' % sensor_id, sensor_id, folder=slot, metadata=metadata)) # Persist the raft-level overscan correlation plot. metadata = dict(LSST_NUM=raft_id, TESTTYPE='FE55', TEST_CATEGORY='EO') results.extend(siteUtils.persist_png_files('%s*.png' % raft_id, raft_id, metadata=metadata))
def validate_cte(results, det_names): """Validate the CTE task results.""" run = siteUtils.getRunNumber() missing_det_names = [] for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) superflats \ = sorted(glob.glob('{}_superflat_*.fits'.format(file_prefix))) if not superflats: missing_det_names.append(det_name) continue for item in superflats: eotestUtils.addHeaderData(item, FILENAME=item, DATE=eotestUtils.utc_now_isoformat()) results.extend([siteUtils.make_fileref(x) for x in superflats]) results_file = '%s_eotest_results.fits' % file_prefix data = sensorTest.EOTestResults(results_file) amps = data['AMP'] cti_high_serial = data['CTI_HIGH_SERIAL'] cti_high_serial_error = data['CTI_HIGH_SERIAL_ERROR'] cti_high_parallel = data['CTI_HIGH_PARALLEL'] cti_high_parallel_error = data['CTI_HIGH_PARALLEL_ERROR'] cti_low_serial = data['CTI_LOW_SERIAL'] cti_low_serial_error = data['CTI_LOW_SERIAL_ERROR'] cti_low_parallel = data['CTI_LOW_PARALLEL'] cti_low_parallel_error = data['CTI_LOW_PARALLEL_ERROR'] for values in zip(amps, cti_high_serial, cti_high_serial_error, cti_high_parallel, cti_high_parallel_error, cti_low_serial, cti_low_serial_error, cti_low_parallel, cti_low_parallel_error): results.append( lcatr.schema.valid(lcatr.schema.get('cte_BOT'), amp=values[0], cti_high_serial=values[1], cti_high_serial_error=values[2], cti_high_parallel=values[3], cti_high_parallel_error=values[4], cti_low_serial=values[5], cti_low_serial_error=values[6], cti_low_parallel=values[7], cti_low_parallel_error=values[8], slot=slot, raft=raft)) # Persist the png files. png_file_list = '{}_cte_task_png_files.txt'.format(det_name) with open(png_file_list, 'r') as input_: png_files = [x.strip() for x in input_] metadata = dict(DETECTOR=det_name, RUN=run, TESTTYPE='SFLAT_500', TEST_CATEGORY='EO') results.extend( siteUtils.persist_png_files('', file_prefix, png_files=png_files, metadata=metadata)) report_missing_data("validate_cte", missing_det_names) return results
def validate_fe55(results, det_names): """Validate and persist fe55 gain and psf results.""" run = siteUtils.getRunNumber() missing_det_names = [] for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) # The output files from producer script. gain_file = '%(file_prefix)s_eotest_results.fits' % locals() psf_results_files \ = glob.glob('%(file_prefix)s_psf_results*.fits' % locals()) if not os.path.isfile(gain_file) or not psf_results_files: # Results for this detector are not available so note # that and continue with the others. missing_det_names.append(det_name) continue psf_results = psf_results_files[0] rolloff_mask = '%(file_prefix)s_edge_rolloff_mask.fits' % locals() output_files = psf_results, rolloff_mask # Add/update the metadata to the primary HDU of these files. for fitsfile in output_files: eotestUtils.addHeaderData(fitsfile, TESTTYPE='FE55', DATE=eotestUtils.utc_now_isoformat()) results.extend([lcatr.schema.fileref.make(x) for x in output_files]) # Persist the median bias FITS file. bias_frame \ = glob.glob('%(file_prefix)s_median_bias.fits' % locals())[0] results.append(siteUtils.make_fileref(bias_frame)) # Persist the png files. png_file_list = '{}_fe55_task_png_files.txt'.format(det_name) with open(png_file_list, 'r') as input_: png_files = [x.strip() for x in input_] metadata = dict(TESTTYPE='FE55', TEST_CATEGORY='EO', DETECTOR=det_name, RUN=run) results.extend( siteUtils.persist_png_files('', file_prefix, png_files=png_files, metadata=metadata)) data = sensorTest.EOTestResults(gain_file) amps = data['AMP'] gain_data = data['GAIN'] gain_errors = data['GAIN_ERROR'] sigmas = data['PSF_SIGMA'] for amp, gain_value, gain_error, sigma in zip(amps, gain_data, gain_errors, sigmas): if not np.isfinite(gain_error): gain_error = -1 results.append( lcatr.schema.valid(lcatr.schema.get('fe55_BOT_analysis'), amp=amp, gain=gain_value, gain_error=gain_error, psf_sigma=sigma, slot=slot, raft=raft)) report_missing_data('validate_fe55', missing_det_names) return results
results = [] for slot, sensor_id in raft.items(): print("Processing:", slot, sensor_id) if 'ccd2' in slot : continue wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id] ccd_vendor = sensor_id.split('-')[0].upper() # superflats = glob.glob('%(sensor_id)s_superflat_*.fits' % locals()) superflats = glob.glob('%s_superflat_*.fits' % wgSlotName) for item in superflats: eotestUtils.addHeaderData(item, FILENAME=item, DATE=eotestUtils.utc_now_isoformat()) results.extend([siteUtils.make_fileref(x, folder=slot) for x in superflats]) results_file = '%s_eotest_results.fits' % wgSlotName data = sensorTest.EOTestResults(results_file) amps = data['AMP'] cti_high_serial = data['CTI_HIGH_SERIAL'] cti_high_serial_error = data['CTI_HIGH_SERIAL_ERROR'] cti_high_parallel = data['CTI_HIGH_PARALLEL'] cti_high_parallel_error = data['CTI_HIGH_PARALLEL_ERROR'] cti_low_serial = data['CTI_LOW_SERIAL'] cti_low_serial_error = data['CTI_LOW_SERIAL_ERROR'] cti_low_parallel = data['CTI_LOW_PARALLEL'] cti_low_parallel_error = data['CTI_LOW_PARALLEL_ERROR']
def validate_flat_pairs(results, det_names): """Validate the flat pair analysis results.""" run = siteUtils.getRunNumber() missing_det_names = set() for det_name in det_names: raft, slot = det_name.split('_') file_prefix = make_file_prefix(run, det_name) det_resp_data = '%s_det_response.fits' % file_prefix if not os.path.isfile(det_resp_data): missing_det_names.add(det_name) continue eotestUtils.addHeaderData(det_resp_data, DETECTOR=det_name, TESTTYPE='FLAT', DATE=eotestUtils.utc_now_isoformat()) results.append(siteUtils.make_fileref(det_resp_data)) results_file = '%s_eotest_results.fits' % file_prefix data = sensorTest.EOTestResults(results_file) amps = data['AMP'] max_observed_signal_data = data['MAX_OBSERVED_SIGNAL'] max_frac_dev_data = data['MAX_FRAC_DEV'] row_mean_var_slope_data = data['ROW_MEAN_VAR_SLOPE'] linearity_turnoff_data = data['LINEARITY_TURNOFF'] for amp, max_observed_signal, max_frac_dev, row_mean_var_slope, \ linearity_turnoff in zip(amps, max_observed_signal_data, max_frac_dev_data, row_mean_var_slope_data, linearity_turnoff_data): results.append( lcatr.schema.valid(lcatr.schema.get('flat_pairs_BOT'), amp=amp, max_observed_signal=max_observed_signal, max_frac_dev=max_frac_dev, row_mean_var_slope=row_mean_var_slope, linearity_turnoff=linearity_turnoff, slot=slot, raft=raft)) # Persist the png files. metadata = dict(DETECTOR=det_name, RUN=run, TESTTYPE='FLAT', TEST_CATEGORY='EO') results.extend( siteUtils.persist_png_files(('%s_linearity*.png' % file_prefix), file_prefix, metadata=metadata)) results.extend( siteUtils.persist_png_files( ('%s_row_means_variance.png' % file_prefix), file_prefix, metadata=metadata)) # Persist the raft-level imaging region correlation plots. missing_raft_names = set() for raft in camera_info.get_installed_raft_names(): metadata = dict(TESTTYPE='FLAT', TEST_CATEGORY='EO', RAFT=raft, RUN=run) file_prefix = make_file_prefix(run, raft) filename = f'{file_prefix}_imaging_region_correlations.png' if not os.path.isfile(filename): missing_raft_names.add(raft) continue results.extend( siteUtils.persist_png_files(filename, file_prefix, metadata=metadata)) # Persist any pd correction file specified in the lcatr.cfg file. pd_corrections_file_env = 'LCATR_PD_CORRECTIONS_FILE' if pd_corrections_file_env in os.environ: pd_corrections_file = os.environ[pd_corrections_file_env] shutil.copy(pd_corrections_file, '.') fileref = siteUtils.make_fileref(os.path.basename(pd_corrections_file)) results.append(fileref) report_missing_data("validate_flat_pairs", missing_det_names) report_missing_data("validate_flat_pairs", sorted(list(missing_raft_names)), components='rafts', total=21) return results