def persist_tearing_png_files(png_files, folder=None, metadata=None):
"""
Create the lcatr.schema.filerefs for persisting the png files
with the DataCatalog.
Parameters
----------
png_files: list
A list of png files for the tearing profile plots.
folder: str [None]
Folder under which to persist the png file. For raft-level
analysis, this would be the slot number of the CCD.
metadata: dict [None]
Any additional metadata to persist with the png files.
Returns
-------
list: This is a list of lcatr.schema.filerefs.
"""
if metadata is None:
metadata = dict()
md = siteUtils.DataCatalogMetadata(**metadata)
png_filerefs = []
for png_file in png_files:
dp = 'tearing_profiles'
lsst_id = os.path.basename(png_file).split('_')[0]
png_filerefs.append(
siteUtils.make_fileref(png_file,
folder=folder,
metadata=md(DATA_PRODUCT=dp,
LsstId=lsst_id)))
return png_filerefs
def validate_raft_results(results, raft_names):
"""Validate the raft level results."""
run = siteUtils.getRunNumber()
slot_names = camera_info.get_slot_names()
md = siteUtils.DataCatalogMetadata(ORIGIN=siteUtils.getSiteName(),
TEST_CATEGORY='EO',
DATA_PRODUCT='EOTEST_RESULTS')
missing_raft_names = []
for raft_name in raft_names:
for slot_name in slot_names:
det_name = make_file_prefix(raft_name, slot_name)
file_prefix = make_file_prefix(run, det_name)
results_file = '{}_eotest_results.fits'.format(file_prefix)
if not os.path.isfile(results_file):
if raft_name not in missing_raft_names:
missing_raft_names.append(raft_name)
continue
eotestUtils.addHeaderData(results_file,
DETECTOR=det_name,
DATE=eotestUtils.utc_now_isoformat(),
RUNNUM=run)
results.append(
lcatr.schema.fileref.make(results_file,
metadata=md(SLOT=slot_name,
RAFT=raft_name)))
# Persist the png files.
png_file_list = '{}_raft_results_task_png_files.txt'.format(raft_name)
if not os.path.isfile(png_file_list):
continue
with open(png_file_list, 'r') as input_:
png_files = [x.strip() for x in input_]
metadata = dict(TEST_CATEGORY='EO', DETECTOR=det_name, RUN=run)
results.extend(
siteUtils.persist_png_files('',
file_prefix,
png_files=png_files,
metadata=metadata))
report_missing_data("validate_raft_results",
missing_raft_names,
components='rafts',
total=21)
return results
LSST_NUM=sensor_id,
TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
#
# Persist the mean bias FITS file.
#
bias_mean_file = glob.glob('%(sensor_id)s_mean_bias_*.fits' % locals())[0]
results = [lcatr.schema.fileref.make(bias_mean_file)]
#
# Common metadata for persisted non-FITS files.
#
md = siteUtils.DataCatalogMetadata(CCD_MANU=siteUtils.getCcdVendor().upper(),
LSST_NUM=sensor_id,
producer='SR-EOT-1',
TESTTYPE='FE55',
TEST_CATEGORY='EO')
#
# Persist various png files.
#
png_files = glob.glob('%(sensor_id)s_fe55*.png' % locals())
png_filerefs = []
for png_file in png_files:
dp = eotestUtils.png_data_product(png_file, sensor_id)
png_filerefs.append(
lcatr.schema.fileref.make(png_file, metadata=md(DATA_PRODUCT=dp)))
results.extend(png_filerefs)
data = sensorTest.EOTestResults(gain_file)
amps = data['AMP']
= aliveness_utils.raft_channel_statuses(fits_files)
for slot in channel_status:
bad_channels = 0
for amp, status in channel_status[slot].items():
if channel_status[slot][amp] == 'bad':
bad_channels += 1
signal = channel_signal[slot][amp]
channel = imutils.channelIds[amp]
output.write(row_template % locals())
results.append(lcatr.schema.valid(job_schema,
exptime=exptime,
slot=slot,
bad_channels=bad_channels))
results.append(lcatr.schema.fileref.make(outfile))
# Persist the sequencer file that was used.
seq_file = glob.glob('*.seq')[0]
results.append(lcatr.schema.fileref.make(seq_file))
# Add png files.
raft_id = siteUtils.getUnitId()
md = siteUtils.DataCatalogMetadata(ORIGIN=siteUtils.getSiteName(),
TEST_CATEGORY='EO')
results.extend(siteUtils.persist_png_files('*.png', raft_id, metadata=md))
results.extend(siteUtils.jobInfo())
lcatr.schema.write_file(results)
lcatr.schema.validate_file()
#!/usr/bin/env python
"""
Validator script for collect_raft_results job.
"""
from __future__ import print_function
import lcatr.schema
import siteUtils
import eotestUtils
import camera_components
results = []
run_number = siteUtils.getRunNumber()
md = siteUtils.DataCatalogMetadata(ORIGIN=siteUtils.getSiteName(),
TEST_CATEGORY='EO',
DATA_PRODUCT='EOTEST_RESULTS')
# Persist eotest_results files for each sensor.
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
for slot, sensor_id in raft.items():
ccd_vendor = sensor_id.split('-')[0].upper()
if 'ccd2' in slot :
continue
wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id]
results_file = '%s_eotest_results.fits' % wgSlotName
eotestUtils.addHeaderData(results_file, LSST_NUM=sensor_id,
#!/usr/bin/env python
import os
import glob
import shutil
import lcatr.schema
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
md = siteUtils.DataCatalogMetadata(CCD_MANU=siteUtils.getCcdVendor(),
LSST_NUM=sensor_id,
PRODUCER='SR-EOT-1',
ORIGIN=siteUtils.getSiteName(),
TEST_CATEGORY='EO')
results_file = '%s_eotest_results.fits' % sensor_id
eotestUtils.addHeaderData(results_file, LSST_NUM=sensor_id,
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results = [lcatr.schema.fileref.make(results_file,
metadata=md(DATA_PRODUCT='EOTEST_RESULTS'))]
png_files = glob.glob('*.png')
results.extend([lcatr.schema.fileref.make(item,
metadata=md(DATA_PRODUCT=eotestUtils.png_data_product(item, sensor_id)))
for item in png_files])
test_report = '%s_eotest_report.pdf' % sensor_id
results.append(lcatr.schema.fileref.make(test_report,
metadata=md(DATA_PRODUCT='EOTEST_REPORT')))