def aggregate_filerefs(producer, testtype, origin=None, dp_mapping=None):
"""
Aggregate the filerefs for the metrology data products and return
them as a python list.
"""
if origin is None:
origin = siteUtils.getSiteName()
if dp_mapping is None:
dp_mapping = dict(BOXPLOT='boxplot',
HISTOGRAM='hist',
POINT_CLOUD_10='point_cloud_azim_10',
POINT_CLOUD_45='point_cloud_azim_45',
RESIDUALS='residuals',
QUANTILE_TABLE='quantile_table')
# Common metadata fields for all data products.
ccd_vendor = siteUtils.getCcdVendor()
sensor_id = siteUtils.getUnitId()
md = siteUtils.DataCatalogMetadata(CCD_MANU=ccd_vendor,
LSST_NUM=sensor_id,
PRODUCER=producer,
ORIGIN=origin,
TESTTYPE=testtype,
TEST_CATEGORY='MET')
# Create filerefs for each data product, adding the file-specific
# data product metadata.
tt_ext = testtype.lower()
results = []
for dp, ext in dp_mapping.items():
pattern = '%(sensor_id)s_%(tt_ext)s_*%(ext)s.*'% locals()
dp_files = glob.glob(pattern)
results.extend([lcatr.schema.fileref.make(dp_file,
metadata=md(DATA_PRODUCT=dp))
for dp_file in dp_files])
return results
import glob
import pyfits
import numpy as np
import lsst.eotest.sensor as sensorTest
import lcatr.schema
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
# Save the output file from producer script.
persistence_file = '%(sensor_id)s_persistence.fits' % locals()
eotestUtils.addHeaderData(persistence_file, LSST_NUM=sensor_id,
TESTTYPE='PERSISTENCE',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results = [lcatr.schema.fileref.make(persistence_file)]
# Save the deferred charge for each amplifier from the first post-flat
# dark frame as the image persistence metric.
#
# Read the results from the FITS file.
persistence = pyfits.open(persistence_file)
times = persistence[1].data.field('TIME')
# Times of dark frames are relative to the MJD-OBS of the flat.
post_flat_times = times[np.where(times > 0)]
index = np.where(times == min(post_flat_times))[0][0]
# Loop over amplifiers and record the deferred charge (median pixel
sensor_id = siteUtils.getUnitId()
# The output files from producer script.
gain_file = '%(sensor_id)s_eotest_results.fits' % locals()
psf_results = glob.glob('%(sensor_id)s_psf_results*.fits' % locals())[0]
rolloff_mask = '%(sensor_id)s_rolloff_defects_mask.fits' % locals()
output_files = gain_file, psf_results, rolloff_mask
# Add/update the metadata to the primary HDU of these files.
for fitsfile in output_files:
eotestUtils.addHeaderData(fitsfile,
LSST_NUM=sensor_id,
TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
#
# Persist the mean bias FITS file.
#
bias_mean_file = glob.glob('%(sensor_id)s_mean_bias_*.fits' % locals())[0]
results = [lcatr.schema.fileref.make(bias_mean_file)]
#
# Common metadata for persisted non-FITS files.
#
md = siteUtils.DataCatalogMetadata(CCD_MANU=siteUtils.getCcdVendor().upper(),
LSST_NUM=sensor_id,
producer='SR-EOT-1',
TESTTYPE='FE55',
TEST_CATEGORY='EO')
#
sensor_id = siteUtils.getUnitId()
# The output files from producer script.
gain_file = '%(sensor_id)s_eotest_results.fits' % locals()
psf_results = glob.glob('%(sensor_id)s_psf_results*.fits' % locals())[0]
rolloff_mask = '%(sensor_id)s_rolloff_defects_mask.fits' % locals()
output_files = gain_file, psf_results, rolloff_mask
# Add/update the metadata to the primary HDU of these files.
for fitsfile in output_files:
eotestUtils.addHeaderData(fitsfile,
LSST_NUM=sensor_id,
TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results = []
data = sensorTest.EOTestResults(gain_file)
amps = data['AMP']
gain_data = data['GAIN']
gain_errors = data['GAIN_ERROR']
sigmas = data['PSF_SIGMA']
for amp, gain_value, gain_error, sigma in zip(amps, gain_data, gain_errors,
sigmas):
results.append(
lcatr.schema.valid(lcatr.schema.get('fe55_analysis'),
amp=amp,
gain=gain_value,
gain_error=gain_error,
#!/usr/bin/env python
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
ccd_vendor = siteUtils.getCcdVendor()
flat_files = siteUtils.datacatalog_glob('*_flat*flat?_*.fits',
testtype='FLAT',
imgtype='FLAT',
description='Flat files:')
mask_files = eotestUtils.glob_mask_files()
gains = eotestUtils.getSensorGains(jobname='fe55_offline')
task = sensorTest.FlatPairTask()
task.run(sensor_id, flat_files, mask_files, gains)
if ccd_vendor == 'ITL':
#
# Perform linearity analysis using special dataset from ITL
try:
flat_files = siteUtils.datacatalog_glob('*_linearity_flat*.fits',
testtype='LINEARITY',
imgtype='FLAT',
description='ITL linearity files:')
if flat_files:
task = sensorTest.LinearityTask()
task.run(sensor_id, flat_files, mask_files, gains)
except:
#!/usr/bin/env python
import os
import glob
import lsst.eotest.sensor as sensorTest
import lcatr.schema
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
ccd_vendor = siteUtils.getCcdVendor()
det_resp_data = '%s_det_response.fits' % sensor_id
eotestUtils.addHeaderData(det_resp_data,
LSST_NUM=sensor_id,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results = [lcatr.schema.fileref.make(det_resp_data)]
if ccd_vendor == 'ITL':
# Persist detector response for special linearity dataset from ITL.
det_resp_data = '%s_det_response_linearity.fits' % sensor_id
if os.path.isfile(det_resp_data):
# File will only exist if analyzing vendor data directly.
eotestUtils.addHeaderData(det_resp_data,
LSST_NUM=sensor_id,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results.append(lcatr.schema.fileref.make(det_resp_data))
pylab.savefig('%s_psf_dists.png' % sensor_id)
# Photon Transfer Curves
ptc_file = processName_dependencyGlob('%s_ptc.fits' % sensor_id,
jobname='ptc_offline')[0]
plots.ptcs(ptc_file=ptc_file)
pylab.savefig('%s_ptcs.png' % sensor_id)
detresp_file = processName_dependencyGlob('%s_det_response.fits' % sensor_id,
jobname='flat_pairs_offline')[0]
# Full well plots
plots.full_well(ptc_file=ptc_file, detresp_file=detresp_file)
pylab.savefig('%s_full_well.png' % sensor_id)
# Linearity plots
if siteUtils.getCcdVendor() == 'ITL':
# Use special linearity dataset for ITL data
try:
detresp_file = processName_dependencyGlob(
'%s_det_response_linearity.fits' % sensor_id,
jobname='flat_pairs_offline')[0]
except:
pass
plots.linearity(detresp_file=detresp_file)
pylab.savefig('%s_linearity.png' % sensor_id)
plots.linearity_resids(detresp_file=detresp_file)
pylab.savefig('%s_linearity_resids.png' % sensor_id)
# System Gain per segment
#!/usr/bin/env python
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
ccd_vendor = siteUtils.getCcdVendor()
flat_files = siteUtils.datacatalog_glob('*_flat*flat?_*.fits',
testtype='FLAT',
imgtype='FLAT',
description='Flat files:')
mask_files = eotestUtils.glob_mask_files()
gains = eotestUtils.getSensorGains(jobname='fe55_offline')
task = sensorTest.FlatPairTask()
task.run(sensor_id, flat_files, mask_files, gains)
if ccd_vendor == 'ITL':
#
# Perform linearity analysis using special dataset from ITL
try:
flat_files = siteUtils.datacatalog_glob(
'*_linearity_flat*.fits',
testtype='LINEARITY',
imgtype='FLAT',
description='ITL linearity files:')
if flat_files:
task = sensorTest.LinearityTask()
task.run(sensor_id, flat_files, mask_files, gains)
results = []
for band in qe_results:
results.append(validate(job, band=band,
QE=np.average(qe_results[band])))
return results
def metrology(self):
return []
if __name__ == '__main__':
results = [siteUtils.packageVersions()]
lsstnum = siteUtils.getUnitId()
vendorDataDir = os.readlink('vendorData')
print 'Vendor data location:', vendorDataDir
if siteUtils.getCcdVendor() == 'ITL':
vendor = ItlResults(vendorDataDir)
translator = ItlFitsTranslator(lsstnum, vendorDataDir, '.')
else:
vendor = e2vResults(vendorDataDir)
translator = e2vFitsTranslator(lsstnum, vendorDataDir, '.')
results.extend(vendor.run_all())
translator.run_all()
results.extend([lcatr.schema.fileref.make(x) for x in translator.outfiles])
lcatr.schema.write_file(results)
lcatr.schema.validate_file()
import siteUtils
import metUtils
from MetrologyData import md_factory
producer = 'SR-MET-07'
testtype = 'FLATNESS'
results = metUtils.aggregate_filerefs_ts5(producer, testtype)
# Add the QA plot to the results
raft_id = siteUtils.getUnitId()
qafile = glob.glob('*_qa_plot.png')[0]
#print('qafile: %s' % qafile)
#print('raft_id: %s' % raft_id)
md = siteUtils.DataCatalogMetadata(CCD_MANU=siteUtils.getCcdVendor(),
LSST_NUM=siteUtils.getUnitId(),
PRODUCER=producer,
ORIGIN=siteUtils.getSiteName(),
TESTTYPE=testtype,
TEST_CATEGORY='MET')
results.extend([lcatr.schema.fileref.make(qafile, metadata=md(DATA_PRODUCT='QA_PLOT'))])
raftData = md_factory.load('flatness_ts5_delta.pickle')
peak_valley_95 = raftData.quantiles['0.975'] - raftData.quantiles['0.025']
peak_valley_100 = raftData.quantiles['1.000'] - raftData.quantiles['0.000']
# Make strings out of the quantile information
quantiles = raftData.quantiles
quantile_levels = quantiles.keys()
#!/usr/bin/env python
import os
import glob
import lsst.eotest.sensor as sensorTest
import lcatr.schema
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
ccd_vendor = siteUtils.getCcdVendor()
det_resp_data = '%s_det_response.fits' % sensor_id
eotestUtils.addHeaderData(det_resp_data, LSST_NUM=sensor_id, TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results = [lcatr.schema.fileref.make(det_resp_data)]
if ccd_vendor == 'ITL':
# Persist detector response for special linearity dataset from ITL.
det_resp_data = '%s_det_response_linearity.fits' % sensor_id
if os.path.isfile(det_resp_data):
# File will only exist if analyzing vendor data directly.
eotestUtils.addHeaderData(det_resp_data, LSST_NUM=sensor_id,
TESTTYPE='FLAT',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results.append(lcatr.schema.fileref.make(det_resp_data))
results_file = '%s_eotest_results.fits' % sensor_id
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
