def run_qe_task(sensor_id):
try:
if hw_objects[sensor_id] :
return
except:
hw_objects[sensor_id] = True
print("hw_objects = ",hw_objects)
"Single sensor execution of the QE task."
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
lambda_files = siteUtils.dependency_glob('S*/%s_lambda_flat_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('qe_raft_acq'),
description='Lambda files:')
pd_ratio_file = eotestUtils.getPhotodiodeRatioFile()
if pd_ratio_file is None:
message = ("The test-stand specific photodiode ratio file is " +
"not given in config/%s/eotest_calibrations.cfg."
% siteUtils.getSiteName())
raise RuntimeError(message)
correction_image = eotestUtils.getIlluminationNonUniformityImage()
if correction_image is None:
print()
print("WARNING: The correction image file is not given in")
print("config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName())
print("No correction for non-uniform illumination will be applied.")
print()
sys.stdout.flush()
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
task = sensorTest.QeTask()
task.config.temp_set_point = -100.
task.run(sensor_id, lambda_files, pd_ratio_file, mask_files, gains,
correction_image=correction_image)
results_file \
= siteUtils.dependency_glob('%s_eotest_results.fits' % sensor_id,
jobname='fe55_raft_analysis',
description='Fe55 results file')[0]
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
siteUtils.make_png_file(plots.qe,
'%s_qe.png' % file_prefix,
qe_file='%s_QE.fits' % sensor_id)
try:
plots.flat_fields(os.path.dirname(lambda_files[0]),
annotation='e-/pixel, gain-corrected, bias-subtracted')
except Exception as eobj:
print("Exception raised while creating flat fields:")
print(str(eobj))
def __init__(self, configFile, sys_paths=()):
"""
configFile contains the names of the site-specific
configuration files. File basenames are provided in
configFile, and the full paths are constructed in the
_read(...) method.
"""
super(CcsSetup, self).__init__()
self.commands = []
self['tsCWD'] = os.getcwd()
self['labname'] = siteUtils.getSiteName()
self['jobname'] = siteUtils.getJobName()
self['CCDID'] = siteUtils.getUnitId()
self['UNITID'] = siteUtils.getUnitId()
self['LSSTID'] = siteUtils.getLSSTId()
try:
self['RUNNUM'] = siteUtils.getRunNumber()
except Exception:
self['RUNNUM'] = "no_lcatr_run_number"
self['ts'] = os.getenv('CCS_TS', default='ts')
self['archon'] = os.getenv('CCS_ARCHON', default='archon')
# The following are only available for certain contexts.
if 'CCS_VAC_OUTLET' in os.environ:
self['vac_outlet'] = os.getenv('CCS_VAC_OUTLET')
if 'CCS_CRYO_OUTLET' in os.environ:
self['cryo_outlet'] = os.getenv('CCS_CRYO_OUTLET')
if 'CCS_PUMP_OUTLET' in os.environ:
self['pump_outlet'] = os.getenv('CCS_PUMP_OUTLET')
self._read(os.path.join(siteUtils.getJobDir(), configFile))
self.sys_paths = sys_paths
def qe_jh_task(det_name):
"""JH version of single sensor execution of the QE task."""
run = siteUtils.getRunNumber()
file_prefix = make_file_prefix(run, det_name)
acq_jobname = siteUtils.getProcessName('BOT_acq')
lambda_files = siteUtils.dependency_glob(glob_pattern('qe', det_name),
acq_jobname=acq_jobname)
if not lambda_files:
print("qe_task: QE scan files not found for detector", det_name)
return None
pd_ratio_file = eotestUtils.getPhotodiodeRatioFile()
if pd_ratio_file is None:
message = ("The BOT photodiode ratio file is " +
"not given in config/%s/eotest_calibrations.cfg."
% siteUtils.getSiteName())
raise RuntimeError(message)
# correction_image = eotestUtils.getIlluminationNonUniformityImage()
# if correction_image is None:
# print()
# print("WARNING: The correction image file is not given in")
# print("config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName())
# print("No correction for non-uniform illumination will be applied.")
# print()
# sys.stdout.flush()
mask_files = get_mask_files(det_name)
eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
gains = get_amplifier_gains(eotest_results_file)
bias_frame = bias_filename(run, det_name)
return qe_task(run, det_name, lambda_files, pd_ratio_file, gains,
mask_files=mask_files, bias_frame=bias_frame,
mondiode_func=mondiode_value)
def __init__(self, configFile):
"""
configFile contains the names of the site-specific
configuration files. File basenames are provided in
configFile, and the full paths are constructed in the
_read(...) method.
"""
super(CcsSetup, self).__init__()
if os.environ.has_key('CCS_TS'):
self['ts']=_quote(os.getenv('CCS_TS'))
else:
self['ts'] = _quote('ts')
self['tsCWD'] = _quote(os.getcwd())
self['labname'] = _quote(siteUtils.getSiteName())
self['CCDID'] = _quote(siteUtils.getUnitId())
# self._read(os.path.join(siteUtils.getJobDir(), configFile))
CCDTYPE = _quote(siteUtils.getUnitType())
def run_flat_pair_task(sensor_id):
file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
flat_files = siteUtils.dependency_glob(
'S*/%s_flat*flat?_*.fits' % sensor_id,
jobname=siteUtils.getProcessName('flat_pair_raft_acq'),
description='Flat files:')
mask_files = \
eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
sensor_id=sensor_id)
use_exptime = True
if siteUtils.getSiteName() == 'SLAC':
# Since slit-width can be set individually for each exposure
# on TS-8 at IR-2 (LSSTTD-1231), we need to use the MONDIODE
# keyword for computing the integrated incident flux.
use_exptime = False
task = sensorTest.FlatPairTask()
task.run(sensor_id,
flat_files,
mask_files,
gains,
linearity_spec_range=(1e4, 9e4),
use_exptime=use_exptime)
results_file = '%s_eotest_results.fits' % sensor_id
plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)
Ne_bounds = (1e4, 9e4)
detresp_file = '%s_det_response.fits' % sensor_id
siteUtils.make_png_file(plots.linearity,
'%s_linearity.png' % file_prefix,
detresp_file=detresp_file,
max_dev=0.03,
use_exptime=use_exptime,
Ne_bounds=Ne_bounds)
siteUtils.make_png_file(plots.linearity_resids,
'%s_linearity_resids.png' % file_prefix,
detresp_file=detresp_file,
max_dev=0.03,
Ne_bounds=Ne_bounds,
use_exptime=use_exptime)
def validate_raft_results(results, raft_names):
"""Validate the raft level results."""
run = siteUtils.getRunNumber()
slot_names = camera_info.get_slot_names()
md = siteUtils.DataCatalogMetadata(ORIGIN=siteUtils.getSiteName(),
TEST_CATEGORY='EO',
DATA_PRODUCT='EOTEST_RESULTS')
missing_raft_names = []
for raft_name in raft_names:
for slot_name in slot_names:
det_name = make_file_prefix(raft_name, slot_name)
file_prefix = make_file_prefix(run, det_name)
results_file = '{}_eotest_results.fits'.format(file_prefix)
if not os.path.isfile(results_file):
if raft_name not in missing_raft_names:
missing_raft_names.append(raft_name)
continue
eotestUtils.addHeaderData(results_file,
DETECTOR=det_name,
DATE=eotestUtils.utc_now_isoformat(),
RUNNUM=run)
results.append(
lcatr.schema.fileref.make(results_file,
metadata=md(SLOT=slot_name,
RAFT=raft_name)))
# Persist the png files.
png_file_list = '{}_raft_results_task_png_files.txt'.format(raft_name)
if not os.path.isfile(png_file_list):
continue
with open(png_file_list, 'r') as input_:
png_files = [x.strip() for x in input_]
metadata = dict(TEST_CATEGORY='EO', DETECTOR=det_name, RUN=run)
results.extend(
siteUtils.persist_png_files('',
file_prefix,
png_files=png_files,
metadata=metadata))
report_missing_data("validate_raft_results",
missing_raft_names,
components='rafts',
total=21)
return results
def aggregate_filerefs(producer, testtype, origin=None, dp_mapping=None):
"""
Aggregate the filerefs for the metrology data products and return
them as a python list.
"""
if origin is None:
origin = siteUtils.getSiteName()
if dp_mapping is None:
dp_mapping = dict(BOXPLOT='boxplot',
HISTOGRAM='hist',
POINT_CLOUD_10='point_cloud_azim_10',
POINT_CLOUD_45='point_cloud_azim_45',
RESIDUALS='residuals',
QUANTILE_TABLE='quantile_table')
# Common metadata fields for all data products.
ccd_vendor = siteUtils.getCcdVendor()
sensor_id = siteUtils.getUnitId()
md = siteUtils.DataCatalogMetadata(CCD_MANU=ccd_vendor,
LSST_NUM=sensor_id,
PRODUCER=producer,
ORIGIN=origin,
TESTTYPE=testtype,
TEST_CATEGORY='MET')
# Create filerefs for each data product, adding the file-specific
# data product metadata.
tt_ext = testtype.lower()
results = []
for dp, ext in dp_mapping.items():
pattern = '%(sensor_id)s_%(tt_ext)s_*%(ext)s.*'% locals()
dp_files = glob.glob(pattern)
results.extend([lcatr.schema.fileref.make(dp_file,
metadata=md(DATA_PRODUCT=dp))
for dp_file in dp_files])
return results
#!/usr/bin/env python
import sys
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
spot_files = siteUtils.dependency_glob(
'*_spot_*.fits',
jobname=siteUtils.getProcessName('xtalk_acq'),
description='Spot files:')
mask_files = eotestUtils.glob_mask_files()
system_xtalk_file = eotestUtils.getSystemCrosstalkFile()
if system_xtalk_file is None:
print
print "WARNING: The system crosstalk file is not given in"
print "config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName()
print "No correction for system crosstalk will be performed."
print
sys.stdout.flush()
task = sensorTest.CrosstalkTask()
task.run(sensor_id,
spot_files,
mask_files,
system_xtalk_file=system_xtalk_file)
def __init__(self, configFile):
"""
configFile contains the names of the site-specific
configuration files. File basenames are provided in
configFile, and the full paths are constructed in the
_read(...) method.
"""
super(CcsSetup, self).__init__()
if os.environ.has_key('CCS_TS8'):
self['ts8']=_quote(os.getenv('CCS_TS8'))
else:
self['ts8'] = _quote('ts8')
if os.environ.has_key('CCS_JYTH'):
self['jyth']=_quote(os.getenv('CCS_JYTH'))
else:
self['jyth'] = _quote('JythonInterpreterConsole')
if os.environ.has_key('CCS_JSON_PORT'):
self['jsonport']=os.getenv('CCS_JSON_PORT')
else:
self['jsonport'] = 4444
if os.environ.has_key('CCS_PS'):
self['ps']=_quote(os.getenv('CCS_PS'))
else:
self['ps'] = _quote('ccs-rebps')
if os.environ.has_key('CCS_TS'):
self['ts']=_quote(os.getenv('CCS_TS'))
else:
self['ts'] = _quote('ts')
if os.environ.has_key('CCS_ARCHON'):
self['archon']=_quote(os.getenv('CCS_ARCHON'))
else:
self['archon'] = _quote('archon')
if os.environ.has_key('CCS_VAC_OUTLET'):
self['vac_outlet']=os.getenv('CCS_VAC_OUTLET')
# there is no default for vac_outlet - if there is a script that needs
# it and it has not been defined then I want it to crash
if os.environ.has_key('CCS_CRYO_OUTLET'):
self['cryo_outlet']=os.getenv('CCS_CRYO_OUTLET')
# there is no default for cryo_outlet - if there is a script that needs
# it and it has not been defined then I want it to crash
if os.environ.has_key('CCS_PUMP_OUTLET'):
self['pump_outlet']=os.getenv('CCS_PUMP_OUTLET')
# there is no default for pump_outlet - if there is a script that needs
# it and it has not been defined then I want it to crash
self['tsCWD'] = _quote(os.getcwd())
self['labname'] = _quote(siteUtils.getSiteName())
self['jobname'] = _quote(siteUtils.getJobName())
self['CCDID'] = _quote(siteUtils.getUnitId())
self['UNITID'] = _quote(siteUtils.getUnitId())
self['LSSTID'] = _quote(siteUtils.getLSSTId())
unitid = siteUtils.getUnitId()
CCDTYPE = _quote(siteUtils.getUnitType())
ccdnames = {}
ccdmanunames = {}
ccdnames,ccdmanunames = siteUtils.getCCDNames()
print "retrieved the following LSST CCD names list"
print ccdnames
print "retrieved the following Manufacturers CCD names list"
print ccdmanunames
for slot in ccdnames :
print "CCD %s is in slot %s" % (ccdnames[slot],slot)
self['CCD%s'%slot] = _quote(ccdnames[slot])
if 'itl' in ccdnames[slot].lower() :
CCDTYPE = 'itl'
if 'e2v' in ccdnames[slot].lower() :
CCDTYPE = 'e2v'
for slot in ccdmanunames :
print "CCD %s is in slot %s" % (ccdmanunames[slot],slot)
self['CCDMANU%s'%slot] = _quote(ccdmanunames[slot])
try:
self['RUNNUM'] = _quote(siteUtils.getRunNumber())
except:
self['RUNNUM'] = "no_lcatr_run_number"
pass
self._read(os.path.join(siteUtils.getJobDir(), configFile))
print "CCDTYPE = %s" % CCDTYPE
self['sequence_file'] = _quote("NA")
self['acffile'] = self['itl_acffile']
# set default type
self['CCSCCDTYPE'] = _quote("ITL")
if ("RTM" in unitid.upper() or "ETU" in unitid.upper() or "RSA" in unitid.upper()) :
if ("e2v" in CCDTYPE) :
self['CCSCCDTYPE'] = _quote("E2V")
self['acffile'] = self['e2v_acffile']
self['sequence_file'] = self['e2v_seqfile']
else :
self['CCSCCDTYPE'] = _quote("ITL")
self['acffile'] = self['itl_acffile']
self['sequence_file'] = self['itl_seqfile']
os.system("export | grep -i seq")
seqdir = ""
if os.environ.has_key('SEQUENCERFILESDIR') :
seqdir = os.getenv('SEQUENCERFILESDIR')
print "seqdir=",seqdir
self['sequence_file'] = self['sequence_file'].replace('${SEQUENCERFILESDIR}',seqdir)
os.system("cp -vp %s %s" % (self['sequence_file'],self['tsCWD']))
# now use the local copy
# bb = self['sequence_file'].split("/")
# self['sequence_file'] = _quote("%s/%s" % (os.getcwd(),bb[len(bb)-1].split("'")[0]))
print "The sequence file to be used is %s" % self['sequence_file']
else :
if ("ITL" in CCDTYPE) :
self['CCSCCDTYPE'] = _quote("ITL")
self['acffile'] = self['itl_acffile']
if ("e2v" in CCDTYPE) :
self['CCSCCDTYPE'] = _quote("E2V")
self['acffile'] = self['e2v_acffile']
print "The acffile to be used is %s" % self['acffile']
= aliveness_utils.raft_channel_statuses(fits_files)
for slot in channel_status:
bad_channels = 0
for amp, status in channel_status[slot].items():
if channel_status[slot][amp] == 'bad':
bad_channels += 1
signal = channel_signal[slot][amp]
channel = imutils.channelIds[amp]
output.write(row_template % locals())
results.append(lcatr.schema.valid(job_schema,
exptime=exptime,
slot=slot,
bad_channels=bad_channels))
results.append(lcatr.schema.fileref.make(outfile))
# Persist the sequencer file that was used.
seq_file = glob.glob('*.seq')[0]
results.append(lcatr.schema.fileref.make(seq_file))
# Add png files.
raft_id = siteUtils.getUnitId()
md = siteUtils.DataCatalogMetadata(ORIGIN=siteUtils.getSiteName(),
TEST_CATEGORY='EO')
results.extend(siteUtils.persist_png_files('*.png', raft_id, metadata=md))
results.extend(siteUtils.jobInfo())
lcatr.schema.write_file(results)
lcatr.schema.validate_file()
#!/usr/bin/env python
import sys
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
spot_files = siteUtils.dependency_glob('*_spot_*.fits',
jobname=siteUtils.getProcessName('xtalk_acq'),
description='Spot files:')
mask_files = eotestUtils.glob_mask_files()
system_xtalk_file = eotestUtils.getSystemCrosstalkFile()
if system_xtalk_file is None:
print
print "WARNING: The system crosstalk file is not given in"
print "config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName()
print "No correction for system crosstalk will be performed."
print
sys.stdout.flush()
task = sensorTest.CrosstalkTask()
task.run(sensor_id, spot_files, mask_files, system_xtalk_file=system_xtalk_file)
producer = 'SR-MET-07'
testtype = 'FLATNESS'
results = metUtils.aggregate_filerefs_ts5(producer, testtype)
# Add the QA plot to the results
raft_id = siteUtils.getUnitId()
qafile = glob.glob('*_qa_plot.png')[0]
#print('qafile: %s' % qafile)
#print('raft_id: %s' % raft_id)
md = siteUtils.DataCatalogMetadata(CCD_MANU=siteUtils.getCcdVendor(),
LSST_NUM=siteUtils.getUnitId(),
PRODUCER=producer,
ORIGIN=siteUtils.getSiteName(),
TESTTYPE=testtype,
TEST_CATEGORY='MET')
results.extend([lcatr.schema.fileref.make(qafile, metadata=md(DATA_PRODUCT='QA_PLOT'))])
raftData = md_factory.load('flatness_ts5_delta.pickle')
peak_valley_95 = raftData.quantiles['0.975'] - raftData.quantiles['0.025']
peak_valley_100 = raftData.quantiles['1.000'] - raftData.quantiles['0.000']
# Make strings out of the quantile information
quantiles = raftData.quantiles
quantile_levels = quantiles.keys()
quantile_str = ''
z_str = ''
for key in sorted(quantile_levels):
#!/usr/bin/env python
import os
import glob
import shutil
import lcatr.schema
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
md = siteUtils.DataCatalogMetadata(CCD_MANU=siteUtils.getCcdVendor(),
LSST_NUM=sensor_id,
PRODUCER='SR-EOT-1',
ORIGIN=siteUtils.getSiteName(),
TEST_CATEGORY='EO')
results_file = '%s_eotest_results.fits' % sensor_id
eotestUtils.addHeaderData(results_file, LSST_NUM=sensor_id,
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results = [lcatr.schema.fileref.make(results_file,
metadata=md(DATA_PRODUCT='EOTEST_RESULTS'))]
png_files = glob.glob('*.png')
results.extend([lcatr.schema.fileref.make(item,
metadata=md(DATA_PRODUCT=eotestUtils.png_data_product(item, sensor_id)))
for item in png_files])
test_report = '%s_eotest_report.pdf' % sensor_id
results.append(lcatr.schema.fileref.make(test_report,
metadata=md(DATA_PRODUCT='EOTEST_REPORT')))