def __init__(self, configFile, sys_paths=()):
"""
configFile contains the names of the site-specific
configuration files. File basenames are provided in
configFile, and the full paths are constructed in the
_read(...) method.
"""
super(CcsSetup, self).__init__()
self.commands = []
self['tsCWD'] = os.getcwd()
self['labname'] = siteUtils.getSiteName()
self['jobname'] = siteUtils.getJobName()
self['CCDID'] = siteUtils.getUnitId()
self['UNITID'] = siteUtils.getUnitId()
self['LSSTID'] = siteUtils.getLSSTId()
try:
self['RUNNUM'] = siteUtils.getRunNumber()
except Exception:
self['RUNNUM'] = "no_lcatr_run_number"
self['ts'] = os.getenv('CCS_TS', default='ts')
self['archon'] = os.getenv('CCS_ARCHON', default='archon')
# The following are only available for certain contexts.
if 'CCS_VAC_OUTLET' in os.environ:
self['vac_outlet'] = os.getenv('CCS_VAC_OUTLET')
if 'CCS_CRYO_OUTLET' in os.environ:
self['cryo_outlet'] = os.getenv('CCS_CRYO_OUTLET')
if 'CCS_PUMP_OUTLET' in os.environ:
self['pump_outlet'] = os.getenv('CCS_PUMP_OUTLET')
self._read(os.path.join(siteUtils.getJobDir(), configFile))
self.sys_paths = sys_paths
def persist_traps_analysis():
"""Persist the results from the traps analysis."""
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
ccd_vendor = sensor_id.split('-')[0].upper()
trap_file = '%s_traps.fits' % sensor_id
eotestUtils.addHeaderData(trap_file,
LSST_NUM=sensor_id,
TESTTYPE='TRAP',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
results.append(siteUtils.make_fileref(trap_file, folder=slot))
mask_file = '%s_traps_mask.fits' % sensor_id
results.append(siteUtils.make_fileref(mask_file, folder=slot))
results_file = '%s_eotest_results.fits' % sensor_id
data = sensorTest.EOTestResults(results_file)
amps = data['AMP']
num_traps = data['NUM_TRAPS']
for amp, ntrap in zip(amps, num_traps):
results.append(
lcatr.schema.valid(lcatr.schema.get('traps_raft'),
amp=amp,
num_traps=ntrap,
slot=slot,
sensor_id=sensor_id))
return results
def validate_flat_gain_stability(results, det_names):
"""Valdiate the output files from the flat_gain_stability analysis"""
if 'gainstability' not in get_analysis_types():
return results
run = siteUtils.getRunNumber()
missing_det_names = set()
for det_name in det_names:
file_prefix = make_file_prefix(run, det_name)
results_file = f'{file_prefix}_flat_signal_sequence.pickle'
if not os.path.isfile(results_file):
missing_det_names.add(det_name)
else:
md = dict(DATA_PRODUCT='flat_gain_stability_results')
results.append(siteUtils.make_fileref(results_file, metadata=md))
report_missing_data('validate_flat_gain_stability', missing_det_names)
unit_id = siteUtils.getUnitId()
png_files = glob.glob('*flat_gain_stability.png')
for png_file in png_files:
md = dict(DATA_PRODUCT='flat_gain_stability_plot')
if unit_id in png_file:
md['LsstId'] = unit_id
results.append(siteUtils.make_fileref(png_file, metadata=md))
return results
def _get_ccd_and_reb_names(self):
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
for slot in raft.slot_names:
sensor = raft.sensor(slot)
self.set_item('ccd_names["%s"]' % slot, 'SensorInfo("%s", "%s")'
% (str(sensor.sensor_id),
str(sensor.manufacturer_sn)))
# # aliveness bench reb serial numbers:
# for slot, reb_sn in zip(('REB0', 'REB1', 'REB2'),
# [412220615, 412162821, 305879976]):
# raft.rebs[slot].manufacturer_sn = '%x' % reb_sn
for slot, reb in raft.rebs.items():
self.set_item('reb_eT_info["%s"]' % slot,
'RebInfo("%s", "%s", "%s")'
% (reb.reb_id, reb.manufacturer_sn,
reb.firmware_version))
ccd_type = str(raft.sensor_type.split('-')[0])
self['ccd_type'] = ccd_type
if ccd_type == 'ITL':
self.set_item('sequence_file', self['itl_seqfile'])
elif ccd_type.upper() == 'E2V':
self.set_item('sequence_file', self['e2v_seqfile'])
else:
raise RuntimeError('Invalid ccd_type: %s' % ccd_type)
shutil.copy(self['sequence_file'].strip("'"),
self['tsCWD'].strip("'"))
def plot_all_rafts(run, results_dir='.', cut=None, y_range=(0.998, 1.002),
divide_by_flux=True, figsize=(18, 18)):
"""
Plot the flat gain stability curve for all 25 rafts in a 5x5 grid.
"""
raft_files = defaultdict(list)
files = sorted(glob.glob(os.path.join(results_dir,
'*flat_signal_sequence.pickle')))
if not files:
return
for item in files:
raft_name = os.path.basename(item)[:len('R22')]
raft_files[raft_name].append(item)
figure = plt.figure(figsize=figsize)
rafts = sorted(list(raft_files.keys()))
for i, raft in enumerate(rafts):
figure.add_subplot(5, 5, i+1)
plot_raft(raft_files[raft], cut=cut, divide_by_flux=divide_by_flux,
y_range=y_range)
plt.tight_layout(rect=(0, 0, 1, 0.95))
suptitle = append_acq_run(f'Flat gain stability, Run {run}')
plt.suptitle(suptitle)
unit_id = siteUtils.getUnitId()
plt.savefig(f'{unit_id}_{run}_flat_gain_stability.png')
def persist_fe55_gains():
"""Persist only the Fe55 gains from the results file."""
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
# Save eotest results file with nominal gains.
ccd_vendor = sensor_id.split('-')[0].upper()
results_file = '%s_eotest_results.fits' % sensor_id
eotestUtils.addHeaderData(results_file,
LSST_NUM=sensor_id,
TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
results.append(lcatr.schema.fileref.make(results_file))
# Persist nominal values to eT results database.
amps = imutils.allAmps()
gain_data = np.ones(len(amps))
gain_errors = np.zeros(len(amps))
sigmas = np.zeros(len(amps))
for amp, gain_value, gain_error, sigma in zip(amps, gain_data,
gain_errors, sigmas):
if not np.isfinite(gain_error):
gain_error = -1
results.append(
lcatr.schema.valid(lcatr.schema.get('fe55_raft_analysis'),
amp=amp,
gain=gain_value,
gain_error=gain_error,
psf_sigma=sigma,
slot=slot,
sensor_id=sensor_id))
return results
def getSensorGains(jobname='fe55_analysis', sensor_id=None):
if sensor_id is None:
sensor_id = siteUtils.getUnitId()
try:
gain_file = dependency_glob('%s_eotest_results.fits' % sensor_id,
jobname=jobname)[0]
except IndexError:
raise RuntimeError('eotestUtils.getSensorGains: %s %s'
% (sensor_id, jobname))
data = sensorTest.EOTestResults(gain_file)
amps = data['AMP']
gains = data['GAIN']
sensorGains = dict([(amp, gains[amp-1]) for amp in amps])
return sensorGains
def plot_all_raft_fe55_gains(raft_files, figsize=(18, 18), y_range=None):
"""
Plot the flat gain stability curve for all 25 rafts in a 5x5 grid.
"""
figure = plt.figure(figsize=figsize)
rafts = sorted(list(raft_files.keys()))
for i, raft in enumerate(rafts, 1):
figure.add_subplot(5, 5, i)
plot_raft_fe55_gains_by_ccd(raft_files[raft], y_range=y_range)
plt.tight_layout(rect=(0, 0, 1, 0.95))
run = os.path.basename(raft_files[rafts[0]][0]).split('_')[2]
suptitle = append_acq_run(f'Fe55 gain stability, Run {run}')
plt.suptitle(suptitle)
unit_id = siteUtils.getUnitId()
plt.savefig(f'{unit_id}_{run}_fe55_gain_stability.png')
def __init__(self, configFile):
"""
configFile contains the names of the site-specific
configuration files. File basenames are provided in
configFile, and the full paths are constructed in the
_read(...) method.
"""
super(CcsSetup, self).__init__()
if os.environ.has_key('CCS_TS'):
self['ts']=_quote(os.getenv('CCS_TS'))
else:
self['ts'] = _quote('ts')
self['tsCWD'] = _quote(os.getcwd())
self['labname'] = _quote(siteUtils.getSiteName())
self['CCDID'] = _quote(siteUtils.getUnitId())
# self._read(os.path.join(siteUtils.getJobDir(), configFile))
CCDTYPE = _quote(siteUtils.getUnitType())
def parsl_sensor_analyses(run_task_func,
raft_id=None,
processes=None,
cwd=None,
walltime=3600):
"""
Run a sensor-level analysis task implemented as a pickleable
function that takes the desired sensor id as its single argument.
Parameters
----------
run_task_func : function
A pickleable function that takes the sensor_id string as
its argument.
raft_id : str [None]
The RTM (or RSA) LSST ID. If None (default), the LCATR_UNIT_ID
is used.
processes : int [None]
The maximum number of processes to have running at once.
If None (default), then set to 1 or one less than
the number of cores, whichever is larger.
cwd : str [None]
The working directory to cd to at the remote node. Nominally, this
is a location on the shared file system.
walltime: float [3600]
Walltime in seconds for python app execution. If the python app
does not return within walltime, a parsl.app.errors.AppTimeout
exception will be thrown.
Raises
------
parsl.app.errors.AppTimeout
"""
load_ir2_dc_config()
if raft_id is None:
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
return parsl_device_analysis_pool(run_task_func,
raft.sensor_names,
processes=processes,
cwd=cwd,
walltime=walltime)
def getSensorGains(jobname='fe55_analysis', sensor_id=None):
if (os.environ.get('LCATR_USE_UNIT_GAINS', 'False') == 'True'
or os.environ.get("LCATR_SKIP_FE55_ANALYSIS", "False") == "True"):
return {amp: 1 for amp in range(1, 17)}
if sensor_id is None:
sensor_id = siteUtils.getUnitId()
try:
gain_file = dependency_glob('%s_eotest_results.fits' % sensor_id,
jobname=jobname)[0]
except IndexError:
raise RuntimeError('eotestUtils.getSensorGains: %s %s' %
(sensor_id, jobname))
data = sensorTest.EOTestResults(gain_file)
amps = data['AMP']
gains = data['GAIN']
sensorGains = dict([(amp, gains[amp - 1]) for amp in amps])
return sensorGains
def sensor_analyses(run_task_func,
raft_id=None,
processes=None,
cwd=None,
walltime=3600):
"""
Run a sensor-level analysis task implemented as a pickleable
function that takes the desired sensor id as its single argument.
Parameters
----------
run_task_func: function
A pickleable function that takes the sensor_id string as
its argument.
raft_id: str, optional
The RTM (or RSA) LSST ID. If None (default), the LCATR_UNIT_ID
is used.
processes: int [None]
The maximum number of processes to have running at once.
If None, then set to 1 or one less than the number of cores,
whichever is larger.
cwd: str [None] Deprecated.
Working directory to cd to for parsl multi-node processing.
If None, then use `cwd = os.path.abspath('.')`.
walltime: float [3600] Deprecated.
Walltime in seconds for parsl app execution. If the app does not
return within walltime, a parsl.app.errors.AppTimeout exception
will be raised. This is not used for non-parsl processing.
"""
if raft_id is None:
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
run_device_analysis_pool(run_task_func,
raft.sensor_names,
processes=processes)
return None
def aggregate_filerefs(producer, testtype, origin=None, dp_mapping=None):
"""
Aggregate the filerefs for the metrology data products and return
them as a python list.
"""
if origin is None:
origin = siteUtils.getSiteName()
if dp_mapping is None:
dp_mapping = dict(BOXPLOT='boxplot',
HISTOGRAM='hist',
POINT_CLOUD_10='point_cloud_azim_10',
POINT_CLOUD_45='point_cloud_azim_45',
RESIDUALS='residuals',
QUANTILE_TABLE='quantile_table')
# Common metadata fields for all data products.
ccd_vendor = siteUtils.getCcdVendor()
sensor_id = siteUtils.getUnitId()
md = siteUtils.DataCatalogMetadata(CCD_MANU=ccd_vendor,
LSST_NUM=sensor_id,
PRODUCER=producer,
ORIGIN=origin,
TESTTYPE=testtype,
TEST_CATEGORY='MET')
# Create filerefs for each data product, adding the file-specific
# data product metadata.
tt_ext = testtype.lower()
results = []
for dp, ext in dp_mapping.items():
pattern = '%(sensor_id)s_%(tt_ext)s_*%(ext)s.*'% locals()
dp_files = glob.glob(pattern)
results.extend([lcatr.schema.fileref.make(dp_file,
metadata=md(DATA_PRODUCT=dp))
for dp_file in dp_files])
return results
#!/usr/bin/env python
import os, sys, glob
import siteUtils
print "executing producer_test_job.py"
IRODS = False
uid = siteUtils.getUnitId()
print "Running ASPIC PRODUCER on ", uid
uid = uid.split("LCA-11721-ASPIC-")[1]
uid = uid.replace("P", "0")
if not IRODS:
basedir = "/sps/lsst/DataBE/ASPIC_production"
logdir = os.path.join(basedir, "Logs")
chipdir = os.path.join(basedir, "CHIP%s" % uid)
try:
input_file = os.environ["ASPIC_LOGFILE"]
except KeyError:
try:
input_file = glob.glob(os.path.join(logdir, "log-%s-*.txt" % uid))[0]
except:
raise Exception("Input file %s not found in %s" % ("log-%s-*.txt" % uid, logdir))
os.system("cp %s ." % input_file)
os.system("ln -s %s ." % chipdir)
else:
basedir = "/lsst-fr/home/lsstcamera/ASPIC_production"
logdir = os.path.join(basedir, "Logs")
chipdir = os.path.join(basedir, "CHIP%s" % uid)
input_file = os.environ["ASPIC_LOGFILE"]
def __init__(self, configFile):
"""
configFile contains the names of the site-specific
configuration files. File basenames are provided in
configFile, and the full paths are constructed in the
_read(...) method.
"""
super(CcsSetup, self).__init__()
if os.environ.has_key('CCS_TS8'):
self['ts8']=_quote(os.getenv('CCS_TS8'))
else:
self['ts8'] = _quote('ts8')
if os.environ.has_key('CCS_JYTH'):
self['jyth']=_quote(os.getenv('CCS_JYTH'))
else:
self['jyth'] = _quote('JythonInterpreterConsole')
if os.environ.has_key('CCS_JSON_PORT'):
self['jsonport']=os.getenv('CCS_JSON_PORT')
else:
self['jsonport'] = 4444
if os.environ.has_key('CCS_PS'):
self['ps']=_quote(os.getenv('CCS_PS'))
else:
self['ps'] = _quote('ccs-rebps')
if os.environ.has_key('CCS_TS'):
self['ts']=_quote(os.getenv('CCS_TS'))
else:
self['ts'] = _quote('ts')
if os.environ.has_key('CCS_ARCHON'):
self['archon']=_quote(os.getenv('CCS_ARCHON'))
else:
self['archon'] = _quote('archon')
if os.environ.has_key('CCS_VAC_OUTLET'):
self['vac_outlet']=os.getenv('CCS_VAC_OUTLET')
# there is no default for vac_outlet - if there is a script that needs
# it and it has not been defined then I want it to crash
if os.environ.has_key('CCS_CRYO_OUTLET'):
self['cryo_outlet']=os.getenv('CCS_CRYO_OUTLET')
# there is no default for cryo_outlet - if there is a script that needs
# it and it has not been defined then I want it to crash
if os.environ.has_key('CCS_PUMP_OUTLET'):
self['pump_outlet']=os.getenv('CCS_PUMP_OUTLET')
# there is no default for pump_outlet - if there is a script that needs
# it and it has not been defined then I want it to crash
self['tsCWD'] = _quote(os.getcwd())
self['labname'] = _quote(siteUtils.getSiteName())
self['jobname'] = _quote(siteUtils.getJobName())
self['CCDID'] = _quote(siteUtils.getUnitId())
self['UNITID'] = _quote(siteUtils.getUnitId())
self['LSSTID'] = _quote(siteUtils.getLSSTId())
unitid = siteUtils.getUnitId()
CCDTYPE = _quote(siteUtils.getUnitType())
ccdnames = {}
ccdmanunames = {}
ccdnames,ccdmanunames = siteUtils.getCCDNames()
print "retrieved the following LSST CCD names list"
print ccdnames
print "retrieved the following Manufacturers CCD names list"
print ccdmanunames
for slot in ccdnames :
print "CCD %s is in slot %s" % (ccdnames[slot],slot)
self['CCD%s'%slot] = _quote(ccdnames[slot])
if 'itl' in ccdnames[slot].lower() :
CCDTYPE = 'itl'
if 'e2v' in ccdnames[slot].lower() :
CCDTYPE = 'e2v'
for slot in ccdmanunames :
print "CCD %s is in slot %s" % (ccdmanunames[slot],slot)
self['CCDMANU%s'%slot] = _quote(ccdmanunames[slot])
try:
self['RUNNUM'] = _quote(siteUtils.getRunNumber())
except:
self['RUNNUM'] = "no_lcatr_run_number"
pass
self._read(os.path.join(siteUtils.getJobDir(), configFile))
print "CCDTYPE = %s" % CCDTYPE
self['sequence_file'] = _quote("NA")
self['acffile'] = self['itl_acffile']
# set default type
self['CCSCCDTYPE'] = _quote("ITL")
if ("RTM" in unitid.upper() or "ETU" in unitid.upper() or "RSA" in unitid.upper()) :
if ("e2v" in CCDTYPE) :
self['CCSCCDTYPE'] = _quote("E2V")
self['acffile'] = self['e2v_acffile']
self['sequence_file'] = self['e2v_seqfile']
else :
self['CCSCCDTYPE'] = _quote("ITL")
self['acffile'] = self['itl_acffile']
self['sequence_file'] = self['itl_seqfile']
os.system("export | grep -i seq")
seqdir = ""
if os.environ.has_key('SEQUENCERFILESDIR') :
seqdir = os.getenv('SEQUENCERFILESDIR')
print "seqdir=",seqdir
self['sequence_file'] = self['sequence_file'].replace('${SEQUENCERFILESDIR}',seqdir)
os.system("cp -vp %s %s" % (self['sequence_file'],self['tsCWD']))
# now use the local copy
# bb = self['sequence_file'].split("/")
# self['sequence_file'] = _quote("%s/%s" % (os.getcwd(),bb[len(bb)-1].split("'")[0]))
print "The sequence file to be used is %s" % self['sequence_file']
else :
if ("ITL" in CCDTYPE) :
self['CCSCCDTYPE'] = _quote("ITL")
self['acffile'] = self['itl_acffile']
if ("e2v" in CCDTYPE) :
self['CCSCCDTYPE'] = _quote("E2V")
self['acffile'] = self['e2v_acffile']
print "The acffile to be used is %s" % self['acffile']
def make_focal_plane_plots():
#
# Focal plane heat maps
#
fp_configs = {
'fe55_BOT_analysis':
(('gain', 'psf_sigma'), ('clipped_autoscale', (3, 5.5)),
('e-/ADU', 'micron'), (False, False), ('1', '1')),
'read_noise_BOT':
(('read_noise', ), ((0, 20), ), ('e-', ), (False, ), ('1', )),
'bright_defects_BOT':
(('bright_pixels', 'bright_columns'), (None, None), (None, None),
(True, True), ('1', '1')),
'dark_defects_BOT': (('dark_pixels', 'dark_columns'), (None, None),
(None, None), (True, True), ('1', '1')),
'dark_current_BOT': (('dark_current_95CL', ), ((0, 0.3), ),
('e-/pix/s', ), (False, ), ('1', )),
'dark_current_fit_BOT':
(('dark_current_slope', 'dark_current_intercept'), ((0, 0.1), (-1, 1)),
('e-/pix/s', 'e-/pix'), (False, False), ('1', '1')),
'traps_BOT': (('num_traps', ), (None, ), (None, ), (False, ), ('1', )),
'cte_BOT': (('cti_high_serial', 'cti_high_parallel', 'cti_low_serial',
'cti_low_parallel'), ((0, 1e-5), (0, 1e-5), (0, 1e-5),
(0, 1e-5)),
(None, None, None, None), (False, False, False, False),
('1e-6', '1e-6', '1e-6', '1e-6')),
'flat_pairs_BOT':
(('max_frac_dev', 'max_observed_signal', 'row_mean_var_slope',
'linearity_turnoff'), ((0, 0.05), (5e4, 2.e5), (0, 2), (5e4, 2.e5)),
(None, 'ADU', None, 'ADU'), (False, False, False, False), ('1', '1',
'1', '1')),
'ptc_BOT': (('ptc_gain', 'ptc_a00',
'ptc_turnoff'), ('clipped_autoscale', (0, 5e-6),
(5e4, 2e5)), ('e-/ADU', None, 'ADU'),
(False, False, False), ('1', '1', '1')),
'brighter_fatter_BOT': (('bf_xcorr', 'bf_ycorr'), ((-5, 5), (-5, 5)),
(None, None), (False, False), ('1', '1')),
'tearing_stats_BOT': (('tearing_detections', ), (None, ), (None, ),
(False, ), ('1', )),
'divisadero_tearing_BOT': (('divisadero_max_dev', ), ((0, 0.05), ),
(None, ), (False, ), ('1', ))
}
camera = camera_info.camera_object
run = siteUtils.getRunNumber()
acq_run = os.environ.get('LCATR_ACQ_RUN', None)
unit_id = siteUtils.getUnitId()
et_results = siteUtils.ETResults(run)
for schema_name, configs in fp_configs.items():
for column, z_range, units, use_log10, scale_factor in zip(*configs):
fig = plt.figure(figsize=(12, 10))
ax = fig.add_subplot(1, 1, 1)
try:
amp_data = et_results.get_amp_data(schema_name, column)
except KeyError:
print("No focal plane results for {}: {} in eTraveler.".format(
schema_name, column))
else:
if units is not None:
title = 'Run {}, {} ({})'.format(run, column, units)
else:
title = 'Run {}, {}'.format(run, column)
if acq_run is not None:
title += f' (acq {acq_run})'
plot_focal_plane(ax,
amp_data,
camera=camera,
z_range=z_range,
use_log10=use_log10,
scale_factor=scale_factor)
plt.title(title)
outfile = '{}_{}_{}.png'.format(unit_id, run, column)
plt.savefig(outfile)
# Histogram the amp-level data.
plt.figure()
hist_range = None if z_range == 'clipped_autoscale' \
else z_range
hist_amp_data(amp_data,
column,
hist_range=hist_range,
use_log10=use_log10,
scale_factor=scale_factor)
plt.title(title)
outfile = '{}_{}_{}_hist.png'.format(unit_id, run, column)
plt.savefig(outfile)
#!/usr/bin/env python
import sys
import os
import siteUtils
import metUtils
from flatnessTask import flatnessTask
raft_id = siteUtils.getUnitId()
# Find the TS5 metrology scan data by constructing the name of the data-taking step
acqjobname = siteUtils.getJobName().replace('_Analysis', '')
if ("Room_Temp_Measurement" in acqjobname and not "After" in acqjobname):
acqjobname = "Pump_and_" + acqjobname
print('flatness_ts5:')
print(acqjobname)
print(siteUtils.getProcessName(acqjobname))
print('----')
# siteUtils returns a list with one member;
# here take the first (and only) member
infile = siteUtils.dependency_glob('*.csv',
jobname=siteUtils.getProcessName(acqjobname),
description='')[0]
print "infile = %s" % infile
flatnessTask(raft_id, infile, dtype='TS5', pickle_file='flatness_ts5.pickle')
host = os.getenv('LCATR_CCS_HOST', 'lsst-mcm')
time_axis = ccs_trending.TimeAxis(dt=0.5, nbins=200)
config_file = os.path.join(os.environ['CRJOBSDIR'], 'harnessed_jobs',
'rtm_aliveness_power_on', 'v0',
'rtm_aliveness_power_plots.cfg')
config = ccs_trending.ccs_trending_config(config_file)
print("Making trending plots")
local_time = datetime.datetime.now().isoformat()[:len('2017-01-24T10:44:00')]
for section in config.sections():
print(" processing", section)
try:
plotter = ccs_trending.TrendingPlotter(ts8, host, time_axis=time_axis)
plotter.read_config(config, section)
plotter.plot()
plt.savefig('%s_%s_%s.png' %
(section, local_time, siteUtils.getUnitId()))
except StandardError as eobj:
print("Exception caught while producing trending plot:")
print(str(eobj))
print("continuing...")
results = []
files = glob.glob("*.txt")
files = files + glob.glob("*summary*")
files = files + glob.glob("*png")
files = files + glob.glob("*log*")
data_products = [lcatr.schema.fileref.make(item) for item in files]
results.extend(data_products)
for k in rsp:
print ('For key %s returned value is %s\n' % (k, rsp[k]) )
# sys.exit(0)
except Exception,msg:
print 'Operation failed with exception: '
print msg
sys.exit(1)
rsp = []
try:
# rsp = conn.getHardwareHierarchy(experimentSN='LCA-10753_RSA-002_CTE_ETU',
# htype='LCA-10753_RSA',
# noBatched='false')
rsp = conn.getHardwareHierarchy(experimentSN=siteUtils.getUnitId(),
htype=siteUtils.getUnitType(),
noBatched='false')
print "Results from getHardwareHierarchy unfiltered:"
iDict = 0
for d in rsp:
print('Examining array element %d' % (iDict))
isaccd = False
ccd_sn = ""
ccd_slot = ""
ccd_htype = ""
got_manu_info = False
for k in d:
if "ccd" in k.lower() :
print('For key {0} value is {1}'.format(k, d[k]))
if ('child_hardwareTypeName value' in k and ('itl-ccd' in d[k].lower() or 'e2v-ccd' in d[k].lower()) ) :
print('Results from getRunInfo for activity %d\n' % (act))
for k in rsp:
print('For key %s returned value is %s\n' % (k, rsp[k]))
# sys.exit(0)
except Exception, msg:
print 'Operation failed with exception: '
print msg
sys.exit(1)
rsp = []
try:
# rsp = conn.getHardwareHierarchy(experimentSN='LCA-10753_RSA-002_CTE_ETU',
# htype='LCA-10753_RSA',
# noBatched='false')
rsp = conn.getHardwareHierarchy(experimentSN=siteUtils.getUnitId(),
htype=siteUtils.getUnitType(),
noBatched='false')
print "Results from getHardwareHierarchy unfiltered:"
iDict = 0
for d in rsp:
print('Examining array element %d' % (iDict))
isaccd = False
ccd_sn = ""
ccd_slot = ""
ccd_htype = ""
got_manu_info = False
for k in d:
if "ccd" in k.lower():
print('For key {0} value is {1}'.format(k, d[k]))
if ('child_hardwareTypeName value' in k and
import glob
import lcatr.schema
import lsst.eotest.image_utils as imutils
import siteUtils
import aliveness_utils
results = []
job_schema = lcatr.schema.get('rtm_aliveness_exposure')
# Infer the sequence numbers from the files for the sensor in slot S00.
seqnos = [x.split('_')[-3] for x in sorted(glob.glob('S00/*.fits'))]
row_template = "%(seqno)s %(slot)s %(channel)s %(signal)s %(status)s\n"
outfile = '%s_%s_rtm_aliveness_bad_channels.txt' % (siteUtils.getUnitId(),
siteUtils.getRunNumber())
with open(outfile, 'w') as output:
for seqno in seqnos:
fits_files = sorted(glob.glob('S??/*_%s_*.fits' % seqno))
for fits_file in fits_files:
results.append(lcatr.schema.fileref.make(fits_file))
channel_signal, channel_status, exptime \
= aliveness_utils.raft_channel_statuses(fits_files)
for slot in channel_status:
bad_channels = 0
for amp, status in channel_status[slot].items():
if channel_status[slot][amp] == 'bad':
bad_channels += 1
signal = channel_signal[slot][amp]
channel = imutils.channelIds[amp]
value is not None and
wl >= wl_range[0] and
wl <= wl_range[1]):
qe_results[band].append(value)
results = []
for band in qe_results:
results.append(validate(job, band=band,
QE=np.average(qe_results[band])))
return results
def metrology(self):
return []
if __name__ == '__main__':
results = [siteUtils.packageVersions()]
lsstnum = siteUtils.getUnitId()
vendorDataDir = os.readlink('vendorData')
print 'Vendor data location:', vendorDataDir
if siteUtils.getCcdVendor() == 'ITL':
vendor = ItlResults(vendorDataDir)
translator = ItlFitsTranslator(lsstnum, vendorDataDir, '.')
else:
vendor = e2vResults(vendorDataDir)
translator = e2vFitsTranslator(lsstnum, vendorDataDir, '.')
results.extend(vendor.run_all())
translator.run_all()
results.extend([lcatr.schema.fileref.make(x) for x in translator.outfiles])
#!/usr/bin/env python
import glob
import pyfits
import numpy as np
import lsst.eotest.sensor as sensorTest
import lcatr.schema
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
# Save the output file from producer script.
persistence_file = '%(sensor_id)s_persistence.fits' % locals()
eotestUtils.addHeaderData(persistence_file, LSST_NUM=sensor_id,
TESTTYPE='PERSISTENCE',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results = [lcatr.schema.fileref.make(persistence_file)]
# Save the deferred charge for each amplifier from the first post-flat
# dark frame as the image persistence metric.
#
# Read the results from the FITS file.
persistence = pyfits.open(persistence_file)
times = persistence[1].data.field('TIME')
# Times of dark frames are relative to the MJD-OBS of the flat.
post_flat_times = times[np.where(times > 0)]
index = np.where(times == min(post_flat_times))[0][0]
#!/usr/bin/env python
"""
Validator script for raft-level CTE analysis.
"""
from __future__ import print_function, absolute_import
import glob
import lsst.cr_eotest.sensor as sensorTest
import lcatr.schema
import siteUtils
import eotestUtils
import camera_components
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
print("Processing:", slot, sensor_id)
if 'ccd2' in slot :
continue
wgSlotName = siteUtils.getWGSlotNames(raft)[sensor_id]
ccd_vendor = sensor_id.split('-')[0].upper()
# superflats = glob.glob('%(sensor_id)s_superflat_*.fits' % locals())
superflats = glob.glob('%s_superflat_*.fits' % wgSlotName)
for item in superflats:
eotestUtils.addHeaderData(item, FILENAME=item,
DATE=eotestUtils.utc_now_isoformat())
results.extend([siteUtils.make_fileref(x, folder=slot) for x in superflats])
def sensor_analyses(run_task_func, raft_id=None, processes=None):
"""
Run a sensor-level analysis task implemented as a pickleable
function that takes the desired sensor id as its single argument.
Parameters
----------
run_task_func : function
A pickleable function that takes the sensor_id string as
its argument.
raft_id : str, optional
The RTM (or RSA) LSST ID. If None (default), the LCATR_UNIT_ID
is used.
processes : int, optional
The maximum number of processes to have running at once.
If None (default), then set to 1 or one less than
the number of cores, whichever is larger.
Notes
-----
Exceptions from subprocesses will be buffered until all of the
subprocesses have finished running. If any exceptions are thrown
and are uncaught, a non-zero exit code will be generated for the
overall process. Output to stdout or stderr from the subprocesses
will be interleaved.
Users can override the default or keyword argument values by setting
the LCATR_PARALLEL_PROCESSES environment variable.
"""
if raft_id is None:
raft_id = siteUtils.getUnitId()
if processes is None:
processes = max(1, multiprocessing.cpu_count() - 1)
processes = int(os.environ.get('LCATR_PARALLEL_PROCESSES', processes))
print("raft_id = ", raft_id)
raft = camera_components.Raft.create_from_etrav(raft_id)
# print("sensor_id = ",raft.sensor_id)
print("sensor_names = ", raft.sensor_names)
print("slots = ", raft.slot_names)
wgslot = siteUtils.getWGSlotNames(raft)
# for slot,sensor_id in zip(raft.slot_names,raft.sensor_names):
# if "ccd1" in slot or "ccd2" in slot :
# wgslot[sensor_id] = 'WREB0'
# elif "guidesensor1" in slot :
# wgslot[sensor_id] = 'GREB0'
# elif "guidesensor2" in slot :
# wgslot[sensor_id] = 'GREB1'
# for sln = raft.slot_names :
# if 'ccd1' in sln:
print("wgslot = ", wgslot)
if processes == 1:
# For cases where only one process will be run at a time, it's
# faster to run serially instead of using a
# multiprocessing.Pool since the pickling that occurs can
# cause significant overhead.
for sensor_id in raft.sensor_names:
run_task_func(wgslot[sensor_id])
else:
pool = multiprocessing.Pool(processes=processes)
results = [
pool.apply_async(run_task_func, (wgslot[sensor_id], ))
for sensor_id in raft.sensor_names
]
pool.close()
pool.join()
for res in results:
res.get()
#!/usr/bin/env python
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils
sensor_id = siteUtils.getUnitId()
lambda_files = siteUtils.datacatalog_glob('*_lambda_flat_*.fits',
testtype='LAMBDA',
imgtype='FLAT',
description='Lambda files:')
mask_files = eotestUtils.glob_mask_files()
gains = eotestUtils.getSensorGains(jobname='fe55_offline')
# @todo Set correction image when it becomes available.
correction_image = None
task = sensorTest.PrnuTask()
task.run(sensor_id, lambda_files, mask_files, gains, correction_image)
def persist_fe55_analysis_results():
"""Persist the results from the full analysis."""
raft_id = siteUtils.getUnitId()
raft = camera_components.Raft.create_from_etrav(raft_id)
results = []
for slot, sensor_id in raft.items():
ccd_vendor = sensor_id.split('-')[0].upper()
# The output files from producer script.
gain_file = '%(sensor_id)s_eotest_results.fits' % locals()
psf_results = glob.glob('%(sensor_id)s_psf_results*.fits' %
locals())[0]
rolloff_mask = '%(sensor_id)s_rolloff_defects_mask.fits' % locals()
output_files = gain_file, psf_results, rolloff_mask
# Add/update the metadata to the primary HDU of these files.
for fitsfile in output_files:
eotestUtils.addHeaderData(fitsfile,
LSST_NUM=sensor_id,
TESTTYPE='FE55',
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=ccd_vendor)
#
# Persist the median bias FITS file.
#
bias_median_file = glob.glob(f'{sensor_id}_*_median_bias.fits')[0]
results.append(siteUtils.make_fileref(bias_median_file, folder=slot))
# Persist the png files.
metadata = dict(CCD_MANU=ccd_vendor,
LSST_NUM=sensor_id,
TESTTYPE='FE55',
TEST_CATEGORY='EO')
results.extend(
siteUtils.persist_png_files('%s*.png' % sensor_id,
sensor_id,
folder=slot,
metadata=metadata))
data = sensorTest.EOTestResults(gain_file)
amps = data['AMP']
gain_data = data['GAIN']
gain_errors = data['GAIN_ERROR']
sigmas = data['PSF_SIGMA']
for amp, gain_value, gain_error, sigma in zip(amps, gain_data,
gain_errors, sigmas):
if not np.isfinite(gain_error):
gain_error = -1
results.append(
lcatr.schema.valid(lcatr.schema.get('fe55_raft_analysis'),
amp=amp,
gain=gain_value,
gain_error=gain_error,
psf_sigma=sigma,
slot=slot,
sensor_id=sensor_id))
results.extend([lcatr.schema.fileref.make(x) for x in output_files])
return results
#!/usr/bin/env python
import os, sys, glob
import siteUtils
print "executing producer_test_job.py"
IRODS = False
uid = siteUtils.getUnitId()
print "Running ASPIC PRODUCER on ", uid
uid = uid.split('LCA-11721-ASPIC-')[1]
uid = uid.replace('P', '0')
if not IRODS:
basedir = "/sps/lsst/DataBE/ASPIC_production"
logdir = os.path.join(basedir, "Logs")
chipdir = os.path.join(basedir, "CHIP%s" % uid)
try:
input_file = os.environ['ASPIC_LOGFILE']
except KeyError:
try:
input_file = glob.glob(os.path.join(logdir,
"log-%s-*.txt" % uid))[0]
except:
raise Exception('Input file %s not found in %s' %
("log-%s-*.txt" % uid, logdir))
os.system('cp %s .' % input_file)
os.system('ln -s %s .' % chipdir)
else:
basedir = "/lsst-fr/home/lsstcamera/ASPIC_production"
logdir = os.path.join(basedir, "Logs")
#!/usr/bin/env python
import glob
import lcatr.schema
import siteUtils
import metUtils
from MetrologyData import md_factory
producer = 'SR-MET-07'
testtype = 'FLATNESS'
results = metUtils.aggregate_filerefs_ts5(producer, testtype)
# Add the QA plot to the results
raft_id = siteUtils.getUnitId()
qafile = glob.glob('*_qa_plot.png')[0]
#print('qafile: %s' % qafile)
#print('raft_id: %s' % raft_id)
md = siteUtils.DataCatalogMetadata(CCD_MANU=siteUtils.getCcdVendor(),
LSST_NUM=siteUtils.getUnitId(),
PRODUCER=producer,
ORIGIN=siteUtils.getSiteName(),
TESTTYPE=testtype,
TEST_CATEGORY='MET')
results.extend([lcatr.schema.fileref.make(qafile, metadata=md(DATA_PRODUCT='QA_PLOT'))])
raftData = md_factory.load('flatness_ts5_delta.pickle')
peak_valley_95 = raftData.quantiles['0.975'] - raftData.quantiles['0.025']
peak_valley_100 = raftData.quantiles['1.000'] - raftData.quantiles['0.000']
