def run_validator(*det_task_names):
"""
Driver function to run the validator function for the desired
detector-level EO task.
"""
results = []
for det_task_name in det_task_names:
validator = eval('validate_{}'.format(det_task_name))
results = validator(results, camera_info.get_det_names())
results.extend(siteUtils.jobInfo())
# Persist the bot_eo_acq_cfg file so that the analysis
# configuration for this job is saved.
acq_config = siteUtils.get_job_acq_configs()
bot_eo_acq_cfg = os.path.basename(acq_config['bot_eo_acq_cfg'])
if os.path.isfile(bot_eo_acq_cfg):
results.append(lcatr.schema.fileref.make(bot_eo_acq_cfg))
# Check if an actual path to a curated gains file has been
# specified. If so, then persist it.
gains_file = siteUtils.\
get_analysis_run('gain', bot_eo_config_file=bot_eo_acq_cfg)
if gains_file is not None and os.path.isfile(gains_file):
shutil.copy(gains_file, '.')
results.append(lcatr.schema.fileref.make(os.path.basename(gains_file)))
lcatr.schema.write_file(results)
lcatr.schema.validate_file()
def ccsValidator(results=None):
"""
Persist standard file patterns, e.g., '*.fits', 'pd-values*.txt',
using lcatr.schema.
"""
if results is None:
results = []
files = glob.glob('*/*.fits')
files += glob.glob('pd-values*.txt')
files += glob.glob('*.png')
files += glob.glob('*.seq')
unique_files = set(files)
results.extend([lcatr.schema.fileref.make(item) for item in unique_files])
results.extend(siteUtils.jobInfo())
results = siteUtils.persist_ccs_versions(results)
results = siteUtils.persist_reb_info(results)
lcatr.schema.write_file(results)
lcatr.schema.validate_file()
def run_validator(*det_task_names):
"""
Driver function to run the validator function for the desired
detector-level EO task.
"""
results = []
for det_task_name in det_task_names:
validator = eval('validate_{}'.format(det_task_name))
results = validator(results, camera_info.get_det_names())
results.extend(siteUtils.jobInfo())
# Persist the bot_eo_acq_cfg file so that the analysis
# configuration for this job is saved.
acq_config = siteUtils.get_job_acq_configs()
bot_eo_acq_cfg = os.path.basename(acq_config['bot_eo_acq_cfg'])
if os.path.isfile(bot_eo_acq_cfg):
results.append(lcatr.schema.fileref.make(bot_eo_acq_cfg))
lcatr.schema.write_file(results)
lcatr.schema.validate_file()
DATE=eotestUtils.utc_now_isoformat(),
CCD_MANU=siteUtils.getCcdVendor().upper())
results = [lcatr.schema.fileref.make(persistence_file)]
# Save the deferred charge for each amplifier from the first post-flat
# dark frame as the image persistence metric.
#
# Read the results from the FITS file.
persistence = pyfits.open(persistence_file)
times = persistence[1].data.field('TIME')
# Times of dark frames are relative to the MJD-OBS of the flat.
post_flat_times = times[np.where(times > 0)]
index = np.where(times == min(post_flat_times))[0][0]
# Loop over amplifiers and record the deferred charge (median pixel
# value and stdev) in the first post-flat dark frame.
gains = eotestUtils.getSensorGains()
for amp in gains:
median_flux = persistence[1].data.field('MEDIAN%02i' % amp)[index]
stdev = persistence[1].data.field('STDEV%02i' % amp)[index]
results.append(lcatr.schema.valid(lcatr.schema.get('persistence'), amp=amp,
deferred_charge_median=median_flux,
deferred_charge_stdev=stdev))
results.extend(siteUtils.jobInfo())
lcatr.schema.write_file(results)
lcatr.schema.validate_file()
cti_low_parallel = data['CTI_LOW_PARALLEL']
cti_low_parallel_error = data['CTI_LOW_PARALLEL_ERROR']
for values in zip(amps,
cti_high_serial, cti_high_serial_error,
cti_high_parallel, cti_high_parallel_error,
cti_low_serial, cti_low_serial_error,
cti_low_parallel, cti_low_parallel_error):
results.append(lcatr.schema.valid(lcatr.schema.get('cte_raft'),
amp=values[0],
cti_high_serial=values[1],
cti_high_serial_error=values[2],
cti_high_parallel=values[3],
cti_high_parallel_error=values[4],
cti_low_serial=values[5],
cti_low_serial_error=values[6],
cti_low_parallel=values[7],
cti_low_parallel_error=values[8],
slot=slot,
sensor_id=wgSlotName))
# Persist the png files.
metadata = dict(CCD_MANU=ccd_vendor, LSST_NUM=sensor_id,
TESTTYPE='SFLAT_500', TEST_CATEGORY='EO')
results.extend(siteUtils.persist_png_files('%s*.png' % sensor_id,
sensor_id, folder=slot,
metadata=metadata))
results.extend(siteUtils.jobInfo())
lcatr.schema.write_file(results)
lcatr.schema.validate_file()
#!/usr/bin/env python
import glob
from collections import OrderedDict
import numpy as np
import pyfits
import lcatr.schema
import siteUtils
import eotestUtils
results = siteUtils.jobInfo()
results.append(eotestUtils.eotestCalibrations())
sensor_id = siteUtils.getUnitId()
qe_data = pyfits.open('%s_QE.fits' % sensor_id)['QE_BANDS'].data
QE = OrderedDict((band, []) for band in qe_data.field('BAND'))
for amp in range(1, 17):
values = qe_data.field('AMP%02i' % amp)
for band, value in zip(QE, values):
QE[band].append(value)
for band in QE:
results.append(lcatr.schema.valid(lcatr.schema.get('qe_analysis'),
band=band, QE=np.mean(QE[band])))
qe_acq_job_id = siteUtils.get_prerequisite_job_id('*_lambda_flat_*.fits',
jobname=siteUtils.getProcessName('qe_acq'))
md = dict(photodiode_ratio_file=dict(JOB_ID=qe_acq_job_id),
illumination_non_uniformity_file=dict(JOB_ID=qe_acq_job_id))
results.extend(eotestUtils.eotestCalibsPersist('photodiode_ratio_file',
'illumination_non_uniformity_file',
metadata=md))
def ccsValidator(jobName, acqfilelist='acqfilelist', statusFlags=('stat','teststand_version','teststand_revision','archon_version','archon_revision','ts8_version','ts8_revision','power_version','power_revision')):
try:
hdrtools.updateFitsHeaders(acqfilelist)
except IOError:
pass
# ccsTrendingPlots(('ts/ccdtemperature', 'ts/dewarpressure'), jobName,
# os.path.join(siteUtils.configDir(), 'ccs_trending.cfg'))
results = []
statusAssignments = {}
try:
statusFile = open("status.out")
for flag in statusFlags:
if (flag=='stat') :
value = int(statusFile.readline().strip())
statusAssignments[flag] = value
else :
strval = statusFile.readline().strip()
statusAssignments[flag] = strval
except IOError:
for flag in statusFlags:
statusAssignments[flag] = -1
print "jobName = %s" % jobName
print "schema = %s" % str(lcatr.schema.get(jobName))
results.append(lcatr.schema.valid(lcatr.schema.get(jobName),
**statusAssignments))
# results.append(siteUtils.packageVersions())
# @todo Fix this. Copying these files should not be necessary.
# jobdir = siteUtils.getJobDir()
# os.system("cp -vp %s/*.fits ." % jobdir)
# @todo Sort out which files really need to be curated.
files = glob.glob('*.fits')
files = files+glob.glob('*/*.fits')
files = files+glob.glob('*/*.fits.gz')
files = files+glob.glob('*log*')
files = files+glob.glob('*.txt')
files = files+glob.glob('*.png')
files = files+glob.glob('*.dat')
files = files+glob.glob('*.seq')
files = files+glob.glob('*.xml')
files = files+glob.glob('*.csv')
files = files+glob.glob('*.pickles')
fileset = set(files)
uniquefiles = list(fileset)
print "The files that will be registered in lims from %s are:" % os.getcwd()
print "files = ",uniquefiles
for line in uniquefiles :
print "%s" % line
data_products = [lcatr.schema.fileref.make(item) for item in uniquefiles]
results.extend(data_products)
print "results=",results
results.extend(siteUtils.jobInfo())
lcatr.schema.write_file(results)
lcatr.schema.validate_file()
# now lets crash if that status file wasn't present
# we do this so that the traveler will know that something bad happened
statusFileCheck = open("status.out")
#!/usr/bin/env python
import glob
from collections import OrderedDict
import numpy as np
import pyfits
import lcatr.schema
import siteUtils
import eotestUtils
results = siteUtils.jobInfo()
results.append(eotestUtils.eotestCalibrations())
sensor_id = siteUtils.getUnitId()
qe_data = pyfits.open('%s_QE.fits' % sensor_id)['QE_BANDS'].data
QE = OrderedDict((band, []) for band in qe_data.field('BAND'))
for amp in range(1, 17):
values = qe_data.field('AMP%02i' % amp)
for band, value in zip(QE, values):
QE[band].append(value)
for band in QE:
results.append(
lcatr.schema.valid(lcatr.schema.get('qe_analysis'),
band=band,
QE=np.mean(QE[band])))
qe_acq_job_id = siteUtils.get_prerequisite_job_id(
'*_lambda_flat_*.fits', jobname=siteUtils.getProcessName('qe_acq'))
md = dict(photodiode_ratio_file=dict(JOB_ID=qe_acq_job_id),
illumination_non_uniformity_file=dict(JOB_ID=qe_acq_job_id))
results.extend(
