def run_validator(*det_task_names): """ Driver function to run the validator function for the desired detector-level EO task. """ results = [] for det_task_name in det_task_names: validator = eval('validate_{}'.format(det_task_name)) results = validator(results, camera_info.get_det_names()) results.extend(siteUtils.jobInfo()) # Persist the bot_eo_acq_cfg file so that the analysis # configuration for this job is saved. acq_config = siteUtils.get_job_acq_configs() bot_eo_acq_cfg = os.path.basename(acq_config['bot_eo_acq_cfg']) if os.path.isfile(bot_eo_acq_cfg): results.append(lcatr.schema.fileref.make(bot_eo_acq_cfg)) # Check if an actual path to a curated gains file has been # specified. If so, then persist it. gains_file = siteUtils.\ get_analysis_run('gain', bot_eo_config_file=bot_eo_acq_cfg) if gains_file is not None and os.path.isfile(gains_file): shutil.copy(gains_file, '.') results.append(lcatr.schema.fileref.make(os.path.basename(gains_file))) lcatr.schema.write_file(results) lcatr.schema.validate_file()
def ccsValidator(results=None): """ Persist standard file patterns, e.g., '*.fits', 'pd-values*.txt', using lcatr.schema. """ if results is None: results = [] files = glob.glob('*/*.fits') files += glob.glob('pd-values*.txt') files += glob.glob('*.png') files += glob.glob('*.seq') unique_files = set(files) results.extend([lcatr.schema.fileref.make(item) for item in unique_files]) results.extend(siteUtils.jobInfo()) results = siteUtils.persist_ccs_versions(results) results = siteUtils.persist_reb_info(results) lcatr.schema.write_file(results) lcatr.schema.validate_file()
def run_validator(*det_task_names): """ Driver function to run the validator function for the desired detector-level EO task. """ results = [] for det_task_name in det_task_names: validator = eval('validate_{}'.format(det_task_name)) results = validator(results, camera_info.get_det_names()) results.extend(siteUtils.jobInfo()) # Persist the bot_eo_acq_cfg file so that the analysis # configuration for this job is saved. acq_config = siteUtils.get_job_acq_configs() bot_eo_acq_cfg = os.path.basename(acq_config['bot_eo_acq_cfg']) if os.path.isfile(bot_eo_acq_cfg): results.append(lcatr.schema.fileref.make(bot_eo_acq_cfg)) lcatr.schema.write_file(results) lcatr.schema.validate_file()
DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=siteUtils.getCcdVendor().upper()) results = [lcatr.schema.fileref.make(persistence_file)] # Save the deferred charge for each amplifier from the first post-flat # dark frame as the image persistence metric. # # Read the results from the FITS file. persistence = pyfits.open(persistence_file) times = persistence[1].data.field('TIME') # Times of dark frames are relative to the MJD-OBS of the flat. post_flat_times = times[np.where(times > 0)] index = np.where(times == min(post_flat_times))[0][0] # Loop over amplifiers and record the deferred charge (median pixel # value and stdev) in the first post-flat dark frame. gains = eotestUtils.getSensorGains() for amp in gains: median_flux = persistence[1].data.field('MEDIAN%02i' % amp)[index] stdev = persistence[1].data.field('STDEV%02i' % amp)[index] results.append(lcatr.schema.valid(lcatr.schema.get('persistence'), amp=amp, deferred_charge_median=median_flux, deferred_charge_stdev=stdev)) results.extend(siteUtils.jobInfo()) lcatr.schema.write_file(results) lcatr.schema.validate_file()
cti_low_parallel = data['CTI_LOW_PARALLEL'] cti_low_parallel_error = data['CTI_LOW_PARALLEL_ERROR'] for values in zip(amps, cti_high_serial, cti_high_serial_error, cti_high_parallel, cti_high_parallel_error, cti_low_serial, cti_low_serial_error, cti_low_parallel, cti_low_parallel_error): results.append(lcatr.schema.valid(lcatr.schema.get('cte_raft'), amp=values[0], cti_high_serial=values[1], cti_high_serial_error=values[2], cti_high_parallel=values[3], cti_high_parallel_error=values[4], cti_low_serial=values[5], cti_low_serial_error=values[6], cti_low_parallel=values[7], cti_low_parallel_error=values[8], slot=slot, sensor_id=wgSlotName)) # Persist the png files. metadata = dict(CCD_MANU=ccd_vendor, LSST_NUM=sensor_id, TESTTYPE='SFLAT_500', TEST_CATEGORY='EO') results.extend(siteUtils.persist_png_files('%s*.png' % sensor_id, sensor_id, folder=slot, metadata=metadata)) results.extend(siteUtils.jobInfo()) lcatr.schema.write_file(results) lcatr.schema.validate_file()
#!/usr/bin/env python import glob from collections import OrderedDict import numpy as np import pyfits import lcatr.schema import siteUtils import eotestUtils results = siteUtils.jobInfo() results.append(eotestUtils.eotestCalibrations()) sensor_id = siteUtils.getUnitId() qe_data = pyfits.open('%s_QE.fits' % sensor_id)['QE_BANDS'].data QE = OrderedDict((band, []) for band in qe_data.field('BAND')) for amp in range(1, 17): values = qe_data.field('AMP%02i' % amp) for band, value in zip(QE, values): QE[band].append(value) for band in QE: results.append(lcatr.schema.valid(lcatr.schema.get('qe_analysis'), band=band, QE=np.mean(QE[band]))) qe_acq_job_id = siteUtils.get_prerequisite_job_id('*_lambda_flat_*.fits', jobname=siteUtils.getProcessName('qe_acq')) md = dict(photodiode_ratio_file=dict(JOB_ID=qe_acq_job_id), illumination_non_uniformity_file=dict(JOB_ID=qe_acq_job_id)) results.extend(eotestUtils.eotestCalibsPersist('photodiode_ratio_file', 'illumination_non_uniformity_file', metadata=md))
def ccsValidator(jobName, acqfilelist='acqfilelist', statusFlags=('stat','teststand_version','teststand_revision','archon_version','archon_revision','ts8_version','ts8_revision','power_version','power_revision')): try: hdrtools.updateFitsHeaders(acqfilelist) except IOError: pass # ccsTrendingPlots(('ts/ccdtemperature', 'ts/dewarpressure'), jobName, # os.path.join(siteUtils.configDir(), 'ccs_trending.cfg')) results = [] statusAssignments = {} try: statusFile = open("status.out") for flag in statusFlags: if (flag=='stat') : value = int(statusFile.readline().strip()) statusAssignments[flag] = value else : strval = statusFile.readline().strip() statusAssignments[flag] = strval except IOError: for flag in statusFlags: statusAssignments[flag] = -1 print "jobName = %s" % jobName print "schema = %s" % str(lcatr.schema.get(jobName)) results.append(lcatr.schema.valid(lcatr.schema.get(jobName), **statusAssignments)) # results.append(siteUtils.packageVersions()) # @todo Fix this. Copying these files should not be necessary. # jobdir = siteUtils.getJobDir() # os.system("cp -vp %s/*.fits ." % jobdir) # @todo Sort out which files really need to be curated. files = glob.glob('*.fits') files = files+glob.glob('*/*.fits') files = files+glob.glob('*/*.fits.gz') files = files+glob.glob('*log*') files = files+glob.glob('*.txt') files = files+glob.glob('*.png') files = files+glob.glob('*.dat') files = files+glob.glob('*.seq') files = files+glob.glob('*.xml') files = files+glob.glob('*.csv') files = files+glob.glob('*.pickles') fileset = set(files) uniquefiles = list(fileset) print "The files that will be registered in lims from %s are:" % os.getcwd() print "files = ",uniquefiles for line in uniquefiles : print "%s" % line data_products = [lcatr.schema.fileref.make(item) for item in uniquefiles] results.extend(data_products) print "results=",results results.extend(siteUtils.jobInfo()) lcatr.schema.write_file(results) lcatr.schema.validate_file() # now lets crash if that status file wasn't present # we do this so that the traveler will know that something bad happened statusFileCheck = open("status.out")
#!/usr/bin/env python import glob from collections import OrderedDict import numpy as np import pyfits import lcatr.schema import siteUtils import eotestUtils results = siteUtils.jobInfo() results.append(eotestUtils.eotestCalibrations()) sensor_id = siteUtils.getUnitId() qe_data = pyfits.open('%s_QE.fits' % sensor_id)['QE_BANDS'].data QE = OrderedDict((band, []) for band in qe_data.field('BAND')) for amp in range(1, 17): values = qe_data.field('AMP%02i' % amp) for band, value in zip(QE, values): QE[band].append(value) for band in QE: results.append( lcatr.schema.valid(lcatr.schema.get('qe_analysis'), band=band, QE=np.mean(QE[band]))) qe_acq_job_id = siteUtils.get_prerequisite_job_id( '*_lambda_flat_*.fits', jobname=siteUtils.getProcessName('qe_acq')) md = dict(photodiode_ratio_file=dict(JOB_ID=qe_acq_job_id), illumination_non_uniformity_file=dict(JOB_ID=qe_acq_job_id)) results.extend(