def aggregate_filerefs(producer, testtype, origin=None, dp_mapping=None): """ Aggregate the filerefs for the metrology data products and return them as a python list. """ if origin is None: origin = siteUtils.getSiteName() if dp_mapping is None: dp_mapping = dict(BOXPLOT='boxplot', HISTOGRAM='hist', POINT_CLOUD_10='point_cloud_azim_10', POINT_CLOUD_45='point_cloud_azim_45', RESIDUALS='residuals', QUANTILE_TABLE='quantile_table') # Common metadata fields for all data products. ccd_vendor = siteUtils.getCcdVendor() sensor_id = siteUtils.getUnitId() md = siteUtils.DataCatalogMetadata(CCD_MANU=ccd_vendor, LSST_NUM=sensor_id, PRODUCER=producer, ORIGIN=origin, TESTTYPE=testtype, TEST_CATEGORY='MET') # Create filerefs for each data product, adding the file-specific # data product metadata. tt_ext = testtype.lower() results = [] for dp, ext in dp_mapping.items(): pattern = '%(sensor_id)s_%(tt_ext)s_*%(ext)s.*'% locals() dp_files = glob.glob(pattern) results.extend([lcatr.schema.fileref.make(dp_file, metadata=md(DATA_PRODUCT=dp)) for dp_file in dp_files]) return results
import glob import pyfits import numpy as np import lsst.eotest.sensor as sensorTest import lcatr.schema import siteUtils import eotestUtils sensor_id = siteUtils.getUnitId() # Save the output file from producer script. persistence_file = '%(sensor_id)s_persistence.fits' % locals() eotestUtils.addHeaderData(persistence_file, LSST_NUM=sensor_id, TESTTYPE='PERSISTENCE', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=siteUtils.getCcdVendor().upper()) results = [lcatr.schema.fileref.make(persistence_file)] # Save the deferred charge for each amplifier from the first post-flat # dark frame as the image persistence metric. # # Read the results from the FITS file. persistence = pyfits.open(persistence_file) times = persistence[1].data.field('TIME') # Times of dark frames are relative to the MJD-OBS of the flat. post_flat_times = times[np.where(times > 0)] index = np.where(times == min(post_flat_times))[0][0] # Loop over amplifiers and record the deferred charge (median pixel
sensor_id = siteUtils.getUnitId() # The output files from producer script. gain_file = '%(sensor_id)s_eotest_results.fits' % locals() psf_results = glob.glob('%(sensor_id)s_psf_results*.fits' % locals())[0] rolloff_mask = '%(sensor_id)s_rolloff_defects_mask.fits' % locals() output_files = gain_file, psf_results, rolloff_mask # Add/update the metadata to the primary HDU of these files. for fitsfile in output_files: eotestUtils.addHeaderData(fitsfile, LSST_NUM=sensor_id, TESTTYPE='FE55', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=siteUtils.getCcdVendor().upper()) # # Persist the mean bias FITS file. # bias_mean_file = glob.glob('%(sensor_id)s_mean_bias_*.fits' % locals())[0] results = [lcatr.schema.fileref.make(bias_mean_file)] # # Common metadata for persisted non-FITS files. # md = siteUtils.DataCatalogMetadata(CCD_MANU=siteUtils.getCcdVendor().upper(), LSST_NUM=sensor_id, producer='SR-EOT-1', TESTTYPE='FE55', TEST_CATEGORY='EO') #
sensor_id = siteUtils.getUnitId() # The output files from producer script. gain_file = '%(sensor_id)s_eotest_results.fits' % locals() psf_results = glob.glob('%(sensor_id)s_psf_results*.fits' % locals())[0] rolloff_mask = '%(sensor_id)s_rolloff_defects_mask.fits' % locals() output_files = gain_file, psf_results, rolloff_mask # Add/update the metadata to the primary HDU of these files. for fitsfile in output_files: eotestUtils.addHeaderData(fitsfile, LSST_NUM=sensor_id, TESTTYPE='FE55', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=siteUtils.getCcdVendor().upper()) results = [] data = sensorTest.EOTestResults(gain_file) amps = data['AMP'] gain_data = data['GAIN'] gain_errors = data['GAIN_ERROR'] sigmas = data['PSF_SIGMA'] for amp, gain_value, gain_error, sigma in zip(amps, gain_data, gain_errors, sigmas): results.append( lcatr.schema.valid(lcatr.schema.get('fe55_analysis'), amp=amp, gain=gain_value, gain_error=gain_error,
#!/usr/bin/env python import lsst.eotest.sensor as sensorTest import siteUtils import eotestUtils sensor_id = siteUtils.getUnitId() ccd_vendor = siteUtils.getCcdVendor() flat_files = siteUtils.datacatalog_glob('*_flat*flat?_*.fits', testtype='FLAT', imgtype='FLAT', description='Flat files:') mask_files = eotestUtils.glob_mask_files() gains = eotestUtils.getSensorGains(jobname='fe55_offline') task = sensorTest.FlatPairTask() task.run(sensor_id, flat_files, mask_files, gains) if ccd_vendor == 'ITL': # # Perform linearity analysis using special dataset from ITL try: flat_files = siteUtils.datacatalog_glob('*_linearity_flat*.fits', testtype='LINEARITY', imgtype='FLAT', description='ITL linearity files:') if flat_files: task = sensorTest.LinearityTask() task.run(sensor_id, flat_files, mask_files, gains) except:
#!/usr/bin/env python import os import glob import lsst.eotest.sensor as sensorTest import lcatr.schema import siteUtils import eotestUtils sensor_id = siteUtils.getUnitId() ccd_vendor = siteUtils.getCcdVendor() det_resp_data = '%s_det_response.fits' % sensor_id eotestUtils.addHeaderData(det_resp_data, LSST_NUM=sensor_id, TESTTYPE='FLAT', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=siteUtils.getCcdVendor().upper()) results = [lcatr.schema.fileref.make(det_resp_data)] if ccd_vendor == 'ITL': # Persist detector response for special linearity dataset from ITL. det_resp_data = '%s_det_response_linearity.fits' % sensor_id if os.path.isfile(det_resp_data): # File will only exist if analyzing vendor data directly. eotestUtils.addHeaderData(det_resp_data, LSST_NUM=sensor_id, TESTTYPE='FLAT', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=siteUtils.getCcdVendor().upper()) results.append(lcatr.schema.fileref.make(det_resp_data))
pylab.savefig('%s_psf_dists.png' % sensor_id) # Photon Transfer Curves ptc_file = processName_dependencyGlob('%s_ptc.fits' % sensor_id, jobname='ptc_offline')[0] plots.ptcs(ptc_file=ptc_file) pylab.savefig('%s_ptcs.png' % sensor_id) detresp_file = processName_dependencyGlob('%s_det_response.fits' % sensor_id, jobname='flat_pairs_offline')[0] # Full well plots plots.full_well(ptc_file=ptc_file, detresp_file=detresp_file) pylab.savefig('%s_full_well.png' % sensor_id) # Linearity plots if siteUtils.getCcdVendor() == 'ITL': # Use special linearity dataset for ITL data try: detresp_file = processName_dependencyGlob( '%s_det_response_linearity.fits' % sensor_id, jobname='flat_pairs_offline')[0] except: pass plots.linearity(detresp_file=detresp_file) pylab.savefig('%s_linearity.png' % sensor_id) plots.linearity_resids(detresp_file=detresp_file) pylab.savefig('%s_linearity_resids.png' % sensor_id) # System Gain per segment
#!/usr/bin/env python import lsst.eotest.sensor as sensorTest import siteUtils import eotestUtils sensor_id = siteUtils.getUnitId() ccd_vendor = siteUtils.getCcdVendor() flat_files = siteUtils.datacatalog_glob('*_flat*flat?_*.fits', testtype='FLAT', imgtype='FLAT', description='Flat files:') mask_files = eotestUtils.glob_mask_files() gains = eotestUtils.getSensorGains(jobname='fe55_offline') task = sensorTest.FlatPairTask() task.run(sensor_id, flat_files, mask_files, gains) if ccd_vendor == 'ITL': # # Perform linearity analysis using special dataset from ITL try: flat_files = siteUtils.datacatalog_glob( '*_linearity_flat*.fits', testtype='LINEARITY', imgtype='FLAT', description='ITL linearity files:') if flat_files: task = sensorTest.LinearityTask() task.run(sensor_id, flat_files, mask_files, gains)
results = [] for band in qe_results: results.append(validate(job, band=band, QE=np.average(qe_results[band]))) return results def metrology(self): return [] if __name__ == '__main__': results = [siteUtils.packageVersions()] lsstnum = siteUtils.getUnitId() vendorDataDir = os.readlink('vendorData') print 'Vendor data location:', vendorDataDir if siteUtils.getCcdVendor() == 'ITL': vendor = ItlResults(vendorDataDir) translator = ItlFitsTranslator(lsstnum, vendorDataDir, '.') else: vendor = e2vResults(vendorDataDir) translator = e2vFitsTranslator(lsstnum, vendorDataDir, '.') results.extend(vendor.run_all()) translator.run_all() results.extend([lcatr.schema.fileref.make(x) for x in translator.outfiles]) lcatr.schema.write_file(results) lcatr.schema.validate_file()
import siteUtils import metUtils from MetrologyData import md_factory producer = 'SR-MET-07' testtype = 'FLATNESS' results = metUtils.aggregate_filerefs_ts5(producer, testtype) # Add the QA plot to the results raft_id = siteUtils.getUnitId() qafile = glob.glob('*_qa_plot.png')[0] #print('qafile: %s' % qafile) #print('raft_id: %s' % raft_id) md = siteUtils.DataCatalogMetadata(CCD_MANU=siteUtils.getCcdVendor(), LSST_NUM=siteUtils.getUnitId(), PRODUCER=producer, ORIGIN=siteUtils.getSiteName(), TESTTYPE=testtype, TEST_CATEGORY='MET') results.extend([lcatr.schema.fileref.make(qafile, metadata=md(DATA_PRODUCT='QA_PLOT'))]) raftData = md_factory.load('flatness_ts5_delta.pickle') peak_valley_95 = raftData.quantiles['0.975'] - raftData.quantiles['0.025'] peak_valley_100 = raftData.quantiles['1.000'] - raftData.quantiles['0.000'] # Make strings out of the quantile information quantiles = raftData.quantiles quantile_levels = quantiles.keys()
#!/usr/bin/env python import os import glob import lsst.eotest.sensor as sensorTest import lcatr.schema import siteUtils import eotestUtils sensor_id = siteUtils.getUnitId() ccd_vendor = siteUtils.getCcdVendor() det_resp_data = '%s_det_response.fits' % sensor_id eotestUtils.addHeaderData(det_resp_data, LSST_NUM=sensor_id, TESTTYPE='FLAT', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=siteUtils.getCcdVendor().upper()) results = [lcatr.schema.fileref.make(det_resp_data)] if ccd_vendor == 'ITL': # Persist detector response for special linearity dataset from ITL. det_resp_data = '%s_det_response_linearity.fits' % sensor_id if os.path.isfile(det_resp_data): # File will only exist if analyzing vendor data directly. eotestUtils.addHeaderData(det_resp_data, LSST_NUM=sensor_id, TESTTYPE='FLAT', DATE=eotestUtils.utc_now_isoformat(), CCD_MANU=siteUtils.getCcdVendor().upper()) results.append(lcatr.schema.fileref.make(det_resp_data)) results_file = '%s_eotest_results.fits' % sensor_id data = sensorTest.EOTestResults(results_file) amps = data['AMP']