def run_qe_task(sensor_id):

    try:
        if hw_objects[sensor_id] :
            return
    except:
        hw_objects[sensor_id] = True
        print("hw_objects = ",hw_objects)


    "Single sensor execution of the QE task."
    file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
    lambda_files = siteUtils.dependency_glob('S*/%s_lambda_flat_*.fits' % sensor_id,
                                             jobname=siteUtils.getProcessName('qe_raft_acq'),
                                             description='Lambda files:')

    pd_ratio_file = eotestUtils.getPhotodiodeRatioFile()
    if pd_ratio_file is None:
        message = ("The test-stand specific photodiode ratio file is " +
                   "not given in config/%s/eotest_calibrations.cfg."
                   % siteUtils.getSiteName())
        raise RuntimeError(message)

    correction_image = eotestUtils.getIlluminationNonUniformityImage()
    if correction_image is None:
        print()
        print("WARNING: The correction image file is not given in")
        print("config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName())
        print("No correction for non-uniform illumination will be applied.")
        print()
        sys.stdout.flush()

    mask_files = \
        eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
    gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
                                       sensor_id=sensor_id)

    task = sensorTest.QeTask()
    task.config.temp_set_point = -100.
    task.run(sensor_id, lambda_files, pd_ratio_file, mask_files, gains,
             correction_image=correction_image)

    results_file \
        = siteUtils.dependency_glob('%s_eotest_results.fits' % sensor_id,
                                    jobname='fe55_raft_analysis',
                                    description='Fe55 results file')[0]
    plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)

    siteUtils.make_png_file(plots.qe,
                            '%s_qe.png' % file_prefix,
                            qe_file='%s_QE.fits' % sensor_id)

    try:
        plots.flat_fields(os.path.dirname(lambda_files[0]),
                          annotation='e-/pixel, gain-corrected, bias-subtracted')
    except Exception as eobj:
        print("Exception raised while creating flat fields:")
        print(str(eobj))
    def __init__(self, configFile, sys_paths=()):
        """
        configFile contains the names of the site-specific
        configuration files.  File basenames are provided in
        configFile, and the full paths are constructed in the
        _read(...) method.
        """
        super(CcsSetup, self).__init__()
        self.commands = []
        self['tsCWD'] = os.getcwd()
        self['labname'] = siteUtils.getSiteName()
        self['jobname'] = siteUtils.getJobName()
        self['CCDID'] = siteUtils.getUnitId()
        self['UNITID'] = siteUtils.getUnitId()
        self['LSSTID'] = siteUtils.getLSSTId()
        try:
            self['RUNNUM'] = siteUtils.getRunNumber()
        except Exception:
            self['RUNNUM'] = "no_lcatr_run_number"

        self['ts'] = os.getenv('CCS_TS', default='ts')
        self['archon'] = os.getenv('CCS_ARCHON', default='archon')

        # The following are only available for certain contexts.
        if 'CCS_VAC_OUTLET' in os.environ:
            self['vac_outlet'] = os.getenv('CCS_VAC_OUTLET')
        if 'CCS_CRYO_OUTLET' in os.environ:
            self['cryo_outlet'] = os.getenv('CCS_CRYO_OUTLET')
        if 'CCS_PUMP_OUTLET' in os.environ:
            self['pump_outlet'] = os.getenv('CCS_PUMP_OUTLET')

        self._read(os.path.join(siteUtils.getJobDir(), configFile))

        self.sys_paths = sys_paths
def qe_jh_task(det_name):
    """JH version of single sensor execution of the QE task."""
    run = siteUtils.getRunNumber()
    file_prefix = make_file_prefix(run, det_name)
    acq_jobname = siteUtils.getProcessName('BOT_acq')

    lambda_files = siteUtils.dependency_glob(glob_pattern('qe', det_name),
                                             acq_jobname=acq_jobname)
    if not lambda_files:
        print("qe_task: QE scan files not found for detector", det_name)
        return None

    pd_ratio_file = eotestUtils.getPhotodiodeRatioFile()
    if pd_ratio_file is None:
        message = ("The BOT photodiode ratio file is " +
                   "not given in config/%s/eotest_calibrations.cfg."
                   % siteUtils.getSiteName())
        raise RuntimeError(message)

#    correction_image = eotestUtils.getIlluminationNonUniformityImage()
#    if correction_image is None:
#        print()
#        print("WARNING: The correction image file is not given in")
#        print("config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName())
#        print("No correction for non-uniform illumination will be applied.")
#        print()
#        sys.stdout.flush()
    mask_files = get_mask_files(det_name)
    eotest_results_file = '{}_eotest_results.fits'.format(file_prefix)
    gains = get_amplifier_gains(eotest_results_file)
    bias_frame = bias_filename(run, det_name)

    return qe_task(run, det_name, lambda_files, pd_ratio_file, gains,
                   mask_files=mask_files, bias_frame=bias_frame,
                   mondiode_func=mondiode_value)
    def __init__(self, configFile):
        """
        configFile contains the names of the site-specific
        configuration files.  File basenames are provided in
        configFile, and the full paths are constructed in the
        _read(...) method.
        """
        super(CcsSetup, self).__init__()
        if os.environ.has_key('CCS_TS'):
            self['ts']=_quote(os.getenv('CCS_TS'))
        else:
            self['ts'] = _quote('ts')
        self['tsCWD'] = _quote(os.getcwd())
        self['labname'] = _quote(siteUtils.getSiteName())
        self['CCDID'] = _quote(siteUtils.getUnitId())
#        self._read(os.path.join(siteUtils.getJobDir(), configFile))
        CCDTYPE = _quote(siteUtils.getUnitType())
Exemple #5
0
def run_flat_pair_task(sensor_id):

    file_prefix = '%s_%s' % (sensor_id, siteUtils.getRunNumber())
    flat_files = siteUtils.dependency_glob(
        'S*/%s_flat*flat?_*.fits' % sensor_id,
        jobname=siteUtils.getProcessName('flat_pair_raft_acq'),
        description='Flat files:')
    mask_files = \
        eotestUtils.glob_mask_files(pattern='%s_*mask.fits' % sensor_id)
    gains = eotestUtils.getSensorGains(jobname='fe55_raft_analysis',
                                       sensor_id=sensor_id)

    use_exptime = True
    if siteUtils.getSiteName() == 'SLAC':
        # Since slit-width can be set individually for each exposure
        # on TS-8 at IR-2 (LSSTTD-1231), we need to use the MONDIODE
        # keyword for computing the integrated incident flux.
        use_exptime = False

    task = sensorTest.FlatPairTask()
    task.run(sensor_id,
             flat_files,
             mask_files,
             gains,
             linearity_spec_range=(1e4, 9e4),
             use_exptime=use_exptime)

    results_file = '%s_eotest_results.fits' % sensor_id
    plots = sensorTest.EOTestPlots(sensor_id, results_file=results_file)

    Ne_bounds = (1e4, 9e4)

    detresp_file = '%s_det_response.fits' % sensor_id
    siteUtils.make_png_file(plots.linearity,
                            '%s_linearity.png' % file_prefix,
                            detresp_file=detresp_file,
                            max_dev=0.03,
                            use_exptime=use_exptime,
                            Ne_bounds=Ne_bounds)
    siteUtils.make_png_file(plots.linearity_resids,
                            '%s_linearity_resids.png' % file_prefix,
                            detresp_file=detresp_file,
                            max_dev=0.03,
                            Ne_bounds=Ne_bounds,
                            use_exptime=use_exptime)
def validate_raft_results(results, raft_names):
    """Validate the raft level results."""
    run = siteUtils.getRunNumber()
    slot_names = camera_info.get_slot_names()
    md = siteUtils.DataCatalogMetadata(ORIGIN=siteUtils.getSiteName(),
                                       TEST_CATEGORY='EO',
                                       DATA_PRODUCT='EOTEST_RESULTS')
    missing_raft_names = []
    for raft_name in raft_names:
        for slot_name in slot_names:
            det_name = make_file_prefix(raft_name, slot_name)
            file_prefix = make_file_prefix(run, det_name)
            results_file = '{}_eotest_results.fits'.format(file_prefix)
            if not os.path.isfile(results_file):
                if raft_name not in missing_raft_names:
                    missing_raft_names.append(raft_name)
                continue
            eotestUtils.addHeaderData(results_file,
                                      DETECTOR=det_name,
                                      DATE=eotestUtils.utc_now_isoformat(),
                                      RUNNUM=run)
            results.append(
                lcatr.schema.fileref.make(results_file,
                                          metadata=md(SLOT=slot_name,
                                                      RAFT=raft_name)))

        # Persist the png files.
        png_file_list = '{}_raft_results_task_png_files.txt'.format(raft_name)
        if not os.path.isfile(png_file_list):
            continue
        with open(png_file_list, 'r') as input_:
            png_files = [x.strip() for x in input_]
        metadata = dict(TEST_CATEGORY='EO', DETECTOR=det_name, RUN=run)
        results.extend(
            siteUtils.persist_png_files('',
                                        file_prefix,
                                        png_files=png_files,
                                        metadata=metadata))

    report_missing_data("validate_raft_results",
                        missing_raft_names,
                        components='rafts',
                        total=21)

    return results
def aggregate_filerefs(producer, testtype, origin=None, dp_mapping=None):
    """
    Aggregate the filerefs for the metrology data products and return
    them as a python list.
    """
    if origin is None:
        origin = siteUtils.getSiteName()

    if dp_mapping is None:
        dp_mapping = dict(BOXPLOT='boxplot',
                          HISTOGRAM='hist',
                          POINT_CLOUD_10='point_cloud_azim_10',
                          POINT_CLOUD_45='point_cloud_azim_45',
                          RESIDUALS='residuals',
                          QUANTILE_TABLE='quantile_table')

    # Common metadata fields for all data products.
    ccd_vendor = siteUtils.getCcdVendor()
    sensor_id = siteUtils.getUnitId()
    md = siteUtils.DataCatalogMetadata(CCD_MANU=ccd_vendor,
                                       LSST_NUM=sensor_id,
                                       PRODUCER=producer,
                                       ORIGIN=origin,
                                       TESTTYPE=testtype,
                                       TEST_CATEGORY='MET')

    # Create filerefs for each data product, adding the file-specific
    # data product metadata.
    tt_ext = testtype.lower()
    results = []
    for dp, ext in dp_mapping.items():
        pattern = '%(sensor_id)s_%(tt_ext)s_*%(ext)s.*'% locals()
        dp_files = glob.glob(pattern)
        results.extend([lcatr.schema.fileref.make(dp_file,
                                                  metadata=md(DATA_PRODUCT=dp))
                        for dp_file in dp_files])

    return results
Exemple #8
0
#!/usr/bin/env python
import sys
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils

sensor_id = siteUtils.getUnitId()
spot_files = siteUtils.dependency_glob(
    '*_spot_*.fits',
    jobname=siteUtils.getProcessName('xtalk_acq'),
    description='Spot files:')
mask_files = eotestUtils.glob_mask_files()
system_xtalk_file = eotestUtils.getSystemCrosstalkFile()
if system_xtalk_file is None:
    print
    print "WARNING: The system crosstalk file is not given in"
    print "config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName()
    print "No correction for system crosstalk will be performed."
    print
    sys.stdout.flush()

task = sensorTest.CrosstalkTask()
task.run(sensor_id,
         spot_files,
         mask_files,
         system_xtalk_file=system_xtalk_file)
Exemple #9
0
    def __init__(self, configFile):
        """
        configFile contains the names of the site-specific
        configuration files.  File basenames are provided in
        configFile, and the full paths are constructed in the
        _read(...) method.
        """
        super(CcsSetup, self).__init__()
        if os.environ.has_key('CCS_TS8'):
            self['ts8']=_quote(os.getenv('CCS_TS8'))
        else:
            self['ts8'] = _quote('ts8')
        if os.environ.has_key('CCS_JYTH'):
            self['jyth']=_quote(os.getenv('CCS_JYTH'))
        else:
            self['jyth'] = _quote('JythonInterpreterConsole')
        if os.environ.has_key('CCS_JSON_PORT'):
            self['jsonport']=os.getenv('CCS_JSON_PORT')
        else:
            self['jsonport'] = 4444
        if os.environ.has_key('CCS_PS'):
            self['ps']=_quote(os.getenv('CCS_PS'))
        else:
            self['ps'] = _quote('ccs-rebps')
        if os.environ.has_key('CCS_TS'):
            self['ts']=_quote(os.getenv('CCS_TS'))
        else:
            self['ts'] = _quote('ts')
        if os.environ.has_key('CCS_ARCHON'):
            self['archon']=_quote(os.getenv('CCS_ARCHON'))
        else:
            self['archon'] = _quote('archon')
        if os.environ.has_key('CCS_VAC_OUTLET'):
            self['vac_outlet']=os.getenv('CCS_VAC_OUTLET')
# there is no default for vac_outlet - if there is a script that needs
# it and it has not been defined then I want it to crash
        if os.environ.has_key('CCS_CRYO_OUTLET'):
            self['cryo_outlet']=os.getenv('CCS_CRYO_OUTLET')
# there is no default for cryo_outlet - if there is a script that needs
# it and it has not been defined then I want it to crash
        if os.environ.has_key('CCS_PUMP_OUTLET'):
            self['pump_outlet']=os.getenv('CCS_PUMP_OUTLET')
# there is no default for pump_outlet - if there is a script that needs
# it and it has not been defined then I want it to crash
        self['tsCWD'] = _quote(os.getcwd())
        self['labname'] = _quote(siteUtils.getSiteName())
        self['jobname'] = _quote(siteUtils.getJobName())
        self['CCDID'] = _quote(siteUtils.getUnitId())
        self['UNITID'] = _quote(siteUtils.getUnitId())
        self['LSSTID'] = _quote(siteUtils.getLSSTId())

        unitid = siteUtils.getUnitId()
        CCDTYPE = _quote(siteUtils.getUnitType())
        ccdnames = {}
        ccdmanunames = {}
        ccdnames,ccdmanunames = siteUtils.getCCDNames()

        print "retrieved the following LSST CCD names list"
        print ccdnames
        print "retrieved the following Manufacturers CCD names list"
        print ccdmanunames
        for slot in ccdnames :
            print "CCD %s is in slot %s" % (ccdnames[slot],slot)
            self['CCD%s'%slot] = _quote(ccdnames[slot])
            if 'itl' in ccdnames[slot].lower() :
                CCDTYPE = 'itl'
            if 'e2v' in ccdnames[slot].lower() :
                CCDTYPE = 'e2v'
        for slot in ccdmanunames :
            print "CCD %s is in slot %s" % (ccdmanunames[slot],slot)
            self['CCDMANU%s'%slot] = _quote(ccdmanunames[slot])
        try:
            self['RUNNUM'] = _quote(siteUtils.getRunNumber())
        except:
            self['RUNNUM'] = "no_lcatr_run_number"
            pass
        self._read(os.path.join(siteUtils.getJobDir(), configFile))

        print "CCDTYPE = %s" % CCDTYPE
        self['sequence_file'] = _quote("NA")
        self['acffile'] = self['itl_acffile']
# set default type
        self['CCSCCDTYPE'] = _quote("ITL")
        if ("RTM" in unitid.upper() or "ETU" in unitid.upper() or "RSA" in unitid.upper()) :
            if ("e2v" in CCDTYPE) :
                self['CCSCCDTYPE'] = _quote("E2V")
                self['acffile'] = self['e2v_acffile']
                self['sequence_file'] = self['e2v_seqfile']
            else :
                self['CCSCCDTYPE'] = _quote("ITL")
                self['acffile'] = self['itl_acffile']
                self['sequence_file'] = self['itl_seqfile']
            os.system("export | grep -i seq")
            seqdir = ""
            if os.environ.has_key('SEQUENCERFILESDIR') :
                seqdir = os.getenv('SEQUENCERFILESDIR')
                print "seqdir=",seqdir
                self['sequence_file'] = self['sequence_file'].replace('${SEQUENCERFILESDIR}',seqdir)
            os.system("cp -vp %s %s" % (self['sequence_file'],self['tsCWD']))

            # now use the local copy
#            bb = self['sequence_file'].split("/")
#            self['sequence_file'] = _quote("%s/%s" % (os.getcwd(),bb[len(bb)-1].split("'")[0]))
            print "The sequence file to be used is %s" % self['sequence_file']
        else :
            if ("ITL" in CCDTYPE) :
                self['CCSCCDTYPE'] = _quote("ITL")
                self['acffile'] = self['itl_acffile']
            if ("e2v" in CCDTYPE) :
                self['CCSCCDTYPE'] = _quote("E2V")
                self['acffile'] = self['e2v_acffile']
            print "The acffile to be used is %s" % self['acffile']
            = aliveness_utils.raft_channel_statuses(fits_files)
        for slot in channel_status:
            bad_channels = 0
            for amp, status in channel_status[slot].items():
                if channel_status[slot][amp] == 'bad':
                    bad_channels += 1
                    signal = channel_signal[slot][amp]
                    channel = imutils.channelIds[amp]
                    output.write(row_template % locals())
            results.append(lcatr.schema.valid(job_schema,
                                              exptime=exptime,
                                              slot=slot,
                                              bad_channels=bad_channels))

results.append(lcatr.schema.fileref.make(outfile))

# Persist the sequencer file that was used.
seq_file = glob.glob('*.seq')[0]
results.append(lcatr.schema.fileref.make(seq_file))

# Add png files.
raft_id = siteUtils.getUnitId()
md = siteUtils.DataCatalogMetadata(ORIGIN=siteUtils.getSiteName(),
                                   TEST_CATEGORY='EO')
results.extend(siteUtils.persist_png_files('*.png', raft_id, metadata=md))

results.extend(siteUtils.jobInfo())

lcatr.schema.write_file(results)
lcatr.schema.validate_file()
#!/usr/bin/env python
import sys
import lsst.eotest.sensor as sensorTest
import siteUtils
import eotestUtils

sensor_id = siteUtils.getUnitId()
spot_files = siteUtils.dependency_glob('*_spot_*.fits',
                                       jobname=siteUtils.getProcessName('xtalk_acq'),
                                       description='Spot files:')
mask_files = eotestUtils.glob_mask_files()
system_xtalk_file = eotestUtils.getSystemCrosstalkFile()
if system_xtalk_file is None:
    print 
    print "WARNING: The system crosstalk file is not given in"
    print "config/%s/eotest_calibrations.cfg." % siteUtils.getSiteName()
    print "No correction for system crosstalk will be performed."
    print
    sys.stdout.flush()

task = sensorTest.CrosstalkTask()
task.run(sensor_id, spot_files, mask_files, system_xtalk_file=system_xtalk_file)
producer = 'SR-MET-07'
testtype = 'FLATNESS'

results = metUtils.aggregate_filerefs_ts5(producer, testtype)

# Add the QA plot to the results
raft_id = siteUtils.getUnitId()
qafile = glob.glob('*_qa_plot.png')[0]
#print('qafile:  %s' % qafile)
#print('raft_id:  %s' % raft_id)

md = siteUtils.DataCatalogMetadata(CCD_MANU=siteUtils.getCcdVendor(),
                                   LSST_NUM=siteUtils.getUnitId(),
                                   PRODUCER=producer,
                                   ORIGIN=siteUtils.getSiteName(),
                                   TESTTYPE=testtype,
                                   TEST_CATEGORY='MET')

results.extend([lcatr.schema.fileref.make(qafile, metadata=md(DATA_PRODUCT='QA_PLOT'))])

raftData = md_factory.load('flatness_ts5_delta.pickle')
peak_valley_95 = raftData.quantiles['0.975'] - raftData.quantiles['0.025']
peak_valley_100 = raftData.quantiles['1.000'] - raftData.quantiles['0.000']

# Make strings out of the quantile information
quantiles = raftData.quantiles
quantile_levels = quantiles.keys()
quantile_str = '' 
z_str = ''
for key in sorted(quantile_levels):
Exemple #13
0
#!/usr/bin/env python
import os
import glob
import shutil
import lcatr.schema
import siteUtils
import eotestUtils

sensor_id = siteUtils.getUnitId()

md = siteUtils.DataCatalogMetadata(CCD_MANU=siteUtils.getCcdVendor(),
                                   LSST_NUM=sensor_id,
                                   PRODUCER='SR-EOT-1',
                                   ORIGIN=siteUtils.getSiteName(),
                                   TEST_CATEGORY='EO')

results_file = '%s_eotest_results.fits' % sensor_id
eotestUtils.addHeaderData(results_file, LSST_NUM=sensor_id,
                          DATE=eotestUtils.utc_now_isoformat(),
                          CCD_MANU=siteUtils.getCcdVendor().upper())
results = [lcatr.schema.fileref.make(results_file,
                                     metadata=md(DATA_PRODUCT='EOTEST_RESULTS'))]

png_files = glob.glob('*.png')
results.extend([lcatr.schema.fileref.make(item,
                                          metadata=md(DATA_PRODUCT=eotestUtils.png_data_product(item, sensor_id)))
                for item in png_files])

test_report = '%s_eotest_report.pdf' % sensor_id
results.append(lcatr.schema.fileref.make(test_report,
                                         metadata=md(DATA_PRODUCT='EOTEST_REPORT')))