def __init__(self, name, inptype, outtype, config, logger=None):
if logger is None:
qll = qllogger.QLLogger()
self.m_log = qll.getlog(name)
else:
self.m_log = logger
self.__inpType__ = type(inptype)
self.__outType__ = type(outtype)
self.name = name
self.config = config
self.m_log.debug("initializing Monitoring alg {}".format(name))
def __init__(self, thislist=None, algorithms=None, flavor=None, mode=None):
"""
thislist: given list of PAs
algorithms: Algorithm list coming from config file: e.g lvmspec/data/quicklook/qlconfig_dark.yaml
flavor: only needed if new list is to be built.
mode: online offline?
"""
self.flavor = flavor
self.mode = mode
self.thislist = thislist
self.algorithms = algorithms
self.palist = self._palist()
self.qalist = self._qalist()
qlog = qllogger.QLLogger(name="QLConfig")
self.log = qlog.getlog()
def check_config(outconfig):
"""
Given the expanded config, check for all possible file existence etc....
"""
qlog = qllogger.QLLogger(name="QLConfig")
log = qlog.getlog()
log.info("Checking if all the necessary files exist.")
calib_flavors = ['arcs', 'dark', 'bias']
if outconfig["Flavor"] == 'science':
files = [
outconfig["RawImage"], outconfig["FiberMap"],
outconfig["FiberFlatFile"], outconfig["PSFFile"]
]
for thisfile in files:
if not os.path.exists(thisfile):
sys.exit("File does not exist: {}".format(thisfile))
else:
log.info("File check: Okay: {}".format(thisfile))
elif outconfig["Flavor"] in calib_flavors:
files = [outconfig["RawImage"], outconfig["FiberMap"]]
for thisfile in files:
if not os.path.exists(thisfile):
sys.exit("File does not exist: {}".format(thisfile))
else:
log.info("File check: Okay: {}".format(thisfile))
elif outconfig["Flavor"] == "flat":
files = [
outconfig["RawImage"], outconfig["FiberMap"], outconfig["PSFFile"]
]
for thisfile in files:
if not os.path.exists(thisfile):
sys.exit("File does not exist: {}".format(thisfile))
else:
log.info("File check: Okay: {}".format(thisfile))
log.info("All necessary files exist for {} configuration.".format(
outconfig["Flavor"]))
return
def ql_main(args=None):
from lvmspec.quicklook import quicklook, qllogger, qlconfig
import lvmspec.image as image
import lvmspec.frame as frame
import lvmspec.io.frame as frIO
import lvmspec.io.image as imIO
if args is None:
args = parse()
qlog = qllogger.QLLogger(name="QuickLook", loglevel=args.loglvl)
log = qlog.getlog()
# Sami
# quiet down DESI logs. We don't want LVM_LOGGER to print messages unless they are important
# initalize singleton with WARNING level
quietDesiLogger(args.loglvl + 10)
if args.config is not None:
if args.rawdata_dir:
rawdata_dir = args.rawdata_dir
else:
if 'QL_SPEC_DATA' not in os.environ:
sys.exit(
"must set ${} environment variable or provide rawdata_dir".
format('QL_SPEC_DATA'))
rawdata_dir = os.getenv('QL_SPEC_DATA')
if args.specprod_dir:
specprod_dir = args.specprod_dir
else:
if 'QL_SPEC_REDUX' not in os.environ:
sys.exit(
"must set ${} environment variable or provide specprod_dir"
.format('QL_SPEC_REDUX'))
specprod_dir = os.getenv('QL_SPEC_REDUX')
log.debug("Running Quicklook using configuration file {}".format(
args.config))
if os.path.exists(args.config):
if "yaml" in args.config:
config = qlconfig.Config(args.config,
args.night,
args.camera,
args.expid,
rawdata_dir=rawdata_dir,
specprod_dir=specprod_dir)
configdict = config.expand_config()
else:
log.critical("Can't open config file {}".format(args.config))
sys.exit("Can't open config file")
else:
sys.exit("File does not exist: {}".format(args.config))
elif args.fullconfig is not None: #- This is mostly for development/debugging purpose
log.debug("Running Quicklook using full configuration file {}".format(
args.fullconfig))
if os.path.exists(args.fullconfig):
if "yaml" in args.fullconfig:
configdict = yaml.load(open(args.fullconfig, "r"))
else:
log.critical("Can't open config file {}".format(args.config))
sys.exit("Can't open config file")
else:
sys.exit("File does not exist: {}".format(args.config))
else:
sys.exit(
"Must provide a valid config file. See lvmspec/data/quicklook for an example"
)
#- save the expanded config to a file
if args.save:
if "yaml" in args.save:
f = open(args.save, "w")
yaml.dump(configdict, f)
log.info("Output saved for this configuration to {}".format(
args.save))
f.close()
else:
log.warning(
"Can save config to only yaml output. Put a yaml in the argument"
)
pipeline, convdict = quicklook.setup_pipeline(configdict)
res = quicklook.runpipeline(pipeline,
convdict,
configdict,
mergeQA=args.mergeQA)
inpname = configdict["RawImage"]
night = configdict["Night"]
camera = configdict["Camera"]
expid = configdict["Expid"]
if configdict["OutputFile"] is None:
log.warning(
"Output filename is None and has a object of {}. SKIPPING FINAL OUTPUT"
.format(type(res)))
return
if isinstance(res, image.Image):
if configdict["OutputFile"]:
finalname = configdict["OutputFile"]
else:
finalname = "image-{}-{:08d}.fits".format(camera, expid)
log.critical(
"No final outputname given. Writing to a image file {}".format(
finalname))
imIO.write_image(finalname, res, meta=None)
elif isinstance(res, frame.Frame):
if configdict["OutputFile"]:
finalname = configdict["OutputFile"]
else:
finalname = "frame-{}-{:08d}.fits".format(camera, expid)
log.critical(
"No final outputname given. Writing to a frame file {}".format(
finalname))
frIO.write_frame(finalname, res, header=None)
elif configdict["Flavor"] == 'arcs':
if configdict["OutputFile"]:
finalname = configdict["OutputFile"]
else:
finalname = "psfnight-{}.fits".format(camera)
elif configdict["Flavor"] == 'flat':
if configdict["OutputFile"]:
finalname = configdict["OutputFile"]
else:
finalname = "fiberflat-{}-{:08d}.fits".format(camera, expid)
else:
log.error(
"Result of pipeline is an unknown type {}. Don't know how to write"
.format(type(res)))
sys.exit("Unknown pipeline result type {}.".format(type(res)))
log.info("Pipeline completed. Final result is in {}".format(finalname))
"""
tests for Quicklook Pipeline steps in lvmspec.quicklook.procalgs
"""
import unittest
import numpy as np
import os
import lvmspec
from lvmspec.quicklook import procalgs as PA
from pkg_resources import resource_filename
from lvmspec.test.test_ql_qa import xy2hdr
from lvmspec.preproc import _parse_sec_keyword
import astropy.io.fits as fits
from lvmspec.quicklook import qllogger
qlog = qllogger.QLLogger("QuickLook", 0)
log = qlog.getlog()
class TestQL_PA(unittest.TestCase):
def tearDown(self):
self.rawimage.close()
for filename in [self.rawfile, self.pixfile]:
if os.path.exists(filename):
os.remove(filename)
#- Create some test data
def setUp(self):
self.rawfile = 'test-raw-abcd.fits'
self.pixfile = 'test-pix-abcd.fits'
def setup_pipeline(config):
"""
Given a configuration from QLF, this sets up a pipeline [pa,qa] and also returns a
conversion dictionary from the configuration dictionary so that Pipeline steps (PA) can
take them. This is required for runpipeline.
"""
import astropy.io.fits as fits
import lvmspec.io.fibermap as fibIO
import lvmspec.io.sky as skyIO
import lvmspec.io.fiberflat as ffIO
import lvmspec.fiberflat as ff
import lvmspec.io.image as imIO
import lvmspec.image as im
import lvmspec.io.frame as frIO
import lvmspec.frame as dframe
from lvmspec.quicklook import procalgs
from lvmspec.boxcar import do_boxcar
qlog = qllogger.QLLogger()
log = qlog.getlog()
if config is None:
return None
log.debug("Reading Configuration")
if "RawImage" not in config:
log.critical("Config is missing \"RawImage\" key.")
sys.exit("Missing \"RawImage\" key.")
inpname = config["RawImage"]
if "FiberMap" not in config:
log.critical("Config is missing \"FiberMap\" key.")
sys.exit("Missing \"FiberMap\" key.")
fibname = config["FiberMap"]
proctype = "Exposure"
if "Camera" in config:
camera = config["Camera"]
if "DataType" in config:
proctype = config["DataType"]
debuglevel = 20
if "DebugLevel" in config:
debuglevel = config["DebugLevel"]
log.setLevel(debuglevel)
hbeat = QLHB.QLHeartbeat(log, config["Period"], config["Timeout"])
if config["Timeout"] > 200.0:
log.warning("Heartbeat timeout exceeding 200.0 seconds")
dumpintermediates = False
if "DumpIntermediates" in config:
dumpintermediates = config["DumpIntermediates"]
biasimage = None #- This will be the converted dictionary key
biasfile = None
if "BiasImage" in config:
biasfile = config["BiasImage"]
darkimage = None
darkfile = None
if "DarkImage" in config:
darkfile = config["DarkImage"]
pixelflatfile = None
pixflatimage = None
if "PixelFlat" in config:
pixelflatfile = config["PixelFlat"]
fiberflatimagefile = None
fiberflatimage = None
if "FiberFlatImage" in config:
fiberflatimagefile = config["FiberFlatImage"]
arclampimagefile = None
arclampimage = None
if "ArcLampImage" in config:
arclampimagefile = config["ArcLampImage"]
fiberflatfile = None
fiberflat = None
if config["Flavor"] == 'science':
if "FiberFlatFile" in config:
fiberflatfile = config["FiberFlatFile"]
skyfile = None
skyimage = None
if "SkyFile" in config:
skyfile = config["SkyFile"]
psf = None
if config["Flavor"] == 'dark' or config["Flavor"] == 'bias':
pass
elif config["Flavor"] == 'arcs':
if not os.path.exists(
os.path.join(os.environ['QL_SPEC_REDUX'], 'calib2d', 'psf',
config["Night"])):
os.mkdir(
os.path.join(os.environ['QL_SPEC_REDUX'], 'calib2d', 'psf',
config["Night"]))
pass
elif config["Flavor"] == 'science' or config["Flavor"] == 'flat':
#from specter.psf import load_psf
if "PSFFile" in config:
import lvmspec.psf
psf = lvmspec.psf.PSF(config["PSFFile"])
#psf=load_psf(config["PSFFile"])
if "basePath" in config:
basePath = config["basePath"]
hbeat.start("Reading input file {}".format(inpname))
inp = fits.open(
inpname) #- reading raw image directly from astropy.io.fits
hbeat.start("Reading fiberMap file {}".format(fibname))
fibfile = fibIO.read_fibermap(fibname)
fibhdr = fibfile.meta
convdict = {"FiberMap": fibfile}
if psf is not None:
convdict["PSFFile"] = psf
if biasfile is not None:
hbeat.start("Reading Bias Image {}".format(biasfile))
biasimage = imIO.read_image(biasfile)
convdict["BiasImage"] = biasimage
if darkfile is not None:
hbeat.start("Reading Dark Image {}".format(darkfile))
darkimage = imIO.read_image(darkfile)
convdict["DarkImage"] = darkimage
if pixelflatfile:
hbeat.start("Reading PixelFlat Image {}".format(pixelflatfile))
pixelflatimage = imIO.read_image(pixelflatfile)
convdict["PixelFlat"] = pixelflatimage
if fiberflatimagefile:
hbeat.start("Reading FiberFlat Image {}".format(fiberflatimagefile))
fiberflatimage = imIO.read_image(fiberflatimagefile)
convdict["FiberFlatImage"] = fiberflatimage
if arclampimagefile:
hbeat.start("Reading ArcLampImage {}".format(arclampimagefile))
arclampimage = imIO.read_image(arclampimagefile)
convdict["ArcLampImage"] = arclampimage
if fiberflatfile:
hbeat.start("Reading FiberFlat {}".format(fiberflatfile))
fiberflat = ffIO.read_fiberflat(fiberflatfile)
convdict["FiberFlatFile"] = fiberflat
if skyfile:
hbeat.start("Reading SkyModel file {}".format(skyfile))
skymodel = skyIO.read_sky(skyfile)
convdict["SkyFile"] = skymodel
if dumpintermediates:
convdict["DumpIntermediates"] = dumpintermediates
hbeat.stop("Finished reading all static files")
img = inp
convdict["rawimage"] = img
pipeline = []
for step in config["PipeLine"]:
pa = getobject(step["PA"], log)
if len(pipeline) == 0:
if not pa.is_compatible(type(img)):
log.critical(
"Pipeline configuration is incorrect! check configuration {} {}"
.format(img, pa.is_compatible(img)))
sys.exit("Wrong pipeline configuration")
else:
if not pa.is_compatible(pipeline[-1][0].get_output_type()):
log.critical(
"Pipeline configuration is incorrect! check configuration")
log.critical(
"Can't connect input of {} to output of {}. Incompatible types"
.format(pa.name, pipeline[-1][0].name))
sys.exit("Wrong pipeline configuration")
qas = []
for q in step["QAs"]:
qa = getobject(q, log)
if not qa.is_compatible(pa.get_output_type()):
log.warning(
"QA {} can not be used for output of {}. Skipping expecting {} got {} {}"
.format(qa.name, pa.name, qa.__inpType__,
pa.get_output_type(),
qa.is_compatible(pa.get_output_type())))
else:
qas.append(qa)
pipeline.append([pa, qas])
return pipeline, convdict
def runpipeline(pl, convdict, conf, mergeQA=False):
"""
Runs the quicklook pipeline as configured
Args:
pl: is a list of [pa,qas] where pa is a pipeline step and qas the corresponding
qas for that pa
convdict: converted dictionary e.g : conf["IMAGE"] is the real psf file
but convdict["IMAGE"] is like lvmspec.image.Image object and so on.
details in setup_pipeline method below for examples.
conf: a configured dictionary, read from the configuration yaml file.
e.g: conf=configdict=yaml.load(open('configfile.yaml','rb'))
mergedQA: if True, outputs the merged QA after the execution of pipeline. Perhaps, this
should always be True, but leaving as option, until configuration and IO settles.
"""
qlog = qllogger.QLLogger()
log = qlog.getlog()
hb = QLHB.QLHeartbeat(log, conf["Period"], conf["Timeout"])
inp = convdict["rawimage"]
paconf = conf["PipeLine"]
qlog = qllogger.QLLogger()
log = qlog.getlog()
passqadict = None #- pass this dict to QAs downstream
schemaMerger = QL_QAMerger(conf['Night'], conf['Expid'], conf['Flavor'],
conf['Camera'])
QAresults = [
] #- merged QA list for the whole pipeline. This will be reorganized for databasing after the pipeline executes
for s, step in enumerate(pl):
log.info("Starting to run step {}".format(paconf[s]["StepName"]))
pa = step[0]
pargs = mapkeywords(step[0].config["kwargs"], convdict)
schemaStep = schemaMerger.addPipelineStep(paconf[s]["StepName"])
try:
hb.start("Running {}".format(step[0].name))
oldinp = inp #- copy for QAs that need to see earlier input
inp = pa(inp, **pargs)
except Exception as e:
log.critical("Failed to run PA {} error was {}".format(
step[0].name, e),
exc_info=True)
sys.exit("Failed to run PA {}".format(step[0].name))
qaresult = {}
for qa in step[1]:
try:
qargs = mapkeywords(qa.config["kwargs"], convdict)
hb.start("Running {}".format(qa.name))
qargs[
"dict_countbins"] = passqadict #- pass this to all QA downstream
if qa.name == "RESIDUAL" or qa.name == "Sky_Residual":
res = qa(inp[0], inp[1], **qargs)
else:
if isinstance(inp, tuple):
res = qa(inp[0], **qargs)
else:
res = qa(inp, **qargs)
if qa.name == "COUNTBINS" or qa.name == "CountSpectralBins": #TODO -must run this QA for now. change this later.
passqadict = res
if "qafile" in qargs:
qawriter.write_qa_ql(qargs["qafile"], res)
log.debug("{} {}".format(qa.name, inp))
qaresult[qa.name] = res
schemaStep.addParams(res['PARAMS'])
schemaStep.addMetrics(res['METRICS'])
except Exception as e:
log.warning("Failed to run QA {}. Got Exception {}".format(
qa.name, e),
exc_info=True)
if len(qaresult):
if conf["DumpIntermediates"]:
f = open(paconf[s]["OutputFile"], "w")
f.write(yaml.dump(yamlify(qaresult)))
hb.stop("Step {} finished. Output is in {} ".format(
paconf[s]["StepName"], paconf[s]["OutputFile"]))
else:
hb.stop("Step {} finished.".format(paconf[s]["StepName"]))
QAresults.append([pa.name, qaresult])
hb.stop("Pipeline processing finished. Serializing result")
#- merge QAs for this pipeline execution
if mergeQA is True:
# from lvmspec.quicklook.util import merge_QAs
# log.info("Merging all the QAs for this pipeline execution")
# merge_QAs(QAresults,conf)
log.debug("Dumping mergedQAs")
from lvmspec.io import findfile
ftype = 'ql_mergedQA_file'
specprod_dir = os.environ[
'QL_SPEC_REDUX'] if 'QL_SPEC_REDUX' in os.environ else ""
if conf['Flavor'] == 'arcs':
ftype = 'ql_mergedQAarc_file'
destFile = findfile(ftype,
night=conf['Night'],
expid=conf['Expid'],
camera=conf['Camera'],
specprod_dir=specprod_dir)
# this will overwrite the file. above function returns same name for different QL executions
# results will be erased.
schemaMerger.writeToFile(destFile)
log.info("Wrote merged QA file {}".format(destFile))
if isinstance(inp, tuple):
return inp[0]
else:
return inp
def testconfig(outfilename="qlconfig.yaml"):
"""
Make a test Config file, should be provided by the QL framework
Below the %% variables are replaced by actual object when the respective
algorithm is executed.
"""
qlog = qllogger.QLLogger()
log = qlog.getlog()
url = None #- QA output will be posted to QLF if set true
conf = {
'BiasImage':
os.environ['BIASIMAGE'], # path to bias image
'DarkImage':
os.environ['DARKIMAGE'], # path to dark image
'DataType':
'Exposure', # type of input ['Exposure','Arc','Dark']
'DebugLevel':
20, # debug level
'Period':
5.0, # Heartbeat Period (Secs)
'Timeout':
120.0, # Heartbeat Timeout (Secs)
'DumpIntermediates':
False, # whether to dump output of each step
'FiberFlatFile':
os.environ['FIBERFLATFILE'], # path to fiber flat field file
'FiberFlatImage':
os.environ['FIBERFLATIMAGE'], # for psf calibration
'ArcLampImage':
os.environ['ARCLAMPIMAGE'], # for psf calibration
'SkyFile':
os.environ['SKYFILE'], # path to Sky file
'FiberMap':
os.environ['FIBERMAP'], # path to fiber map
'RawImage':
os.environ['PIXIMAGE'], #path to input image
'PixelFlat':
os.environ['PIXELFLAT'], #path to pixel flat image
'PSFFile':
os.environ[
'PSFFILE'], # for boxcar this can be bootcalib psf or specter psf file
#'PSFFile_sp':os.environ['PSFFILE_sp'], # .../lvmmodel/data/specpsf/psf-r.fits (for running 2d extraction)
'basePath':
os.environ['LVMMODEL'],
'OutputFile':
'lastframe_QL-r0-00000004.fits', # output file from last pipeline step. Need to output intermediate steps? Most likely after boxcar extraction?
'PipeLine': [
{
'PA': {
"ModuleName": "lvmspec.quicklook.procalgs",
"ClassName": "BiasSubtraction",
"Name": "Bias Subtraction",
"kwargs": {
"BiasImage": "%%BiasImage"
}
},
'QAs': [{
"ModuleName": "lvmspec.qa.qa_quicklook",
"ClassName": "Get_RMS",
"Name": "Get RMS",
"kwargs": {},
}, {
"ModuleName": "lvmspec.qa.qa_quicklook",
"ClassName": "Count_Pixels",
"Name": "Count Pixels",
"kwargs": {
'Width': 3.
}
}],
"StepName":
"Preprocessing-Bias Subtraction",
"OutputFile":
"QA_biassubtraction.yaml"
},
{
'PA': {
"ModuleName": "lvmspec.quicklook.procalgs",
"ClassName": "DarkSubtraction",
"Name": "Dark Subtraction",
"kwargs": {
"DarkImage": "%%DarkImage"
}
},
'QAs': [{
"ModuleName": "lvmspec.qa.qa_quicklook",
"ClassName": "Get_RMS",
"Name": "Get RMS",
"kwargs": {},
}, {
"ModuleName": "lvmspec.qa.qa_quicklook",
"ClassName": "Count_Pixels",
"Name": "Count Pixels",
"kwargs": {
'Width': 3.
},
}],
"StepName":
"Preprocessing-Dark Subtraction",
"OutputFile":
"QA_darksubtraction.yaml"
},
{
'PA': {
"ModuleName": "lvmspec.quicklook.procalgs",
"ClassName": "PixelFlattening",
"Name": "Pixel Flattening",
"kwargs": {
"PixelFlat": "%%PixelFlat"
}
},
'QAs': [{
"ModuleName": "lvmspec.qa.qa_quicklook",
"ClassName": "Get_RMS",
"Name": "Get RMS",
"kwargs": {},
}, {
"ModuleName": "lvmspec.qa.qa_quicklook",
"ClassName": "Count_Pixels",
"Name": "Count Pixels",
"kwargs": {
'Width': 3.
},
}],
"StepName":
"Preprocessing-Pixel Flattening",
"OutputFile":
"QA_pixelflattening.yaml"
},
#{'PA':{"ModuleName":"lvmspec.quicklook.procalgs",
# "ClassName":"BoxcarExtraction",
# "Name":"Boxcar Extraction",
# "kwargs":{"PSFFile":"%%PSFFile",
# "BoxWidth":2.5,
# "DeltaW":0.5,
# "Nspec":500
# }
# },
# 'QAs':[],
# "StepName":"Boxcar Extration",
# "OutputFile":"QA_boxcarextraction.yaml"
# },
{
'PA': {
"ModuleName": "lvmspec.quicklook.procalgs",
"ClassName": "Extraction_2d",
"Name": "2D Extraction",
"kwargs": {
"PSFFile_sp":
"/home/govinda/Desi/lvmmodel/data/specpsf/psf-r.fits",
"Nspec": 10,
"Wavelength": "5630,7740,0.5",
"FiberMap":
"%%FiberMap" #need this for qa_skysub downstream as well.
}
},
'QAs': [{
"ModuleName": "lvmspec.qa.qa_quicklook",
"ClassName": "CountSpectralBins",
"Name": "Count Bins above n",
"kwargs": {
'thresh': 100,
'camera': "r0",
'expid': "%08d" % 2,
'url': url
}
}],
"StepName":
"2D Extraction",
"OutputFile":
"qa-extract-r0-00000002.yaml"
},
{
'PA': {
"ModuleName": "lvmspec.quicklook.procalgs",
"ClassName": "ApplyFiberFlat",
"Name": "Apply Fiberflat",
"kwargs": {
"FiberFlatFile": "%%FiberFlatFile"
}
},
'QAs': [],
"StepName": "Apply Fiberflat",
"Outputfile": "apply_fiberflat_QA.yaml"
},
{
'PA': {
"ModuleName": "lvmspec.quicklook.procalgs",
"ClassName": "SubtractSky",
"Name": "Sky Subtraction",
"kwargs": {
"SkyFile": "%%SkyFile"
}
},
'QAs': [{
"ModuleName": "lvmspec.qa.qa_quicklook",
"ClassName": "Calculate_SNR",
"Name": "Calculate Signal-to-Noise ratio",
"kwargs": {
'SkyFile': "%%SkyFile",
'camera': "r0",
'expid': "%08d" % 2,
'url': url
}
}],
"StepName":
"Sky Subtraction",
"OutputFile":
"qa-r0-00000002.yaml"
}
]
}
if "yaml" in outfilename:
f = open(outfilename, "w")
yaml.dump(conf, f)
f.close()
else:
log.warning(
"Only yaml defined. Use yaml format in the output config file")
sys.exit(0)
def __init__(self,
configfile,
night,
camera,
expid,
amps=True,
rawdata_dir=None,
specprod_dir=None,
outdir=None,
qlf=False):
"""
configfile: a configuration file for QL eg: lvmspec/data/quicklook/qlconfig_dark.yaml
night: night for the data to process, eg.'20191015'
camera: which camera to process eg 'r0'
expid: exposure id for the image to be processed
amps: for outputing amps level QA
Note:
rawdata_dir and specprod_dir: if not None, overrides the standard DESI convention
"""
#- load the config file and extract
self.conf = yaml.load(open(configfile, "r"))
self.night = night
self.expid = expid
self.camera = camera
self.amps = amps
self.rawdata_dir = rawdata_dir
self.specprod_dir = specprod_dir
self.outdir = outdir
self.dumpintermediates = self.conf["WriteIntermediatefiles"]
self.writepixfile = self.conf["WritePixfile"]
self.writeskymodelfile = self.conf["WriteSkyModelfile"]
self.writestaticplots = self.conf["WriteStaticPlots"]
self.usesigma = self.conf["UseResolution"]
self.pipeline = self.conf["Pipeline"]
self.algorithms = self.conf["Algorithms"]
self._palist = Palist(self.pipeline, self.algorithms)
self.pamodule = self._palist.pamodule
self.qamodule = self._palist.qamodule
if "BoxcarExtract" in self.algorithms.keys():
if "wavelength" in self.algorithms["BoxcarExtract"].keys():
self.wavelength = self.algorithms["BoxcarExtract"][
"wavelength"][self.camera[0]]
else:
self.wavelength = None
if "SkySub_QL" in self.algorithms.keys():
if "Calculate_SNR" in self.algorithms["SkySub_QL"]["QA"].keys():
if "QSO_SNR_Residuals" in self.algorithms["SkySub_QL"]["QA"][
"Calculate_SNR"].keys():
self.qso_snr_resid = self.algorithms["SkySub_QL"]["QA"][
"Calculate_SNR"]["QSO_SNR_Residuals"]
else:
self.qso_snr_resid = None
self._qlf = qlf
qlog = qllogger.QLLogger(name="QLConfig")
self.log = qlog.getlog()
self._qaRefKeys = {
"Bias_From_Overscan": "BIAS_AMP",
"Get_RMS": "NOISE_AMP",
"Count_Pixels": "NPIX_AMP",
"Calc_XWSigma": "XWSIGMA",
"CountSpectralBins": "NGOODFIB",
"Sky_Peaks": "PEAKCOUNT",
"Sky_Continuum": "SKYCONT",
"Integrate_Spec": "MAGDIFF_TGT",
"Sky_Residual": "RESIDRMS",
"Calculate_SNR": "FIDSNR_TGT"
}
