def __init__(self, specprod_dir=None, qaprod_dir=None):
""" Class to organize and execute QA for a DESI production
Args:
specprod_dir(str): Path containing the exposures/ directory to use. If the value
is None, then the value of :func:`specprod_root` is used instead.
qaprod_dir(str): Path containing the root path for QA output
Notes:
Attributes:
qa_exps : list
List of QA_Exposure classes, one per exposure in production
data : dict
"""
# Init
if specprod_dir is None:
specprod_dir = specprod_root()
if qaprod_dir is None:
qaprod_dir = qaprod_root()
#
self.specprod_dir = specprod_dir
self.qaprod_dir = qaprod_dir
tmp = specprod_dir.split('/')
self.prod_name = tmp[-1] if (len(tmp[-1]) > 0) else tmp[-2]
# Exposure dict stored as [night][exposure]
self.mexp_dict = {}
# QA Exposure objects
self.qa_exps = []
# dict to hold QA data
# Data Model : key1 = Night(s); key2 = Expids
self.data = {}
#
self.qaexp_outroot = None
def __init__(self, specprod_dir=None, qaprod_dir=None):
""" Class to organize and execute QA for a DESI production
Args:
specprod_dir(str): Path containing the exposures/ directory to use. If the value
is None, then the value of :func:`specprod_root` is used instead.
qaprod_dir(str): Path containing the root path for QA output
Notes:
Attributes:
qa_exps : list
List of QA_Exposure classes, one per exposure in production
data : dict
"""
# Init
if specprod_dir is None:
specprod_dir = specprod_root()
if qaprod_dir is None:
qaprod_dir = qaprod_root()
#
self.specprod_dir = specprod_dir
self.qaprod_dir = qaprod_dir
tmp = specprod_dir.split('/')
self.prod_name = tmp[-1] if (len(tmp[-1]) > 0) else tmp[-2]
# Exposure dict stored as [night][exposure]
self.mexp_dict = {}
# QA Exposure objects
self.qa_exps = []
# dict to hold QA data
# Data Model : key1 = Night(s); key2 = Expids
self.data = {}
#
self.qaexp_outroot = None
def __init__(self,
expid,
night,
flavor=None,
specprod_dir=None,
in_data=None,
qaprod_dir=None,
no_load=False,
multi_root=None,
**kwargs):
"""
Class to organize and execute QA for a DESI Exposure
x.flavor, x.data
Args:
expid: int -- Exposure ID
night: str -- YYYYMMDD
specprod_dir(str): Path containing the exposures/ directory to use. If the value
is None, then the value of :func:`specprod_root` is used instead.
in_data: dict, optional -- Input data
Mainly for reading from disk
no_load: bool, optional -- Do not load QA data (rare)
multi_root: str, optional
Load QA from a slurped file.
This is the root and the path is qaprod_dir
Notes:
Attributes:
All input args become object attributes.
"""
# Init
if not isinstance(expid, int):
raise IOError("expid must be an int at instantiation")
self.expid = expid
self.night = night
self.meta = {}
# Paths
self.specprod_dir = specprod_dir
if qaprod_dir is None:
qaprod_dir = qaprod_root(self.specprod_dir)
self.qaprod_dir = qaprod_dir
# Load meta from frame (ideally)
frames_dict = get_files(filetype=str('frame'),
night=night,
expid=expid,
specprod_dir=self.specprod_dir)
if len(frames_dict) > 0:
frame_file = list(frames_dict.items())[0][1] # Any one will do
frame_meta = read_meta_frame(frame_file)
self.load_meta(frame_meta)
flavor = self.meta['FLAVOR'] # Over-rides any input value
else:
flavor = flavor
assert flavor in desi_params[
'frame_types'], "Unknown flavor {} for night {} expid {}".format(
flavor, night, expid)
if flavor in ['science']:
self.type = 'data'
else:
self.type = 'calib'
self.flavor = flavor
# Internal dicts
self.data = dict(flavor=self.flavor,
expid=self.expid,
night=self.night,
frames={})
# Load?
if no_load:
return
if in_data is None:
self.load_qa_data(multi_root=multi_root, **kwargs)
else:
assert isinstance(in_data, dict)
self.data = in_data
# Others
self.qa_s2n = None
def main(args) :
# imports
import glob
from desispec.io import findfile, makepath
from desispec.io import get_exposures
from desispec.io import get_files, get_nights
from desispec.io import get_reduced_frames
from desispec.io import specprod_root
from desispec.io import qaprod_root
from desispec.qa import utils as qa_utils
import copy
import pdb
# Init
specprod_dir = specprod_root()
# Log
log=get_logger()
log.info("starting")
# Path
if args.qaprod_dir is not None:
qaprod_dir = args.qaprod_dir
else:
qaprod_dir = qaprod_root()
# Channels
if args.channels is not None:
channels = [iarg for iarg in args.channels.split(',')]
else:
channels = ['b','r','z']
# Sky dict
sky_dict = dict(wave=[], skyflux=[], res=[], count=0)
channel_dict = dict(b=copy.deepcopy(sky_dict),
r=copy.deepcopy(sky_dict),
z=copy.deepcopy(sky_dict),
)
# Nights
if args.nights is not None:
nights = [iarg for iarg in args.nights.split(',')]
else:
nights = None
# Exposure plot?
if args.expid is not None:
# Nights
if nights is None:
nights = get_nights()
nights.sort()
# Find the exposure
for night in nights:
if args.expid in get_exposures(night, specprod_dir=specprod_dir):
frames_dict = get_files(filetype=str('cframe'), night=night,
expid=args.expid, specprod_dir=specprod_dir)
# Loop on channel
#for channel in ['b','r','z']:
for channel in ['z']:
channel_dict[channel]['cameras'] = []
for camera, cframe_fil in frames_dict.items():
if channel in camera:
sky_file = findfile(str('sky'), night=night, camera=camera,
expid=args.expid, specprod_dir=specprod_dir)
wave, flux, res, _ = qa_utils.get_skyres(cframe_fil)
# Append
channel_dict[channel]['wave'].append(wave)
channel_dict[channel]['skyflux'].append(np.log10(np.maximum(flux,1e-1)))
channel_dict[channel]['res'].append(res)
channel_dict[channel]['cameras'].append(camera)
channel_dict[channel]['count'] += 1
if channel_dict[channel]['count'] > 0:
from desispec.qa.qa_plots import skysub_resid_series # Hidden to help with debugging
skysub_resid_series(channel_dict[channel], 'wave',
outfile=qaprod_dir+'/QA_skyresid_wave_expid_{:d}{:s}.png'.format(args.expid, channel))
skysub_resid_series(channel_dict[channel], 'flux',
outfile=qaprod_dir+'/QA_skyresid_flux_expid_{:d}{:s}.png'.format(args.expid, channel))
return
# Skyline
if args.skyline:
from desispec.qa.qa_plots import skyline_resid
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
log.info("Loading sky residuals for {:d} cframes".format(len(cframes)))
if len(cframes) == 1:
log.error('len(cframes)==1; starting debugging')
pdb.set_trace() # Need to call differently
else:
sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(
cframes, flatten=False)
# Plot
outfile=args.outdir+'/skyline_{:s}.png'.format(channel)
log.info("Plotting to {:s}".format(outfile))
skyline_resid(channel, sky_wave, sky_flux, sky_res, sky_ivar,
outfile=outfile)
return
# Full Prod Plot?
if args.prod:
from desispec.qa.qa_plots import skysub_resid_dual
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
log.info("Loading sky residuals for {:d} cframes".format(len(cframes)))
sky_wave, sky_flux, sky_res, _ = qa_utils.get_skyres(cframes)
# Plot
outfile=qaprod_dir+'/skyresid_prod_dual_{:s}.png'.format(channel)
makepath(outfile)
log.info("Plotting to {:s}".format(outfile))
skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile=outfile)
return
# Test sky noise for Gaussianity
if args.gauss:
from desispec.qa.qa_plots import skysub_gauss
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
# Cut down for debugging
#cframes = [cframes[ii] for ii in range(15)]
#
log.info("Loading sky residuals for {:d} cframes".format(len(cframes)))
sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(cframes)
# Plot
log.info("Plotting..")
outfile=qaprod_dir+'/skyresid_prod_gauss_{:s}.png'.format(channel)
makepath(outfile)
skysub_gauss(sky_wave, sky_flux, sky_res, sky_ivar,
outfile=outfile)
return