def __init__(self, specprod_dir=None, **kwargs):
""" Class to organize and execute QA for a DESI production
Args:
specprod_dir(str): Path containing the exposures/ directory to use. If the value
is None, then the value of :func:`specprod_root` is used instead.
Notes:
Attributes:
qa_exps : list
List of QA_Exposure classes, one per exposure in production
data : dict
"""
if specprod_dir is None:
specprod_dir = specprod_root()
self.specprod_dir = specprod_dir
# Init
QA_MultiExp.__init__(self, specprod_dir=specprod_dir, **kwargs)
# Load up exposures for the full production
nights = get_nights(specprod_dir=self.specprod_dir)
for night in nights:
self.mexp_dict[night] = {}
'''
for exposure in get_exposures(night, specprod_dir = self.specprod_dir):
# Object only??
frames_dict = get_files(filetype = str('frame'), night = night,
expid = exposure, specprod_dir = self.specprod_dir)
self.mexp_dict[night][exposure] = frames_dict
'''
# Output file names
self.qaexp_outroot = self.qaprod_dir+'/'+self.prod_name+'_qa'
# Nights list
self.qa_nights = []
def __init__(self, specprod_dir=None, **kwargs):
""" Class to organize and execute QA for a DESI production
Args:
specprod_dir(str): Path containing the exposures/ directory to use. If the value
is None, then the value of :func:`specprod_root` is used instead.
Notes:
Attributes:
qa_exps : list
List of QA_Exposure classes, one per exposure in production
data : dict
"""
if specprod_dir is None:
specprod_dir = specprod_root()
self.specprod_dir = specprod_dir
# Init
QA_MultiExp.__init__(self, specprod_dir=specprod_dir, **kwargs)
# Load up exposures for the full production
nights = get_nights(specprod_dir=self.specprod_dir)
for night in nights:
self.mexp_dict[night] = {}
for exposure in get_exposures(night,
specprod_dir=self.specprod_dir):
# Object only??
frames_dict = get_files(filetype=str('frame'),
night=night,
expid=exposure,
specprod_dir=self.specprod_dir)
self.mexp_dict[night][exposure] = frames_dict
# Output file names
self.qaexp_outroot = self.qaprod_dir + '/' + self.prod_name + '_qa'
# Nights list
self.qa_nights = []
def __init__(self, night, **kwargs):
""" Class to organize and execute QA for a DESI production
Args:
specprod_dir(str): Path containing the exposures/ directory to use. If the value
is None, then the value of :func:`specprod_root` is used instead.
Notes:
**kwargs are passed to QA_MultiExp
Attributes:
qa_exps : list
List of QA_Exposure classes, one per exposure in production
data : dict
"""
# Init
self.night = night
# Instantiate
QA_MultiExp.__init__(self, **kwargs)
# Load up exposures for the full production
nights = get_nights(specprod_dir=self.specprod_dir)
# Check the night exists
if self.night not in nights:
raise IOError("Bad input night for this production")
# Load up
self.mexp_dict[self.night] = {}
for exposure in get_exposures(self.night,
specprod_dir=self.specprod_dir):
# Object only??
frames_dict = get_files(filetype=str('frame'),
night=self.night,
expid=exposure,
specprod_dir=self.specprod_dir)
self.mexp_dict[self.night][exposure] = frames_dict
# Output file names
self.qaexp_outroot = self.qaprod_dir + '/' + self.night + '_qa'
def make_exposures():
""" Generate HTML to organize exposure HTML
Parameters
----------
Returns
-------
links : str
body : str
"""
# Organized HTML
html_file = meta.findfile('qa_exposures_html')
html_path, _ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Exposures QA')
# Loop on Nights
nights = get_nights()
nights.sort()
links = ''
body = ''
for night in nights:
# HTML
f.write('<h2> Night -- {:s} </h2>\n'.format(night))
f.write('<h3><ul>\n')
# Loop on expsoures
for expid in get_exposures(night):
if not os.path.exists(html_path + '/' + night +
'/{:08d}'.format(expid)):
continue
# Link
f.write(
'<li><a href="{:s}/{:08d}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'
.format(night, expid, expid, expid))
# Generate Exposure html
make_exposure(night, expid)
f.write('</ul></h3>\n')
# Finish
finish(f, body)
def calib(qaprod_dir=None, specprod_dir=None):
""" Generate HTML to orgainze calib HTML
"""
# Organized HTML
html_file = meta.findfile('qa_calib_html', qaprod_dir=qaprod_dir)
html_path, _ = os.path.split(html_file)
makepath(html_file)
# Open
f = open(html_file, 'w')
init(f, 'Calibration QA')
# Loop on Nights
nights = get_nights(sub_folder='calibnight', specprod_dir=specprod_dir)
nights.sort()
links = ''
body = ''
for night in nights:
all_png = glob.glob(html_path + '/' + night + '/qa*.png')
if len(all_png) == 0:
continue
# Find expid
expids = []
for png in all_png:
expids.append(int(png[-12:-4])) # A bit risky
expids = np.unique(expids)
expids.sort()
f.write('<h2> Night -- {:s} </h2>\n'.format(night))
f.write('<h3><ul>\n')
for expid in expids:
# Link
f.write(
'<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.
format(night, expid, expid))
# Generate Exposure html
calib_exp(night, expid, qaprod_dir=qaprod_dir)
f.write('</ul></h3>\n')
# Finish
finish(f, body)
# Return
return links, body
def make_exposures(qaprod_dir=None):
""" Generate HTML to organize exposure HTML
Parameters
----------
Returns
-------
links : str
body : str
"""
# Organized HTML
html_file = meta.findfile('qa_exposures_html', qaprod_dir=qaprod_dir)
html_path,_ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Exposures QA')
# Loop on Nights
nights = get_nights()
nights.sort()
links = ''
body = ''
for night in nights:
# HTML
f.write('<h2> Night -- {:s} </h2>\n'.format(night))
f.write('<h3><ul>\n')
# Loop on expsoures
for expid in get_exposures(night):
if not os.path.exists(html_path+'/'+night+'/{:08d}'.format(expid)):
continue
# Link
f.write('<li><a href="{:s}/{:08d}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid, expid))
# Generate Exposure html
make_exposure(night, expid, qaprod_dir=qaprod_dir)
f.write('</ul></h3>\n')
# Finish
finish(f,body)
def calib(qaprod_dir=None):
""" Generate HTML to orgainze calib HTML
"""
# Organized HTML
html_file = meta.findfile('qa_calib_html', qaprod_dir=qaprod_dir)
html_path,_ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Calibration QA')
# Loop on Nights
nights = get_nights(sub_folder='calibnight')
nights.sort()
links = ''
body = ''
for night in nights:
all_png = glob.glob(html_path+'/'+night+'/qa*.png')
if len(all_png) == 0:
continue
# Find expid
expids = []
for png in all_png:
expids.append(int(png[-12:-4])) # A bit risky
expids = np.unique(expids)
expids.sort()
f.write('<h2> Night -- {:s} </h2>\n'.format(night))
f.write('<h3><ul>\n')
for expid in expids:
# Link
f.write('<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid))
# Generate Exposure html
calib_exp(night, expid, qaprod_dir=qaprod_dir)
f.write('</ul></h3>\n')
# Finish
finish(f,body)
# Return
return links, body
def main(args):
# imports
import glob
from desispec.io import findfile
from desispec.io import get_exposures
from desispec.io import get_files, get_nights
from desispec.io import read_frame
from desispec.io import get_reduced_frames
from desispec.io.sky import read_sky
from desispec.io import specprod_root
from desispec.qa import utils as qa_utils
import copy
import pdb
# Log
log = get_logger()
log.info("starting")
# Path
if args.reduxdir is not None:
specprod_dir = args.reduxdir
else:
specprod_dir = specprod_root()
# Channels
if args.channels is not None:
channels = [iarg for iarg in args.channels.split(',')]
else:
channels = ['b', 'r', 'z']
# Sky dict
sky_dict = dict(wave=[], skyflux=[], res=[], count=0)
channel_dict = dict(
b=copy.deepcopy(sky_dict),
r=copy.deepcopy(sky_dict),
z=copy.deepcopy(sky_dict),
)
# Exposure plot?
if args.expid is not None:
# Nights
path_nights = glob.glob(specprod_dir + '/exposures/*')
if nights is None:
nights = get_nights()
nights.sort()
# Find the exposure
for night in nights:
if args.expid in get_exposures(night, specprod_dir=specprod_dir):
frames_dict = get_files(filetype=str('cframe'),
night=night,
expid=args.expid,
specprod_dir=specprod_dir)
# Loop on channel
#for channel in ['b','r','z']:
for channel in ['z']:
channel_dict[channel]['cameras'] = []
for camera, cframe_fil in frames_dict.items():
if channel in camera:
sky_file = findfile(str('sky'),
night=night,
camera=camera,
expid=args.expid,
specprod_dir=specprod_dir)
wave, flux, res, _ = qa_utils.get_skyres(
cframe_fil)
# Append
channel_dict[channel]['wave'].append(wave)
channel_dict[channel]['skyflux'].append(
np.log10(np.maximum(flux, 1e-1)))
channel_dict[channel]['res'].append(res)
channel_dict[channel]['cameras'].append(camera)
channel_dict[channel]['count'] += 1
if channel_dict[channel]['count'] > 0:
from desispec.qa.qa_plots import skysub_resid_series # Hidden to help with debugging
skysub_resid_series(
channel_dict[channel],
'wave',
outfile='QA_skyresid_wave_expid_{:d}{:s}.png'.
format(args.expid, channel))
skysub_resid_series(
channel_dict[channel],
'flux',
outfile='QA_skyresid_flux_expid_{:d}{:s}.png'.
format(args.expid, channel))
return
# Nights
if args.nights is not None:
nights = [iarg for iarg in args.nights.split(',')]
else:
nights = None
# Full Prod Plot?
if args.prod:
from desispec.qa.qa_plots import skysub_resid_dual
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
log.info("Loading sky residuals for {:d} cframes".format(
len(cframes)))
sky_wave, sky_flux, sky_res, _ = qa_utils.get_skyres(cframes)
# Plot
outfile = 'QA/skyresid_prod_dual_{:s}.png'.format(channel)
log.info("Plotting to {:s}".format(outfile))
skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile=outfile)
return
# Full Prod Plot?
if args.gauss:
from desispec.qa.qa_plots import skysub_gauss
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
# Cut down for debugging
#cframes = [cframes[ii] for ii in range(15)]
#
log.info("Loading sky residuals for {:d} cframes".format(
len(cframes)))
sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(
cframes)
# Plot
log.info("Plotting..")
skysub_gauss(
sky_wave,
sky_flux,
sky_res,
sky_ivar,
outfile='skyresid_prod_gauss_{:s}.png'.format(channel))
return
def main(args) :
# imports
import glob
from desispec.io import findfile, makepath
from desispec.io import get_exposures
from desispec.io import get_files, get_nights
from desispec.io import get_reduced_frames
from desispec.io import specprod_root
from desispec.io import qaprod_root
from desispec.qa import utils as qa_utils
import copy
import pdb
# Init
specprod_dir = specprod_root()
# Log
log=get_logger()
log.info("starting")
# Path
if args.qaprod_dir is not None:
qaprod_dir = args.qaprod_dir
else:
qaprod_dir = qaprod_root()
# Channels
if args.channels is not None:
channels = [iarg for iarg in args.channels.split(',')]
else:
channels = ['b','r','z']
# Sky dict
sky_dict = dict(wave=[], skyflux=[], res=[], count=0)
channel_dict = dict(b=copy.deepcopy(sky_dict),
r=copy.deepcopy(sky_dict),
z=copy.deepcopy(sky_dict),
)
# Nights
if args.nights is not None:
nights = [iarg for iarg in args.nights.split(',')]
else:
nights = None
# Exposure plot?
if args.expid is not None:
# Nights
if nights is None:
nights = get_nights()
nights.sort()
# Find the exposure
for night in nights:
if args.expid in get_exposures(night, specprod_dir=specprod_dir):
frames_dict = get_files(filetype=str('cframe'), night=night,
expid=args.expid, specprod_dir=specprod_dir)
# Loop on channel
#for channel in ['b','r','z']:
for channel in ['z']:
channel_dict[channel]['cameras'] = []
for camera, cframe_fil in frames_dict.items():
if channel in camera:
sky_file = findfile(str('sky'), night=night, camera=camera,
expid=args.expid, specprod_dir=specprod_dir)
wave, flux, res, _ = qa_utils.get_skyres(cframe_fil)
# Append
channel_dict[channel]['wave'].append(wave)
channel_dict[channel]['skyflux'].append(np.log10(np.maximum(flux,1e-1)))
channel_dict[channel]['res'].append(res)
channel_dict[channel]['cameras'].append(camera)
channel_dict[channel]['count'] += 1
if channel_dict[channel]['count'] > 0:
from desispec.qa.qa_plots import skysub_resid_series # Hidden to help with debugging
skysub_resid_series(channel_dict[channel], 'wave',
outfile=qaprod_dir+'/QA_skyresid_wave_expid_{:d}{:s}.png'.format(args.expid, channel))
skysub_resid_series(channel_dict[channel], 'flux',
outfile=qaprod_dir+'/QA_skyresid_flux_expid_{:d}{:s}.png'.format(args.expid, channel))
return
# Skyline
if args.skyline:
from desispec.qa.qa_plots import skyline_resid
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
log.info("Loading sky residuals for {:d} cframes".format(len(cframes)))
if len(cframes) == 1:
log.error('len(cframes)==1; starting debugging')
pdb.set_trace() # Need to call differently
else:
sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(
cframes, flatten=False)
# Plot
outfile=args.outdir+'/skyline_{:s}.png'.format(channel)
log.info("Plotting to {:s}".format(outfile))
skyline_resid(channel, sky_wave, sky_flux, sky_res, sky_ivar,
outfile=outfile)
return
# Full Prod Plot?
if args.prod:
from desispec.qa.qa_plots import skysub_resid_dual
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
log.info("Loading sky residuals for {:d} cframes".format(len(cframes)))
sky_wave, sky_flux, sky_res, _ = qa_utils.get_skyres(cframes)
# Plot
outfile=qaprod_dir+'/skyresid_prod_dual_{:s}.png'.format(channel)
makepath(outfile)
log.info("Plotting to {:s}".format(outfile))
skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile=outfile)
return
# Test sky noise for Gaussianity
if args.gauss:
from desispec.qa.qa_plots import skysub_gauss
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
# Cut down for debugging
#cframes = [cframes[ii] for ii in range(15)]
#
log.info("Loading sky residuals for {:d} cframes".format(len(cframes)))
sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(cframes)
# Plot
log.info("Plotting..")
outfile=qaprod_dir+'/skyresid_prod_gauss_{:s}.png'.format(channel)
makepath(outfile)
skysub_gauss(sky_wave, sky_flux, sky_res, sky_ivar,
outfile=outfile)
return