def exposure_fiberflat(channel, expid, metric, outfile=None):
""" Generate an Exposure level plot of a FiberFlat metric
Args:
channel: str, e.g. 'b', 'r', 'z'
expid: int
metric: str, allowed entires are: ['meanflux']
Returns:
"""
from desispec.io.meta import find_exposure_night, findfile
from desispec.io.frame import read_meta_frame, read_frame
from desispec.io.fiberflat import read_fiberflat
from desimodel.focalplane import fiber_area_arcsec2
log = get_logger()
# Find exposure
night = find_exposure_night(expid)
# Search for frames with the input channel
frame0 = findfile('frame', camera=channel+'0', night=night, expid=expid)
if not os.path.exists(frame0):
log.fatal("No Frame 0 for channel={:s} and expid={:d}".format(channel, expid))
# Confirm frame is a Flat
fmeta = read_meta_frame(frame0)
assert fmeta['FLAVOR'].strip() == 'flat'
# Load up all the frames
x,y,metrics = [],[],[]
for wedge in range(10):
# Load
frame_file = findfile('frame', camera=channel+'{:d}'.format(wedge), night=night, expid=expid)
fiber_file = findfile('fiberflat', camera=channel+'{:d}'.format(wedge), night=night, expid=expid)
try:
frame = read_frame(frame_file)
except:
continue
else:
fiberflat = read_fiberflat(fiber_file)
fibermap = frame.fibermap
gdp = fiberflat.mask == 0
# X,Y
x.append([fibermap['X_TARGET']])
y.append([fibermap['Y_TARGET']])
area = fiber_area_arcsec2(x[-1], y[-1])
mean_area = np.mean(area)
# Metric
if metric == 'meanflux':
mean_norm = np.mean(fiberflat.fiberflat*gdp,axis=1) / (area / mean_area)
metrics.append([mean_norm])
# Cocatenate
x = np.concatenate(x)
y = np.concatenate(y)
metrics = np.concatenate(metrics)
# Plot
if outfile is None:
outfile='qa_{:08d}_{:s}_fiberflat.png'.format(expid, channel)
exposure_map(x,y,metrics, mlbl='Mean Flux',
title='Mean Flux for Exposure {:08d}, Channel {:s}'.format(expid, channel),
outfile=outfile)
def make_exposure(night, expid, qaprod_dir=None):
""" Generate HTML for exposure PNGs
Parameters
----------
setup : str
cbset : str
det : int
Returns
-------
links : str
body : str
"""
# File name
html_file = meta.findfile('qa_exposure_html',
night=night,
expid=expid,
qaprod_dir=qaprod_dir)
html_path, _ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Exposure QA')
links = ''
body = ''
# Loop on Nights
for ctype in ['sky', 'flux']:
#
all_png = glob.glob(html_path +
'/qa-{:s}-*-{:08d}.png'.format(ctype, expid))
all_png.sort()
if len(all_png) == 0:
continue
# Type
links += '<h2> {:s} Calib</h2>\n'.format(ctype)
for png in all_png:
_, png_file = os.path.split(png)
# Image
href = "{:s}".format(png_file[:-4])
links += '<li><a class="reference internal" href="#{:s}">{:s}</a></li>\n'.format(
href, href)
body += '<div class="section" id="{:s}">\n'.format(href)
body += '<img class ="research" src="{:s}" width="100%" height="auto"/>\n'.format(
png_file)
#f.write('<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid))
f.write('<ul>\n')
f.write(links)
f.write('</ul>\n')
f.write(body)
# Finish
finish(f, '')
# Return
return links, body
def make_exposure(night, expid, qaprod_dir=None):
""" Generate HTML for exposure PNGs
Parameters
----------
setup : str
cbset : str
det : int
Returns
-------
links : str
body : str
"""
# File name
html_file = meta.findfile('qa_exposure_html', night=night, expid=expid, qaprod_dir=qaprod_dir)
html_path,_ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Exposure QA')
links = ''
body = ''
# Loop on Nights
for ctype in ['sky', 'flux']:
#
all_png = glob.glob(html_path+'/qa-{:s}-*-{:08d}.png'.format(ctype,expid))
all_png.sort()
if len(all_png) == 0:
continue
# Type
links += '<h2> {:s} Calib</h2>\n'.format(ctype)
for png in all_png:
_,png_file = os.path.split(png)
# Image
href="{:s}".format(png_file[:-4])
links += '<li><a class="reference internal" href="#{:s}">{:s}</a></li>\n'.format(href, href)
body += '<div class="section" id="{:s}">\n'.format(href)
body += '<img class ="research" src="{:s}" width="100%" height="auto"/>\n'.format(png_file)
#f.write('<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid))
f.write('<ul>\n')
f.write(links)
f.write('</ul>\n')
f.write(body)
# Finish
finish(f,'')
# Return
return links, body
def calib_exp(night, expid):
""" Geneate HTML for calib exposure PNGs
Args:
night:
expid:
Returns:
"""
# File name
html_file = meta.findfile('qa_calib_exp_html', night=night, expid=expid)
html_path, _ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Calibration Exposure QA')
# Loop on Nights
for ctype in ['flat']:
links = ''
body = ''
#
all_png = glob.glob(html_path +
'/qa-{:s}-*-{:08d}.png'.format(ctype, expid))
all_png.sort()
if len(all_png) == 0:
continue
# Type
links += '<h2> {:s} Calib</h2>\n'.format(ctype)
for png in all_png:
_, png_file = os.path.split(png)
# Image
href = "{:s}".format(png_file[:-4])
links += '<li><a class="reference internal" href="#{:s}">{:s}</a></li>\n'.format(
href, href)
body += '<div class="section" id="{:s}">\n'.format(href)
body += '<img class ="research" src="{:s}" width="100%" height="auto"/>\n'.format(
png_file)
#f.write('<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid))
f.write('<ul>\n')
f.write(links)
f.write('</ul>\n')
f.write(body)
# Finish
finish(f, '')
# Return
return links, body
def make_exposures():
""" Generate HTML to organize exposure HTML
Parameters
----------
Returns
-------
links : str
body : str
"""
# Organized HTML
html_file = meta.findfile('qa_exposures_html')
html_path, _ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Exposures QA')
# Loop on Nights
nights = get_nights()
nights.sort()
links = ''
body = ''
for night in nights:
# HTML
f.write('<h2> Night -- {:s} </h2>\n'.format(night))
f.write('<h3><ul>\n')
# Loop on expsoures
for expid in get_exposures(night):
if not os.path.exists(html_path + '/' + night +
'/{:08d}'.format(expid)):
continue
# Link
f.write(
'<li><a href="{:s}/{:08d}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'
.format(night, expid, expid, expid))
# Generate Exposure html
make_exposure(night, expid)
f.write('</ul></h3>\n')
# Finish
finish(f, body)
def calib(qaprod_dir=None, specprod_dir=None):
""" Generate HTML to orgainze calib HTML
"""
# Organized HTML
html_file = meta.findfile('qa_calib_html', qaprod_dir=qaprod_dir)
html_path, _ = os.path.split(html_file)
makepath(html_file)
# Open
f = open(html_file, 'w')
init(f, 'Calibration QA')
# Loop on Nights
nights = get_nights(sub_folder='calibnight', specprod_dir=specprod_dir)
nights.sort()
links = ''
body = ''
for night in nights:
all_png = glob.glob(html_path + '/' + night + '/qa*.png')
if len(all_png) == 0:
continue
# Find expid
expids = []
for png in all_png:
expids.append(int(png[-12:-4])) # A bit risky
expids = np.unique(expids)
expids.sort()
f.write('<h2> Night -- {:s} </h2>\n'.format(night))
f.write('<h3><ul>\n')
for expid in expids:
# Link
f.write(
'<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.
format(night, expid, expid))
# Generate Exposure html
calib_exp(night, expid, qaprod_dir=qaprod_dir)
f.write('</ul></h3>\n')
# Finish
finish(f, body)
# Return
return links, body
def make_exposures(qaprod_dir=None):
""" Generate HTML to organize exposure HTML
Parameters
----------
Returns
-------
links : str
body : str
"""
# Organized HTML
html_file = meta.findfile('qa_exposures_html', qaprod_dir=qaprod_dir)
html_path,_ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Exposures QA')
# Loop on Nights
nights = get_nights()
nights.sort()
links = ''
body = ''
for night in nights:
# HTML
f.write('<h2> Night -- {:s} </h2>\n'.format(night))
f.write('<h3><ul>\n')
# Loop on expsoures
for expid in get_exposures(night):
if not os.path.exists(html_path+'/'+night+'/{:08d}'.format(expid)):
continue
# Link
f.write('<li><a href="{:s}/{:08d}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid, expid))
# Generate Exposure html
make_exposure(night, expid, qaprod_dir=qaprod_dir)
f.write('</ul></h3>\n')
# Finish
finish(f,body)
def calib(qaprod_dir=None):
""" Generate HTML to orgainze calib HTML
"""
# Organized HTML
html_file = meta.findfile('qa_calib_html', qaprod_dir=qaprod_dir)
html_path,_ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Calibration QA')
# Loop on Nights
nights = get_nights(sub_folder='calibnight')
nights.sort()
links = ''
body = ''
for night in nights:
all_png = glob.glob(html_path+'/'+night+'/qa*.png')
if len(all_png) == 0:
continue
# Find expid
expids = []
for png in all_png:
expids.append(int(png[-12:-4])) # A bit risky
expids = np.unique(expids)
expids.sort()
f.write('<h2> Night -- {:s} </h2>\n'.format(night))
f.write('<h3><ul>\n')
for expid in expids:
# Link
f.write('<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid))
# Generate Exposure html
calib_exp(night, expid, qaprod_dir=qaprod_dir)
f.write('</ul></h3>\n')
# Finish
finish(f,body)
# Return
return links, body
def assemble_fibermap(night, expid, badamps=None, force=False):
"""
Create a fibermap for a given night and expid
Args:
night (int): YEARMMDD night of sunset
expid (int): exposure ID
Options:
badamps (str): comma separated list of "{camera}{petal}{amp}", i.e. "[brz][0-9][ABCD]". Example: 'b7D,z8A'
force (bool): create fibermap even if missing coordinates/guide files
"""
log = get_logger()
#- raw data file for header
rawfile = findfile('raw', night, expid)
try:
rawheader = fits.getheader(rawfile, 'SPEC')
except KeyError:
rawheader = fits.getheader(rawfile, 'SPS')
#- Find fiberassign file
fafile = find_fiberassign_file(night, expid)
#- Find coordinates file in same directory
dirname, filename = os.path.split(fafile)
globfiles = glob.glob(dirname + '/coordinates-*.fits')
if len(globfiles) == 1:
coordfile = globfiles[0]
elif len(globfiles) == 0:
message = f'No coordinates*.fits file in fiberassign dir {dirname}'
if force:
log.error(message + '; continuing anyway')
coordfile = None
else:
raise FileNotFoundError(message)
elif len(globfiles) > 1:
raise RuntimeError(
f'Multiple coordinates*.fits files in fiberassign dir {dirname}')
#- And guide file
dirname, filename = os.path.split(fafile)
globfiles = glob.glob(dirname + '/guide-????????.fits.fz')
if len(globfiles) == 0:
#- try falling back to acquisition image
globfiles = glob.glob(dirname + '/guide-????????-0000.fits.fz')
if len(globfiles) == 1:
guidefile = globfiles[0]
elif len(globfiles) == 0:
message = f'No guide-*.fits.fz file in fiberassign dir {dirname}'
if force:
log.error(message + '; continuing anyway')
guidefile = None
else:
raise FileNotFoundError(message)
elif len(globfiles) > 1:
raise RuntimeError(
f'Multiple guide-*.fits.fz files in fiberassign dir {dirname}')
#- Preflight announcements
log.info(f'Night {night} spectro expid {expid}')
log.info(f'Raw data file {rawfile}')
log.info(f'Fiberassign file {fafile}')
log.info(f'Platemaker coordinates file {coordfile}')
log.info(f'Guider file {guidefile}')
#----
#- Read and assemble
fa = Table.read(fafile, 'FIBERASSIGN')
fa.sort('LOCATION')
#- also read extra keywords from HDU 0
fa_hdr0 = fits.getheader(fafile, 0)
if 'OUTDIR' in fa_hdr0:
fa_hdr0.rename_keyword('OUTDIR', 'FAOUTDIR')
skipkeys = ['SIMPLE', 'EXTEND', 'COMMENT', 'EXTNAME', 'BITPIX', 'NAXIS']
addkeys(fa.meta, fa_hdr0, skipkeys=skipkeys)
#- Read platemaker (pm) coordinates file; 3 formats to support:
# 1. has FLAGS_CNT/EXP_n and DX_n, DX_n (e.g. 20201214/00067678)
# 2. has FLAGS_CNT/EXP_n but not DX_n, DY_n (e.g. 20210402/00083144)
# 3. doesn't have any of these (e.g. 20201220/00069029)
# Notes:
# * don't use FIBER_DX/DY because some files are missing those
# (e.g. 20210224/00077902)
# * don't use FLAGS_COR_n because some files are missing that
# (e.g. 20210402/00083144)
pm = None
numiter = 0
if coordfile is None:
log.error('No coordinates file, thus no info on fiber positioning')
else:
pm = Table.read(coordfile, 'DATA') #- PM = PlateMaker
#- If missing columns *and* not the first in a (split) sequence,
#- try again with the first expid in the sequence
#- (e.g. 202010404/00083419 -> 83418)
if 'DX_0' not in pm.colnames:
log.error(f'Missing DX_0 in {coordfile}')
if 'VISITIDS' in rawheader:
firstexp = int(rawheader['VISITIDS'].split(',')[0])
if firstexp != rawheader['EXPID']:
origcorrdfile = coordfile
coordfile = findfile('coordinates', night, firstexp)
log.info(f'trying again with {coordfile}')
pm = Table.read(coordfile, 'DATA')
else:
log.error(f'no earlier coordinates file for this tile')
else:
log.error(
'Missing VISITIDS header keywords to find earlier coordinates file'
)
if 'FLAGS_CNT_0' not in pm.colnames:
log.error(
f'Missing spotmatch FLAGS_CNT_0 in {coordfile}; no positioner offset info'
)
pm = None
numiter = 0
else:
#- Count number of iterations in file
numiter = len(
[col for col in pm.colnames if col.startswith('FLAGS_CNT_')])
log.info(f'Using FLAGS_CNT_{numiter-1} in {coordfile}')
#- Now let's merge that platemaker coordinates table (pm) with fiberassign
if pm is not None:
pm['LOCATION'] = 1000 * pm['PETAL_LOC'] + pm['DEVICE_LOC']
keep = np.in1d(pm['LOCATION'], fa['LOCATION'])
pm = pm[keep]
pm.sort('LOCATION')
log.info('{}/{} fibers in coordinates file'.format(len(pm), len(fa)))
#- Create fibermap table to merge with fiberassign file
fibermap = Table()
fibermap['LOCATION'] = pm['LOCATION']
fibermap['NUM_ITER'] = numiter
#- Sometimes these columns are missing in the coordinates files, maybe
#- only when numiter=1, i.e. only a blind move but not corrections?
if f'FPA_X_{numiter-1}' in pm.colnames:
fibermap['FIBER_X'] = pm[f'FPA_X_{numiter-1}']
fibermap['FIBER_Y'] = pm[f'FPA_Y_{numiter-1}']
fibermap['DELTA_X'] = pm[f'DX_{numiter-1}']
fibermap['DELTA_Y'] = pm[f'DY_{numiter-1}']
else:
log.warning(
'No FIBER_X/Y or DELTA_X/Y information from platemaker')
fibermap['FIBER_X'] = np.zeros(len(pm))
fibermap['FIBER_Y'] = np.zeros(len(pm))
fibermap['DELTA_X'] = np.zeros(len(pm))
fibermap['DELTA_Y'] = np.zeros(len(pm))
#- Bit definitions at https://desi.lbl.gov/trac/wiki/FPS/PositionerFlags
#- FLAGS_EXP bit 2 is for positioners (not FIF, GIF, ...)
#- These should match what is in fiberassign
expflags = pm[f'FLAGS_EXP_{numiter-1}']
good = ((expflags & 4) == 4)
if np.any(~good):
badloc = list(pm['LOCATION'][~good])
log.error(
f'Flagging {len(badloc)} locations without POS_POS bit set: {badloc}'
)
#- Keep only matched positioners (FLAGS_CNT_n bit 0)
cntflags = pm[f'FLAGS_CNT_{numiter-1}']
spotmatched = ((cntflags & 1) == 1)
num_nomatch = np.sum(good & ~spotmatched)
if num_nomatch > 0:
badloc = list(pm['LOCATION'][good & ~spotmatched])
log.error(
f'Flagging {num_nomatch} unmatched fiber locations: {badloc}')
good &= spotmatched
#- Add our own requirement on good positioning
if ((f'DX_{numiter-1}' in pm.colnames)
and (f'DY_{numiter-1}' in pm.colnames)):
#- offset in cm -> um
dr = np.sqrt(pm[f'DX_{numiter-1}']**2 +
pm[f'DY_{numiter-1}']**2) * 1000
goodpos = (dr < 100) #- HARDCODE
num_badpos = np.sum(good & ~goodpos)
if num_badpos > 0:
log.error(
f'Flagging {num_badpos} positioners >100 um off target')
good &= goodpos
bad = ~good
fibermap['_BADPOS'] = np.zeros(len(fibermap), dtype=bool)
fibermap['_BADPOS'][bad] = True
#- Missing columns from coordinates file...
if ('FIBER_RA' in pm.colnames) and ('FIBER_DEC' in pm.colnames):
fibermap['FIBER_RA'] = pm['FIBER_RA']
fibermap['FIBER_DEC'] = pm['FIBER_DEC']
else:
log.warning('No FIBER_RA or FIBER_DEC from platemaker')
fibermap['FIBER_RA'] = np.zeros(len(pm))
fibermap['FIBER_DEC'] = np.zeros(len(pm))
fibermap = join(fa, fibermap, join_type='left')
#- Set fiber status bits
missing = np.in1d(fibermap['LOCATION'], pm['LOCATION'], invert=True)
fibermap['FIBERSTATUS'][missing] |= fibermask.MISSINGPOSITION
badpos = fibermap['_BADPOS']
fibermap['FIBERSTATUS'][badpos] |= fibermask.BADPOSITION
fibermap.remove_column('_BADPOS')
else:
#- No coordinates file or no positioning iterations;
#- just use fiberassign + dummy columns
log.error(
'Unable to find useful coordinates file; proceeding with fiberassign + dummy columns'
)
fibermap = fa
fibermap['NUM_ITER'] = 0
fibermap['FIBER_X'] = 0.0
fibermap['FIBER_Y'] = 0.0
fibermap['DELTA_X'] = 0.0
fibermap['DELTA_Y'] = 0.0
fibermap['FIBER_RA'] = 0.0
fibermap['FIBER_DEC'] = 0.0
# Update data types to be consistent with updated value if coord file was used.
for val in ['FIBER_X', 'FIBER_Y', 'DELTA_X', 'DELTA_Y']:
old_col = fibermap[val]
fibermap.replace_column(
val, Table.Column(name=val, data=old_col.data, dtype='>f8'))
for val in ['LOCATION', 'NUM_ITER']:
old_col = fibermap[val]
fibermap.replace_column(
val, Table.Column(name=val, data=old_col.data, dtype=np.int64))
#- Update SKY and STD target bits to be in both CMX_TARGET and DESI_TARGET
#- i.e. if they are set in one, also set in the other. Ditto for SV*
for targetcol in ['CMX_TARGET', 'SV0_TARGET', 'SV1_TARGET', 'SV2_TARGET']:
if targetcol in fibermap.colnames:
for mask in [
desi_mask.SKY, desi_mask.STD_FAINT, desi_mask.STD_BRIGHT
]:
ii = (fibermap[targetcol] & mask) != 0
iidesi = (fibermap['DESI_TARGET'] & mask) != 0
fibermap[targetcol][iidesi] |= mask
fibermap['DESI_TARGET'][ii] |= mask
#- Add header information from rawfile
log.debug(f'Adding header keywords from {rawfile}')
skipkeys = [
'EXPTIME',
]
addkeys(fibermap.meta, rawheader, skipkeys=skipkeys)
fibermap['EXPTIME'] = rawheader['EXPTIME']
#- Add header info from guide file
#- sometimes full header is in HDU 0, other times HDU 1...
if guidefile is not None:
log.debug(f'Adding header keywords from {guidefile}')
guideheader = fits.getheader(guidefile, 0)
if 'TILEID' not in guideheader:
guideheader = fits.getheader(guidefile, 1)
if fibermap.meta['TILEID'] != guideheader['TILEID']:
raise RuntimeError('fiberassign tile {} != guider tile {}'.format(
fibermap.meta['TILEID'], guideheader['TILEID']))
addkeys(fibermap.meta, guideheader, skipkeys=skipkeys)
fibermap.meta['EXTNAME'] = 'FIBERMAP'
#- Early data raw headers had bad >8 char 'FIBERASSIGN' keyword
if 'FIBERASSIGN' in fibermap.meta:
log.warning('Renaming header keyword FIBERASSIGN -> FIBASSGN')
fibermap.meta['FIBASSGN'] = fibermap.meta['FIBERASSIGN']
del fibermap.meta['FIBERASSIGN']
#- similarly for early splits in raw data file
if 'USESPLITS' in fibermap.meta:
log.warning('Renaming header keyword USESPLITS -> USESPLIT')
fibermap.meta['USESPLIT'] = fibermap.meta['USESPLITS']
del fibermap.meta['USESPLITS']
#- Record input guide and coordinates files
if guidefile is not None:
fibermap.meta['GUIDEFIL'] = os.path.basename(guidefile)
else:
fibermap.meta['GUIDEFIL'] = 'MISSING'
if coordfile is not None:
fibermap.meta['COORDFIL'] = os.path.basename(coordfile)
else:
fibermap.meta['COORDFIL'] = 'MISSING'
#- Lastly, mask the fibers defined by badamps
if badamps is not None:
maskbits = {
'b': fibermask.BADAMPB,
'r': fibermask.BADAMPR,
'z': fibermask.BADAMPZ
}
ampoffsets = {'A': 0, 'B': 250, 'C': 0, 'D': 250}
for (camera, petal, amplifier) in parse_badamps(badamps):
maskbit = maskbits[camera]
ampoffset = ampoffsets[amplifier]
fibermin = int(petal) * 500 + ampoffset
fibermax = fibermin + 250
ampfibs = np.arange(fibermin, fibermax)
truefmax = fibermax - 1
log.info(f'Masking fibers from {fibermin} to {truefmax} for camera {camera} because of badamp entry '+\
f'{camera}{petal}{amplifier}')
ampfiblocs = np.in1d(fibermap['FIBER'], ampfibs)
fibermap['FIBERSTATUS'][ampfiblocs] |= maskbit
#- Some code incorrectly relies upon the fibermap being sorted by
#- fiber number, so accomodate that before returning the table
fibermap.sort('FIBER')
return fibermap
def toplevel(qaprod_dir=None):
""" Generate HTML to top level QA
Mainly generates the highest level HTML file
which has links to the Exposure and Calib QA.
This also slurps any .png files in the top-level
Parameters
----------
setup : str
cbset : str
det : int
Returns
-------
links : str
body : str
"""
# Organized HTML
html_file = meta.findfile('qa_toplevel_html', qaprod_dir=qaprod_dir)
html_path,_ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Top Level QA')
# Calib?
calib2d_file = meta.findfile('qa_calib_html', qaprod_dir=qaprod_dir)
if os.path.exists(calib2d_file):
# Truncate the path
c2d_path, fname = os.path.split(calib2d_file)
last_slash = c2d_path.rfind('/')
f.write('<h2><a href="{:s}">Calibration QA</a></h2>\n'.format(c2d_path[last_slash+1:]+'/'+fname))
# Full path
#f.write('<h2><a href="{:s}">Calibration QA</a></h2>\n'.format(calib2d_file))
# Exposures?
exposures_file = meta.findfile('qa_exposures_html', qaprod_dir=qaprod_dir)
if os.path.exists(exposures_file):
# Truncated path
exp_path, fname = os.path.split(exposures_file)
last_slash = exp_path.rfind('/')
f.write('<h2><a href="{:s}">Exposures QA</a></h2>\n'.format(exp_path[last_slash+1:]+'/'+fname))
# Full path
#f.write('<h2><a href="{:s}">Exposures QA</a></h2>\n'.format(exposures_file))
# Existing PNGs
f.write('<hr>\n')
f.write('<h2>PNGs</h2>\n')
all_png = glob.glob(html_path+'/*.png')
all_png.sort()
# Type
links = ''
body = ''
for png in all_png:
_, png_file = os.path.split(png)
# Image
href="{:s}".format(png_file[:-4])
links += '<li><a class="reference internal" href="#{:s}">{:s}</a></li>\n'.format(href, href)
body += '<div class="section" id="{:s}">\n'.format(href)
body += '<img class ="research" src="{:s}" width="100%" height="auto"/>\n'.format(png_file)
f.write('<h3><ul>\n')
f.write(links)
f.write('</ul></h3>\n')
f.write(body)
# Finish
finish(f,'')
# Return
return
def make_frameqa(self, make_plots=False, clobber=True):
""" Work through the Production and make QA for all frames
Parameters:
make_plots: bool, optional
Remake the plots too?
clobber: bool, optional
Returns:
"""
# imports
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.fiberflat import read_fiberflat
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots
from desispec.io.fluxcalibration import read_stdstar_models
# Loop on nights
path_nights = glob.glob(self.specprod_dir+'/exposures/*')
nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights]
for night in nights:
for exposure in get_exposures(night, specprod_dir = self.specprod_dir):
# Object only??
frames_dict = get_files(filetype = str('frame'), night = night,
expid = exposure, specprod_dir = self.specprod_dir)
for camera,frame_fil in frames_dict.items():
# Load frame
frame = read_frame(frame_fil)
spectro = int(frame.meta['CAMERA'][-1])
if frame.meta['FLAVOR'] in ['flat','arc']:
qatype = 'qa_calib'
else:
qatype = 'qa_data'
qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
if (not clobber) & os.path.isfile(qafile):
log.info("qafile={:s} exists. Not over-writing. Consider clobber=True".format(qafile))
continue
# Load
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame.meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
skymodel = read_sky(sky_fil)
qaframe.run_qa('SKYSUB', (frame, skymodel))
if make_plots:
qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir,
spectrograph=spectro)
model_tuple=read_stdstar_models(stdstar_fil)
flux_fil = meta.findfile('calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
fluxcalib = read_flux_calibration(flux_fil)
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple))#, indiv_stars))
if make_plots:
qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple)
# Write
write_qa_frame(qafile, qaframe)
def ql_main(args=None):
qlog = qllogger.QLLogger("QuickLook", 20)
log = qlog.getlog()
if args is None:
args = parse()
if args.config is not None:
if args.rawdata_dir:
rawdata_dir = args.rawdata_dir
else:
if 'QL_SPEC_DATA' not in os.environ:
sys.exit(
"must set ${} environment variable or provide rawdata_dir".
format('QL_SPEC_DATA'))
rawdata_dir = os.getenv('QL_SPEC_DATA')
if args.specprod_dir:
specprod_dir = args.specprod_dir
else:
if 'QL_SPEC_REDUX' not in os.environ:
sys.exit(
"must set ${} environment variable or provide specprod_dir"
.format('QL_SPEC_REDUX'))
specprod_dir = os.getenv('QL_SPEC_REDUX')
log.info("Running Quicklook using configuration file {}".format(
args.config))
if os.path.exists(args.config):
if "yaml" in args.config:
config = qlconfig.Config(args.config,
args.night,
args.camera,
args.expid,
rawdata_dir=rawdata_dir,
specprod_dir=specprod_dir)
configdict = config.expand_config()
else:
log.critical("Can't open config file {}".format(args.config))
sys.exit("Can't open config file")
else:
sys.exit("File does not exist: {}".format(args.config))
elif args.fullconfig is not None: #- This is mostly for development/debugging purpose
log.info("Running Quicklook using full configuration file {}".format(
args.fullconfig))
if os.path.exists(args.fullconfig):
if "yaml" in args.fullconfig:
configdict = yaml.load(open(args.fullconfig, "r"))
else:
log.critical("Can't open config file {}".format(args.config))
sys.exit("Can't open config file")
else:
sys.exit("File does not exist: {}".format(args.config))
else:
sys.exit(
"Must provide a valid config file. See desispec/data/quicklook for an example"
)
#- save the expanded config to a file
if args.save:
if "yaml" in args.save:
f = open(args.save, "w")
yaml.dump(configdict, f)
log.info("Output saved for this configuration to {}".format(
args.save))
f.close()
else:
log.info(
"Can save config to only yaml output. Put a yaml in the argument"
)
pipeline, convdict = quicklook.setup_pipeline(configdict)
res = quicklook.runpipeline(pipeline,
convdict,
configdict,
mergeQA=args.mergeQA)
inpname = configdict["RawImage"]
night = configdict["Night"]
camera = configdict["Camera"]
expid = configdict["Expid"]
if isinstance(res, image.Image):
if configdict["OutputFile"]:
finalname = configdict["OutputFile"]
else:
finalname = "image-{}-{:08d}.fits".format(camera, expid)
log.critical(
"No final outputname given. Writing to a image file {}".format(
finalname))
imIO.write_image(finalname, res, meta=None)
elif isinstance(res, frame.Frame):
if configdict["Flavor"] == 'arcs':
from desispec.io.meta import findfile
finalname = "psfnight-{}.fits".format(camera)
finalframe = findfile('frame',
night=night,
expid=expid,
camera=camera)
frIO.write_frame(finalframe, res, header=None)
else:
if configdict["OutputFile"]:
finalname = configdict["OutputFile"]
else:
finalname = "frame-{}-{:08d}.fits".format(camera, expid)
log.critical(
"No final outputname given. Writing to a frame file {}".
format(finalname))
frIO.write_frame(finalname, res, header=None)
else:
log.error(
"Result of pipeline is an unknown type {}. Don't know how to write"
.format(type(res)))
sys.exit("Unknown pipeline result type {}.".format(type(res)))
log.info("Pipeline completed. Final result is in {}".format(finalname))
def qaframe_from_frame(frame_file,
specprod_dir=None,
make_plots=False,
qaprod_dir=None,
output_dir=None,
clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots_ql
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file)
# Load frame
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file,
qaprod_dir=qaprod_dir,
output_dir=output_dir)
if os.path.isfile(qafile) and (not clobber):
write = False
else:
write = True
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path, _ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber):
write = True
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig',
night=night,
camera=camera,
expid=expid,
qaprod_dir=qaprod_dir,
specprod_dir=specprod_dir,
outdir=output_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
fiberflat_fil = meta.findfile('fiberflatnight',
night=night,
camera=camera)
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(
os.path.join(path, '*', 'fiberflat-' + camera + '*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path, _ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(
os.path.join(path, 'fiberflat-' + camera + '*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn(
"Sky file {:s} not found. Skipping..".format(sky_fil))
else:
if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber):
write = True
if make_plots:
qafig = meta.findfile('qa_sky_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
qafig2 = meta.findfile('qa_skychi_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# S/N QA on cframe
if qatype == 'qa_data':
# cframe
cframe_file = frame_file.replace('frame-', 'cframe-')
cframe = read_frame(cframe_file)
if qaframe.run_qa('S2N', (cframe, ), clobber=clobber):
write = True
# Figure?
if make_plots:
s2n_dict = copy.deepcopy(qaframe.qa_data['S2N'])
qafig = meta.findfile('qa_s2n_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
#badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist()
#sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE')
coeff = s2n_dict['METRICS']['FITCOEFF_TGT'] #[sci_idx]
# Add an item or two for the QL method
s2n_dict['CAMERA'] = camera
s2n_dict['EXPID'] = expid
s2n_dict['PANAME'] = 'SNRFit'
s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA']
s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC']
objlist = s2n_dict['METRICS']['OBJLIST']
# Deal with YAML list instead of ndarray
s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(
s2n_dict['METRICS']['MEDIAN_SNR'])
# Generate
if (not os.path.isfile(qafig)) or clobber:
qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist,
[[]] * len(objlist), coeff)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('calib',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn(
"Flux file {:s} not found. Skipping..".format(flux_fil))
else:
if qaframe.run_qa(
'FLUXCALIB', (frame, fluxcalib),
clobber=clobber): # , model_tuple))#, indiv_stars))
write = True
if make_plots:
qafig = meta.findfile('qa_flux_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib) # , model_tuple)
# Write
if write:
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe
if flavor == 'science':
parts = prog.split(' ')
if parts[0] == 'SV1':
if parts[1] == targets:
parts = x.split('/')
night = parts[9]
expid = np.int(parts[10])
name = parts[-1]
parts = name.split('-')
camera = parts[1]
calib = findfile('fluxcalib', night=night, expid=expid, camera=camera, specprod_dir=None)
# findfile('cframe', night=night, expid=expid, camera=camera, specprod_dir=blanc)
cframe = fits.open(x)
hdr = cframe[0].header['FIBERFLT']
hdr = hdr.replace('SPECPROD', blanc)
tileid = cframe[0].header['TILEID']
if tileid != tile:
print('Skipping tile {}'.format(tileid))
continue
iin = x.replace('cframe', 'frame')
sky = x.replace('cframe', 'sky')
psf = sky.replace('sky', 'psf')
def toplevel():
""" Generate HTML to top level QA
Parameters
----------
setup : str
cbset : str
det : int
Returns
-------
links : str
body : str
"""
# Organized HTML
html_file = meta.findfile('qa_toplevel_html')
html_path, _ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Top Level QA')
# Calib?
calib2d_file = meta.findfile('qa_calib_html')
if os.path.exists(calib2d_file):
# Truncate the path
c2d_path, fname = os.path.split(calib2d_file)
last_slash = c2d_path.rfind('/')
f.write('<h2><a href="{:s}">Calibration QA</a></h2>\n'.format(
c2d_path[last_slash + 1:] + '/' + fname))
# Full path
#f.write('<h2><a href="{:s}">Calibration QA</a></h2>\n'.format(calib2d_file))
# Exposures?
exposures_file = meta.findfile('qa_exposures_html')
if os.path.exists(exposures_file):
# Truncated path
exp_path, fname = os.path.split(exposures_file)
last_slash = exp_path.rfind('/')
f.write('<h2><a href="{:s}">Exposures QA</a></h2>\n'.format(
exp_path[last_slash + 1:] + '/' + fname))
# Full path
#f.write('<h2><a href="{:s}">Exposures QA</a></h2>\n'.format(exposures_file))
# Existing PNGs
f.write('<hr>\n')
f.write('<h2>PNGs</h2>\n')
all_png = glob.glob(html_path + '/*.png')
all_png.sort()
# Type
links = ''
body = ''
for png in all_png:
_, png_file = os.path.split(png)
# Image
href = "{:s}".format(png_file[:-4])
links += '<li><a class="reference internal" href="#{:s}">{:s}</a></li>\n'.format(
href, href)
body += '<div class="section" id="{:s}">\n'.format(href)
body += '<img class ="research" src="{:s}" width="100%" height="auto"/>\n'.format(
png_file)
f.write('<h3><ul>\n')
f.write(links)
f.write('</ul></h3>\n')
f.write(body)
# Finish
finish(f, '')
# Return
return
def SNRFit(frame, night, camera, expid, objlist, params, fidboundary=None):
"""
Signal vs. Noise With fitting
Take flux and inverse variance arrays and calculate S/N for individual
targets (ELG, LRG, QSO, STD) and for each amplifier of the camera.
then fit the log(snr)=a+b*mag or log(snr)=poly(mag)
see http://arXiv.org/abs/0706.1062v2 for proper fitting of power-law distributions
it is not implemented here!
Args:
frame: desispec.Frame object
params: parameters dictionary
{
"Func": "linear", # Fit function type one of ["linear","poly"]
"FIDMAG": 22.0, # magnitude to evaluate the fit
"Filter":"DECAM_R", #filter name
}
fidboundary : list of slices indicating where to select in fiber
and wavelength directions for each amp (output of slice_fidboundary function)
Returns a dictionary similar to SignalVsNoise
"""
#- Get imaging magnitudes and calculate S/N
magnitudes = frame.fibermap['MAG']
fmag = 22.0
if "FIDMAG" in params:
fmag = params["FIDMAG"]
filters = frame.fibermap['FILTER']
mediansnr = SN_ratio(frame.flux, frame.ivar)
qadict = {"MEDIAN_SNR": mediansnr}
if camera[0] == 'b':
thisfilter = 'DECAM_G' #- should probably come from param. Hard coding for now
elif camera[0] == 'r':
thisfilter = 'DECAM_R'
else:
thisfilter = 'DECAM_Z'
if "Filter" in params:
thisfilter = params["Filter"]
#- This is a very basic (and temporary!!) flux calibration
#- used to convert sky background and noise to proper flux units
#- NOTE: the B+R**2 term in the S/N equation is being multipled
#- by 1e-33 based on simple unit conversion analysis comparing
#- imaging magnitude converted to flux and integrated spectral counts
try:
#- Get sky background and read noise from previous QAs
cfile = findfile('ql_skycont_file',
int(night),
int(expid),
camera,
specprod_dir=os.environ['QL_SPEC_REDUX'])
with open(cfile) as cf:
contfile = yaml.load(cf)
contval = contfile["METRICS"]["SKYCONT"]
rfile = findfile('ql_getrms_file',
int(night),
int(expid),
camera,
specprod_dir=os.environ['QL_SPEC_REDUX'])
with open(rfile) as rf:
rmsfile = yaml.load(rf)
rmsval = rmsfile["METRICS"]["NOISE_OVER"]
#- NOTE: the 1e-3 used here is because scipy doesn't converge with
#- such small flux values (thus multiplying the flux and conversion
#- factor of 1e-33 by 1e30, which does not change any output)
br2 = 1e-3 * (len(frame.wave) * (contval + rmsval**2))
#- Set up fit of SNR vs. Mag.
#- Using astronomical S/N equation, fitting only 'a'
funcMap = {
"linear": lambda x, a, b: a + b * x,
"poly": lambda x, a, b, c: a + b * x + c * x**2,
"astro": lambda x, a: (a * x) / np.sqrt(a * x + br2)
}
except:
br2 = None
log.info(
"Was not able to calculate B+R**2 from prior knowledge, fitting this too..."
)
#- Fitting 'a' and 'B+R**2' because the necessary information wasn't available)
funcMap = {
"linear": lambda x, a, b: a + b * x,
"poly": lambda x, a, b, c: a + b * x + c * x**2,
"astro": lambda x, a, b: (a * x) / np.sqrt(a * x + b)
}
fitfunc = funcMap["astro"]
if br2 is None:
initialParams = [0.3, 600.]
else:
initialParams = [0.3]
neg_snr_tot = []
#- neg_snr_tot counts the number of times a fiber has a negative median SNR. This should
#- not happen for non-sky fibers with actual flux in them. However, it does happen rarely
#- in sims. To avoid this, we omit such fibers in the fit, but keep count for diagnostic
#- purposes.
#- Loop over each target type, and associate SNR and image magnitudes for each type.
ra = []
dec = []
resid_snr = []
fidsnr_tgt = []
fitcoeff = []
fitcovar = []
snrmag = []
badfibs = []
fitsnr = []
for T in objlist:
fibers = np.where(frame.fibermap['OBJTYPE'] == T)[0]
medsnr = mediansnr[fibers]
mags = np.zeros(medsnr.shape)
if T == "STD": # this should be fixed "STAR" or "STD" should be used consistently everywhere!
T = "STAR"
for ii, fib in enumerate(fibers):
if thisfilter not in filters[fib]:
print(
"WARNING!!! {} magnitude is not available filter for fiber {}"
.format(thisfilter, fib))
mags[ii] = None
else:
mags[ii] = magnitudes[fib][filters[fib] == thisfilter]
try:
#- Determine invalid SNR and mag values and remove
m = mags
s = medsnr
neg_snr = len(np.where(medsnr <= 0.0)[0])
neg_snr_tot.append(neg_snr)
neg_val = np.where(medsnr <= 0.0)[0]
inf_mag = np.where(m == np.inf)[0]
nan_mag = np.where(np.isnan(m))[0]
none_mag = np.where(m == np.array(None))[0]
cut = []
cuts = [neg_val, inf_mag, nan_mag, none_mag]
for cc in range(len(cuts)):
if len(cuts[cc]) > 0:
for ci in range(len(cuts[cc])):
cut.append(cuts[cc][ci])
if len(cut) > 0:
m = list(m)
s = list(s)
makecut = sorted(list(set(cut)))
for nn in range(len(makecut)):
m.remove(m[makecut[nn]])
s.remove(s[makecut[nn]])
for ni in range(len(makecut)):
makecut[ni] -= 1
m = np.array(m)
s = np.array(s)
log.warning(
"In fit of {}, had to remove NANs from data for fitting!".
format(T))
badfibs.append(fibers[sorted(list(set(cut)))])
xs = m.argsort()
#- Using AB magnitude system for conversion to flux
x = 1e30 * (10**(-0.4 * (m[xs] + 48.6)))
med_snr = s[xs]
y = med_snr
#- Fit SNR vs. Mag. to fit function, evaluate at fiducial magnitude,
#- and store results in METRICS
out = optimize.curve_fit(fitfunc, x, y, p0=initialParams)
vs = list(out[0])
cov = list(out[1])
fitcoeff.append(vs)
fitcovar.append(cov)
fidsnr_tgt.append(fitfunc(fmag, *out[0]))
except RuntimeError:
log.warning("In fit of {}, Fit minimization failed!".format(T))
vs = np.array(initialParams)
vs.fill(np.nan)
cov = np.empty((len(initialParams), len(initialParams)))
cov.fill(np.nan)
vs = list(vs)
cov = list(cov)
fitcoeff.append(vs)
fitcovar.append(cov)
fidsnr_tgt.append(np.nan)
except scipy.optimize.OptimizeWarning:
log.warning(
"WARNING!!! {} Covariance estimation failed!".format(T))
vs = out[0]
cov = np.empty((len(initialParams), len(initialParams)))
cov.fill(np.nan)
vs = list(vs)
cov = list(cov)
fitcoeff.append(vs)
fitcovar.append(cov)
fidsnr_tgt.append(np.nan)
qadict["%s_FIBERID" % T] = fibers.tolist()
snr_mag = [medsnr, mags]
snrmag.append(snr_mag)
#- Calculate residual S/N for focal plane plots
fit_snr = []
for mm in range(len(x)):
snr = fitfunc(x[mm], *vs)
fit_snr.append(snr)
for rr in range(len(fit_snr)):
resid = (med_snr[rr] - fit_snr[rr]) / fit_snr[rr]
resid_snr.append(resid)
fitsnr.append(fit_snr)
qadict["NUM_NEGATIVE_SNR"] = sum(neg_snr_tot)
qadict["SNR_MAG_TGT"] = snrmag
qadict["FITCOEFF_TGT"] = fitcoeff
qadict["FITCOVAR_TGT"] = fitcovar
qadict["SNR_RESID"] = resid_snr
qadict["FIDSNR_TGT"] = fidsnr_tgt
qadict["RA"] = frame.fibermap['RA_TARGET']
qadict["DEC"] = frame.fibermap['DEC_TARGET']
return qadict, badfibs, fitsnr
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None,
output_dir=None, clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots_ql
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file)
# Load frame
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir)
if os.path.isfile(qafile) and (not clobber):
write = False
else:
write = True
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path,_ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber):
write = True
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid,
qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir)
fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera)
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(os.path.join(path,'*','fiberflat-'+camera+'*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path,_ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(os.path.join(path,'fiberflat-'+camera+'*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn("Sky file {:s} not found. Skipping..".format(sky_fil))
else:
if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber):
write=True
if make_plots:
qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# S/N QA on cframe
if qatype == 'qa_data':
# cframe
cframe_file = frame_file.replace('frame-', 'cframe-')
cframe = read_frame(cframe_file)
if qaframe.run_qa('S2N', (cframe,), clobber=clobber):
write=True
# Figure?
if make_plots:
s2n_dict = copy.deepcopy(qaframe.qa_data['S2N'])
qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
#badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist()
#sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE')
coeff = s2n_dict['METRICS']['FITCOEFF_TGT']#[sci_idx]
# Add an item or two for the QL method
s2n_dict['CAMERA'] = camera
s2n_dict['EXPID'] = expid
s2n_dict['PANAME'] = 'SNRFit'
s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA']
s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC']
objlist = s2n_dict['METRICS']['OBJLIST']
# Deal with YAML list instead of ndarray
s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(s2n_dict['METRICS']['MEDIAN_SNR'])
# Generate
if (not os.path.isfile(qafig)) or clobber:
qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]]*len(objlist), coeff)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn("Flux file {:s} not found. Skipping..".format(flux_fil))
else:
if qaframe.run_qa('FLUXCALIB', (frame, fluxcalib), clobber=clobber): # , model_tuple))#, indiv_stars))
write = True
if make_plots:
qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple)
# Write
if write:
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe
flux = pickle.load(handle)['ELG']
with open(
os.environ['CSCRATCH'] +
'/radlss/test/ensemble/template-elg-ensemble-objmeta.fits',
'rb') as handle:
objmeta = pickle.load(handle)['ELG']
for band in ['b', 'r', 'z']:
cam = band + petal
# E.g. ~/andes/exposures/20200315/00055642/cframe-b0-00055642.fits
cframes[cam] = read_frame(
findfile('cframe',
night=night,
expid=expid,
camera=cam,
specprod_dir='/global/homes/m/mjwilson/andes/'))
cframes[cam].flux[fiber, :] = flux[band][115]
rrz = meta['REDSHIFT'][115]
z = 1.0
v = 75.0
r = 0.7
lnA = 2.5
_, mod = doublet(z, twave, sigmav=v, r=r)
postages = cframe_postage(cframes, fiber, rrz)
postage = postages[3]
def make_frameqa(self, make_plots=False, clobber=True):
""" Work through the Production and make QA for all frames
Parameters:
make_plots: bool, optional
Remake the plots too?
clobber: bool, optional
Returns:
"""
# imports
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.fiberflat import read_fiberflat
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots
from desispec.io.fluxcalibration import read_stdstar_models
# Loop on nights
path_nights = glob.glob(self.specprod_dir + '/exposures/*')
nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights]
for night in nights:
for exposure in get_exposures(night,
specprod_dir=self.specprod_dir):
# Object only??
frames_dict = get_files(filetype=str('frame'),
night=night,
expid=exposure,
specprod_dir=self.specprod_dir)
for camera, frame_fil in frames_dict.items():
# Load frame
frame = read_frame(frame_fil)
spectro = int(frame.meta['CAMERA'][-1])
if frame.meta['FLAVOR'] in ['flat', 'arc']:
qatype = 'qa_calib'
else:
qatype = 'qa_data'
qafile = meta.findfile(qatype,
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
if (not clobber) & os.path.isfile(qafile):
log.info(
"qafile={:s} exists. Not over-writing. Consider clobber=True"
.format(qafile))
continue
# Load
qaframe = load_qa_frame(qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Flat QA
if frame.meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat),
clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile(
'qa_flat_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame,
fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
skymodel = read_sky(sky_fil)
qaframe.run_qa('SKYSUB', (frame, skymodel))
if make_plots:
qafig = meta.findfile(
'qa_sky_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_skyres(qafig, frame, skymodel,
qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile(
'stdstars',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir,
spectrograph=spectro)
model_tuple = read_stdstar_models(stdstar_fil)
flux_fil = meta.findfile(
'calib',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
fluxcalib = read_flux_calibration(flux_fil)
qaframe.run_qa(
'FLUXCALIB',
(frame, fluxcalib, model_tuple)) #, indiv_stars))
if make_plots:
qafig = meta.findfile(
'qa_flux_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib, model_tuple)
# Write
write_qa_frame(qafile, qaframe)
def qaframe_from_frame(frame_file,
specprod_dir=None,
make_plots=False,
qaprod_dir=None,
output_dir=None,
clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file)
# Load frame
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file,
qaprod_dir=qaprod_dir,
output_dir=output_dir)
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path, _ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
fiberflat_fil = meta.findfile('fiberflatnight',
night=night,
camera=camera)
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(
os.path.join(path, '*', 'fiberflat-' + camera + '*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path, _ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(
os.path.join(path, 'fiberflat-' + camera + '*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn(
"Sky file {:s} not found. Skipping..".format(sky_fil))
else:
qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber)
if make_plots:
qafig = meta.findfile('qa_sky_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qafig2 = meta.findfile('qa_skychi_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('calib',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn(
"Flux file {:s} not found. Skipping..".format(flux_fil))
else:
qaframe.run_qa(
'FLUXCALIB',
(frame, fluxcalib)) # , model_tuple))#, indiv_stars))
if make_plots:
qafig = meta.findfile('qa_flux_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib) # , model_tuple)
# Write
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe
