def bgs_write_simdata(sim, obs_conds, simdir, obsrand, overwrite=False):
"""Build and write a metadata table with the simulation inputs.
Currently, the only quantities that can be varied are moonfrac,
moonsep, and exptime, but more choices can be added as needed.
"""
from desispec.io.util import makepath
simdatafile = os.path.join(simdir, sim['suffix'],
'bgs-{}-simdata.fits'.format(sim['suffix']))
makepath(simdatafile)
cols = [('SEED', 'S20'), ('NSPEC', 'i4'), ('EXPTIME', 'f4'),
('AIRMASS', 'f4'), ('SEEING', 'f4'), ('MOONFRAC', 'f4'),
('MOONSEP', 'f4'), ('MOONALT', 'f4')]
simdata = Table(np.zeros(sim['nsim'], dtype=cols))
simdata['EXPTIME'].unit = 's'
simdata['SEEING'].unit = 'arcsec'
simdata['MOONSEP'].unit = 'deg'
simdata['MOONALT'].unit = 'deg'
simdata['SEED'] = sim['seed']
simdata['NSPEC'] = sim['nspec']
simdata['AIRMASS'] = obs_conds['AIRMASS']
simdata['SEEING'] = obs_conds['SEEING']
simdata['MOONALT'] = obs_conds['MOONALT']
if 'moonfracmin' in obs_conds.keys():
simdata['MOONFRAC'] = obsrand.uniform(obs_conds['moonfracmin'],
obs_conds['moonfracmax'],
sim['nsim'])
else:
simdata['MOONFRAC'] = obs_conds['MOONFRAC']
if 'moonsepmin' in obs_conds.keys():
simdata['MOONSEP'] = obsrand.uniform(obs_conds['moonsepmin'],
obs_conds['moonsepmax'],
sim['nsim'])
else:
simdata['MOONSEP'] = obs_conds['MOONSEP']
if 'exptimemin' in obs_conds.keys():
simdata['EXPTIME'] = obsrand.uniform(obs_conds['exptimemin'],
obs_conds['exptimemax'],
sim['nsim'])
else:
simdata['EXPTIME'] = obs_conds['EXPTIME']
if overwrite or not os.path.isfile(simdatafile):
print('Writing {}'.format(simdatafile))
write_bintable(simdatafile,
simdata,
extname='SIMDATA',
clobber=overwrite)
return simdata
def write_fiberflat(outfile, fiberflat, header=None):
"""Write fiberflat object to outfile
Args:
outfile: filepath string or (night, expid, camera) tuple
fiberflat: FiberFlat object
header: (optional) dict or fits.Header object to use as HDU 0 header
Returns:
filepath of file that was written
"""
outfile = makepath(outfile, 'fiberflat')
if header is None:
hdr = fitsheader(fiberflat.header)
else:
hdr = fitsheader(header)
if fiberflat.chi2pdf is not None:
hdr['chi2pdf'] = float(fiberflat.chi2pdf)
add_dependencies(hdr)
ff = fiberflat #- shorthand
hdus = fits.HDUList()
hdus.append(fits.PrimaryHDU(ff.fiberflat.astype('f4'), header=hdr))
hdus.append(fits.ImageHDU(ff.ivar.astype('f4'), name='IVAR'))
hdus.append(fits.CompImageHDU(ff.mask, name='MASK'))
hdus.append(fits.ImageHDU(ff.meanspec.astype('f4'), name='MEANSPEC'))
hdus.append(fits.ImageHDU(ff.wave.astype('f4'), name='WAVELENGTH'))
hdus.writeto(outfile + '.tmp', clobber=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def write_templates(outfile,
flux,
wave,
meta,
objtype=None,
comments=None,
units=None):
"""Write out simulated galaxy templates. (Incomplete documentation...)
Args:
outfile (str): Output file name.
"""
from astropy.io import fits
from desispec.io.util import fitsheader, write_bintable, makepath
# Create the path to OUTFILE if necessary.
outfile = makepath(outfile)
header = dict(OBJTYPE=(objtype, 'Object type'),
CUNIT=('Angstrom', 'units of wavelength array'),
CRPIX1=(1, 'reference pixel number'),
CRVAL1=(wave[0], 'Starting wavelength [Angstrom]'),
CDELT1=(wave[1] - wave[0], 'Wavelength step [Angstrom]'),
LOGLAM=(0, 'linear wavelength steps, not log10'),
AIRORVAC=('vac', 'wavelengths in vacuum (vac) or air'),
BUNIT=('1e-17 erg/(s cm2 Angstrom)', 'spectrum flux units'))
hdr = fitsheader(header)
fits.writeto(outfile, flux.astype(np.float32), header=hdr, clobber=True)
write_bintable(outfile,
meta,
header=hdr,
comments=comments,
units=units,
extname='METADATA')
def write_qa_frame(outfile, qaframe, verbose=False):
"""Write QA for a given frame
Args:
outfile : str
filename
qa_exp : QA_Frame object, with the following attributes
qa_data: dict of QA info
"""
log = get_logger()
outfile = makepath(outfile, 'qa')
# Generate the dict
odict = {
qaframe.night: {
qaframe.expid: {
qaframe.camera: {},
'flavor': qaframe.flavor
}
}
}
odict[qaframe.night][qaframe.expid][qaframe.camera] = qaframe.qa_data
ydict = yamlify(odict)
# Simple yaml
with open(outfile, 'w') as yamlf:
yamlf.write(yaml.dump(ydict))
if verbose:
log.info("Wrote QA frame file: {:s}".format(outfile))
return outfile
def write_qa_prod(outroot, qaprod, indent=True):
"""Write QA for a given production
Args:
outroot : str
filename without format extension
qa_prod : QA_Prod object
Returns:
outfile: str
output filename
"""
from desiutil.io import combine_dicts
log = get_logger()
outfile = outroot + '.json'
outfile = makepath(outfile, 'qa')
# Loop on exposures
odict = {}
for qaexp in qaprod.qa_exps:
# Get the exposure dict
idict = write_qa_exposure('foo', qaexp, ret_dict=True)
odict = combine_dicts(odict, idict)
ydict = yamlify(odict) # This works well for JSON too
# Simple json
with open(outfile, 'wt') as fh:
json.dump(ydict, fh, indent=indent)
log.info('Wrote QA_Prod file: {:s}'.format(outfile))
return outfile
def write_qa_frame(outfile, qaframe):
"""Write QA for a given frame
Args:
outfile : str
filename
qa_exp : QA_Frame object, with the following attributes
qa_data: dict of QA info
"""
outfile = makepath(outfile, 'qa')
# Generate the dict
odict = {
qaframe.night: {
qaframe.expid: {
qaframe.camera: {},
'flavor': qaframe.flavor
}
}
}
odict[qaframe.night][qaframe.expid][qaframe.camera] = qaframe.qa_data
ydict = yamlify(odict)
# Simple yaml
with open(outfile, 'w') as yamlf:
yamlf.write(yaml.dump(ydict)) #, default_flow_style=True) )
return outfile
def write_qa_multiexp(outroot, qa_mexp, indent=True, skip_rebuild=False):
"""Write QA for a given production
Args:
outroot : str
filename without format extension
qa_prod : QA_Prod object
skip_rebuild : bool, optional
Do not rebuild the data dict
Returns:
outfile: str
output filename
"""
log = get_logger()
outfile = outroot + '.json'
outfile = makepath(outfile, 'qa')
if not skip_rebuild:
qa_mexp.build_data()
ydict = yamlify(qa_mexp.data) # This works well for JSON too
# Simple json
with open(outfile, 'wt') as fh:
json.dump(ydict, fh, indent=indent)
log.info('Wrote QA_Prod file: {:s}'.format(outfile))
return outfile
def write_fibermap(outfile, fibermap, header=None, clobber=True, extname='FIBERMAP'):
"""Write fibermap binary table to outfile.
Args:
outfile (str): output filename
fibermap: astropy Table of fibermap data
header: header data to include in same HDU as fibermap
clobber (bool, optional): overwrite outfile if it exists
extname (str, optional): set the extension name.
Returns:
write_fibermap (str): full path to filename of fibermap file written.
"""
outfile = makepath(outfile)
#- astropy.io.fits incorrectly generates warning about 2D arrays of strings
#- Temporarily turn off warnings to avoid this; desispec.test.test_io will
#- catch it if the arrays actually are written incorrectly.
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(fibermap.meta)
add_dependencies(hdr)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
write_bintable(outfile, fibermap, hdr, comments=fibermap_comments,
extname=extname, clobber=clobber)
return outfile
def write_fibermap(outfile, fibermap, header=None):
"""Write fibermap binary table to outfile.
Args:
outfile (str): output filename
fibermap: astropy Table of fibermap data
header: header data to include in same HDU as fibermap
Returns:
write_fibermap (str): full path to filename of fibermap file written.
"""
outfile = makepath(outfile)
#- astropy.io.fits incorrectly generates warning about 2D arrays of strings
#- Temporarily turn off warnings to avoid this; desispec.test.test_io will
#- catch it if the arrays actually are written incorrectly.
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(fibermap.meta)
add_dependencies(hdr)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
write_bintable(outfile,
fibermap,
hdr,
comments=fibermap_comments,
extname="FIBERMAP",
clobber=True)
return outfile
def write_fiberflat(outfile,fiberflat,header=None):
"""Write fiberflat object to outfile
Args:
outfile: filepath string or (night, expid, camera) tuple
fiberflat: FiberFlat object
header: (optional) dict or fits.Header object to use as HDU 0 header
Returns:
filepath of file that was written
"""
outfile = makepath(outfile, 'fiberflat')
if header is None:
hdr = fitsheader(fiberflat.header)
else:
hdr = fitsheader(header)
if fiberflat.chi2pdf is not None:
hdr['chi2pdf'] = float(fiberflat.chi2pdf)
add_dependencies(hdr)
ff = fiberflat #- shorthand
hdus = fits.HDUList()
hdus.append(fits.PrimaryHDU(ff.fiberflat.astype('f4'), header=hdr))
hdus.append(fits.ImageHDU(ff.ivar.astype('f4'), name='IVAR'))
hdus.append(fits.CompImageHDU(ff.mask, name='MASK'))
hdus.append(fits.ImageHDU(ff.meanspec.astype('f4'), name='MEANSPEC'))
hdus.append(fits.ImageHDU(ff.wave.astype('f4'), name='WAVELENGTH'))
hdus.writeto(outfile+'.tmp', clobber=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
return outfile
def write_sky(outfile, skymodel, header=None):
"""Write sky model.
Args:
outfile : filename or (night, expid, camera) tuple
skymodel : SkyModel object, with the following attributes
wave : 1D wavelength in vacuum Angstroms
flux : 2D[nspec, nwave] sky flux
ivar : 2D inverse variance of sky flux
mask : 2D mask for sky flux
header : optional fits header data (fits.Header, dict, or list)
"""
outfile = makepath(outfile, 'sky')
#- Convert header to fits.Header if needed
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(skymodel.header)
add_dependencies(hdr)
hx = fits.HDUList()
hdr['EXTNAME'] = ('SKY', 'no dimension')
hx.append( fits.PrimaryHDU(skymodel.flux.astype('f4'), header=hdr) )
hx.append( fits.ImageHDU(skymodel.ivar.astype('f4'), name='IVAR') )
hx.append( fits.CompImageHDU(skymodel.mask, name='MASK') )
hx.append( fits.ImageHDU(skymodel.wave.astype('f4'), name='WAVELENGTH') )
hx.writeto(outfile+'.tmp', clobber=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
return outfile
def write_fibermap(outfile, fibermap, header=None):
"""Write fibermap binary table to outfile.
Args:
outfile (str): output filename
fibermap: ndarray with named columns of fibermap data
header: header data to include in same HDU as fibermap
Returns:
write_fibermap (str): full path to filename of fibermap file written.
"""
outfile = makepath(outfile)
#- astropy.io.fits incorrectly generates warning about 2D arrays of strings
#- Temporarily turn off warnings to avoid this; desispec.test.test_io will
#- catch it if the arrays actually are written incorrectly.
hdr = fitsheader(header)
add_dependencies(hdr)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
write_bintable(outfile, fibermap, hdr, comments=fibermap_comments,
extname="FIBERMAP", clobber=True)
return outfile
def write_qa_prod(outroot, qaprod):
"""Write QA for a given production
Args:
outroot : str
filename without format extension
qa_prod : QA_Prod object
Returns:
outfile or odict : str or dict
"""
from desiutil.io import combine_dicts
outfile = outroot+'.yaml'
outfile = makepath(outfile, 'qa')
# Loop on exposures
odict = {}
for qaexp in qaprod.qa_exps:
# Get the exposure dict
idict = write_qa_exposure('foo', qaexp, ret_dict=True)
odict = combine_dicts(odict, idict)
ydict = yamlify(odict)
# Simple yaml
with open(outfile, 'w') as yamlf:
yamlf.write( yaml.dump(ydict))#, default_flow_style=True) )
log.info('Wrote QA_Prod file: {:s}'.format(outfile))
return outfile
def write_fiberflat(outfile, fiberflat, header=None):
"""Write fiberflat object to outfile
Args:
outfile: filepath string or (night, expid, camera) tuple
header: (optional) dict or fits.Header object to use as HDU 0 header
Returns:
filepath of file that was written
"""
outfile = makepath(outfile, 'fiberflat')
if header is None:
hdr = fitsheader(fiberflat.header)
else:
hdr = fitsheader(header)
ff = fiberflat #- shorthand
hdus = fits.HDUList()
hdus.append(fits.PrimaryHDU(ff.fiberflat, header=hdr))
hdus.append(fits.ImageHDU(ff.ivar, name='IVAR'))
hdus.append(fits.ImageHDU(ff.mask, name='MASK'))
hdus.append(fits.ImageHDU(ff.meanspec, name='MEANSPEC'))
hdus.append(fits.ImageHDU(ff.wave, name='WAVELENGTH'))
hdus.writeto(outfile, clobber=True)
return outfile
def write_qa_exposure(outroot, qaexp, ret_dict=False):
"""Write QA for a given exposure
Args:
outroot : str
filename without format extension
qa_exp : QA_Exposure object
ret_dict : bool, optional
Return dict only? [for qa_prod, mainly]
Returns:
outfile or odict : str or dict
"""
# Generate the dict
odict = {qaexp.night: {qaexp.expid: {}}}
odict[qaexp.night][qaexp.expid]['flavor'] = qaexp.flavor
odict[qaexp.night][qaexp.expid]['meta'] = qaexp.meta
cameras = list(qaexp.data['frames'].keys())
for camera in cameras:
odict[qaexp.night][qaexp.expid][camera] = qaexp.data['frames'][camera]
# Return dict only?
if ret_dict:
return odict
# Simple yaml
ydict = yamlify(odict)
outfile = outroot+'.yaml'
outfile = makepath(outfile, 'qa')
with open(outfile, 'w') as yamlf:
yamlf.write( yaml.dump(ydict))#, default_flow_style=True) )
return outfile
def write_qa_exposure(outroot, qaexp, ret_dict=False):
"""Write QA for a given exposure
Args:
outroot : str
filename without format extension
qa_exp : QA_Exposure object
ret_dict : bool, optional
Return dict only? [for qa_prod, mainly]
Returns:
outfile or odict : str or dict
"""
# Generate the dict
odict = {qaexp.night: {qaexp.expid: {}}}
odict[qaexp.night][qaexp.expid]['flavor'] = qaexp.flavor
cameras = qaexp.data['frames'].keys()
for camera in cameras:
odict[qaexp.night][qaexp.expid][camera] = qaexp.data['frames'][camera]
# Return dict only?
if ret_dict:
return odict
# Simple yaml
ydict = yamlify(odict)
outfile = outroot+'.yaml'
outfile = makepath(outfile, 'qa')
with open(outfile, 'w') as yamlf:
yamlf.write( yaml.dump(ydict))#, default_flow_style=True) )
return outfile
def write_templates(outfile, flux, wave, meta, objtype=None,
comments=None, units=None):
"""Write out simulated galaxy templates. (Incomplete documentation...)
Args:
outfile (str): Output file name.
Returns:
Raises
"""
from astropy.io import fits
from desispec.io.util import fitsheader, write_bintable, makepath
# Create the path to OUTFILE if necessary.
outfile = makepath(outfile)
header = dict(
OBJTYPE = (objtype, 'Object type'),
CUNIT = ('Angstrom', 'units of wavelength array'),
CRPIX1 = (1, 'reference pixel number'),
CRVAL1 = (wave[0], 'Starting wavelength [Angstrom]'),
CDELT1 = (wave[1]-wave[0], 'Wavelength step [Angstrom]'),
LOGLAM = (0, 'linear wavelength steps, not log10'),
AIRORVAC = ('vac', 'wavelengths in vacuum (vac) or air'),
BUNIT = ('erg/s/cm2/A', 'spectrum flux units')
)
hdr = fitsheader(header)
fits.writeto(outfile,flux.astype(np.float32),header=hdr,clobber=True)
write_bintable(outfile, meta, header=hdr, comments=comments,
units=units, extname='METADATA')
def bgs_write_simdata(sim, overwrite=False):
"""Create a metadata table with simulation inputs.
Parameters
----------
sim : dict
Simulation parameters from command line.
overwrite : bool
Overwrite simulation data file.
Returns
-------
simdata : Table
Data table written to disk.
"""
from desispec.io.util import makepath
from desispec.io.util import write_bintable
simdatafile = os.path.join(sim.simdir,
'bgs_{}_simdata.fits'.format(sim.simid))
makepath(simdatafile)
cols = [('SEED', 'S20'), ('NSPEC', 'i4'), ('EXPTIME', 'f4'),
('AIRMASS', 'f4'), ('SEEING', 'f4'), ('MOONFRAC', 'f4'),
('MOONSEP', 'f4'), ('MOONALT', 'f4')]
simdata = Table(np.zeros(sim.nsim, dtype=cols))
simdata['EXPTIME'].unit = 's'
simdata['SEEING'].unit = 'arcsec'
simdata['MOONSEP'].unit = 'deg'
simdata['MOONALT'].unit = 'deg'
simdata['SEED'] = sim.seed
simdata['NSPEC'] = sim.nspec
simdata['AIRMASS'] = sim.airmass
simdata['SEEING'] = sim.seeing
simdata['MOONALT'] = sim.moonalt
simdata['MOONSEP'] = sim.moonsep
simdata['MOONFRAC'] = sim.moonfrac
simdata['EXPTIME'] = sim.exptime
if overwrite or not os.path.isfile(simdatafile):
print('Writing {}'.format(simdatafile))
write_bintable(simdatafile, simdata, extname='SIMDATA', clobber=True)
return simdata
def calib(qaprod_dir=None, specprod_dir=None):
""" Generate HTML to orgainze calib HTML
"""
# Organized HTML
html_file = meta.findfile('qa_calib_html', qaprod_dir=qaprod_dir)
html_path, _ = os.path.split(html_file)
makepath(html_file)
# Open
f = open(html_file, 'w')
init(f, 'Calibration QA')
# Loop on Nights
nights = get_nights(sub_folder='calibnight', specprod_dir=specprod_dir)
nights.sort()
links = ''
body = ''
for night in nights:
all_png = glob.glob(html_path + '/' + night + '/qa*.png')
if len(all_png) == 0:
continue
# Find expid
expids = []
for png in all_png:
expids.append(int(png[-12:-4])) # A bit risky
expids = np.unique(expids)
expids.sort()
f.write('<h2> Night -- {:s} </h2>\n'.format(night))
f.write('<h3><ul>\n')
for expid in expids:
# Link
f.write(
'<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.
format(night, expid, expid))
# Generate Exposure html
calib_exp(night, expid, qaprod_dir=qaprod_dir)
f.write('</ul></h3>\n')
# Finish
finish(f, body)
# Return
return links, body
def write_frame(outfile, frame, header=None, fibermap=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
frame: desispec.frame.Frame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a Frame object to pass into write_frame,
frame = Frame(wave, flux, ivar, resolution_data)
"""
outfile = makepath(outfile, 'frame')
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(frame.meta)
add_dependencies(hdr)
hdus = fits.HDUList()
x = fits.PrimaryHDU(frame.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
hdus.append(x)
hdus.append(fits.ImageHDU(frame.ivar.astype('f4'), name='IVAR'))
hdus.append(fits.CompImageHDU(frame.mask, name='MASK'))
hdus.append(fits.ImageHDU(frame.wave.astype('f4'), name='WAVELENGTH'))
hdus.append(
fits.ImageHDU(frame.resolution_data.astype('f4'), name='RESOLUTION'))
if fibermap is not None:
hdus.append(fits.BinTableHDU(np.asarray(fibermap), name='FIBERMAP'))
elif frame.fibermap is not None:
hdus.append(
fits.BinTableHDU(np.asarray(frame.fibermap), name='FIBERMAP'))
elif frame.spectrograph is not None:
x.header[
'FIBERMIN'] = 500 * frame.spectrograph # Hard-coded (as in desispec.frame)
else:
log.error(
"You are likely writing a frame without sufficient fiber info")
hdus.writeto(outfile + '.tmp', clobber=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def write_frame(outfile, frame, header=None, fibermap=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
frame: desispec.frame.Frame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a Frame object to pass into write_frame,
frame = Frame(wave, flux, ivar, resolution_data)
"""
outfile = makepath(outfile, 'frame')
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(frame.meta)
add_dependencies(hdr)
hdus = fits.HDUList()
x = fits.PrimaryHDU(frame.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
hdus.append(x)
hdus.append( fits.ImageHDU(frame.ivar.astype('f4'), name='IVAR') )
hdus.append( fits.CompImageHDU(frame.mask, name='MASK') )
hdus.append( fits.ImageHDU(frame.wave.astype('f4'), name='WAVELENGTH') )
hdus.append( fits.ImageHDU(frame.resolution_data.astype('f4'), name='RESOLUTION' ) )
if fibermap is not None:
hdus.append( fits.BinTableHDU(np.asarray(fibermap), name='FIBERMAP' ) )
elif frame.fibermap is not None:
hdus.append( fits.BinTableHDU(np.asarray(frame.fibermap), name='FIBERMAP' ) )
elif frame.spectrograph is not None:
x.header['FIBERMIN'] = 500*frame.spectrograph # Hard-coded (as in desispec.frame)
else:
log.error("You are likely writing a frame without sufficient fiber info")
if frame.chi2pix is not None:
hdus.append( fits.ImageHDU(frame.chi2pix.astype('f4'), name='CHI2PIX' ) )
hdus.writeto(outfile+'.tmp', clobber=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
return outfile
def write_qa_brick(outfile, qabrick):
"""Write QA for a given exposure
Args:
outfile : filename
qabrick : QA_Brick object
_data: dict of QA info
"""
outfile = makepath(outfile, 'qa')
# Simple yaml
ydict = yamlify(qabrick.data)
with open(outfile, 'w') as yamlf:
yamlf.write( yaml.dump(ydict))#, default_flow_style=True) )
return outfile
def write_qa_brick(outfile, qabrick):
"""Write QA for a given exposure
Args:
outfile : filename
qabrick : QA_Brick object
_data: dict of QA info
"""
outfile = makepath(outfile, 'qa')
# Simple yaml
ydict = yamlify(qabrick.data)
with open(outfile, 'w') as yamlf:
yamlf.write(yaml.dump(ydict))#, default_flow_style=True) )
return outfile
def write_fibermap(outfile,
fibermap,
header=None,
clobber=True,
extname='FIBERMAP'):
"""Write fibermap binary table to outfile.
Args:
outfile (str): output filename
fibermap: astropy Table of fibermap data
header: header data to include in same HDU as fibermap
clobber (bool, optional): overwrite outfile if it exists
extname (str, optional): set the extension name.
Returns:
write_fibermap (str): full path to filename of fibermap file written.
"""
log = get_logger()
outfile = makepath(outfile)
#- astropy.io.fits incorrectly generates warning about 2D arrays of strings
#- Temporarily turn off warnings to avoid this; desispec.test.test_io will
#- catch it if the arrays actually are written incorrectly.
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(fibermap.meta)
add_dependencies(hdr)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
t0 = time.time()
write_bintable(outfile,
fibermap,
hdr,
comments=fibermap_comments,
extname=extname,
clobber=clobber)
duration = time.time() - t0
log.info(iotime.format('write', outfile, duration))
return outfile
def write_qa_frame(outfile, qaframe):
"""Write QA for a given frame
Args:
outfile : str
filename
qa_exp : QA_Frame object, with the following attributes
qa_data: dict of QA info
"""
outfile = makepath(outfile, 'qa')
# Generate the dict
odict = {qaframe.night: {qaframe.expid: {qaframe.camera: {}, 'flavor': qaframe.flavor}}}
odict[qaframe.night][qaframe.expid][qaframe.camera] = qaframe.qa_data
ydict = yamlify(odict)
# Simple yaml
with open(outfile, 'w') as yamlf:
yamlf.write( yaml.dump(ydict))#, default_flow_style=True) )
return outfile
def write_frame(outfile, frame, header=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
frame: desispec.frame.Frame object with wave, flux, ivar...
header: optional astropy.io.fits.Header or dict to override frame.header
Returns:
full filepath of output file that was written
Note:
to create a Frame object to pass into write_frame,
frame = Frame(wave, flux, ivar, resolution_data)
"""
outfile = makepath(outfile, 'frame')
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(frame.header)
if 'SPECMIN' not in hdr:
hdr['SPECMIN'] = 0
if 'SPECMAX' not in hdr:
hdr['SPECMAX'] = hdr['SPECMIN'] + frame.nspec
hdus = fits.HDUList()
x = fits.PrimaryHDU(frame.flux, header=hdr)
x.header['EXTNAME'] = 'FLUX'
hdus.append(x)
hdus.append( fits.ImageHDU(frame.ivar, name='IVAR') )
hdus.append( fits.ImageHDU(frame.mask, name='MASK') )
hdus.append( fits.ImageHDU(frame.wave, name='WAVELENGTH') )
hdus.append( fits.ImageHDU(frame.resolution_data, name='RESOLUTION' ) )
hdus.writeto(outfile, clobber=True)
return outfile
def write_qa_multiexp(outroot, mdict, indent=True):
"""Write QA for a given production
Args:
outroot : str
filename without format extension
mdict : dict
Returns:
outfile: str
output filename
"""
log=get_logger()
outfile = outroot+'.json'
outfile = makepath(outfile, 'qa')
ydict = yamlify(mdict) # This works well for JSON too
# Simple json
with open(outfile, 'wt') as fh:
json.dump(ydict, fh, indent=indent)
log.info('Wrote QA Multi-Exposure file: {:s}'.format(outfile))
return outfile
def write_qa_multiexp(outroot, mdict, indent=True):
"""Write QA for a given production
Args:
outroot : str
filename without format extension
mdict : dict
Returns:
outfile: str
output filename
"""
log = get_logger()
outfile = outroot + '.json'
outfile = makepath(outfile, 'qa')
ydict = yamlify(mdict) # This works well for JSON too
# Simple json
with open(outfile, 'wt') as fh:
json.dump(ydict, fh, indent=indent)
log.info('Wrote QA Multi-Exposure file: {:s}'.format(outfile))
return outfile
def write_templates(outfile, flux, wave, meta):
"""Write out simulated galaxy templates.
Args:
outfile (str): Output file name.
flux (numpy.ndarray): Flux vector (1e-17 erg/s/cm2/A)
wave (numpy.ndarray): Wavelength vector (Angstrom).
meta (astropy.table.Table): metadata table.
"""
from astropy.io import fits
from desispec.io.util import makepath
# Create the path to OUTFILE if necessary.
outfile = makepath(outfile)
hx = fits.HDUList()
hdu_wave = fits.PrimaryHDU(wave)
hdu_wave.header['EXTNAME'] = 'WAVE'
hdu_wave.header['BUNIT'] = 'Angstrom'
hdu_wave.header['AIRORVAC'] = ('vac', 'Vacuum wavelengths')
hx.append(hdu_wave)
hdu_flux = fits.ImageHDU(flux)
hdu_flux.header['EXTNAME'] = 'FLUX'
hdu_flux.header['BUNIT'] = str(fluxunits)
hx.append(hdu_flux)
hdu_meta = fits.table_to_hdu(meta)
hdu_meta.header['EXTNAME'] = 'METADATA'
hx.append(hdu_meta)
log.info('Writing {}'.format(outfile))
try:
hx.writeto(outfile, overwrite=True)
except:
hx.writeto(outfile, clobber=True)
def write_sky(outfile, skymodel, header=None):
"""Write sky model.
Args:
outfile : filename or (night, expid, camera) tuple
skymodel : SkyModel object, with the following attributes
wave : 1D wavelength in vacuum Angstroms
flux : 2D[nspec, nwave] sky flux
ivar : 2D inverse variance of sky flux
mask : 2D mask for sky flux
header : optional fits header data (fits.Header, dict, or list)
"""
outfile = makepath(outfile, 'sky')
#- Convert header to fits.Header if needed
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(skymodel.header)
hdr['EXTNAME'] = ('SKY', 'no dimension')
fits.writeto(outfile, skymodel.flux, header=hdr, clobber=True)
hdr['EXTNAME'] = ('IVAR', 'no dimension')
hdu = fits.ImageHDU(skymodel.ivar, header=hdr)
fits.append(outfile, hdu.data, header=hdu.header)
hdr['EXTNAME'] = ('MASK', 'no dimension')
hdu = fits.ImageHDU(skymodel.mask, header=hdr)
fits.append(outfile, hdu.data, header=hdu.header)
hdr['EXTNAME'] = ('WAVELENGTH', '[Angstroms]')
hdu = fits.ImageHDU(skymodel.wave, header=hdr)
fits.append(outfile, hdu.data, header=hdu.header)
return outfile
def write_sky(outfile, skymodel, header=None):
"""Write sky model.
Args:
outfile : filename or (night, expid, camera) tuple
skymodel : SkyModel object, with the following attributes
wave : 1D wavelength in vacuum Angstroms
flux : 2D[nspec, nwave] sky flux
ivar : 2D inverse variance of sky flux
mask : 2D mask for sky flux
header : optional fits header data (fits.Header, dict, or list)
"""
outfile = makepath(outfile, 'sky')
#- Convert header to fits.Header if needed
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(skymodel.header)
hdr['EXTNAME'] = ('SKY', 'no dimension')
fits.writeto(outfile, skymodel.flux,header=hdr, clobber=True)
hdr['EXTNAME'] = ('IVAR', 'no dimension')
hdu = fits.ImageHDU(skymodel.ivar, header=hdr)
fits.append(outfile, hdu.data, header=hdu.header)
hdr['EXTNAME'] = ('MASK', 'no dimension')
hdu = fits.ImageHDU(skymodel.mask, header=hdr)
fits.append(outfile, hdu.data, header=hdu.header)
hdr['EXTNAME'] = ('WAVELENGTH', '[Angstroms]')
hdu = fits.ImageHDU(skymodel.wave, header=hdr)
fits.append(outfile, hdu.data, header=hdu.header)
return outfile
def main():
"""Runs the process
"""
# Check environmental variables are set
assert "DESI_SPECTRO_DATA" in os.environ, "Missing $DESI_SPECTRO_DATA environment variable"
assert "PRODNAME" in os.environ, "Missing $PRODNAME environment variable"
assert "DESI_SPECTRO_REDUX" in os.environ, "Missing $DESI_SPECTRO_REDUX environment variable"
# Grab nights in data redux
nights = glob.glob(dio_meta.specprod_root() + "/exposures/*")
if len(nights) == 0:
raise ValueError("No nights in exposures!")
# Build up list of fibermap files
fibermap_files = []
for night in nights:
onight = night[night.rfind("/") :]
files = glob.glob(dio_meta.data_root() + "/" + onight + "/fibermap*")
#
fibermap_files += files
# Get list of zbest files
zbest_files = glob.glob(dio_meta.specprod_root() + "/bricks/*/zbest*")
if len(zbest_files) == 0:
raise ValueError("No redshifts?!")
# Meta
meta = get_meta()
# Load+write table
simtab_fil = dio_meta.specprod_root() + "/QA/sim_z_table.fits"
dio_util.makepath(simtab_fil)
simz_tab = dsqa_z.load_z(fibermap_files, zbest_files)
simz_tab.meta = meta
simz_tab.write(simtab_fil, overwrite=True)
# Summary stats
summ_file = dio_meta.specprod_root() + "/QA/sim_z_summ.yaml"
dio_util.makepath(summ_file)
summ_dict = dsqa_z.summ_stats(simz_tab)
# Write
with open(summ_file, "w") as outfile:
outfile.write(yaml.dump(meta)) # , default_flow_style=True) )
outfile.write(yaml.dump(summ_dict, default_flow_style=False))
# import pdb
# pdb.set_trace()
"""
# Reorder + cut
summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL']
# Write
summ_tab.meta = meta
summ_tab.write(summ_file,format='ascii.ecsv',
formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,clobber=True)
"""
# QA Figures
fig_file = dio_meta.specprod_root() + "/QA/sim_z.pdf"
dio_util.makepath(fig_file)
pp = PdfPages(fig_file)
# Summ
dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp)
for objtype in ["ELG", "LRG", "QSO_T", "QSO_L"]:
dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp)
# All done
pp.close()
def write_frame(outfile, frame, header=None, fibermap=None, units=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
frame: desispec.frame.Frame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a Frame object to pass into write_frame,
frame = Frame(wave, flux, ivar, resolution_data)
"""
outfile = makepath(outfile, 'frame')
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(frame.meta)
add_dependencies(hdr)
hdus = fits.HDUList()
x = fits.PrimaryHDU(frame.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
if units is not None:
units = str(units)
if 'BUNIT' in hdr and hdr['BUNIT'] != units:
log.warn('BUNIT {bunit} != units {units}; using {units}'.format(
bunit=hdr['BUNIT'], units=units))
x.header['BUNIT'] = units
hdus.append(x)
hdus.append(fits.ImageHDU(frame.ivar.astype('f4'), name='IVAR'))
hdus.append(fits.CompImageHDU(frame.mask, name='MASK'))
hdus.append(fits.ImageHDU(frame.wave.astype('f4'), name='WAVELENGTH'))
hdus[-1].header['BUNIT'] = 'Angstrom'
hdus.append(
fits.ImageHDU(frame.resolution_data.astype('f4'), name='RESOLUTION'))
if fibermap is not None:
fibermap = desiutil.io.encode_table(fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
elif frame.fibermap is not None:
fibermap = desiutil.io.encode_table(
frame.fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
elif frame.spectrograph is not None:
x.header[
'FIBERMIN'] = 500 * frame.spectrograph # Hard-coded (as in desispec.frame)
else:
log.error(
"You are likely writing a frame without sufficient fiber info")
if frame.chi2pix is not None:
hdus.append(fits.ImageHDU(frame.chi2pix.astype('f4'), name='CHI2PIX'))
hdus.writeto(outfile + '.tmp', clobber=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def main():
'''Runs the process
'''
# Check environmental variables are set
assert 'DESI_SPECTRO_DATA' in os.environ, 'Missing $DESI_SPECTRO_DATA environment variable'
assert 'SPECPROD' in os.environ, 'Missing $SPECPROD environment variable'
assert 'DESI_SPECTRO_REDUX' in os.environ, 'Missing $DESI_SPECTRO_REDUX environment variable'
# Grab nights in data redux
nights = glob.glob(dio_meta.specprod_root()+'/exposures/*')
if len(nights) == 0:
raise ValueError('No nights in exposures!')
# Build up list of fibermap files
fibermap_files = []
for night in nights:
onight = night[night.rfind('/'):]
files = glob.glob(dio_meta.rawdata_root()+'/'+onight+'/fibermap*')
#
fibermap_files += files
# Get list of zbest files
zbest_files = glob.glob(dio_meta.specprod_root()+'/bricks/*/zbest*')
if len(zbest_files) == 0:
raise ValueError('No redshifts?!')
# Meta
meta = get_meta()
# Load+write table
simtab_fil = dio_meta.specprod_root()+'/QA/sim_z_table.fits'
dio_util.makepath(simtab_fil)
simz_tab = dsqa_z.load_z(fibermap_files, zbest_files)
simz_tab.meta = meta
simz_tab.write(simtab_fil,overwrite=True)
# Summary stats
summ_file = dio_meta.specprod_root()+'/QA/sim_z_summ.yaml'
dio_util.makepath(summ_file)
summ_dict = dsqa_z.summ_stats(simz_tab)
# Write
with open(summ_file, 'w') as outfile:
outfile.write( yaml.dump(meta))#, default_flow_style=True) )
outfile.write( yaml.dump(summ_dict, default_flow_style=False) )
#import pdb
#pdb.set_trace()
'''
# Reorder + cut
summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL']
# Write
summ_tab.meta = meta
summ_tab.write(summ_file,format='ascii.ecsv',
formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,overwrite=True)
'''
# QA Figures
fig_file = dio_meta.specprod_root()+'/QA/sim_z.pdf'
dio_util.makepath(fig_file)
pp = PdfPages(fig_file)
# Summ
dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp)
for objtype in ['ELG','LRG', 'QSO_T', 'QSO_L']:
dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp)
# All done
pp.close()
def main():
log = get_logger()
key = 'DESI_ROOT'
if key not in os.environ:
log.fatal('Required ${} environment variable not set'.format(key))
return 0
desidir = os.getenv(key)
simsdir = os.path.join(desidir, 'spectro', 'sim', 'bgs-sims1.0')
brickdir = os.path.join(simsdir, 'bricks')
parser = argparse.ArgumentParser()
parser.add_argument('--sim', type=int, default=None, help='Simulation number (see documentation)')
parser.add_argument('--nproc', type=int, default=1, help='Number of processors to use.')
parser.add_argument('--simsdir', default=simsdir, help='Top-level simulation directory')
parser.add_argument('--bricks', action='store_true', help='Generate the brick files.')
parser.add_argument('--zfind', action='store_true', help='Fit for the redshifts.')
parser.add_argument('--results', action='store_true', help='Merge all the relevant results.')
parser.add_argument('--qaplots', action='store_true', help='Generate QAplots.')
args = parser.parse_args()
if args.sim is None:
parser.print_help()
sys.exit(1)
# --------------------------------------------------
# Initialize the parameters of each simulation here.
if args.sim == 1:
seed = 678245
brickname = 'sim01'
nbrick = 50
nspec = 20
phase = 0.25
rmag = (19.5, 19.5)
redshift = (0.1, 0.3)
zenith = 30
exptime = 300
angle = (0, 150)
simoptions = [
'--exptime-range', '{}'.format(exptime), '{}'.format(exptime),
'--rmagrange-bgs', '{}'.format(rmag[0]), '{}'.format(rmag[1]),
'--zrange-bgs', '{}'.format(redshift[0]), '{}'.format(redshift[1]),
'--moon-zenith-range', '{}'.format(zenith), '{}'.format(zenith),
'--moon-phase-range', '{}'.format(phase), '{}'.format(phase),
'--moon-angle-range', '{}'.format(angle[0]), '{}'.format(angle[1])
]
elif args.sim == 2:
seed = 991274
brickname = 'sim02'
nbrick = 50
nspec = 20
phase = (0.0, 1.0)
rmag = (19.5, 19.5)
redshift = (0.1, 0.3)
zenith = 30
exptime = 300
angle = 60
simoptions = [
'--exptime-range', '{}'.format(exptime), '{}'.format(exptime),
'--rmagrange-bgs', '{}'.format(rmag[0]), '{}'.format(rmag[1]),
'--zrange-bgs', '{}'.format(redshift[0]), '{}'.format(redshift[1]),
'--moon-zenith-range', '{}'.format(zenith), '{}'.format(zenith),
'--moon-phase-range', '{}'.format(phase[0]), '{}'.format(phase[1]),
'--moon-angle-range', '{}'.format(angle), '{}'.format(angle)
]
elif args.sim == 3:
seed = 471934
brickname = 'sim03'
nbrick = 50
nspec = 20
phase = (0.0, 1.0)
rmag = (19.5, 19.5)
redshift = (0.1, 0.3)
zenith = 30
exptime = 300
angle = (0, 150)
simoptions = [
'--exptime-range', '{}'.format(exptime), '{}'.format(exptime),
'--rmagrange-bgs', '{}'.format(rmag[0]), '{}'.format(rmag[1]),
'--zrange-bgs', '{}'.format(redshift[0]), '{}'.format(redshift[1]),
'--moon-zenith-range', '{}'.format(zenith), '{}'.format(zenith),
'--moon-phase-range', '{}'.format(phase[0]), '{}'.format(phase[1]),
'--moon-angle-range', '{}'.format(angle[0]), '{}'.format(angle[1])
]
elif args.sim == 4:
seed = 971234
brickname = 'sim04'
nbrick = 100
nspec = 50
phase = (0.0, 1.0)
rmag = (17.5, 20.0)
redshift = (0.1, 0.3)
zenith = 30
exptime = 300
angle = (0, 150)
simoptions = [
'--exptime-range', '{}'.format(exptime), '{}'.format(exptime),
'--rmagrange-bgs', '{}'.format(rmag[0]), '{}'.format(rmag[1]),
'--zrange-bgs', '{}'.format(redshift[0]), '{}'.format(redshift[1]),
'--moon-zenith-range', '{}'.format(zenith), '{}'.format(zenith),
'--moon-phase-range', '{}'.format(phase[0]), '{}'.format(phase[1]),
'--moon-angle-range', '{}'.format(angle[0]), '{}'.format(angle[1])
]
nobj = nbrick*nspec
rand = np.random.RandomState(seed)
# --------------------------------------------------
# Generate the brick and truth files.
if args.bricks:
brightoptions = [
'--brickname', '{}'.format(brickname),
'--nbrick', '{}'.format(nbrick),
'--nspec', '{}'.format(nspec),
'--outdir', '{}'.format(simsdir),
'--brickdir', '{}'.format(brickdir),
'--seed', '{}'.format(seed),
'--objtype', 'BGS']
brightargs = brightsims.parse(np.hstack((brightoptions, simoptions)))
brightargs.verbose = True
brightsims.main(brightargs)
# --------------------------------------------------
# Fit the redshifts
if args.zfind:
inputfile = os.path.join(simsdir, brickname+'-input.fits')
log.info('Reading {}'.format(inputfile))
cat = fits.getdata(inputfile, 1)
log.info('Testing with just one brick!')
#for ib in range(10, 11):
for ib in range(nbrick):
thisbrick = cat['BRICKNAME'][ib]
brickfiles = [os.path.join(brickdir, 'brick-{}-{}.fits'.format(ch, thisbrick)) for ch in ['b', 'r', 'z']]
redoptions = [
'--brick', thisbrick,
'--nproc', '{}'.format(args.nproc),
'--specprod_dir', simsdir,
'--zrange-galaxy', '{}'.format(redshift[0]), '{}'.format(redshift[1]),
'--outfile', os.path.join(brickdir, thisbrick, 'zbest-{}.fits'.format(thisbrick)),
'--objtype', 'ELG,LRG']
redargs = zfind.parse(redoptions)
zfind.main(redargs)
# --------------------------------------------------
# Parse and write out the simulation inputs, brick spectra, and redshifts
if args.results:
inputfile = os.path.join(simsdir, brickname+'-input.fits')
log.info('Reading {}'.format(inputfile))
cat = fits.getdata(inputfile, 1)
# Build a results table.
resultfile = makepath(os.path.join(simsdir, '{}-results.fits'.format(brickname)))
resultcols = [
('EXPTIME', 'f4'),
('AIRMASS', 'f4'),
('MOONPHASE', 'f4'),
('MOONANGLE', 'f4'),
('MOONZENITH', 'f4'),
('SNR_B', 'f4'),
('SNR_R', 'f4'),
('SNR_Z', 'f4'),
('TARGETID', 'i8'),
('RMAG', 'f4'),
('D4000', 'f4'),
('EWHBETA', 'f4'),
('ZTRUE', 'f4'),
('Z', 'f4'),
('ZERR', 'f4'),
('ZWARNING', 'f4')]
result = Table(np.zeros(nobj, dtype=resultcols))
result['EXPTIME'].unit = 's'
result['MOONANGLE'].unit = 'deg'
result['MOONZENITH'].unit = 'deg'
for ib in range(nbrick):
# Copy over some data.
thisbrick = cat['BRICKNAME'][ib]
result['EXPTIME'][nspec*ib:nspec*(ib+1)] = cat['EXPTIME'][ib]
result['AIRMASS'][nspec*ib:nspec*(ib+1)] = cat['AIRMASS'][ib]
result['MOONPHASE'][nspec*ib:nspec*(ib+1)] = cat['MOONPHASE'][ib]
result['MOONANGLE'][nspec*ib:nspec*(ib+1)] = cat['MOONANGLE'][ib]
result['MOONZENITH'][nspec*ib:nspec*(ib+1)] = cat['MOONZENITH'][ib]
# Read the truth file of the first channel to get the metadata.
truthfile = os.path.join(brickdir, thisbrick, 'truth-brick-{}-{}.fits'.format('b', thisbrick))
log.info('Reading {}'.format(truthfile))
truth = io.Brick(truthfile).hdu_list[4].data
result['TARGETID'][nspec*ib:nspec*(ib+1)] = truth['TARGETID']
result['RMAG'][nspec*ib:nspec*(ib+1)] = 22.5-2.5*np.log10(truth['DECAM_FLUX'][:,2])
result['D4000'][nspec*ib:nspec*(ib+1)] = truth['D4000']
result['EWHBETA'][nspec*ib:nspec*(ib+1)] = truth['EWHBETA']
result['ZTRUE'][nspec*ib:nspec*(ib+1)] = truth['TRUEZ']
# Finally read the zbest file.
zbestfile = os.path.join(brickdir, thisbrick, 'zbest-{}.fits'.format(thisbrick))
if os.path.isfile(zbestfile):
log.info('Reading {}'.format(zbestfile))
zbest = read_zbest(zbestfile)
# There's gotta be a better way than looping here!
for ii in range(nspec):
this = np.where(zbest.targetid[ii] == result['TARGETID'])[0]
result['Z'][this] = zbest.z[ii]
result['ZERR'][this] = zbest.zerr[ii]
result['ZWARNING'][this] = zbest.zwarn[ii]
#pdb.set_trace()
# Finally, read the spectra and truth tables, one per channel.
for channel in ('b','r','z'):
brickfile = os.path.join(brickdir, thisbrick, 'brick-{}-{}.fits'.format(channel, thisbrick))
log.info('Reading {}'.format(brickfile))
brick = io.Brick(brickfile)
wave = brick.get_wavelength_grid()
for iobj in range(nspec):
flux = brick.hdu_list[0].data[iobj,:]
ivar = brick.hdu_list[1].data[iobj,:]
these = np.where((wave>np.mean(wave)-50)*(wave<np.mean(wave)+50)*(flux>0))[0]
result['SNR_'+channel.upper()][nspec*ib+iobj] = \
np.median(np.sqrt(flux[these]*ivar[these]))
log.info('Writing {}'.format(resultfile))
write_bintable(resultfile, result, extname='RESULTS', clobber=True)
# --------------------------------------------------
# Build QAplots
if args.qaplots:
phaserange = (-0.05, 1.05)
snrrange = (0, 3)
rmagrange = (17.5, 20)
anglerange = (-5, 155)
d4000range = (0.9, 2.2)
snrinterval = 1.0
cmap = mpl.colors.ListedColormap(sns.color_palette('muted'))
resultfile = os.path.join(simsdir, '{}-results.fits'.format(brickname))
log.info('Reading {}'.format(resultfile))
res = fits.getdata(resultfile, 1)
qafile = os.path.join(simsdir, 'qa-{}.pdf'.format(brickname))
log.info('Writing {}'.format(qafile))
# ------------------------------
# Simulation 1
if args.sim == 1:
fig, ax0 = plt.subplots(1, 1, figsize=(6, 4.5))
ax0.scatter(res['MOONANGLE']+rand.normal(0, 0.2, nobj), res['SNR_B'], label='b channel', c=col[1])
ax0.scatter(res['MOONANGLE']+rand.normal(0, 0.2, nobj), res['SNR_R'], label='r channel', c=col[2])
ax0.scatter(res['MOONANGLE']+rand.normal(0, 0.2, nobj), res['SNR_Z'], label='z channel', c=col[0])
ax0.set_xlabel('Object-Moon Angle (deg)')
ax0.set_ylabel(r'Signal-to-Noise Ratio (pixel$^{-1}$)')
ax0.set_xlim(anglerange)
ax0.set_ylim(snrrange)
plt.legend(loc='upper left', labelspacing=0.25)
plt.text(0.95, 0.7, 't = {:g} s\nr = {:g} mag\nRedshift = {:.2f}-{:.2f}'.format(exptime, rmag,
redshift[0], redshift[1])+\
'\nLunar Zenith Angle = {:g} deg\nLunar Phase = {:g}'.format(zenith, phase),
horizontalalignment='right', transform=ax0.transAxes,
fontsize=11)
plt.subplots_adjust(bottom=0.2, right=0.95, left=0.15)
plt.savefig(qafile)
plt.close()
#pdb.set_trace()
# ------------------------------
# Simulation 2
if args.sim == 2:
fig, ax0 = plt.subplots(1, 1, figsize=(6, 4.5))
ax0.scatter(res['MOONPHASE']+rand.normal(0, 0.02, nobj), res['SNR_B'], label='b channel', c=col[1])
ax0.scatter(res['MOONPHASE']+rand.normal(0, 0.02, nobj), res['SNR_R'], label='r channel', c=col[2])
ax0.scatter(res['MOONPHASE']+rand.normal(0, 0.02, nobj), res['SNR_Z'], label='z channel', c=col[0])
ax0.set_xlabel('Lunar Phase (0=Full, 1=New)')
ax0.set_ylabel(r'Signal-to-Noise Ratio (pixel$^{-1}$)')
ax0.set_xlim(phaserange)
ax0.set_ylim(snrrange)
plt.legend(loc='upper left', labelspacing=0.25)
plt.text(0.95, 0.7, 't = {:g} s\nr = {:g} mag\nRedshift = {:.2f}-{:.2f}'.format(exptime, rmag,
redshift[0], redshift[1])+\
'\nLunar Zenith Angle = {:g} deg\nObject-Moon Angle = {:g} deg'.format(zenith, angle),
horizontalalignment='right', transform=ax0.transAxes,
fontsize=11)
plt.subplots_adjust(bottom=0.2, right=0.95, left=0.15)
plt.savefig(qafile)
plt.close()
# ------------------------------
# Simulation 3
if args.sim == 3:
fig, ax0 = plt.subplots(1, 1, figsize=(6, 4))
im = ax0.scatter(res['MOONANGLE']+rand.normal(0, 0.3, nobj), res['SNR_R'], c=res['MOONPHASE'], vmin=phaserange,
cmap=cmap)
ax0.set_xlabel('Object-Moon Angle (deg)')
ax0.set_ylabel(r'r channel Signal-to-Noise Ratio (pixel$^{-1}$)')
ax0.set_xlim(anglerange)
ax0.set_ylim(snrrange)
ax0.yaxis.set_major_locator(mpl.ticker.MultipleLocator(snrinterval))
plt.text(0.95, 0.7, 't = {:g} s\nr = {:g} mag\nRedshift = {:.2f}-{:.2f}'.format(exptime, rmag,
redshift[0], redshift[1])+\
'\nLunar Zenith Angle = {:g} deg'.format(zenith),
horizontalalignment='right', transform=ax0.transAxes,
fontsize=11)
cbar = fig.colorbar(im)
cbar.set_ticks([0, 0.5, 1])
cbar.ax.set_yticklabels(['Full','Quarter','New'], rotation=90)
cbar.ax.set_ylabel('Lunar Phase')
plt.subplots_adjust(bottom=0.2, right=1.0)
plt.savefig(qafile)
plt.close()
# ------------------------------
# Simulation 4
if args.sim == 4:
bins = 30
zgood = (np.abs(res['Z']-res['ZTRUE'])<5E-5)*(res['ZWARNING']==0)*1
H, xedges, yedges = np.histogram2d(res['RMAG'], res['MOONPHASE'], bins=bins, weights=zgood)
H2, _, _ = np.histogram2d(res['RMAG'], res['MOONPHASE'], bins=bins)
extent = np.array((rmagrange, phaserange)).flatten()
#pdb.set_trace()
fig, ax0 = plt.subplots(1, 1, figsize=(6, 4))
im = ax0.imshow(H/H2, extent=extent, interpolation='nearest')#, cmap=cmap)
ax0.set_xlabel('r (AB mag)')
ax0.set_ylabel('Lunar Phase (0=Full, 1=New)')
ax0.set_xlim(rmagrange)
ax0.set_ylim(phaserange)
#plt.text(0.95, 0.7, 't = {:g} s\nr = {:g} mag\nRedshift = {:.2f}-{:.2f}'.format(exptime, rmag,
# redshift[0], redshift[1])+\
# '\nLunar Zenith Angle = {:g} deg'.format(zenith),
# horizontalalignment='right', transform=ax0.transAxes,
# fontsize=11)
cbar = fig.colorbar(im)
#cbar.set_ticks([0, 0.5, 1])
#cbar.ax.set_yticklabels(['Full','Quarter','New'], rotation=90)
#cbar.ax.set_ylabel('Lunar Phase')
plt.subplots_adjust(bottom=0.2, right=1.0)
plt.savefig(qafile)
plt.close()
# ------------------------------
# Simulation 10
if args.sim == 10:
zgood = (np.abs(res['Z']-res['ZTRUE'])<0.001)*(res['ZWARNING']==0)*1
#pdb.set_trace()
fig, (ax0, ax1, ax2) = plt.subplots(3, 1, figsize=(6,6))
# moon phase vs S/N(r)
im = ax0.scatter(res['RMAG'], res['SNR_R'], c=res['MOONPHASE'], vmin=phaserange,
cmap=cmap)
ax0.set_xlabel('r (AB mag)')
ax0.set_ylabel('S/N (r channel)')
ax0.set_xlim(rmagrange)
ax0.set_ylim(snrrange)
ax0.yaxis.set_major_locator(mpl.ticker.MultipleLocator(snrinterval))
# object-moon angle vs S/N(r)
im = ax1.scatter(res['MOONANGLE'], res['SNR_R'], c=res['MOONPHASE'], vmin=phaserange,
cmap=cmap)
ax1.set_xlabel('Object-Moon Angle (deg)')
ax1.set_ylabel('S/N (r channel)')
ax1.set_xlim(anglerange)
ax1.set_ylim(snrrange)
ax1.yaxis.set_major_locator(mpl.ticker.MultipleLocator(snrinterval))
# D(4000) vs S/N(r)
im = ax2.scatter(res['D4000'], res['SNR_R'], c=res['MOONPHASE'], vmin=phaserange,
cmap=cmap)
ax2.set_xlabel('$D_{n}(4000)$')
ax2.set_ylabel('S/N (r channel)')
ax2.set_xlim(d4000range)
ax2.set_ylim(snrrange)
ax2.yaxis.set_major_locator(mpl.ticker.MultipleLocator(snrinterval))
# Shared colorbar
cbarax = fig.add_axes([0.83, 0.15, 0.03, 0.8])
cbar = fig.colorbar(im, cax=cbarax)
ticks = ['Full','Quarter','New']
cbar.set_ticks([0, 0.5, 1])
cbar.ax.set_yticklabels(ticks, rotation=-45)
plt.tight_layout(pad=0.5)#, h_pad=0.2, w_pad=0.3)
plt.subplots_adjust(right=0.78)
plt.savefig(qafile)
plt.close()
def write_qframe(outfile, qframe, header=None, fibermap=None, units=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
qframe: desispec.qproc.QFrame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a QFrame object to pass into write_qframe,
qframe = QFrame(wave, flux, ivar)
"""
log = get_logger()
outfile = makepath(outfile, 'qframe')
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(qframe.meta)
add_dependencies(hdr)
hdus = fits.HDUList()
x = fits.PrimaryHDU(qframe.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
if units is not None:
units = str(units)
if 'BUNIT' in hdr and hdr['BUNIT'] != units:
log.warning('BUNIT {bunit} != units {units}; using {units}'.format(
bunit=hdr['BUNIT'], units=units))
x.header['BUNIT'] = units
hdus.append(x)
hdus.append( fits.ImageHDU(qframe.ivar.astype('f4'), name='IVAR') )
if qframe.mask is None :
qframe.mask=np.zeros(qframe.flux.shape,dtype=np.uint32)
# hdus.append( fits.CompImageHDU(qframe.mask, name='MASK') )
hdus.append( fits.ImageHDU(qframe.mask, name='MASK') )
if qframe.sigma is None :
qframe.sigma=np.zeros(qframe.flux.shape,dtype=np.float)
hdus.append( fits.ImageHDU(qframe.sigma.astype('f4'), name='YSIGMA') )
hdus.append( fits.ImageHDU(qframe.wave.astype('f8'), name='WAVELENGTH') )
hdus[-1].header['BUNIT'] = 'Angstrom'
if fibermap is not None:
fibermap = encode_table(fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append( fits.convenience.table_to_hdu(fibermap) )
elif qframe.fibermap is not None:
fibermap = encode_table(qframe.fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append( fits.convenience.table_to_hdu(fibermap) )
elif qframe.spectrograph is not None:
x.header['FIBERMIN'] = 500*qframe.spectrograph # Hard-coded (as in desispec.qproc.qframe)
else:
log.error("You are likely writing a qframe without sufficient fiber info")
hdus.writeto(outfile+'.tmp', clobber=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
return outfile
def main(args):
# Set up the logger.
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
objtype = args.objtype.upper()
log.debug('Using OBJTYPE {}'.format(objtype))
log.debug('Simulating {:g} bricks each with {:g} spectra'.format(args.nbrick, args.nspec))
# Draw priors uniformly given the input ranges.
rand = np.random.RandomState(args.seed)
exptime = rand.uniform(args.exptime_range[0], args.exptime_range[1], args.nbrick)
airmass = rand.uniform(args.airmass_range[0], args.airmass_range[1], args.nbrick)
moonphase = rand.uniform(args.moon_phase_range[0], args.moon_phase_range[1], args.nbrick)
moonangle = rand.uniform(args.moon_angle_range[0], args.moon_angle_range[1], args.nbrick)
moonzenith = rand.uniform(args.moon_zenith_range[0], args.moon_zenith_range[1], args.nbrick)
# Build a metadata table with the simulation inputs.
metafile = makepath(os.path.join(args.outdir, '{}-input.fits'.format(args.brickname)))
metacols = [
('BRICKNAME', 'S20'),
('SEED', 'S20'),
('EXPTIME', 'f4'),
('AIRMASS', 'f4'),
('MOONPHASE', 'f4'),
('MOONANGLE', 'f4'),
('MOONZENITH', 'f4')]
meta = Table(np.zeros(args.nbrick, dtype=metacols))
meta['EXPTIME'].unit = 's'
meta['MOONANGLE'].unit = 'deg'
meta['MOONZENITH'].unit = 'deg'
meta['BRICKNAME'] = ['{}-{:03d}'.format(args.brickname, ii) for ii in range(args.nbrick)]
meta['EXPTIME'] = exptime
meta['AIRMASS'] = airmass
meta['MOONPHASE'] = moonphase
meta['MOONANGLE'] = moonangle
meta['MOONZENITH'] = moonzenith
log.debug('Writing {}'.format(metafile))
write_bintable(metafile, meta, extname='METADATA', clobber=True)
# Generate each brick in turn.
for ii in range(args.nbrick):
thisbrick = meta['BRICKNAME'][ii]
log.debug('Building brick {}'.format(thisbrick))
brickargs = ['--brickname', thisbrick,
'--objtype', args.objtype,
'--nspec', '{}'.format(args.nspec),
'--outdir', os.path.join(args.brickdir, thisbrick),
'--outdir-truth', os.path.join(args.brickdir, thisbrick),
'--exptime', '{}'.format(exptime[ii]),
'--airmass', '{}'.format(airmass[ii]),
'--moon-phase', '{}'.format(moonphase[ii]),
'--moon-angle', '{}'.format(moonangle[ii]),
'--moon-zenith', '{}'.format(moonzenith[ii]),
'--zrange-bgs', '{}'.format(args.zrange_bgs[0]), '{}'.format(args.zrange_bgs[1]),
'--rmagrange-bgs', '{}'.format(args.rmagrange_bgs[0]), '{}'.format(args.rmagrange_bgs[1])]
if args.seed is not None:
brickargs.append('--seed')
brickargs.append('{}'.format(args.seed))
quickargs = quickbrick.parse(brickargs)
if args.verbose:
quickargs.verbose = True
quickbrick.main(quickargs)
def main(args):
# Set up the logger.
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
objtype = args.objtype.upper()
log.debug('Using OBJTYPE {}'.format(objtype))
log.debug('Simulating {:g} bricks each with {:g} spectra'.format(
args.nexp, args.nspec))
# Draw priors uniformly given the input ranges.
rand = np.random.RandomState(args.seed)
exptime = rand.uniform(args.exptime_range[0], args.exptime_range[1],
args.nexp)
airmass = rand.uniform(args.airmass_range[0], args.airmass_range[1],
args.nexp)
moonphase = rand.uniform(args.moon_phase_range[0],
args.moon_phase_range[1], args.nexp)
moonangle = rand.uniform(args.moon_angle_range[0],
args.moon_angle_range[1], args.nexp)
moonzenith = rand.uniform(args.moon_zenith_range[0],
args.moon_zenith_range[1], args.nexp)
maglimits = args.rmagrange - bgs
zrange = args.zrange - bgs
# Build a metadata table with the simulation inputs.
metafile = makepath(
os.path.join(args.outdir, '{}-input.fits'.format(args.brickname)))
metacols = [('BRICKNAME', 'S20'), ('SEED', 'S20'), ('EXPTIME', 'f4'),
('AIRMASS', 'f4'), ('MOONPHASE', 'f4'), ('MOONANGLE', 'f4'),
('MOONZENITH', 'f4')]
meta = Table(np.zeros(args.nexp, dtype=metacols))
meta['EXPTIME'].unit = 's'
meta['MOONANGLE'].unit = 'deg'
meta['MOONZENITH'].unit = 'deg'
meta['BRICKNAME'] = [
'{}-{:03d}'.format(args.nexp, ii) for ii in range(args.nexp)
]
meta['EXPTIME'] = exptime
meta['AIRMASS'] = airmass
meta['MOONPHASE'] = moonphase
meta['MOONANGLE'] = moonangle
meta['MOONZENITH'] = moonzenith
log.debug('Writing {}'.format(metafile))
write_bintable(metafile, meta, extname='METADATA', clobber=True)
# Generate each brick in turn.
for ii in range(args.nexp):
thisbrick = meta['BRICKNAME'][ii]
log.debug('Building brick {}'.format(thisbrick))
brickargs = [
'--brickname', thisbrick, '--objtype', args.objtype, '--nspec',
'{}'.format(args.nspec), '--outdir',
os.path.join(args.brickdir, thisbrick), '--outdir-truth',
os.path.join(args.brickdir,
thisbrick), '--exptime', '{}'.format(exptime[ii]),
'--airmass', '{}'.format(airmass[ii]), '--moon-phase', '{}'.format(
moonphase[ii]), '--moon-angle', '{}'.format(moonangle[ii]),
'--moon-zenith', '{}'.format(moonzenith[ii]), '--zrange-bgs',
'{}'.format(args.zrange_bgs[0]), '{}'.format(args.zrange_bgs[1]),
'--rmagrange-bgs', '{}'.format(args.rmagrange_bgs[0]),
'{}'.format(args.rmagrange_bgs[1])
]
if args.seed is not None:
brickargs.append('--seed')
brickargs.append('{}'.format(args.seed))
quickargs = quickbrick.parse(brickargs)
if args.verbose:
quickargs.verbose = True
quickbrick.main(quickargs)
def main():
'''Runs the process
'''
# Check environmental variables are set
assert 'DESI_SPECTRO_DATA' in os.environ, 'Missing $DESI_SPECTRO_DATA environment variable'
assert 'PRODNAME' in os.environ, 'Missing $PRODNAME environment variable'
assert 'DESI_SPECTRO_REDUX' in os.environ, 'Missing $DESI_SPECTRO_REDUX environment variable'
# Grab nights in data redux
nights = glob.glob(dio_meta.specprod_root() + '/exposures/*')
if len(nights) == 0:
raise ValueError('No nights in exposures!')
# Build up list of fibermap files
fibermap_files = []
for night in nights:
onight = night[night.rfind('/'):]
files = glob.glob(dio_meta.rawdata_root() + '/' + onight +
'/fibermap*')
#
fibermap_files += files
# Get list of zbest files
zbest_files = glob.glob(dio_meta.specprod_root() + '/bricks/*/zbest*')
if len(zbest_files) == 0:
raise ValueError('No redshifts?!')
# Meta
meta = get_meta()
# Load+write table
simtab_fil = dio_meta.specprod_root() + '/QA/sim_z_table.fits'
dio_util.makepath(simtab_fil)
simz_tab = dsqa_z.load_z(fibermap_files, zbest_files)
simz_tab.meta = meta
simz_tab.write(simtab_fil, overwrite=True)
# Summary stats
summ_file = dio_meta.specprod_root() + '/QA/sim_z_summ.yaml'
dio_util.makepath(summ_file)
summ_dict = dsqa_z.summ_stats(simz_tab)
# Write
with open(summ_file, 'w') as outfile:
outfile.write(yaml.dump(meta)) #, default_flow_style=True) )
outfile.write(yaml.dump(summ_dict, default_flow_style=False))
#import pdb
#pdb.set_trace()
'''
# Reorder + cut
summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL']
# Write
summ_tab.meta = meta
summ_tab.write(summ_file,format='ascii.ecsv',
formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,clobber=True)
'''
# QA Figures
fig_file = dio_meta.specprod_root() + '/QA/sim_z.pdf'
dio_util.makepath(fig_file)
pp = PdfPages(fig_file)
# Summ
dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp)
for objtype in ['ELG', 'LRG', 'QSO_T', 'QSO_L']:
dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp)
# All done
pp.close()
def write_qframe(outfile, qframe, header=None, fibermap=None, units=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
qframe: desispec.qproc.QFrame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a QFrame object to pass into write_qframe,
qframe = QFrame(wave, flux, ivar)
"""
log = get_logger()
outfile = makepath(outfile, 'qframe')
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(qframe.meta)
add_dependencies(hdr)
hdus = fits.HDUList()
x = fits.PrimaryHDU(qframe.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
if units is not None:
units = str(units)
if 'BUNIT' in hdr and hdr['BUNIT'] != units:
log.warning('BUNIT {bunit} != units {units}; using {units}'.format(
bunit=hdr['BUNIT'], units=units))
x.header['BUNIT'] = units
hdus.append(x)
hdus.append(fits.ImageHDU(qframe.ivar.astype('f4'), name='IVAR'))
if qframe.mask is None:
qframe.mask = np.zeros(qframe.flux.shape, dtype=np.uint32)
# hdus.append( fits.CompImageHDU(qframe.mask, name='MASK') )
hdus.append(fits.ImageHDU(qframe.mask, name='MASK'))
if qframe.sigma is None:
qframe.sigma = np.zeros(qframe.flux.shape, dtype=np.float)
hdus.append(fits.ImageHDU(qframe.sigma.astype('f4'), name='YSIGMA'))
hdus.append(fits.ImageHDU(qframe.wave.astype('f8'), name='WAVELENGTH'))
hdus[-1].header['BUNIT'] = 'Angstrom'
if fibermap is not None:
fibermap = encode_table(fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
elif qframe.fibermap is not None:
fibermap = encode_table(qframe.fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
elif qframe.spectrograph is not None:
x.header[
'FIBERMIN'] = 500 * qframe.spectrograph # Hard-coded (as in desispec.qproc.qframe)
else:
log.error(
"You are likely writing a qframe without sufficient fiber info")
raise ValueError('no fibermap')
hdus.writeto(outfile + '.tmp', overwrite=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
