def write_qa_prod(outroot, qaprod, indent=True):
"""Write QA for a given production
Args:
outroot : str
filename without format extension
qa_prod : QA_Prod object
Returns:
outfile: str
output filename
"""
from lvmutil.io import combine_dicts
log=get_logger()
outfile = outroot+'.json'
outfile = makepath(outfile, 'qa')
# Loop on exposures
odict = {}
for qaexp in qaprod.qa_exps:
# Get the exposure dict
idict = write_qa_exposure('foo', qaexp, ret_dict=True)
odict = combine_dicts(odict, idict)
ydict = yamlify(odict) # This works well for JSON too
# Simple json
with open(outfile, 'wt') as fh:
json.dump(ydict, fh, indent=indent)
log.info('Wrote QA_Prod file: {:s}'.format(outfile))
return outfile
def write_qa_exposure(outroot, qaexp, ret_dict=False):
"""Write QA for a given exposure
Args:
outroot : str
filename without format extension
qa_exp : QA_Exposure object
ret_dict : bool, optional
Return dict only? [for qa_prod, mainly]
Returns:
outfile or odict : str or dict
"""
# Generate the dict
odict = {qaexp.night: {qaexp.expid: {}}}
odict[qaexp.night][qaexp.expid]['flavor'] = qaexp.flavor
odict[qaexp.night][qaexp.expid]['meta'] = qaexp.meta
cameras = list(qaexp.data['frames'].keys())
for camera in cameras:
odict[qaexp.night][qaexp.expid][camera] = qaexp.data['frames'][camera]
# Return dict only?
if ret_dict:
return odict
# Simple yaml
ydict = yamlify(odict)
outfile = outroot+'.yaml'
outfile = makepath(outfile, 'qa')
with open(outfile, 'w') as yamlf:
yamlf.write( yaml.dump(ydict))#, default_flow_style=True) )
return outfile
def write_fibermap(outfile, fibermap, header=None):
"""Write fibermap binary table to outfile.
Args:
outfile (str): output filename
fibermap: astropy Table of fibermap data
header: header data to include in same HDU as fibermap
Returns:
write_fibermap (str): full path to filename of fibermap file written.
"""
outfile = makepath(outfile)
#- astropy.io.fits incorrectly generates warning about 2D arrays of strings
#- Temporarily turn off warnings to avoid this; lvmspec.test.test_io will
#- catch it if the arrays actually are written incorrectly.
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(fibermap.meta)
add_dependencies(hdr)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
write_bintable(outfile, fibermap, hdr, comments=fibermap_comments,
extname="FIBERMAP", clobber=True)
return outfile
def write_qa_brick(outfile, qabrick):
"""Write QA for a given exposure
Args:
outfile : filename
qabrick : QA_Brick object
_data: dict of QA info
"""
outfile = makepath(outfile, 'qa')
# Simple yaml
ydict = yamlify(qabrick.data)
with open(outfile, 'w') as yamlf:
yamlf.write( yaml.dump(ydict))#, default_flow_style=True) )
return outfile
def write_qa_frame(outfile, qaframe, verbose=False):
"""Write QA for a given frame
Args:
outfile : str
filename
qa_exp : QA_Frame object, with the following attributes
qa_data: dict of QA info
"""
log=get_logger()
outfile = makepath(outfile, 'qa')
# Generate the dict
odict = {qaframe.night: {qaframe.expid: {qaframe.camera: {}, 'flavor': qaframe.flavor}}}
odict[qaframe.night][qaframe.expid][qaframe.camera] = qaframe.qa_data
ydict = yamlify(odict)
# Simple yaml
with open(outfile, 'w') as yamlf:
yamlf.write( yaml.dump(ydict))#, default_flow_style=True) )
if verbose:
log.info("Wrote QA frame file: {:s}".format(outfile))
return outfile
def write_templates(outfile, flux, wave, meta):
"""Write out simulated galaxy templates.
Args:
outfile (str): Output file name.
flux (numpy.ndarray): Flux vector (1e-17 erg/s/cm2/A)
wave (numpy.ndarray): Wavelength vector (Angstrom).
meta (astropy.table.Table): metadata table.
"""
from astropy.io import fits
from lvmspec.io.util import makepath
# Create the path to OUTFILE if necessary.
outfile = makepath(outfile)
hx = fits.HDUList()
hdu_wave = fits.PrimaryHDU(wave)
hdu_wave.header['EXTNAME'] = 'WAVE'
hdu_wave.header['BUNIT'] = 'Angstrom'
hdu_wave.header['AIRORVAC'] = ('vac', 'Vacuum wavelengths')
hx.append(hdu_wave)
hdu_flux = fits.ImageHDU(flux)
hdu_flux.header['EXTNAME'] = 'FLUX'
hdu_flux.header['BUNIT'] = str(fluxunits)
hx.append(hdu_flux)
hdu_meta = fits.table_to_hdu(meta)
hdu_meta.header['EXTNAME'] = 'METADATA'
hx.append(hdu_meta)
log.info('Writing {}'.format(outfile))
try:
hx.writeto(outfile, overwrite=True)
except:
hx.writeto(outfile, clobber=True)
def main(args):
# Set up the logger.
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
objtype = args.objtype.upper()
log.debug('Using OBJTYPE {}'.format(objtype))
log.debug('Simulating {:g} bricks each with {:g} spectra'.format(args.nbrick, args.nspec))
# Draw priors uniformly given the input ranges.
rand = np.random.RandomState(args.seed)
exptime = rand.uniform(args.exptime_range[0], args.exptime_range[1], args.nbrick)
airmass = rand.uniform(args.airmass_range[0], args.airmass_range[1], args.nbrick)
moonphase = rand.uniform(args.moon_phase_range[0], args.moon_phase_range[1], args.nbrick)
moonangle = rand.uniform(args.moon_angle_range[0], args.moon_angle_range[1], args.nbrick)
moonzenith = rand.uniform(args.moon_zenith_range[0], args.moon_zenith_range[1], args.nbrick)
# Build a metadata table with the simulation inputs.
metafile = makepath(os.path.join(args.outdir, '{}-input.fits'.format(args.brickname)))
metacols = [
('BRICKNAME', 'S20'),
('SEED', 'S20'),
('EXPTIME', 'f4'),
('AIRMASS', 'f4'),
('MOONPHASE', 'f4'),
('MOONANGLE', 'f4'),
('MOONZENITH', 'f4')]
meta = Table(np.zeros(args.nbrick, dtype=metacols))
meta['EXPTIME'].unit = 's'
meta['MOONANGLE'].unit = 'deg'
meta['MOONZENITH'].unit = 'deg'
meta['BRICKNAME'] = ['{}-{:03d}'.format(args.brickname, ii) for ii in range(args.nbrick)]
meta['EXPTIME'] = exptime
meta['AIRMASS'] = airmass
meta['MOONPHASE'] = moonphase
meta['MOONANGLE'] = moonangle
meta['MOONZENITH'] = moonzenith
log.debug('Writing {}'.format(metafile))
write_bintable(metafile, meta, extname='METADATA', clobber=True)
# Generate each brick in turn.
for ii in range(args.nbrick):
thisbrick = meta['BRICKNAME'][ii]
log.debug('Building brick {}'.format(thisbrick))
brickargs = ['--brickname', thisbrick,
'--objtype', args.objtype,
'--nspec', '{}'.format(args.nspec),
'--outdir', os.path.join(args.brickdir, thisbrick),
'--outdir-truth', os.path.join(args.brickdir, thisbrick),
'--exptime', '{}'.format(exptime[ii]),
'--airmass', '{}'.format(airmass[ii]),
'--moon-phase', '{}'.format(moonphase[ii]),
'--moon-angle', '{}'.format(moonangle[ii]),
'--moon-zenith', '{}'.format(moonzenith[ii]),
'--zrange-bgs', '{}'.format(args.zrange_bgs[0]), '{}'.format(args.zrange_bgs[1]),
'--rmagrange-bgs', '{}'.format(args.rmagrange_bgs[0]), '{}'.format(args.rmagrange_bgs[1])]
if args.seed is not None:
brickargs.append('--seed')
brickargs.append('{}'.format(args.seed))
quickargs = quickbrick.parse(brickargs)
if args.verbose:
quickargs.verbose = True
quickbrick.main(quickargs)
def main():
'''Runs the process
'''
# Check environmental variables are set
assert 'LVM_SPECTRO_DATA' in os.environ, 'Missing $LVM_SPECTRO_DATA environment variable'
assert 'SPECPROD' in os.environ, 'Missing $SPECPROD environment variable'
assert 'LVM_SPECTRO_REDUX' in os.environ, 'Missing $LVM_SPECTRO_REDUX environment variable'
# Grab nights in data redux
nights = glob.glob(dio_meta.specprod_root() + '/exposures/*')
if len(nights) == 0:
raise ValueError('No nights in exposures!')
# Build up list of fibermap files
fibermap_files = []
for night in nights:
onight = night[night.rfind('/'):]
files = glob.glob(dio_meta.rawdata_root() + '/' + onight +
'/fibermap*')
#
fibermap_files += files
# Get list of zbest files
zbest_files = glob.glob(dio_meta.specprod_root() + '/bricks/*/zbest*')
if len(zbest_files) == 0:
raise ValueError('No redshifts?!')
# Meta
meta = get_meta()
# Load+write table
simtab_fil = dio_meta.specprod_root() + '/QA/sim_z_table.fits'
dio_util.makepath(simtab_fil)
simz_tab = dsqa_z.load_z(fibermap_files, zbest_files)
simz_tab.meta = meta
simz_tab.write(simtab_fil, overwrite=True)
# Summary stats
summ_file = dio_meta.specprod_root() + '/QA/sim_z_summ.yaml'
dio_util.makepath(summ_file)
summ_dict = dsqa_z.summ_stats(simz_tab)
# Write
with open(summ_file, 'w') as outfile:
outfile.write(yaml.dump(meta)) #, default_flow_style=True) )
outfile.write(yaml.dump(summ_dict, default_flow_style=False))
#import pdb
#pdb.set_trace()
'''
# Reorder + cut
summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL']
# Write
summ_tab.meta = meta
summ_tab.write(summ_file,format='ascii.ecsv',
formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,clobber=True)
'''
# QA Figures
fig_file = dio_meta.specprod_root() + '/QA/sim_z.pdf'
dio_util.makepath(fig_file)
pp = PdfPages(fig_file)
# Summ
dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp)
for objtype in ['ELG', 'LRG', 'QSO_T', 'QSO_L']:
dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp)
# All done
pp.close()