def main(args):
import os.path
import sys
import yaml
import numpy as np
import pdb
import matplotlib
matplotlib.use('agg')
import desispec.io
from desiutil.log import get_logger
from desisim.spec_qa import redshifts as dsqa_z
from desiutil.io import yamlify
import desiutil.depend
log = get_logger()
# Initialize
if args.qaprod_dir is not None:
qaprod_dir = args.qaprod_dir
else:
qaprod_dir = desispec.io.meta.qaprod_root()
if args.load_simz_table is not None:
from astropy.table import Table
log.info("Loading simz info from {:s}".format(args.load_simz_table))
simz_tab = Table.read(args.load_simz_table)
else:
# Grab list of fibermap files
fibermap_files = []
zbest_files = []
nights = desispec.io.get_nights()
for night in nights:
for exposure in desispec.io.get_exposures(night, raw=True, rawdata_dir=args.rawdir):
# Ignore exposures with no fibermap, assuming they are calibration data.
fibermap_path = desispec.io.findfile(filetype='fibermap', night=night,
expid=exposure, rawdata_dir=args.rawdir)
if not os.path.exists(fibermap_path):
log.debug('Skipping exposure %08d with no fibermap.' % exposure)
continue
# Load data
fibermap_data = desispec.io.read_fibermap(fibermap_path)
# Skip calib
objtype = fibermap_data['OBJTYPE'][0]
if objtype in ['FLAT', 'FLT', 'ARC', 'BIAS', 'BIA']:
continue
elif objtype in ['SKY','TGT','BAD']:
pass
else:
err_message = 'Unrecognized OBJTYPE {}'.format(objtype)
log.critical(err_message)
raise ValueError(err_message)
# Append fibermap file
fibermap_files.append(fibermap_path)
# Slurp the zbest_files
zbest_files += dsqa_z.find_zbest_files(fibermap_data)
# Cut down zbest_files to unique ones
zbest_files = list(set(zbest_files))
if len(zbest_files) == 0:
log.fatal('No zbest files found')
sys.exit(1)
# Write? Table
simz_tab, zbtab = dsqa_z.load_z(fibermap_files, zbest_files=zbest_files)
dsqa_z.match_truth_z(simz_tab, zbtab)
if args.write_simz_table is not None:
simz_tab.write(args.write_simz_table, overwrite=True)
# Meta data
meta = dict(
DESISIM = desiutil.depend.getdep(simz_tab.meta, 'desisim'),
SPECPROD = os.getenv('SPECPROD', 'unknown'),
PIXPROD = os.getenv('PIXPROD', 'unknown'),
)
# Include specter version if it was used to generate input files
# (it isn't used for specsim inputs so that dependency may not exist)
try:
meta['SPECTER'] = desiutil.depend.getdep(simz_tab.meta, 'specter')
except KeyError:
pass
# Run stats
log.info("Running stats..")
summ_dict = dsqa_z.summ_stats(simz_tab)
if args.yaml_file is not None:
log.info("Generating yaml file of stats: {:s}".format(args.yaml_file))
# yamlify
# Write yaml
desispec.io.util.makepath(args.yaml_file)
with open(args.yaml_file, 'w') as outfile:
outfile.write(yaml.dump(yamlify(meta), default_flow_style=False))
outfile.write(yaml.dump(yamlify(summ_dict), default_flow_style=False))
log.info("Generating QA files")
# Summary for dz of all types
outfile = qaprod_dir+'/QA_dzsumm.png'
desispec.io.util.makepath(outfile)
dsqa_z.dz_summ(simz_tab, outfile=outfile)
# Summary of individual types
#outfile = args.qafig_root+'_summ_fig.png'
#dsqa_z.summ_fig(simz_tab, summ_dict, meta, outfile=outfile)
for objtype in ['BGS', 'MWS', 'ELG','LRG', 'QSO_T', 'QSO_L']:
outfile = qaprod_dir+'/QA_zfind_{:s}.png'.format(objtype)
desispec.io.util.makepath(outfile)
dsqa_z.obj_fig(simz_tab, objtype, summ_dict, outfile=outfile)
def main(args):
import os.path
import sys
import yaml
import numpy as np
import pdb
import matplotlib
matplotlib.use('agg')
import desispec.io
from desiutil.log import get_logger
from desisim.spec_qa import redshifts as dsqa_z
from desiutil.io import yamlify
import desiutil.depend
from desimodel.footprint import radec2pix
log = get_logger()
if args.load_simz_table is not None:
from astropy.table import Table
log.info("Loading simz info from {:s}".format(args.load_simz_table))
simz_tab = Table.read(args.load_simz_table)
else:
# Grab list of fibermap files
fibermap_files = []
zbest_files = []
nights = desispec.io.get_nights()
for night in nights:
for exposure in desispec.io.get_exposures(night,
raw=True,
rawdata_dir=args.rawdir):
# Ignore exposures with no fibermap, assuming they are calibration data.
fibermap_path = desispec.io.findfile(filetype='fibermap',
night=night,
expid=exposure,
rawdata_dir=args.rawdir)
if not os.path.exists(fibermap_path):
log.debug('Skipping exposure %08d with no fibermap.' %
exposure)
continue
'''
# Search for zbest files
fibermap_data = desispec.io.read_fibermap(fibermap_path)
flavor = fibermap_data.meta['FLAVOR']
if flavor.lower() in ('arc', 'flat', 'bias'):
log.debug('Skipping calibration {} exposure {:08d}'.format(flavor, exposure))
continue
brick_names = set(fibermap_data['BRICKNAME'])
import pdb; pdb.set_trace()
for brick in brick_names:
zbest_path=desispec.io.findfile('zbest', groupname=brick, specprod_dir=args.reduxdir)
if os.path.exists(zbest_path):
log.debug('Found {}'.format(os.path.basename(zbest_path)))
zbest_files.append(zbest_path)
else:
log.warn('Missing {}'.format(os.path.basename(zbest_path)))
#pdb.set_trace()
'''
# Load data
fibermap_data = desispec.io.read_fibermap(fibermap_path)
# Skip calib
if fibermap_data['OBJTYPE'][0] in ['FLAT', 'ARC', 'BIAS']:
continue
elif fibermap_data['OBJTYPE'][0] in ['SKY', 'STD', 'SCIENCE']:
pass
else:
pdb.set_trace()
# Append fibermap file
fibermap_files.append(fibermap_path)
# Search for zbest files with healpy
ra_targ = fibermap_data['RA_TARGET'].data
dec_targ = fibermap_data['DEC_TARGET'].data
# Getting some NAN in RA/DEC
good = np.isfinite(ra_targ) & np.isfinite(dec_targ)
pixels = radec2pix(64, ra_targ[good], dec_targ[good])
uni_pixels = np.unique(pixels)
for uni_pix in uni_pixels:
zbest_files.append(
desispec.io.findfile('zbest',
groupname=uni_pix,
nside=64))
# Cut down zbest_files to unique ones
zbest_files = list(set(zbest_files))
if len(zbest_files) == 0:
log.fatal('No zbest files found')
sys.exit(1)
# Write? Table
simz_tab = dsqa_z.load_z(fibermap_files, zbest_files)
if args.write_simz_table is not None:
simz_tab.write(args.write_simz_table, overwrite=True)
# Meta data
meta = dict(
DESISIM=desiutil.depend.getdep(simz_tab.meta, 'desisim'),
SPECTER=desiutil.depend.getdep(simz_tab.meta, 'specter'),
SPECPROD=os.getenv('SPECPROD', 'unknown'),
PIXPROD=os.getenv('PIXPROD', 'unknown'),
)
# Run stats
log.info("Running stats..")
summ_dict = dsqa_z.summ_stats(simz_tab)
if args.qafile is not None:
log.info("Generating yaml file: {:s}".format(args.qafile))
# yamlify
# Write yaml
with open(args.qafile, 'w') as outfile:
outfile.write(yaml.dump(yamlify(meta), default_flow_style=False))
outfile.write(
yaml.dump(yamlify(summ_dict), default_flow_style=False))
if args.qafig_root is not None:
log.info("Generating QA files")
# Summary for dz of all types
outfile = args.qafig_root + '_dzsumm.png'
#dsqa_z.dz_summ(simz_tab, outfile=outfile)
# Summary of individual types
#outfile = args.qafig_root+'_summ_fig.png'
#dsqa_z.summ_fig(simz_tab, summ_dict, meta, outfile=outfile)
for objtype in ['BGS', 'MWS', 'ELG', 'LRG', 'QSO_T', 'QSO_L']:
outfile = args.qafig_root + '_zfind_{:s}.png'.format(objtype)
dsqa_z.obj_fig(simz_tab, objtype, summ_dict, outfile=outfile)
def main(args):
import os.path
import sys
import yaml
import numpy as np
import pdb
import matplotlib
matplotlib.use('agg')
import desispec.io
from desiutil.log import get_logger
from desisim.spec_qa import redshifts as dsqa_z
from desiutil.io import yamlify
import desiutil.depend
log = get_logger()
# Initialize
if args.qaprod_dir is not None:
qaprod_dir = args.qaprod_dir
else:
qaprod_dir = desispec.io.meta.qaprod_root()
if args.load_simz_table is not None:
from astropy.table import Table
log.info("Loading simz info from {:s}".format(args.load_simz_table))
simz_tab = Table.read(args.load_simz_table)
else:
# Grab list of fibermap files
fibermap_files = []
zbest_files = []
nights = desispec.io.get_nights()
for night in nights:
for exposure in desispec.io.get_exposures(night,
raw=True,
rawdata_dir=args.rawdir):
# Ignore exposures with no fibermap, assuming they are calibration data.
fibermap_path = desispec.io.findfile(filetype='fibermap',
night=night,
expid=exposure,
rawdata_dir=args.rawdir)
if not os.path.exists(fibermap_path):
log.debug('Skipping exposure %08d with no fibermap.' %
exposure)
continue
# Load data
fibermap_data = desispec.io.read_fibermap(fibermap_path)
# Skip calib
objtype = fibermap_data['OBJTYPE'][0]
if objtype in ['FLAT', 'FLT', 'ARC', 'BIAS', 'BIA']:
continue
elif objtype in ['SKY', 'TGT', 'BAD']:
pass
else:
err_message = 'Unrecognized OBJTYPE {}'.format(objtype)
log.critical(err_message)
raise ValueError(err_message)
# Append fibermap file
fibermap_files.append(fibermap_path)
# Slurp the zbest_files
zbest_files += dsqa_z.find_zbest_files(fibermap_data)
# Cut down zbest_files to unique ones
zbest_files = list(set(zbest_files))
if len(zbest_files) == 0:
log.fatal('No zbest files found')
sys.exit(1)
# Write? Table
simz_tab, zbtab = dsqa_z.load_z(fibermap_files,
zbest_files=zbest_files)
dsqa_z.match_truth_z(simz_tab, zbtab)
if args.write_simz_table is not None:
simz_tab.write(args.write_simz_table, overwrite=True)
# Meta data
meta = dict(
DESISIM=desiutil.depend.getdep(simz_tab.meta, 'desisim'),
SPECPROD=os.getenv('SPECPROD', 'unknown'),
PIXPROD=os.getenv('PIXPROD', 'unknown'),
)
# Include specter version if it was used to generate input files
# (it isn't used for specsim inputs so that dependency may not exist)
try:
meta['SPECTER'] = desiutil.depend.getdep(simz_tab.meta, 'specter')
except KeyError:
pass
# Run stats
log.info("Running stats..")
summ_dict = dsqa_z.summ_stats(simz_tab)
if args.yaml_file is not None:
log.info("Generating yaml file of stats: {:s}".format(args.yaml_file))
# yamlify
# Write yaml
desispec.io.util.makepath(args.yaml_file)
with open(args.yaml_file, 'w') as outfile:
outfile.write(yaml.dump(yamlify(meta), default_flow_style=False))
outfile.write(
yaml.dump(yamlify(summ_dict), default_flow_style=False))
log.info("Generating QA files")
# Summary for dz of all types
outfile = qaprod_dir + '/QA_dzsumm.png'
desispec.io.util.makepath(outfile)
dsqa_z.dz_summ(simz_tab, outfile=outfile)
# Summary of individual types
#outfile = args.qafig_root+'_summ_fig.png'
#dsqa_z.summ_fig(simz_tab, summ_dict, meta, outfile=outfile)
for objtype in ['BGS', 'MWS', 'ELG', 'LRG', 'QSO_T', 'QSO_L']:
outfile = qaprod_dir + '/QA_zfind_{:s}.png'.format(objtype)
desispec.io.util.makepath(outfile)
dsqa_z.obj_fig(simz_tab, objtype, summ_dict, outfile=outfile)
def main():
'''Runs the process
'''
# Check environmental variables are set
assert 'DESI_SPECTRO_DATA' in os.environ, 'Missing $DESI_SPECTRO_DATA environment variable'
assert 'PRODNAME' in os.environ, 'Missing $PRODNAME environment variable'
assert 'DESI_SPECTRO_REDUX' in os.environ, 'Missing $DESI_SPECTRO_REDUX environment variable'
# Grab nights in data redux
nights = glob.glob(dio_meta.specprod_root() + '/exposures/*')
if len(nights) == 0:
raise ValueError('No nights in exposures!')
# Build up list of fibermap files
fibermap_files = []
for night in nights:
onight = night[night.rfind('/'):]
files = glob.glob(dio_meta.rawdata_root() + '/' + onight +
'/fibermap*')
#
fibermap_files += files
# Get list of zbest files
zbest_files = glob.glob(dio_meta.specprod_root() + '/bricks/*/zbest*')
if len(zbest_files) == 0:
raise ValueError('No redshifts?!')
# Meta
meta = get_meta()
# Load+write table
simtab_fil = dio_meta.specprod_root() + '/QA/sim_z_table.fits'
dio_util.makepath(simtab_fil)
simz_tab = dsqa_z.load_z(fibermap_files, zbest_files)
simz_tab.meta = meta
simz_tab.write(simtab_fil, overwrite=True)
# Summary stats
summ_file = dio_meta.specprod_root() + '/QA/sim_z_summ.yaml'
dio_util.makepath(summ_file)
summ_dict = dsqa_z.summ_stats(simz_tab)
# Write
with open(summ_file, 'w') as outfile:
outfile.write(yaml.dump(meta)) #, default_flow_style=True) )
outfile.write(yaml.dump(summ_dict, default_flow_style=False))
#import pdb
#pdb.set_trace()
'''
# Reorder + cut
summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL']
# Write
summ_tab.meta = meta
summ_tab.write(summ_file,format='ascii.ecsv',
formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,clobber=True)
'''
# QA Figures
fig_file = dio_meta.specprod_root() + '/QA/sim_z.pdf'
dio_util.makepath(fig_file)
pp = PdfPages(fig_file)
# Summ
dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp)
for objtype in ['ELG', 'LRG', 'QSO_T', 'QSO_L']:
dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp)
# All done
pp.close()
def main():
'''Runs the process
'''
# Check environmental variables are set
assert 'DESI_SPECTRO_DATA' in os.environ, 'Missing $DESI_SPECTRO_DATA environment variable'
assert 'SPECPROD' in os.environ, 'Missing $SPECPROD environment variable'
assert 'DESI_SPECTRO_REDUX' in os.environ, 'Missing $DESI_SPECTRO_REDUX environment variable'
# Grab nights in data redux
nights = glob.glob(dio_meta.specprod_root()+'/exposures/*')
if len(nights) == 0:
raise ValueError('No nights in exposures!')
# Build up list of fibermap files
fibermap_files = []
for night in nights:
onight = night[night.rfind('/'):]
files = glob.glob(dio_meta.rawdata_root()+'/'+onight+'/fibermap*')
#
fibermap_files += files
# Get list of zbest files
zbest_files = glob.glob(dio_meta.specprod_root()+'/bricks/*/zbest*')
if len(zbest_files) == 0:
raise ValueError('No redshifts?!')
# Meta
meta = get_meta()
# Load+write table
simtab_fil = dio_meta.specprod_root()+'/QA/sim_z_table.fits'
dio_util.makepath(simtab_fil)
simz_tab = dsqa_z.load_z(fibermap_files, zbest_files)
simz_tab.meta = meta
simz_tab.write(simtab_fil,overwrite=True)
# Summary stats
summ_file = dio_meta.specprod_root()+'/QA/sim_z_summ.yaml'
dio_util.makepath(summ_file)
summ_dict = dsqa_z.summ_stats(simz_tab)
# Write
with open(summ_file, 'w') as outfile:
outfile.write( yaml.dump(meta))#, default_flow_style=True) )
outfile.write( yaml.dump(summ_dict, default_flow_style=False) )
#import pdb
#pdb.set_trace()
'''
# Reorder + cut
summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL']
# Write
summ_tab.meta = meta
summ_tab.write(summ_file,format='ascii.ecsv',
formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,overwrite=True)
'''
# QA Figures
fig_file = dio_meta.specprod_root()+'/QA/sim_z.pdf'
dio_util.makepath(fig_file)
pp = PdfPages(fig_file)
# Summ
dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp)
for objtype in ['ELG','LRG', 'QSO_T', 'QSO_L']:
dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp)
# All done
pp.close()
def main():
"""Runs the process
"""
# Check environmental variables are set
assert "DESI_SPECTRO_DATA" in os.environ, "Missing $DESI_SPECTRO_DATA environment variable"
assert "PRODNAME" in os.environ, "Missing $PRODNAME environment variable"
assert "DESI_SPECTRO_REDUX" in os.environ, "Missing $DESI_SPECTRO_REDUX environment variable"
# Grab nights in data redux
nights = glob.glob(dio_meta.specprod_root() + "/exposures/*")
if len(nights) == 0:
raise ValueError("No nights in exposures!")
# Build up list of fibermap files
fibermap_files = []
for night in nights:
onight = night[night.rfind("/") :]
files = glob.glob(dio_meta.data_root() + "/" + onight + "/fibermap*")
#
fibermap_files += files
# Get list of zbest files
zbest_files = glob.glob(dio_meta.specprod_root() + "/bricks/*/zbest*")
if len(zbest_files) == 0:
raise ValueError("No redshifts?!")
# Meta
meta = get_meta()
# Load+write table
simtab_fil = dio_meta.specprod_root() + "/QA/sim_z_table.fits"
dio_util.makepath(simtab_fil)
simz_tab = dsqa_z.load_z(fibermap_files, zbest_files)
simz_tab.meta = meta
simz_tab.write(simtab_fil, overwrite=True)
# Summary stats
summ_file = dio_meta.specprod_root() + "/QA/sim_z_summ.yaml"
dio_util.makepath(summ_file)
summ_dict = dsqa_z.summ_stats(simz_tab)
# Write
with open(summ_file, "w") as outfile:
outfile.write(yaml.dump(meta)) # , default_flow_style=True) )
outfile.write(yaml.dump(summ_dict, default_flow_style=False))
# import pdb
# pdb.set_trace()
"""
# Reorder + cut
summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL']
# Write
summ_tab.meta = meta
summ_tab.write(summ_file,format='ascii.ecsv',
formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,clobber=True)
"""
# QA Figures
fig_file = dio_meta.specprod_root() + "/QA/sim_z.pdf"
dio_util.makepath(fig_file)
pp = PdfPages(fig_file)
# Summ
dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp)
for objtype in ["ELG", "LRG", "QSO_T", "QSO_L"]:
dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp)
# All done
pp.close()