def spline_fit(output_wave,
input_wave,
input_flux,
required_resolution,
input_ivar=None,
order=3):
"""Performs a spline fit.
"""
if input_ivar is not None:
selection = np.where(input_ivar > 0)[0]
if selection.size < 2:
log = get_logger()
log.error(
"cannot do spline fit because only {0:d} values with ivar>0".
format(selection.size))
raise Error
w1 = input_wave[selection[0]]
w2 = input_wave[selection[-1]]
else:
w1 = input_wave[0]
w2 = input_wave[-1]
res = required_resolution
n = int((w2 - w1) / res)
res = (w2 - w1) / (n + 1)
knots = w1 + res * (0.5 + np.arange(n))
toto = scipy.interpolate.splrep(input_wave,
input_flux,
w=input_ivar,
k=order,
task=-1,
t=knots)
output_flux = scipy.interpolate.splev(output_wave, toto)
return output_flux
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--infile',
type=str,
default=None,
required=True,
help=
'path of DESI frame fits file corresponding to a continuum lamp exposure'
)
parser.add_argument('--outfile',
type=str,
default=None,
required=True,
help='path of DESI fiberflat fits file')
args = parser.parse_args()
log = get_logger()
log.info("starting")
frame = read_frame(args.infile)
fiberflat = compute_fiberflat(frame)
write_fiberflat(args.outfile, fiberflat, frame.header)
log.info("successfully wrote %s" % args.outfile)
def compatible(head1, head2) :
log = get_logger()
for k in ["PSFTYPE","NPIX_X","NPIX_Y","HSIZEX","HSIZEY","BUNDLMIN","BUNDLMAX","FIBERMAX","FIBERMIN","FIBERMAX","NPARAMS","LEGDEG","GHDEGX","GHDEGY"] :
if (head1[k] != head2[k]) :
log.warning("different {} : {}, {}".format(k, head1[k], head2[k]))
return False
return True
def apply_fiberflat(frame, fiberflat):
### def apply_fiberflat(flux,ivar,wave,fiberflat,ffivar,ffmask,ffwave):
"""Apply fiberflat to frame. Modifies frame.flux and frame.ivar
"""
log = get_logger()
log.info("starting")
# check same wavelength, die if not the case
if not np.allclose(frame.wave, fiberflat.wave):
message = "frame and fiberflat do not have the same wavelength arrays"
log.critical(message)
raise ValueError(message)
"""
F'=F/C
Var(F') = Var(F)/C**2 + F**2*( d(1/C)/dC )**2*Var(C)
= 1/(ivar(F)*C**2) + F**2*(1/C**2)**2*Var(C)
= 1/(ivar(F)*C**2) + F**2*Var(C)/C**4
= 1/(ivar(F)*C**2) + F**2/(ivar(C)*C**4)
"""
#- shorthand
ff = fiberflat
sp = frame #- sp=spectra for this frame
sp.flux = sp.flux * (ff.fiberflat > 0) / (ff.fiberflat +
(ff.fiberflat == 0))
sp.ivar = (sp.ivar > 0) * (ff.ivar > 0) * (ff.fiberflat > 0) / (1. / (
(sp.ivar + (sp.ivar == 0)) *
(ff.fiberflat**2 +
(ff.fiberflat == 0))) + sp.flux**2 / (ff.ivar * ff.fiberflat**4 +
(ff.ivar * ff.fiberflat == 0)))
log.info("done")
def main(args) :
log=get_logger()
log.info("starting")
# Process
frame = read_frame(args.infile)
fiberflat = compute_fiberflat(frame)
# QA
if (args.qafile is not None):
log.info("performing fiberflat QA")
# Load
qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
# Run
qaframe.run_qa('FIBERFLAT', (frame, fiberflat))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fiberflat(args.qafig, qaframe, frame, fiberflat)
# Write
write_fiberflat(args.outfile, fiberflat, frame.meta)
log.info("successfully wrote %s"%args.outfile)
def spline_fit(output_wave,input_wave,input_flux,required_resolution,input_ivar=None,order=3):
"""Performs spline fit of input_flux vs. input_wave and resamples at output_wave
Args:
output_wave : 1D array of output wavelength samples
input_wave : 1D array of input wavelengths
input_flux : 1D array of input flux density
required_resolution (float) : resolution for spline knot placement
Options:
input_ivar : 1D array of weights for input_flux
order (int) : spline order
Returns:
output_flux : 1D array of flux sampled at output_wave
"""
if input_ivar is not None :
selection=np.where(input_ivar>0)[0]
if selection.size < 2 :
log=get_logger()
log.error("cannot do spline fit because only {0:d} values with ivar>0".format(selection.size))
raise ValueError
w1=input_wave[selection[0]]
w2=input_wave[selection[-1]]
else :
w1=input_wave[0]
w2=input_wave[-1]
res=required_resolution
n=int((w2-w1)/res)
res=(w2-w1)/(n+1)
knots=w1+res*(0.5+np.arange(n))
toto=scipy.interpolate.splrep(input_wave,input_flux,w=input_ivar,k=order,task=-1,t=knots)
output_flux = scipy.interpolate.splev(output_wave,toto)
return output_flux
def main(args):
log = get_logger()
log.info("starting")
# Process
frame = read_frame(args.infile)
fiberflat = compute_fiberflat(frame)
# QA
if (args.qafile is not None):
log.info("performing fiberflat QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Run
qaframe.run_qa('FIBERFLAT', (frame, fiberflat))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fiberflat(args.qafig, qaframe, frame, fiberflat)
# Write
write_fiberflat(args.outfile, fiberflat, frame.meta)
log.info("successfully wrote %s" % args.outfile)
def apply_fiberflat(frame, fiberflat):
### def apply_fiberflat(flux,ivar,wave,fiberflat,ffivar,ffmask,ffwave):
"""Apply fiberflat to frame. Modifies frame.flux and frame.ivar
"""
log=get_logger()
log.info("starting")
# check same wavelength, die if not the case
if not np.allclose(frame.wave, fiberflat.wave):
message = "frame and fiberflat do not have the same wavelength arrays"
log.critical(message)
raise ValueError(message)
"""
F'=F/C
Var(F') = Var(F)/C**2 + F**2*( d(1/C)/dC )**2*Var(C)
= 1/(ivar(F)*C**2) + F**2*(1/C**2)**2*Var(C)
= 1/(ivar(F)*C**2) + F**2*Var(C)/C**4
= 1/(ivar(F)*C**2) + F**2/(ivar(C)*C**4)
"""
#- shorthand
ff = fiberflat
sp = frame #- sp=spectra for this frame
sp.flux = sp.flux*(ff.fiberflat>0)/(ff.fiberflat+(ff.fiberflat==0))
sp.ivar=(sp.ivar>0)*(ff.ivar>0)*(ff.fiberflat>0)/( 1./((sp.ivar+(sp.ivar==0))*(ff.fiberflat**2+(ff.fiberflat==0))) + sp.flux**2/(ff.ivar*ff.fiberflat**4+(ff.ivar*ff.fiberflat==0)) )
log.info("done")
def __init__(self, wave, flux=None, ivar=None, mask=None, resolution=None):
assert wave.ndim == 1, "Input wavelength should be 1D"
assert (flux is None) or (
flux.shape == wave.shape), "wave and flux should have same shape"
assert (ivar is None) or (
ivar.shape == wave.shape), "wave and ivar should have same shape"
assert (mask is None) or (
mask.shape == wave.shape), "wave and mask should have same shape"
assert (resolution is None) or (isinstance(
resolution, desispec.resolution.Resolution))
assert (resolution is None) or (resolution.shape[0] == len(wave)
), "resolution size mismatch to wave"
self.wave = wave
self.flux = flux
self.ivar = ivar
self.mask = mask
self.resolution = resolution
self.R = resolution #- shorthand
self.log = get_logger()
# Initialize the quantities we will accumulate during co-addition. Note that our
# internal Cinv is a dense matrix.
if ivar is None:
n = len(wave)
self.Cinv = np.zeros((n, n))
self.Cinv_f = np.zeros((n, ))
else:
assert flux is not None and resolution is not None, 'Missing flux and/or resolution.'
diag_ivar = scipy.sparse.dia_matrix((ivar[np.newaxis, :], [0]),
resolution.shape)
self.Cinv = self.resolution.T.dot(diag_ivar.dot(self.resolution))
self.Cinv_f = self.resolution.T.dot(self.ivar * self.flux)
def __init__(self,wave,flux=None,ivar=None,mask=None,resolution=None):
assert wave.ndim == 1, "Input wavelength should be 1D"
assert (flux is None) or (flux.shape == wave.shape), "wave and flux should have same shape"
assert (ivar is None) or (ivar.shape == wave.shape), "wave and ivar should have same shape"
assert (mask is None) or (mask.shape == wave.shape), "wave and mask should have same shape"
assert (resolution is None) or (isinstance(resolution, desispec.resolution.Resolution))
assert (resolution is None) or (resolution.shape[0] == len(wave)), "resolution size mismatch to wave"
self.wave = wave
self.flux = flux
self.ivar = ivar
self.mask = mask
# if mask is None:
# self.mask = np.zeros(self.flux.shape, dtype=np.uint32)
# else:
# self.mask = util.mask32(mask)
self.resolution = resolution
self.R = resolution #- shorthand
self.log = get_logger()
# Initialize the quantities we will accumulate during co-addition. Note that our
# internal Cinv is a dense matrix.
if ivar is None:
n = len(wave)
self.Cinv = np.zeros((n,n))
self.Cinv_f = np.zeros((n,))
else:
assert flux is not None and resolution is not None,'Missing flux and/or resolution.'
diag_ivar = scipy.sparse.dia_matrix((ivar[np.newaxis,:],[0]),resolution.shape)
self.Cinv = self.resolution.T.dot(diag_ivar.dot(self.resolution))
self.Cinv_f = self.resolution.T.dot(self.ivar*self.flux)
def check_env():
"""
Check required environment variables; raise RuntimeException if missing
"""
log = get_logger()
#- template locations
missing_template_env = False
for objtype in ('ELG', 'LRG', 'STD', 'QSO'):
name = 'DESI_'+objtype+'_TEMPLATES'
if name not in os.environ:
log.warning('missing ${0} needed for simulating spectra'.format(name))
missing_template_env = True
if missing_template_env:
log.warning(' e.g. see NERSC:/project/projectdirs/desi/datachallenge/dc2/templates/')
missing_env = False
for name in (
'DESI_SPECTRO_SIM', 'DESI_SPECTRO_DATA', 'DESI_SPECTRO_REDUX', 'PIXPROD', 'PRODNAME'):
if name not in os.environ:
log.warning("missing ${0}".format(name))
missing_env = True
if missing_env:
log.warning("Why are these needed?")
log.warning(" Simulations written to $DESI_SPECTRO_SIM/$PIXPROD/")
log.warning(" Raw data read from $DESI_SPECTRO_DATA/")
log.warning(" Spectro pipeline output written to $DESI_SPECTRO_REDUX/$PRODNAME/")
#- Wait until end to raise exception so that we report everything that
#- is missing before actually failing
if missing_env or missing_template_env:
log.critical("missing env vars; exiting without running pipeline")
sys.exit(1)
def qa_zbest(param, zf, brick):
"""
Args:
param : dict of QA parameters
zf : ZfindBase object
brick : brick dict
Returns:
qa_zbest: dict
"""
log = get_logger()
# Parse brick table
key = list(brick.keys())[0]
btbl = brick[key].hdu_list[4].data
# Output dict
qadict = {}
# Failures
nfail = np.sum(zf.zwarn > 0)
qadict['NFAIL'] = int(nfail) # For yaml
if nfail > param['MAX_NFAIL']:
log.warning("High number of failed redshifts {:d}".format(nfail))
# Simple redshift stats
gdz = zf.zwarn == 0
qadict['MEAN_Z'] = float(np.mean(zf.z[gdz]))
qadict['MEDIAN_Z'] = float(np.median(zf.z[gdz]))
qadict['RMS_Z'] = float(np.std(zf.z[gdz]))
# Match zf ID to brick ID
srt = np.argsort(btbl['TARGETID'])
left = np.searchsorted(btbl['TARGETID'],
zf.targetid,
side='left',
sorter=srt)
# Types (ELG, QSO, LRG, STAR, ??) -- Need to allow for multiple of target options
qadict['NTYPE'] = dict(ELG=0, QSO=0, LRG=0, STAR=0, UNKWN=0, MATCH=0)
for kk, ztype in enumerate(zf.spectype):
# Brick index
idx = srt[left[kk]]
#
if ztype in param['ELG_TYPES']:
qadict['NTYPE']['ELG'] += 1
if btbl[idx]['OBJTYPE'] in param['ELG_TYPES']:
qadict['NTYPE']['MATCH'] += 1
elif ztype in param['QSO_TYPES']:
qadict['NTYPE']['QSO'] += 1
if btbl[idx]['OBJTYPE'] in param['QSO_TYPES']:
qadict['NTYPE']['MATCH'] += 1
elif ztype in param['STAR_TYPES']:
qadict['NTYPE']['STAR'] += 1
if btbl[idx]['OBJTYPE'] in param['STAR_TYPES']:
qadict['NTYPE']['MATCH'] += 1
else:
qadict['NTYPE']['UNKWN'] += 1
# Return
return qadict
def qa_zbest(param, zf, brick):
"""
Args:
param : dict of QA parameters
zf : ZfindBase object
brick : brick dict
Returns:
qa_zbest: dict
"""
log = get_logger()
# Parse brick table
key = brick.keys()[0]
btbl = brick[key].hdu_list[4].data
# Output dict
qadict = {}
# Failures
nfail = np.sum(zf.zwarn > 0)
qadict['NFAIL'] = int(nfail) # For yaml
if nfail > param['MAX_NFAIL']:
log.warn("High number of failed redshifts {:d}".format(nfail))
# Simple redshift stats
gdz = zf.zwarn == 0
qadict['MEAN_Z'] = float(np.mean(zf.z[gdz]))
qadict['MEDIAN_Z'] = float(np.median(zf.z[gdz]))
qadict['RMS_Z'] = float(np.std(zf.z[gdz]))
# Match zf ID to brick ID
srt = np.argsort(btbl['TARGETID'])
left = np.searchsorted(btbl['TARGETID'], zf.targetid,
side='left',sorter=srt)
# Types (ELG, QSO, LRG, STAR, ??) -- Need to allow for multiple of target options
qadict['NTYPE'] = dict(ELG=0, QSO=0, LRG=0, STAR=0, UNKWN=0, MATCH=0)
for kk,ztype in enumerate(zf.spectype):
# Brick index
idx = srt[left[kk]]
#
if ztype in param['ELG_TYPES']:
qadict['NTYPE']['ELG'] += 1
if btbl[idx]['OBJTYPE'] in param['ELG_TYPES']:
qadict['NTYPE']['MATCH'] += 1
elif ztype in param['QSO_TYPES']:
qadict['NTYPE']['QSO'] += 1
if btbl[idx]['OBJTYPE'] in param['QSO_TYPES']:
qadict['NTYPE']['MATCH'] += 1
elif ztype in param['STAR_TYPES']:
qadict['NTYPE']['STAR'] += 1
if btbl[idx]['OBJTYPE'] in param['STAR_TYPES']:
qadict['NTYPE']['MATCH'] += 1
else:
qadict['NTYPE']['UNKWN'] += 1
# Return
return qadict
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile',
type=str,
default=None,
required=True,
help='path of DESI exposure frame fits file')
parser.add_argument('--fibermap',
type=str,
default=None,
required=True,
help='path of DESI exposure frame fits file')
parser.add_argument('--fiberflat',
type=str,
default=None,
required=True,
help='path of DESI fiberflat fits file')
parser.add_argument('--outfile',
type=str,
default=None,
required=True,
help='path of DESI sky fits file')
args = parser.parse_args()
log = get_logger()
log.info("starting")
# read exposure to load data and get range of spectra
frame = read_frame(args.infile)
specmin = frame.header["SPECMIN"]
specmax = frame.header["SPECMAX"]
# read fibermap to locate sky fibers
fibermap = read_fibermap(args.fibermap)
selection = np.where((fibermap["OBJTYPE"] == "SKY")
& (fibermap["FIBER"] >= specmin)
& (fibermap["FIBER"] <= specmax))[0]
if selection.size == 0:
log.error("no sky fiber in fibermap %s" % args.fibermap)
sys.exit(12)
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
# compute sky model
skymodel = compute_sky(frame, fibermap)
# write result
write_sky(args.outfile, skymodel, frame.header)
log.info("successfully wrote %s" % args.outfile)
def main(args) :
log=get_logger()
log.info("read frame")
# read frame
frame = read_frame(args.infile)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux,model_wave,model_fibers=read_stdstar_models(args.models)
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
model_fibers = model_fibers%500
if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'):
for i in model_fibers:
log.error("inconsistency with spectrum %d, OBJTYPE='%s' in fibermap"%(i,fibermap["OBJTYPE"][i]))
sys.exit(12)
fluxcalib = compute_flux_calibration(frame, model_wave, model_flux)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s"%args.outfile)
def bin_bounds(x):
"""Calculates the bin boundaries of an array `x`.
Returns tuple of lower and upper bounds, each with same length as `x`.
"""
if x.size < 2:
get_logger().error("bin_bounds, x.size={0:d}".format(x.size))
exit(12)
tx = np.sort(x)
x_minus = np.roll(tx, 1)
x_minus[0] = x_minus[1] + tx[0] - tx[1]
x_plus = np.roll(tx, -1)
x_plus[-1] = x_plus[-2] + tx[-1] - tx[-2]
x_minus = 0.5 * (x + x_minus)
x_plus = 0.5 * (x + x_plus)
del tx
return x_minus, x_plus
def bin_bounds(x):
"""Calculates the bin boundaries of an array `x`.
Returns tuple of lower and upper bounds, each with same length as `x`.
"""
if x.size<2 :
get_logger().error("bin_bounds, x.size={0:d}".format(x.size))
exit(12)
tx=np.sort(x)
x_minus=np.roll(tx,1)
x_minus[0]=x_minus[1]+tx[0]-tx[1]
x_plus=np.roll(tx,-1)
x_plus[-1]=x_plus[-2]+tx[-1]-tx[-2]
x_minus=0.5*(x+x_minus)
x_plus=0.5*(x+x_plus)
del tx
return x_minus,x_plus
def main(args):
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='ELG archetypes')
parser.add_argument('-o', '--objtype', type=str, default='ELG', help='ELG', metavar='')
# Set up the logger.
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
objtype = args.objtype.upper()
log.debug('Using OBJTYPE {}'.format(objtype))
baseflux, basewave, basemeta = read_basis_templates(objtype=objtype)
pdb.set_trace()
def compatible(head1, head2):
log = get_logger()
for k in [
"PSFTYPE", "NPIX_X", "NPIX_Y", "HSIZEX", "HSIZEY", "BUNDLMIN",
"BUNDLMAX", "FIBERMAX", "FIBERMIN", "FIBERMAX", "NPARAMS",
"LEGDEG", "GHDEGX", "GHDEGY"
]:
if (head1[k] != head2[k]):
log.warning("different {} : {}, {}".format(k, head1[k], head2[k]))
return False
return True
def main() :
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile', type = str, default = None, required=True,
help = 'path of DESI exposure frame fits file')
parser.add_argument('--fiberflat', type = str, default = None,
help = 'path of DESI fiberflat fits file')
parser.add_argument('--sky', type = str, default = None,
help = 'path of DESI sky fits file')
parser.add_argument('--calib', type = str, default = None,
help = 'path of DESI calibration fits file')
parser.add_argument('--outfile', type = str, default = None, required=True,
help = 'path of DESI sky fits file')
# add calibration here when exists
args = parser.parse_args()
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
if args.fiberflat!=None :
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
if args.sky!=None :
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
if args.calib!=None :
log.info("calibrate")
# read calibration
fluxcalib=read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# save output
write_frame(args.outfile, frame)
log.info("successfully wrote %s"%args.outfile)
def main(args):
log = get_logger()
log.info("starting")
qa_prod = QA_Prod(args.specprod_dir)
# Remake Frame QA?
if args.make_frameqa > 0:
log.info("(re)generating QA related to frames")
if (args.make_frameqa % 4) >= 2:
make_frame_plots = True
else:
make_frame_plots = False
# Run
qa_prod.make_frameqa(make_plots=make_frame_plots, clobber=args.clobber)
# Slurp?
if args.slurp:
qa_prod.slurp(make=(args.make_frameqa > 0), remove=args.remove)
# Channel histograms
if args.channel_hist is not None:
# imports
from matplotlib.backends.backend_pdf import PdfPages
from desispec.qa import qa_plots as dqqp
#
qa_prod.load_data()
outfile = qa_prod.prod_name + '_chist.pdf'
pp = PdfPages(outfile)
# Default?
if args.channel_hist == 'default':
dqqp.prod_channel_hist(qa_prod,
'FIBERFLAT',
'MAX_RMS',
pp=pp,
close=False)
dqqp.prod_channel_hist(qa_prod,
'SKYSUB',
'MED_RESID',
xlim=(-1, 1),
pp=pp,
close=False)
dqqp.prod_channel_hist(qa_prod,
'FLUXCALIB',
'MAX_ZP_OFF',
pp=pp,
close=False)
# Finish
print("Writing {:s}".format(outfile))
pp.close()
def test_log(self):
desi_level=os.getenv("DESI_LOGLEVEL")
for level in [None,log.DEBUG,log.INFO,log.WARNING,log.ERROR] :
logger=log.get_logger(level)
print("with the requested debugging level={0}".format(level))
if desi_level is not None and (desi_level != "" ) :
print("(but overuled by env. DESI_LOGLEVEL='{0}')".format(desi_level))
print("--------------------------------------------------")
logger.debug("This is a debugging message")
logger.info("This is an information")
logger.warning("This is an warning")
logger.error("This is an error")
logger.critical("This is a critical error")
def test_log(self):
desi_level = os.getenv("DESI_LOGLEVEL")
for level in [None, log.DEBUG, log.INFO, log.WARNING, log.ERROR]:
logger = log.get_logger(level)
print("with the requested debugging level={0}".format(level))
if desi_level is not None and (desi_level != ""):
print("(but overuled by env. DESI_LOGLEVEL='{0}')".format(
desi_level))
print("--------------------------------------------------")
logger.debug("This is a debugging message")
logger.info("This is an information")
logger.warning("This is an warning")
logger.error("This is an error")
logger.critical("This is a critical error")
def apply_fiberflat(frame, fiberflat):
"""Apply fiberflat to frame. Modifies frame.flux and frame.ivar
Args:
frame : `desispec.Frame` object
fiberflat : `desispec.FiberFlat` object
The frame is divided by the fiberflat, except where the fiberflat=0.
frame.mask gets bit specmask.BADFIBERFLAT set where
* fiberflat.fiberflat == 0
* fiberflat.ivar == 0
* fiberflat.mask != 0
"""
log = get_logger()
log.info("starting")
# check same wavelength, die if not the case
if not np.allclose(frame.wave, fiberflat.wave):
message = "frame and fiberflat do not have the same wavelength arrays"
log.critical(message)
raise ValueError(message)
"""
F'=F/C
Var(F') = Var(F)/C**2 + F**2*( d(1/C)/dC )**2*Var(C)
= 1/(ivar(F)*C**2) + F**2*(1/C**2)**2*Var(C)
= 1/(ivar(F)*C**2) + F**2*Var(C)/C**4
= 1/(ivar(F)*C**2) + F**2/(ivar(C)*C**4)
"""
#- shorthand
ff = fiberflat
sp = frame #- sp=spectra for this frame
#- update sp.ivar first since it depends upon the original sp.flux
sp.ivar = (sp.ivar > 0) * (ff.ivar > 0) * (ff.fiberflat > 0) / (1. / (
(sp.ivar + (sp.ivar == 0)) *
(ff.fiberflat**2 +
(ff.fiberflat == 0))) + sp.flux**2 / (ff.ivar * ff.fiberflat**4 +
(ff.ivar * ff.fiberflat == 0)))
#- Then update sp.flux, taking care not to divide by 0
ii = np.where(ff.fiberflat > 0)
sp.flux[ii] = sp.flux[ii] / ff.fiberflat[ii]
badff = (ff.fiberflat == 0.0) | (ff.ivar == 0) | (ff.mask != 0)
sp.mask[badff] |= specmask.BADFIBERFLAT
log.info("done")
def _redirect_stdout(to):
sys.stdout.close() # + implicit flush()
os.dup2(to.fileno(), fd) # fd writes to 'to' file
sys.stdout = os.fdopen(fd, 'w') # Python writes to fd
sys.stderr.close() # + implicit flush()
os.dup2(to.fileno(), fde) # fd writes to 'to' file
sys.stderr = os.fdopen(fde, 'w') # Python writes to fd
# update desi logging to use new stdout
log = get_logger()
while len(log.handlers) > 0:
h = log.handlers[0]
log.removeHandler(h)
# Add the current stdout.
ch = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('%(levelname)s:%(filename)s:%(lineno)s:%(funcName)s: %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)
def _redirect_stdout(to):
sys.stdout.close() # + implicit flush()
os.dup2(to.fileno(), fd) # fd writes to 'to' file
sys.stdout = os.fdopen(fd, 'w') # Python writes to fd
sys.stderr.close() # + implicit flush()
os.dup2(to.fileno(), fde) # fd writes to 'to' file
sys.stderr = os.fdopen(fde, 'w') # Python writes to fd
# update desi logging to use new stdout
log = get_logger()
while len(log.handlers) > 0:
h = log.handlers[0]
log.removeHandler(h)
# Add the current stdout.
ch = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('%(levelname)s:%(filename)s:%(lineno)s:%(funcName)s: %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)
def apply_fiberflat(frame, fiberflat):
"""Apply fiberflat to frame. Modifies frame.flux and frame.ivar
Args:
frame : `desispec.Frame` object
fiberflat : `desispec.FiberFlat` object
The frame is divided by the fiberflat, except where the fiberflat=0.
frame.mask gets bit specmask.BADFIBERFLAT set where
* fiberflat.fiberflat == 0
* fiberflat.ivar == 0
* fiberflat.mask != 0
"""
log=get_logger()
log.info("starting")
# check same wavelength, die if not the case
if not np.allclose(frame.wave, fiberflat.wave):
message = "frame and fiberflat do not have the same wavelength arrays"
log.critical(message)
raise ValueError(message)
"""
F'=F/C
Var(F') = Var(F)/C**2 + F**2*( d(1/C)/dC )**2*Var(C)
= 1/(ivar(F)*C**2) + F**2*(1/C**2)**2*Var(C)
= 1/(ivar(F)*C**2) + F**2*Var(C)/C**4
= 1/(ivar(F)*C**2) + F**2/(ivar(C)*C**4)
"""
#- shorthand
ff = fiberflat
sp = frame #- sp=spectra for this frame
#- update sp.ivar first since it depends upon the original sp.flux
sp.ivar=(sp.ivar>0)*(ff.ivar>0)*(ff.fiberflat>0)/( 1./((sp.ivar+(sp.ivar==0))*(ff.fiberflat**2+(ff.fiberflat==0))) + sp.flux**2/(ff.ivar*ff.fiberflat**4+(ff.ivar*ff.fiberflat==0)) )
#- Then update sp.flux, taking care not to divide by 0
ii = np.where(ff.fiberflat > 0)
sp.flux[ii] = sp.flux[ii] / ff.fiberflat[ii]
badff = (ff.fiberflat == 0.0) | (ff.ivar == 0) | (ff.mask != 0)
sp.mask[badff] |= specmask.BADFIBERFLAT
log.info("done")
def sim(night, nspec=5, clobber=False):
"""
Simulate data as part of the integration test.
Args:
night (str): YEARMMDD
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
# Create input fibermaps, spectra, and pixel-level raw data
for expid, flavor in zip([0, 1, 2], ['flat', 'arc', 'dark']):
cmd = "newexp-desi --flavor {flavor} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, flavor=flavor, nspec=nspec, night=night)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap),
expid)
inputs = []
outputs = [fibermap, simspec]
if pipe.runcmd(cmd, inputs=inputs, outputs=outputs,
clobber=clobber) != 0:
raise RuntimeError(
'pixsim newexp failed for {} exposure {}'.format(
flavor, expid))
cmd = "pixsim-desi --preproc --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, nspec=nspec, night=night)
inputs = [fibermap, simspec]
outputs = list()
outputs.append(fibermap.replace('fibermap-', 'simpix-'))
for camera in ['b0', 'r0', 'z0']:
pixfile = io.findfile('pix', night, expid, camera)
outputs.append(pixfile)
#outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-')))
if pipe.runcmd(cmd, inputs=inputs, outputs=outputs,
clobber=clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(
flavor, expid))
return
def main() :
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile', type = str, default = None, required=True,
help = 'path of DESI exposure frame fits file')
parser.add_argument('--fibermap', type = str, default = None, required=True,
help = 'path of DESI exposure frame fits file')
parser.add_argument('--fiberflat', type = str, default = None, required=True,
help = 'path of DESI fiberflat fits file')
parser.add_argument('--outfile', type = str, default = None, required=True,
help = 'path of DESI sky fits file')
args = parser.parse_args()
log=get_logger()
log.info("starting")
# read exposure to load data and get range of spectra
frame = read_frame(args.infile)
specmin=frame.header["SPECMIN"]
specmax=frame.header["SPECMAX"]
# read fibermap to locate sky fibers
fibermap = read_fibermap(args.fibermap)
selection=np.where((fibermap["OBJTYPE"]=="SKY")&(fibermap["FIBER"]>=specmin)&(fibermap["FIBER"]<=specmax))[0]
if selection.size == 0 :
log.error("no sky fiber in fibermap %s"%args.fibermap)
sys.exit(12)
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
# compute sky model
skymodel = compute_sky(frame, fibermap)
# write result
write_sky(args.outfile, skymodel, frame.header)
log.info("successfully wrote %s"%args.outfile)
def check_env():
"""
Check required environment variables; raise RuntimeException if missing
"""
log = logging.get_logger()
#- template locations
missing_env = False
if 'DESI_BASIS_TEMPLATES' not in os.environ:
log.warning(
'missing $DESI_BASIS_TEMPLATES needed for simulating spectra'.
format(name))
missing_env = True
if not os.path.isdir(os.getenv('DESI_BASIS_TEMPLATES')):
log.warning('missing $DESI_BASIS_TEMPLATES directory')
log.warning(
'e.g. see NERSC:/project/projectdirs/desi/spectro/templates/basis_templates/v2.2'
)
missing_env = True
for name in ('DESI_SPECTRO_SIM', 'DESI_SPECTRO_REDUX', 'PIXPROD',
'SPECPROD', 'DESIMODEL'):
if name not in os.environ:
log.warning("missing ${0}".format(name))
missing_env = True
if missing_env:
log.warning("Why are these needed?")
log.warning(" Simulations written to $DESI_SPECTRO_SIM/$PIXPROD/")
log.warning(" Raw data read from $DESI_SPECTRO_DATA/")
log.warning(
" Spectro pipeline output written to $DESI_SPECTRO_REDUX/$SPECPROD/"
)
log.warning(" Templates are read from $DESI_BASIS_TEMPLATES")
#- Wait until end to raise exception so that we report everything that
#- is missing before actually failing
if missing_env:
log.critical("missing env vars; exiting without running pipeline")
sys.exit(1)
#- Override $DESI_SPECTRO_DATA to match $DESI_SPECTRO_SIM/$PIXPROD
os.environ['DESI_SPECTRO_DATA'] = os.path.join(
os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD'))
def retry_task(failpath, newopts=None):
log = get_logger()
if not os.path.isfile(failpath):
raise RuntimeError("failure yaml file {} does not exist".format(failpath))
fyml = None
with open(failpath, 'r') as f:
fyml = yaml.load(f)
step = fyml['step']
rawdir = fyml['rawdir']
proddir = fyml['proddir']
name = fyml['task']
grph = fyml['graph']
origopts = fyml['opts']
nproc = fyml['procs']
comm = None
rank = 0
if nproc > 1:
from mpi4py import MPI
comm = MPI.COMM_WORLD
nworld = comm.size
rank = comm.rank
if nworld != nproc:
if rank == 0:
log.warn("WARNING: original task was run with {} processes, re-running with {} instead".format(nproc, nworld))
opts = origopts
if newopts is not None:
log.warn("WARNING: overriding original options")
opts = newopts
try:
run_task(step, rawdir, proddir, grph, opts, comm=comm)
except:
log.error("Retry Failed")
raise
else:
if rank == 0:
os.remove(failpath)
return
def decorrelate(Cinv):
"""Decorrelate an inverse covariance using the matrix square root.
Implements the decorrelation part of the spectroperfectionism algorithm described in
Bolton & Schlegel 2009 (BS) http://arxiv.org/abs/0911.2689, w uses the matrix square root of
Cinv to form a diagonal basis. This is generally a better choice than the eigenvector or
Cholesky bases since it leads to more localized basis vectors, as described in
Hamilton & Tegmark 2000 http://arxiv.org/abs/astro-ph/9905192.
Args:
Cinv(numpy.ndarray): Square array of inverse covariance matrix elements. The input can
either be a scipy.sparse format or else a regular (dense) numpy array, but a
sparse format will be internally converted to a dense matrix so there is no
performance advantage.
Returns:
tuple: Tuple ivar,R of uncorrelated flux inverse variances and the corresponding
resolution matrix. These have shapes (nflux,) and (nflux,nflux) respectively.
The rows of R give the resolution-convolved responses to unit flux for each
wavelength bin. Note that R is returned as a regular (dense) numpy array but
will normally have non-zero values concentrated near the diagonal.
"""
log = get_logger()
# Clean up any roundoff errors by forcing Cinv to be symmetric.
Cinv = 0.5*(Cinv + Cinv.T)
# Convert to a dense matrix if necessary.
if scipy.sparse.issparse(Cinv):
Cinv = Cinv.todense()
# Calculate the matrix square root. Note that we do not use scipy.linalg.sqrtm since
# the method below is about 2x faster for a positive definite matrix.
L,X = scipy.linalg.eigh(Cinv)
# Check for negative eigenvalues.
nbad = np.count_nonzero(L < 0)
if nbad > 0:
log.warning('zeroing {0:d} negative eigenvalue(s).'.format(nbad))
L[L < 0] = 0.
# Calculate the matrix square root Q such that Cinv = Q.Q
Q = X.dot(np.diag(np.sqrt(L)).dot(X.T))
# Calculate and return the corresponding resolution matrix and diagonal flux errors.
s = np.sum(Q,axis=1)
R = Q/s[:,np.newaxis]
ivar = s**2
return ivar,R
def decorrelate(Cinv):
"""Decorrelate an inverse covariance using the matrix square root.
Implements the decorrelation part of the spectroperfectionism algorithm described in
Bolton & Schlegel 2009 (BS) http://arxiv.org/abs/0911.2689, w uses the matrix square root of
Cinv to form a diagonal basis. This is generally a better choice than the eigenvector or
Cholesky bases since it leads to more localized basis vectors, as described in
Hamilton & Tegmark 2000 http://arxiv.org/abs/astro-ph/9905192.
Args:
Cinv(numpy.ndarray): Square array of inverse covariance matrix elements. The input can
either be a scipy.sparse format or else a regular (dense) numpy array, but a
sparse format will be internally converted to a dense matrix so there is no
performance advantage.
Returns:
tuple: Tuple ivar,R of uncorrelated flux inverse variances and the corresponding
resolution matrix. These have shapes (nflux,) and (nflux,nflux) respectively.
The rows of R give the resolution-convolved responses to unit flux for each
wavelength bin. Note that R is returned as a regular (dense) numpy array but
will normally have non-zero values concentrated near the diagonal.
"""
log = get_logger()
# Clean up any roundoff errors by forcing Cinv to be symmetric.
Cinv = 0.5 * (Cinv + Cinv.T)
# Convert to a dense matrix if necessary.
if scipy.sparse.issparse(Cinv):
Cinv = Cinv.todense()
# Calculate the matrix square root. Note that we do not use scipy.linalg.sqrtm since
# the method below is about 2x faster for a positive definite matrix.
L, X = scipy.linalg.eigh(Cinv)
# Check for negative eigenvalues.
nbad = np.count_nonzero(L < 0)
if nbad > 0:
log.warning('zeroing {0:d} negative eigenvalue(s).'.format(nbad))
L[L < 0] = 0.
# Calculate the matrix square root Q such that Cinv = Q.Q
Q = X.dot(np.diag(np.sqrt(L)).dot(X.T))
# Calculate and return the corresponding resolution matrix and diagonal flux errors.
s = np.sum(Q, axis=1)
R = Q / s[:, np.newaxis]
ivar = s**2
return ivar, R
def subtract_sky(frame, skymodel) :
"""Subtract skymodel from frame, altering frame.flux, .ivar, and .mask
"""
assert frame.nspec == skymodel.nspec
assert frame.nwave == skymodel.nwave
log=get_logger()
log.info("starting")
# check same wavelength, die if not the case
if not np.allclose(frame.wave, skymodel.wave):
message = "frame and sky not on same wavelength grid"
log.error(message)
raise ValueError(message)
frame.flux -= skymodel.flux
frame.ivar = util.combine_ivar(frame.ivar, skymodel.ivar)
frame.mask |= skymodel.mask
log.info("done")
def subtract_sky(frame, skymodel):
"""Subtract skymodel from frame, altering frame.flux, .ivar, and .mask
"""
assert frame.nspec == skymodel.nspec
assert frame.nwave == skymodel.nwave
log = get_logger()
log.info("starting")
# check same wavelength, die if not the case
if not np.allclose(frame.wave, skymodel.wave):
message = "frame and sky not on same wavelength grid"
log.error(message)
raise ValueError(message)
frame.flux -= skymodel.flux
frame.ivar = util.combine_ivar(frame.ivar, skymodel.ivar)
frame.mask |= skymodel.mask
log.info("done")
def main() :
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile', type = str, default = None, required=True,
help = 'path of DESI frame fits file corresponding to a continuum lamp exposure')
parser.add_argument('--outfile', type = str, default = None, required=True,
help = 'path of DESI fiberflat fits file')
args = parser.parse_args()
log=get_logger()
log.info("starting")
frame = read_frame(args.infile)
fiberflat = compute_fiberflat(frame)
write_fiberflat(args.outfile, fiberflat, frame.header)
log.info("successfully wrote %s"%args.outfile)
def main(args) :
log=get_logger()
log.info("starting")
qa_prod = QA_Prod(args.specprod_dir)
# Remake Frame QA?
if args.make_frameqa > 0:
log.info("(re)generating QA related to frames")
if (args.make_frameqa % 4) >= 2:
make_frame_plots = True
else:
make_frame_plots = False
# Run
qa_prod.make_frameqa(make_plots=make_frame_plots, clobber=args.clobber)
# Slurp?
if args.slurp:
qa_prod.slurp(make=(args.make_frameqa > 0), remove=args.remove)
# Channel histograms
if args.channel_hist is not None:
# imports
from matplotlib.backends.backend_pdf import PdfPages
from desispec.qa import qa_plots as dqqp
#
qa_prod.load_data()
outfile = qa_prod.prod_name+'_chist.pdf'
pp = PdfPages(outfile)
# Default?
if args.channel_hist == 'default':
dqqp.prod_channel_hist(qa_prod, 'FIBERFLAT', 'MAX_RMS', pp=pp, close=False)
dqqp.prod_channel_hist(qa_prod, 'SKYSUB', 'MED_RESID', xlim=(-1,1), pp=pp, close=False)
dqqp.prod_channel_hist(qa_prod, 'FLUXCALIB', 'MAX_ZP_OFF', pp=pp, close=False)
# Finish
print("Writing {:s}".format(outfile))
pp.close()
def spline_fit(output_wave,input_wave,input_flux,required_resolution,input_ivar=None,order=3):
"""Performs a spline fit.
"""
if input_ivar is not None :
selection=np.where(input_ivar>0)[0]
if selection.size < 2 :
log=get_logger()
log.error("cannot do spline fit because only {0:d} values with ivar>0".format(selection.size))
raise Error
w1=input_wave[selection[0]]
w2=input_wave[selection[-1]]
else :
w1=input_wave[0]
w2=input_wave[-1]
res=required_resolution
n=int((w2-w1)/res)
res=(w2-w1)/(n+1)
knots=w1+res*(0.5+np.arange(n))
toto=scipy.interpolate.splrep(input_wave,input_flux,w=input_ivar,k=order,task=-1,t=knots)
output_flux = scipy.interpolate.splev(output_wave,toto)
return output_flux
def main(args):
log = get_logger()
log.info("starting")
# read exposure to load data and get range of spectra
frame = read_frame(args.infile)
specmin, specmax = np.min(frame.fibers), np.max(frame.fibers)
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
# compute sky model
skymodel = compute_sky(frame)
# QA
if (args.qafile is not None) or (args.qafig is not None):
log.info("performing skysub QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Run
qaframe.run_qa('SKYSUB', (frame, skymodel))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_skyres(args.qafig, frame, skymodel, qaframe)
# write result
write_sky(args.outfile, skymodel, frame.meta)
log.info("successfully wrote %s" % args.outfile)
def sim(night, nspec=5, clobber=False):
"""
Simulate data as part of the integration test.
Args:
night (str): YEARMMDD
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
# Create input fibermaps, spectra, and pixel-level raw data
for expid, flavor in zip([0,1,2], ['flat', 'arc', 'dark']):
cmd = "newexp-desi --flavor {flavor} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, flavor=flavor, nspec=nspec, night=night)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid)
inputs = []
outputs = [fibermap, simspec]
if pipe.runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('pixsim newexp failed for {} exposure {}'.format(flavor, expid))
cmd = "pixsim-desi --preproc --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, nspec=nspec, night=night)
inputs = [fibermap, simspec]
outputs = list()
outputs.append(fibermap.replace('fibermap-', 'simpix-'))
for camera in ['b0', 'r0', 'z0']:
pixfile = io.findfile('pix', night, expid, camera)
outputs.append(pixfile)
#outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-')))
if pipe.runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid))
return
def check_env():
"""
Check required environment variables; raise RuntimeException if missing
"""
log = get_logger()
#- template locations
missing_env = False
if 'DESI_BASIS_TEMPLATES' not in os.environ:
log.warning('missing $DESI_BASIS_TEMPLATES needed for simulating spectra')
missing_env = True
if not os.path.isdir(os.getenv('DESI_BASIS_TEMPLATES')):
log.warning('missing $DESI_BASIS_TEMPLATES directory')
log.warning('e.g. see NERSC:/project/projectdirs/desi/spectro/templates/basis_templates/v1.0')
missing_env = True
for name in (
'DESI_SPECTRO_SIM', 'DESI_SPECTRO_REDUX', 'PIXPROD', 'SPECPROD'):
if name not in os.environ:
log.warning("missing ${0}".format(name))
missing_env = True
if missing_env:
log.warning("Why are these needed?")
log.warning(" Simulations written to $DESI_SPECTRO_SIM/$PIXPROD/")
log.warning(" Raw data read from $DESI_SPECTRO_DATA/")
log.warning(" Spectro pipeline output written to $DESI_SPECTRO_REDUX/$SPECPROD/")
log.warning(" Templates are read from $DESI_BASIS_TEMPLATES")
#- Wait until end to raise exception so that we report everything that
#- is missing before actually failing
if missing_env:
log.critical("missing env vars; exiting without running pipeline")
sys.exit(1)
#- Override $DESI_SPECTRO_DATA to match $DESI_SPECTRO_SIM/$PIXPROD
os.environ['DESI_SPECTRO_DATA'] = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD'))
def main(args) :
log=get_logger()
log.info("starting")
# read exposure to load data and get range of spectra
frame = read_frame(args.infile)
specmin, specmax = np.min(frame.fibers), np.max(frame.fibers)
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
# compute sky model
skymodel = compute_sky(frame)
# QA
if (args.qafile is not None) or (args.qafig is not None):
log.info("performing skysub QA")
# Load
qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
# Run
qaframe.run_qa('SKYSUB', (frame, skymodel))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_skyres(args.qafig, frame, skymodel, qaframe)
# write result
write_sky(args.outfile, skymodel, frame.meta)
log.info("successfully wrote %s"%args.outfile)
def main(args):
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is
None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
if args.fiberflat != None:
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
if args.sky != None:
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
if args.calib != None:
log.info("calibrate")
# read calibration
fluxcalib = read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# save output
write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 A)')
log.info("successfully wrote %s" % args.outfile)
def _test_zbest_io():
"""This should be moved to a separate test file? Yes, it should.
"""
import os
log = get_logger()
nspec, nflux = 10, 20
wave = np.arange(nflux)
flux = np.random.uniform(size=(nspec, nflux))
ivar = np.random.uniform(size=(nspec, nflux))
zfind1 = ZfindBase(wave, flux, ivar)
brickname = '1234p567'
targetids = np.random.randint(0, 12345678, size=nspec)
outfile = 'zbest_test.fits'
write_zbest(outfile, brickname, targetids, zfind1)
zfind2 = read_zbest(outfile)
assert np.all(zfind2.z == zfind1.z)
assert np.all(zfind2.zerr == zfind1.zerr)
assert np.all(zfind2.zwarn == zfind1.zwarn)
assert np.all(zfind2.type == zfind1.type)
assert np.all(zfind2.subtype == zfind1.subtype)
assert np.all(zfind2.brickname == brickname)
assert np.all(zfind2.targetid == targetids)
write_zbest(outfile, brickname, targetids, zfind1, zspec=True)
zfind3 = read_zbest(outfile)
assert np.all(zfind3.wave == zfind1.wave)
assert np.all(zfind3.flux == zfind1.flux)
assert np.all(zfind3.ivar == zfind1.ivar)
assert np.all(zfind3.model == zfind1.model)
log.info("looks OK to me")
os.remove(outfile)
def main(args) :
if args.outfile is not None :
outfile=args.outfile
else :
outfile=args.infile
log = get_logger()
log.info("starting finding cosmics in %s"%args.infile)
img=image.read_image(args.infile)
if args.ignore_cosmic_ccdmask :
log.warning("ignore cosmic ccdmask for test")
log.debug("ccdmask.COSMIC = %d"%ccdmask.COSMIC)
cosmic_ray_prexisting_mask = img.mask & ccdmask.COSMIC
img._mask &= ~ccdmask.COSMIC #- turn off cosmic mask
reject_cosmic_rays(img)
log.info("writing data and new mask in %s"%outfile)
image.write_image(outfile, img, meta=img.meta)
log.info("done")
def _test_zbest_io():
"""This should be moved to a separate test file? Yes, it should.
"""
import os
log=get_logger()
nspec, nflux = 10, 20
wave = np.arange(nflux)
flux = np.random.uniform(size=(nspec, nflux))
ivar = np.random.uniform(size=(nspec, nflux))
zfind1 = ZfindBase(wave, flux, ivar)
brickname = '1234p567'
targetids = np.random.randint(0,12345678, size=nspec)
outfile = 'zbest_test.fits'
write_zbest(outfile, brickname, targetids, zfind1)
zfind2 = read_zbest(outfile)
assert np.all(zfind2.z == zfind1.z)
assert np.all(zfind2.zerr == zfind1.zerr)
assert np.all(zfind2.zwarn == zfind1.zwarn)
assert np.all(zfind2.type == zfind1.type)
assert np.all(zfind2.subtype == zfind1.subtype)
assert np.all(zfind2.brickname == brickname)
assert np.all(zfind2.targetid == targetids)
write_zbest(outfile, brickname, targetids, zfind1, zspec=True)
zfind3 = read_zbest(outfile)
assert np.all(zfind3.wave == zfind1.wave)
assert np.all(zfind3.flux == zfind1.flux)
assert np.all(zfind3.ivar == zfind1.ivar)
assert np.all(zfind3.model == zfind1.model)
log.info("looks OK to me")
os.remove(outfile)
def main(args):
if args.outfile is not None:
outfile = args.outfile
else:
outfile = args.infile
log = get_logger()
log.info("starting finding cosmics in %s" % args.infile)
img = image.read_image(args.infile)
if args.ignore_cosmic_ccdmask:
log.warning("ignore cosmic ccdmask for test")
log.debug("ccdmask.COSMIC = %d" % ccdmask.COSMIC)
cosmic_ray_prexisting_mask = img.mask & ccdmask.COSMIC
img._mask &= ~ccdmask.COSMIC #- turn off cosmic mask
reject_cosmic_rays(img)
log.info("writing data and new mask in %s" % outfile)
image.write_image(outfile, img, meta=img.meta)
log.info("done")
def qa_skysub(param, frame, skymodel, quick_look=False):
"""Calculate QA on SkySubtraction
Note: Pixels rejected in generating the SkyModel (as above), are
not rejected in the stats calculated here. Would need to carry
along current_ivar to do so.
Args:
param : dict of QA parameters
frame : desispec.Frame object
skymodel : desispec.SkyModel object
quick_look : bool, optional
If True, do QuickLook specific QA (or avoid some)
Returns:
qadict: dict of QA outputs
Need to record simple Python objects for yaml (str, float, int)
"""
log = get_logger()
# Output dict
qadict = {}
qadict['NREJ'] = int(skymodel.nrej)
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nfibers = len(skyfibers)
qadict['NSKY_FIB'] = int(nfibers)
current_ivar = frame.ivar[skyfibers].copy()
flux = frame.flux[skyfibers]
# Subtract
res = flux - skymodel.flux[skyfibers] # Residuals
res_ivar = util.combine_ivar(current_ivar, skymodel.ivar[skyfibers])
# Chi^2 and Probability
chi2_fiber = np.sum(res_ivar * (res**2), 1)
chi2_prob = np.zeros(nfibers)
for ii in range(nfibers):
# Stats
dof = np.sum(res_ivar[ii, :] > 0.)
chi2_prob[ii] = scipy.stats.chisqprob(chi2_fiber[ii], dof)
# Bad models
qadict['NBAD_PCHI'] = int(np.sum(chi2_prob < param['PCHI_RESID']))
if qadict['NBAD_PCHI'] > 0:
log.warn("Bad Sky Subtraction in {:d} fibers".format(
qadict['NBAD_PCHI']))
# Median residual
qadict['MED_RESID'] = float(np.median(res)) # Median residual (counts)
log.info("Median residual for sky fibers = {:g}".format(
qadict['MED_RESID']))
# Residual percentiles
perc = dustat.perc(res, per=param['PER_RESID'])
qadict['RESID_PER'] = [float(iperc) for iperc in perc]
# Mean Sky Continuum from all skyfibers
# need to limit in wavelength?
if quick_look:
continuum = scipy.ndimage.filters.median_filter(
flux, 200) # taking 200 bins (somewhat arbitrarily)
mean_continuum = np.zeros(flux.shape[1])
for ii in range(flux.shape[1]):
mean_continuum[ii] = np.mean(continuum[:, ii])
qadict['MEAN_CONTIN'] = mean_continuum
# Median Signal to Noise on sky subtracted spectra
# first do the subtraction:
if quick_look:
fframe = frame # make a copy
sskymodel = skymodel # make a copy
subtract_sky(fframe, sskymodel)
medsnr = np.zeros(fframe.flux.shape[0])
totsnr = np.zeros(fframe.flux.shape[0])
for ii in range(fframe.flux.shape[0]):
signalmask = fframe.flux[ii, :] > 0
# total snr considering bin by bin uncorrelated S/N
snr = fframe.flux[ii, signalmask] * np.sqrt(
fframe.ivar[ii, signalmask])
medsnr[ii] = np.median(snr)
totsnr[ii] = np.sqrt(np.sum(snr**2))
qadict['MED_SNR'] = medsnr # for each fiber
qadict['TOT_SNR'] = totsnr # for each fiber
# Return
return qadict
def compute_fiberflat(frame, nsig_clipping=4.) :
"""Compute fiber flat by deriving an average spectrum and dividing all fiber data by this average.
Input data are expected to be on the same wavelenght grid, with uncorrelated noise.
They however do not have exactly the same resolution.
args:
frame (desispec.Frame): input Frame object with attributes
wave, flux, ivar, resolution_data
nsig_clipping : [optional] sigma clipping value for outlier rejection
returns tuple (fiberflat, ivar, mask, meanspec):
fiberflat : 2D[nwave, nflux] fiberflat (data have to be divided by this to be flatfielded)
ivar : inverse variance of that fiberflat
mask : 0=ok >0 if problems
meanspec : deconvolved mean spectrum
- we first iteratively :
- compute a deconvolved mean spectrum
- compute a fiber flat using the resolution convolved mean spectrum for each fiber
- smooth the fiber flat along wavelength
- clip outliers
- then we compute a fiberflat at the native fiber resolution (not smoothed)
- the routine returns the fiberflat, its inverse variance , mask, and the deconvolved mean spectrum
- the fiberflat is the ratio data/mean , so this flat should be divided to the data
NOTE THAT THIS CODE HAS NOT BEEN TESTED WITH ACTUAL FIBER TRANSMISSION VARIATIONS,
OUTLIER PIXELS, DEAD COLUMNS ...
"""
log=get_logger()
log.info("starting")
#
# chi2 = sum_(fiber f) sum_(wavelenght i) w_fi ( D_fi - F_fi (R_f M)_i )
#
# where
# w = inverse variance
# D = flux data (at the resolution of the fiber)
# F = smooth fiber flat
# R = resolution data
# M = mean deconvolved spectrum
#
# M = A^{-1} B
# with
# A_kl = sum_(fiber f) sum_(wavelenght i) w_fi F_fi^2 (R_fki R_fli)
# B_k = sum_(fiber f) sum_(wavelenght i) w_fi D_fi F_fi R_fki
#
# defining R'_fi = sqrt(w_fi) F_fi R_fi
# and D'_fi = sqrt(w_fi) D_fi
#
# A = sum_(fiber f) R'_f R'_f^T
# B = sum_(fiber f) R'_f D'_f
# (it's faster that way, and we try to use sparse matrices as much as possible)
#
#- Shortcuts
nwave=frame.nwave
nfibers=frame.nspec
wave = frame.wave.copy() #- this will become part of output too
flux = frame.flux
ivar = frame.ivar
# iterative fitting and clipping to get precise mean spectrum
current_ivar=ivar.copy()
smooth_fiberflat=np.ones((frame.flux.shape))
chi2=np.zeros((flux.shape))
sqrtwflat=np.sqrt(current_ivar)*smooth_fiberflat
sqrtwflux=np.sqrt(current_ivar)*flux
# test
#nfibers=20
nout_tot=0
for iteration in range(20) :
# fit mean spectrum
A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr()
B=np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD=scipy.sparse.lil_matrix((nwave,nwave))
# loop on fiber to handle resolution
for fiber in range(nfibers) :
if fiber%10==0 :
log.info("iter %d fiber %d"%(iteration,fiber))
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
# diagonal sparse matrix with content = sqrt(ivar)*flat
SD.setdiag(sqrtwflat[fiber])
sqrtwflatR = SD*R # each row r of R is multiplied by sqrtwflat[r]
A = A+(sqrtwflatR.T*sqrtwflatR).tocsr()
B += sqrtwflatR.T*sqrtwflux[fiber]
log.info("iter %d solving"%iteration)
mean_spectrum=cholesky_solve(A.todense(),B)
log.info("iter %d smoothing"%iteration)
# fit smooth fiberflat and compute chi2
smoothing_res=100. #A
for fiber in range(nfibers) :
#if fiber%10==0 :
# log.info("iter %d fiber %d (smoothing)"%(iteration,fiber))
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
#M = np.array(np.dot(R.todense(),mean_spectrum)).flatten()
M = R.dot(mean_spectrum)
F = flux[fiber]/(M+(M==0))
smooth_fiberflat[fiber]=spline_fit(wave,wave,F,smoothing_res,current_ivar[fiber]*(M!=0))
chi2[fiber]=current_ivar[fiber]*(flux[fiber]-smooth_fiberflat[fiber]*M)**2
log.info("rejecting")
nout_iter=0
if iteration<1 :
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave=np.sum(chi2>nsig_clipping**2,axis=0)
selection=np.where(nout_per_wave>0)[0]
for i in selection :
worst_entry=np.argmax(chi2[:,i])
current_ivar[worst_entry,i]=0
sqrtwflat[worst_entry,i]=0
sqrtwflux[worst_entry,i]=0
nout_iter += 1
else :
# remove all of them at once
bad=(chi2>nsig_clipping**2)
current_ivar *= (bad==0)
sqrtwflat *= (bad==0)
sqrtwflux *= (bad==0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2=float(np.sum(chi2))
ndf=int(np.sum(chi2>0)-nwave-nfibers*(nwave/smoothing_res))
chi2pdf=0.
if ndf>0 :
chi2pdf=sum_chi2/ndf
log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d"%(iteration,sum_chi2,ndf,chi2pdf,nout_iter))
# normalize to get a mean fiberflat=1
mean=np.mean(smooth_fiberflat,axis=0)
smooth_fiberflat = smooth_fiberflat/mean
mean_spectrum = mean_spectrum*mean
if nout_iter == 0 :
break
log.info("nout tot=%d"%nout_tot)
# now use mean spectrum to compute flat field correction without any smoothing
# because sharp feature can arise if dead columns
fiberflat=np.ones((flux.shape))
fiberflat_ivar=np.zeros((flux.shape))
mask=np.zeros((flux.shape)).astype(long) # SOMEONE CHECK THIS !
fiberflat_mask=12 # place holder for actual mask bit when defined
nsig_for_mask=4 # only mask out 4 sigma outliers
for fiber in range(nfibers) :
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = np.array(np.dot(R.todense(),mean_spectrum)).flatten()
fiberflat[fiber] = (M!=0)*flux[fiber]/(M+(M==0)) + (M==0)
fiberflat_ivar[fiber] = ivar[fiber]*M**2
smooth_fiberflat=spline_fit(wave,wave,fiberflat[fiber],smoothing_res,current_ivar[fiber]*M**2*(M!=0))
bad=np.where(fiberflat_ivar[fiber]*(fiberflat[fiber]-smooth_fiberflat)**2>nsig_for_mask**2)[0]
if bad.size>0 :
mask[fiber,bad] += fiberflat_mask
return FiberFlat(wave, fiberflat, fiberflat_ivar, mask, mean_spectrum)
def main(args):
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
if args.night is None:
log.critical('Missing required night argument.')
return -1
# Initialize a dictionary of Brick objects indexed by '<band>_<brick-id>' strings.
bricks = {}
try:
# Loop over exposures available for this night.
for exposure in desispec.io.get_exposures(args.night,
specprod_dir=args.specprod):
# Ignore exposures with no fibermap, assuming they are calibration data.
fibermap_path = desispec.io.findfile(filetype='fibermap',
night=args.night,
expid=exposure,
specprod_dir=args.specprod)
if not os.path.exists(fibermap_path):
log.debug('Skipping exposure %08d with no fibermap.' %
exposure)
continue
# Open the fibermap.
fibermap_data = desispec.io.read_fibermap(fibermap_path)
brick_names = set(fibermap_data['BRICKNAME'])
# Loop over per-camera cframes available for this exposure.
cframes = desispec.io.get_files(filetype='cframe',
night=args.night,
expid=exposure,
specprod_dir=args.specprod)
log.debug(
'Exposure %08d covers %d bricks and has cframes for %s.' %
(exposure, len(brick_names), ','.join(cframes.keys())))
for camera, cframe_path in cframes.iteritems():
band, spectro_id = camera[0], int(camera[1:])
this_camera = (fibermap_data['SPECTROID'] == spectro_id)
# Read this cframe file.
frame = desispec.io.read_frame(cframe_path)
# Loop over bricks.
for brick_name in brick_names:
# Lookup the fibers belong to this brick.
this_brick = (fibermap_data['BRICKNAME'] == brick_name)
brick_data = fibermap_data[this_camera & this_brick]
fibers = np.mod(brick_data['FIBER'], 500)
if len(fibers) == 0:
continue
brick_key = '%s_%s' % (band, brick_name)
# Open the brick file if this is the first time we are using it.
if brick_key not in bricks:
brick_path = desispec.io.findfile('brick',
brickname=brick_name,
band=band)
header = dict(
BRICKNAM=(brick_name, 'Imaging brick name'),
CHANNEL=(band, 'Spectrograph channel [b,r,z]'),
)
bricks[brick_key] = desispec.io.brick.Brick(
brick_path, mode='update', header=header)
# Add these fibers to the brick file. Note that the wavelength array is
# not per-fiber, so we do not slice it before passing it to add_objects().
bricks[brick_key].add_objects(
frame.flux[fibers], frame.ivar[fibers], frame.wave,
frame.resolution_data[fibers], brick_data, args.night,
exposure)
# Close all brick files.
for brick in bricks.itervalues():
log.debug(
'Brick %s now contains %d spectra for %d targets.' %
(brick.path, brick.get_num_spectra(), brick.get_num_targets()))
brick.close()
except RuntimeError as e:
log.critical(str(e))
return -2
def main() :
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--fiberflatexpid', type = int, help = 'fiberflat exposure ID')
parser.add_argument('--fibermap', type = str, help = 'path of fibermap file')
parser.add_argument('--models', type = str, help = 'path of spectro-photometric stellar spectra fits')
parser.add_argument('--spectrograph', type = int, default = 0, help = 'spectrograph number, can go 0-9')
parser.add_argument('--outfile', type = str, help = 'output file for normalized stdstar model flux')
args = parser.parse_args()
log = get_logger()
# Call necessary environment variables. No need if add argument to give full file path.
if 'DESI_SPECTRO_REDUX' not in os.environ:
raise RuntimeError('Set environment DESI_SPECTRO_REDUX. It is needed to read the needed datafiles')
DESI_SPECTRO_REDUX=os.environ['DESI_SPECTRO_REDUX']
PRODNAME=os.environ['PRODNAME']
if 'DESISIM' not in os.environ:
raise RuntimeError('Set environment DESISIM. It will be neede to read the filter transmission files for calibration')
DESISIM=os.environ['DESISIM'] # to read the filter transmission files
if args.fibermap is None or args.models is None or \
args.spectrograph is None or args.outfile is None or \
args.fiberflatexpid is None:
log.critical('Missing a required argument')
parser.print_help()
sys.exit(12)
# read Standard Stars from the fibermap file
# returns the Fiber id, filter names and mags for the standard stars
fiber_tbdata,fiber_header=io.read_fibermap(args.fibermap, header=True)
#- Trim to just fibers on this spectrograph
ii = (500*args.spectrograph <= fiber_tbdata["FIBER"])
ii &= (fiber_tbdata["FIBER"] < 500*(args.spectrograph+1))
fiber_tbdata = fiber_tbdata[ii]
#- Get info for the standard stars
refStarIdx=np.where(fiber_tbdata["OBJTYPE"]=="STD")
refFibers=fiber_tbdata["FIBER"][refStarIdx]
refFilters=fiber_tbdata["FILTER"][refStarIdx]
refMags=fiber_tbdata["MAG"]
fibers={"FIBER":refFibers,"FILTER":refFilters,"MAG":refMags}
NIGHT=fiber_header['NIGHT']
EXPID=fiber_header['EXPID']
filters=fibers["FILTER"]
if 'DESISIM' not in os.environ:
raise RuntimeError('Set environment DESISIM. Can not find filter response files')
basepath=DESISIM+"/data/"
#now load all the skyfiles, framefiles, fiberflatfiles etc
# all three channels files are simultaneously treated for model fitting
skyfile={}
framefile={}
fiberflatfile={}
for i in ["b","r","z"]:
camera = i+str(args.spectrograph)
skyfile[i] = io.findfile('sky', NIGHT, EXPID, camera)
framefile[i] = io.findfile('frame', NIGHT, EXPID, camera)
fiberflatfile[i] = io.findfile('fiberflat', NIGHT, args.fiberflatexpid, camera)
#Read Frames, Flats and Sky files
frameFlux={}
frameIvar={}
frameWave={}
frameResolution={}
framehdr={}
fiberFlat={}
ivarFlat={}
maskFlat={}
meanspecFlat={}
waveFlat={}
headerFlat={}
sky={}
skyivar={}
skymask={}
skywave={}
skyhdr={}
for i in ["b","r","z"]:
#arg=(night,expid,'%s%s'%(i,spectrograph))
#- minimal code change for refactored I/O, while not taking advantage of simplified structure
frame = io.read_frame(framefile[i])
frameFlux[i] = frame.flux
frameIvar[i] = frame.ivar
frameWave[i] = frame.wave
frameResolution[i] = frame.resolution_data
framehdr[i] = frame.header
ff = io.read_fiberflat(fiberflatfile[i])
fiberFlat[i] = ff.fiberflat
ivarFlat[i] = ff.ivar
maskFlat[i] = ff.mask
meanspecFlat[i] = ff.meanspec
waveFlat[i] = ff.wave
headerFlat[i] = ff.header
skymodel = io.read_sky(skyfile[i])
sky[i] = skymodel.flux
skyivar[i] = skymodel.ivar
skymask[i] = skymodel.mask
skywave[i] = skymodel.wave
skyhdr[i] = skymodel.header
# Convolve Sky with Detector Resolution, so as to subtract from data. Convolve for all 500 specs. Subtracting sky this way should be equivalent to sky_subtract
convolvedsky={"b":sky["b"], "r":sky["r"], "z":sky["z"]}
# Read the standard Star data and divide by flat and subtract sky
stars=[]
ivars=[]
for i in fibers["FIBER"]:
#flat and sky should have same wavelength binning as data, otherwise should be rebinned.
stars.append((i,{"b":[frameFlux["b"][i]/fiberFlat["b"][i]-convolvedsky["b"][i],frameWave["b"]],
"r":[frameFlux["r"][i]/fiberFlat["r"][i]-convolvedsky["r"][i],frameWave["r"]],
"z":[frameFlux["z"][i]/fiberFlat["z"][i]-convolvedsky["z"][i],frameWave]},fibers["MAG"][i]))
ivars.append((i,{"b":[frameIvar["b"][i]],"r":[frameIvar["r"][i,:]],"z":[frameIvar["z"][i,:]]}))
stdwave,stdflux,templateid=io.read_stdstar_templates(args.models)
#- Trim standard star wavelengths to just the range we need
minwave = min([min(w) for w in frameWave.values()])
maxwave = max([max(w) for w in frameWave.values()])
ii = (minwave-10 < stdwave) & (stdwave < maxwave+10)
stdwave = stdwave[ii]
stdflux = stdflux[:, ii]
log.info('Number of Standard Stars in this frame: {0:d}'.format(len(stars)))
if len(stars) == 0:
log.critical("No standard stars! Exiting")
sys.exit(1)
# Now for each star, find the best model and normalize.
normflux=[]
bestModelIndex=np.arange(len(stars))
templateID=np.arange(len(stars))
chi2dof=np.zeros(len(stars))
#- TODO: don't use 'l' as a variable name. Can look like a '1'
for k,l in enumerate(stars):
log.info("checking best model for star {0}".format(l[0]))
starindex=l[0]
mags=l[2]
filters=fibers["FILTER"][k]
rflux=stars[k][1]["r"][0]
bflux=stars[k][1]["b"][0]
zflux=stars[k][1]["z"][0]
flux={"b":bflux,"r":rflux,"z":zflux}
#print ivars
rivar=ivars[k][1]["r"][0]
bivar=ivars[k][1]["b"][0]
zivar=ivars[k][1]["z"][0]
ivar={"b":bivar,"r":rivar,"z":zivar}
resol_star={"r":frameResolution["r"][l[0]],"b":frameResolution["b"][l[0]],"z":frameResolution["z"][l[0]]}
# Now find the best Model
bestModelIndex[k],bestmodelWave,bestModelFlux,chi2dof[k]=match_templates(frameWave,flux,ivar,resol_star,stdwave,stdflux)
log.info('Star Fiber: {0}; Best Model Fiber: {1}; TemplateID: {2}; Chisq/dof: {3}'.format(l[0],bestModelIndex[k],templateid[bestModelIndex[k]],chi2dof[k]))
# Normalize the best model using reported magnitude
modelwave,normalizedflux=normalize_templates(stdwave,stdflux[bestModelIndex[k]],mags,filters,basepath)
normflux.append(normalizedflux)
# Now write the normalized flux for all best models to a file
normflux=np.array(normflux)
stdfibers=fibers["FIBER"]
data={}
data['BESTMODEL']=bestModelIndex
data['CHI2DOF']=chi2dof
data['TEMPLATEID']=templateid[bestModelIndex]
norm_model_file=args.outfile
io.write_stdstar_model(norm_model_file,normflux,stdwave,stdfibers,data)
def plot_graph(frame,
fibers,
opt_err=False,
opt_2d=False,
label=None,
subplot=None):
"""Plot graph from a given spectra from a fits file and returns figure
----------
Parameters
----------
frame : File Directory
Where the spectra is collected to be plot.
fibers : fibers to show
"""
log = get_logger()
spectra = frame["FLUX"].data
ivar = frame["IVAR"].data
wave = frame["WAVELENGTH"].data
nfibers = spectra.shape[0]
if np.max(fibers) >= nfibers:
log.warning(
"requested fiber numbers %s exceed number of fibers in file %d" %
(str(fibers), nfibers))
fibers = fibers[fibers < nfibers]
if subplot is None:
subplot = plt.subplot(1, 1, 1)
for fiber in fibers:
if label:
fiber_label = "%s Fiber #%d" % (label, fiber)
else:
fiber_label = "Fiber #%d" % fiber
log.debug("Plotting fiber %03d" % fiber)
if opt_err:
err = np.sqrt(1. / (ivar + (ivar == 0))) * (ivar > 0)
if len(wave.shape) > 1:
subplot.errorbar(wave[fiber],
spectra[fiber],
err[fiber],
fmt="o-",
label=fiber_label)
else:
subplot.errorbar(wave,
spectra[fiber],
err[fiber],
fmt="o-",
label=fiber_label)
else:
if len(wave.shape) > 1:
subplot.plot(wave[fiber],
spectra[fiber],
"-",
label=fiber_label)
else:
subplot.plot(wave, spectra[fiber], "-", label=fiber_label)
subplot.set_xlabel("Wavelength [A]")
if opt_2d:
title = "spectra"
if label is not None:
title = label
plt.figure(title)
if len(wave.shape) == 1:
plt.imshow(spectra[fibers].T,
aspect='auto',
extent=(fibers[0] - 0.5, fibers[-1] + 0.5, wave[0],
wave[-1]),
origin=0.,
interpolation="nearest")
plt.ylabel("Wavelength [A]")
plt.xlabel("Fiber #")
else:
plt.imshow(spectra[fibers].T,
aspect='auto',
extent=(fibers[0] - 0.5, fibers[-1] + 0.5, 0,
spectra.shape[1]),
origin=0.,
interpolation="nearest")
plt.ylabel("Y CCD")
plt.xlabel("Fiber #")
plt.colorbar()
""" Class to organize QA for a full DESI production run
"""
from __future__ import print_function, absolute_import, division, unicode_literals
import numpy as np
import glob, os
from desispec.io import get_exposures
from desispec.io import get_files
from desispec.io import read_frame
from desispec.log import get_logger
log = get_logger()
class QA_Prod(object):
def __init__(self, specprod_dir):
""" Class to organize and execute QA for a DESI production
Args:
specprod_dir(str): Path containing the exposures/ directory to use. If the value
is None, then the value of :func:`specprod_root` is used instead.
Notes:
Attributes:
qa_exps : list
List of QA_Exposure classes, one per exposure in production
data : dict
"""
def main() :
"""
parsezbest.py computes common metrics and makes plots for analyzing results of zdc1 redshift challenge (zbest file).
Metrics:
+ dz = (zbest-ztrue)/(1+ztrue)
+ dv = c*dz
+ pull = (zbest-ztrue)/zerr
+ precision: sigma_z = std(dz), sigma_v = std(dv), nmad_z, nmad_v
+ accuracy (bias): mu_z = mean(dz), mu_v = mean(dv)
+ efficiency
+ purity
+ % of catastrophic failures
+ FOM = purity*efficiency
Results are stored in outfile (default parezbest_results.dat)
Plots:
+ Histograms dz, dv, pull
+ dz as a function of zt and zb for:
- zwarn=0
- zwarn=0 without catastrophic failures
- zwarn!=0
+ dz as a function of average S/N per wavelegnth bin for:
- zwarn=0
- zwarn=0 without catastrophic failures
- zwarn!=0
Color code:
+ zwarn=0: blue filled circles
+ zwarn !=0: red filled circles
+ catastrophic failures: green filled circles
"""
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,prog='parsezbest.py', usage='%(prog)s [options]\n\n parsezbest.py without options runs a demo')
parser.add_argument('--b', type = str, default = None, required=False,
help = 'path of DESI brick in b')
parser.add_argument('--r', type = str, default = None, required=False,
help = 'path of DESI brick in r')
parser.add_argument('--z', type = str, default = None, required=False,
help = 'path of DESI brick in z')
parser.add_argument('--outfile', type = str, default = "parsezbest_results.dat", required=False,
help = 'path of output file')
parser.add_argument('--pathtruth', type = str, default = None, required=False,
help = 'path of truth table if does not exist in bricks')
parser.add_argument('--zbest', type = str, default = None, required=False,
help = 'zbest file')
parser.add_argument('--plot', dest='plots', action='store_true', help = 'Plots are optional. They are put to the screen if --plot is used.')
parser.set_defaults(plots=False)
args = parser.parse_args()
log=get_logger()
file=open(args.outfile,"w")
log.info("starting")
#- if no arguments is passed, parsezbest runs a demo
if ((args.zbest is None) and (args.b is None) and (args.r is None) and (args.z is None)):
args.zbest = "%s/data/zbest-training-elg-100-zztop.fits"%(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(desibest.__file__)))))
args.b = "%s/data/brick-b-elg-100-zztop.fits"%(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(desibest.__file__)))))
args.r = "%s/data/brick-r-elg-100-zztop.fits"%(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(desibest.__file__)))))
args.z = "%s/data/brick-z-elg-100-zztop.fits"%(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(desibest.__file__)))))
elif ((args.zbest is None) or (args.b is None) or (args.r is None) or (args.z is None)):
log.error("Either all files (b, r and z bricks and zbest) or none should be provided")
sys.exit(12)
try:
b_brick=Brick(args.b)
except:
log.error("can not open brick %s"%args.b)
print sys.exc_info()
file.close()
sys.exit(12)
try:
r_brick=Brick(args.r)
except:
log.error("can not open brick %s"%args.r)
print sys.exc_info()
file.close()
sys.exit(12)
try:
z_brick=Brick(args.z)
except:
log.error("can not open brick %s"%args.z)
print sys.exc_info()
file.close()
sys.exit(12)
try :
zb_hdulist=fits.open(args.zbest)
except :
log.error("can not open file %s:"%args.zbest)
print sys.exc_info()
file.close()
sys.exit(12)
log.info("Using zbest file %s"%args.zbest)
log.info("Using %s b brick"%args.b)
log.info("Using %s r brick"%args.r)
log.info("Using %s z brick"%args.z)
log.info(" ")
file.write("Using zbest file %s\n"%args.zbest)
file.write("Using %s b brick\n"%args.b)
file.write("Using %s r brick\n"%args.r)
file.write("Using %s z brick\n"%args.z)
file.write("\n")
brickname = args.b
c=3.e5 # light celerity in vacuum [km/s]
#- Checking zbest structure and format
log.info("Checking zbest structure and format")
log.info("-----------------------------------")
if(zb_hdulist[0].size == 0 and zb_hdulist[1].size != 0):
log.info("zbest file has the required structure")
if (zb_hdulist[1].name != 'ZBEST'):
log.warning("HDU extension 1 has name %s, not ZBEST"%zb_hdulist[1].name)
log.info(" ")
else:
log.error("zbest file does not have the required structure")
log.error("Check ZBEST structure at http://desidatamodel.readthedocs.org/en/latest/DESI_SPECTRO_REDUX/PRODNAME/bricks/BRICKNAME/zbest-BRICKNAME.html")
file.close()
print sys.exc_info()
sys.exit(12)
zb_name = zb_hdulist[1].name
keys=['TARGETID', 'Z', 'ZERR', 'ZWARN', 'TYPE']
subkeys = ['BRICKNAME', 'SUBTYPE']
for k in keys:
try :
zb_hdulist[zb_name].columns.names.index(k)
except :
log.error("Missing column %s in %s"%(k,args.zbest))
log.error("Check ZBEST format at http://desidatamodel.readthedocs.org/en/latest/DESI_SPECTRO_REDUX/PRODNAME/bricks/BRICKNAME/zbest-BRICKNAME.html")
file.close()
print sys.exc_info()
sys.exit(12)
else:
log.info("ZBEST hdu has required column %s"%k)
for sk in subkeys:
try:
zb_hdulist[zb_name].columns.names.index(k)
log.info("ZBEST hdu has optional column %s"%sk)
except:
log.warning("ZBEST hdu misses optional %s column"%sk)
log.info(" ")
#- checking for truth table
log.info("Checking for truth table")
log.info("-----------------------------------")
try:
b_hdulist = fits.open(args.b)
truth_table_hdu=b_hdulist['_TRUTH']
log.info("brick has truth table")
log.info(" ")
except KeyError :
truth_table_hdu=None
if (truth_table_hdu is None):
try:
truthfile = fits.open(args.pathtruth)
truth_table_hdu = truthfile['_TRUTH']
log.info("Found truth table %s"%args.pathtruth)
log.info(" ")
except:
log.error("A truth table should be provided")
file.close()
sys.exit(12)
#- Get results from zbest hdu and infos from truth table
truth = truth_table_hdu.data
zbres=zb_hdulist[zb_name].data
zb = zbres['Z']
zt = truth['TRUEZ']
zw = zbres['ZWARN']
#- joining zbest and truth tables
# checks that targetids have the same dtype in truth and zbres
if (zbres['TARGETID'].dtype != truth['TARGETID'].dtype):
zbres = np.asarray(zbres,dtype=[('BRICKNAME', 'S20'), ('TARGETID', '>i8'), ('Z', '>f8'), ('ZERR', '>f8'), ('ZWARN', '>f8'), ('TYPE', 'S20')])
zb_zt = join(zbres, truth, keys='TARGETID')
truez=zb_zt['TRUEZ']
bestz=zb_zt['Z']
errz=zb_zt['ZERR']
zwarn = zb_zt['ZWARN']
n=bestz.size
if (n == 0):
log.error("target ids in zbest file are not a subset of target ids in truth table")
log.error("did you provide the bricks that correspond to zbest file ?")
file.close()
sys.exit(12)
#- Select objtype
log.info("Target type")
log.info("-----------------------------------")
obj=dict()
objtypes=['ELG','LRG','QSO','QSO_BAD','STAR']
totobj = len(zb_zt['Z'])
for o in objtypes:
index=np.where(zb_zt['OBJTYPE'] == '%s'%o)[0]
obj[o]=len(index)
log.info("%i %s found"%(obj[o],o))
file.write("%i %s found\n"%(obj[o],o))
if (obj[o] != 0):
log.info(" ")
file.write("\n")
#load requirements for target
req= desi_requirement(o)
tz = np.zeros(len(index))
bz = np.zeros(len(index))
zw = np.zeros(len(index))
dv = np.zeros(len(index))
ez = np.zeros(len(index))
if (o == 'ELG'):
trfloii = np.zeros(len(index))
dz=0.
dv=0.
pull=0.
for i,j in zip(index,range(len(index))):
tz[j]=truez[i]
bz[j]=bestz[i]
zw[j]=zwarn[i]
ez[j]=errz[i]
if (o == 'ELG'):
trfloii[j] = zb_zt["OIIFLUX"][i]
dv = c*(bz-tz)/(1+tz)
dz=dv/c
true_pos = np.where((np.abs(dz)<0.0033) & (zw==0))[0]
true_neg = np.where((np.abs(dz)>0.0033) & (zw!=0))[0]
false_pos = np.where((np.abs(dz)>0.0033) & (zw==0))[0]
false_neg = np.where((np.abs(dz)<0.0033) & (zw!=0))[0]
#- total
total = len(true_pos)+len(true_neg)+len(false_pos)+len(false_neg)
#- computes sample efficiency
efficiency = float(len(true_pos))/float(total)
#- computes purity
purity = float(len(true_pos))/float((len(true_pos)+len(false_pos)))
#- catastrophic failures
# cata_fail = float(len(false_pos))/float(total)
cata_fail = float(len(false_pos))/float((len(true_pos)+len(false_pos)))
#- figure of merit
fom = efficiency*purity
# precision
#- sigma
ok = np.where(zw==0)[0]
zerr = np.std(dz[ok])
verr = np.std(dv[ok])
#- quantiles
dz_quant = np.percentile(dz[ok], (2.5, 16, 50, 84, 97.5))
dv_quant = np.percentile(dv[ok], (2.5, 16, 50, 84, 97.5))
dz_err68 = (dz_quant[3] - dz_quant[1])/2.
dz_err95 = (dz_quant[4] - dz_quant[0])/2.
dv_err68 = (dv_quant[3] - dv_quant[1])/2.
dv_err95 = (dv_quant[4] - dv_quant[0])/2.
#accuracy
#- mu
zacc = np.mean(dz[ok])
vacc = np.mean(dv[ok])
#- quantile
dz_acc50 = dz_quant[2]
dv_acc50 = dv_quant[2]
ok_no_cata = np.where((zw==0) & (np.abs(dz)<0.0033))[0]
if (len(ok) != len(ok_no_cata)):
zerr_no_cata = np.std(dz[ok_no_cata])
verr_no_cata = np.std(dv[ok_no_cata])
zacc_no_cata = np.mean(dz[ok_no_cata])
vacc_no_cata = np.mean(dv[ok_no_cata])
dz_quant_no_cata = np.percentile(dz[ok_no_cata], (2.5, 16, 50, 84, 97.5))
dv_quant_no_cata = np.percentile(dv[ok_no_cata], (2.5, 16, 50, 84, 97.5))
dz_err68_no_cata = (dz_quant_no_cata[3] - dz_quant_no_cata[1])/2.
dz_err95_no_cata = (dz_quant_no_cata[4] - dz_quant_no_cata[0])/2.
dv_err68_no_cata = (dv_quant_no_cata[3] - dv_quant_no_cata[1])/2.
dv_err95_no_cata = (dv_quant_no_cata[4] - dv_quant_no_cata[0])/2.
dz_acc50_no_cata = dz_quant_no_cata[2]
dv_acc50_no_cata = dv_quant_no_cata[2]
# NMAD
nmad_z = np.median(np.abs(dz[ok]-np.median(dz[ok])))
nmad_z *= 1.4826
nmad_v = np.median(np.abs(dv[ok]-np.median(dv[ok])))
nmad_v *= 1.4826
#pull
pull_ok_no_cata = np.where((zw==0) & (np.abs(dz)<0.0033) & (ez>0))[0]
pull = (bz[pull_ok_no_cata]-tz[pull_ok_no_cata])/ez[pull_ok_no_cata]
mu_pull = np.mean(pull)
sigma_pull = np.std(pull)
# zwarn
zw0 = len(np.where(zw == 0)[0])
zw_non0 = len(np.where(zw != 0)[0])
# assert (zw0 + zw_non0) != len(index), "zw=0 + zw!=0 not equal to number of objects"
log.info("=====================================")
log.info("%s: Precision and accuracy (zwarn=0)"%o)
log.info("=====================================")
log.info("sigma_z: %f, mu_z: %f"%(zerr,zacc))
log.info("quantile: dz_err68: %f"%(dz_err68))
log.info("quantile: dz_err95: %f"%(dz_err95))
log.info("quantile: dz_acc50: %f"%(dz_acc50))
log.info("NMAD_z: %f"%nmad_z)
log.info("sigma_v: %f, mu_v: %f"%(verr,vacc))
log.info("quantile: dv_err68: %f"%(dv_err68))
log.info("quantile: dv_err95: %f"%(dv_err95))
log.info("quantile: dv_acc50: %f"%(dv_acc50))
log.info("NMAD_v: %f"%nmad_v)
if req is not None:
if (dz_err68>req['SIG_Z']):
log.info("dz_err68 & dv_err68 do not meet DESI requirements on precision for %s"%o)
if (dz_acc50>req['BIAS_Z']):
log.info("dz_acc50 & dv_acc50 do not meet DESI requirements on bias for %s"%o)
log.info(" ")
if (len(ok) != len(ok_no_cata)):
log.info("=====================================")
log.info("%s: Precision and accuracy "%o)
log.info("zwarn=0 without catastrophic failures")
log.info("=====================================")
log.info("sigma_z: %f, mu_z: %f"%(zerr_no_cata,zacc_no_cata))
log.info("quantile: dz_err68: %f"%(dz_err68_no_cata))
log.info("quantile: dz_err95: %f"%(dz_err95_no_cata))
log.info("quantile: dz_acc50: %f"%(dz_acc50_no_cata))
log.info("sigma_v: %f, mu_v: %f"%(verr_no_cata,vacc_no_cata))
log.info("quantile: dv_err68: %f"%(dv_err68_no_cata))
log.info("quantile: dv_err95: %f"%(dv_err95_no_cata))
log.info("quantile: dv_acc50: %f"%(dv_acc50_no_cata))
if req is not None:
if (dz_err68_no_cata>req['SIG_Z']):
log.info("dz_err68 & dv_err68 do not meet DESI requirements on precision for %s"%o)
if (dz_acc50_no_cata>req['BIAS_Z']):
log.info("dz_acc50 & dv_acc50 do not meet DESI requirements on bias for %s"%o)
log.info(" ")
file.write("=====================================\n")
file.write("%s: Precision and accuracy (zwarn=0)\n"%o)
file.write("=====================================\n")
file.write("sigma_z: %f, mu_z: %f\n"%(zerr,zacc))
file.write("quantile: dz_err68: %f\n"%(dz_err68))
file.write("quantile: dz_err95: %f\n"%(dz_err95))
file.write("quantile: dz_acc50: %f\n"%(dz_acc50))
file.write("NMAD_z: %f\n"%nmad_z)
file.write("sigma_v: %f, mu_v: %f\n"%(verr,vacc))
file.write("quantile: dv_err68: %f\n"%(dv_err68))
file.write("quantile: dv_err95: %f\n"%(dv_err95))
file.write("quantile: dv_acc50: %f\n"%(dv_acc50))
file.write("NMAD_v: %f\n"%nmad_v)
if req is not None:
if (dz_err68>req['SIG_Z']):
file.write("dz_err68 & dv_err68 do not meet DESI requirements on precision for %s\n"%o)
if (dz_acc50>req['BIAS_Z']):
file.write("dz_acc50 & dv_acc50 do not meet DESI requirements on bias for %s\n"%o)
file.write("\n")
if (len(ok) != len(ok_no_cata)):
file.write("=====================================\n")
file.write("%s: Precision and accuracy \n"%o)
file.write("zwarn=0 without catastrophic failures\n")
file.write("=====================================\n")
file.write("sigma_z: %f, mu_z: %f\n"%(zerr_no_cata,zacc_no_cata))
file.write("quantile: dz_err68: %f\n"%(dz_err68_no_cata))
file.write("quantile: dz_err95: %f\n"%(dz_err95_no_cata))
file.write("quantile: dz_acc50: %f\n"%(dz_acc50_no_cata))
file.write("sigma_v: %f, mu_v: %f\n"%(verr_no_cata,vacc_no_cata))
file.write("quantile: dv_err68: %f\n"%(dv_err68_no_cata))
file.write("quantile: dv_err95: %f\n"%(dv_err95_no_cata))
file.write("quantile: dv_acc50: %f\n"%(dv_acc50_no_cata))
if req is not None:
if (dz_err68_no_cata>req['SIG_Z']):
file.write("dz_err68 & dv_err68 do not meet DESI requirements on precision for %s\n"%o)
if (dz_acc50_no_cata>req['BIAS_Z']):
file.write("dz_acc50 & dv_acc50 do not meet DESI requirements on bias for %s\n"%o)
file.write("\n")
log.info("=====================================")
log.info("%s: Pull (zwarn=0, no cata. fail.)"%o)
log.info("=====================================")
log.info("mu: %f, sigma: %f"%(mu_pull,sigma_pull))
log.info(" ")
file.write("=====================================\n")
file.write("%s: Pull (zwarn=0, no cata. fail.)\n"%o)
file.write("=====================================\n")
file.write("mu: %f, sigma: %f\n"%(mu_pull,sigma_pull))
file.write("\n")
if (o == 'ELG'):
true_pos_oII = np.where((np.abs(dz)<0.0033) & (zw==0) & (trfloii>8e-17))[0]
true_neg_oII = np.where((np.abs(dz)>0.0033) & (zw!=0) & (trfloii>8e-17))[0]
false_pos_oII = np.where((np.abs(dz)>0.0033) & (zw==0) & (trfloii>8e-17))[0]
false_neg_oII = np.where((np.abs(dz)<0.0033) & (zw!=0) & (trfloii>8e-17))[0]
#- total
total_oII = len(true_pos_oII)+len(true_neg_oII)+len(false_pos_oII)+len(false_neg_oII)
#- computes sample efficiency
efficiency_oII = float(len(true_pos_oII))/float(total_oII)
#- computes purity
purity_oII = float(len(true_pos_oII))/float((len(true_pos_oII)+len(false_pos_oII)))
#- catastrophic failures
# cata_fail_oII = float(len(false_pos_oII))/float(total_oII)
cata_fail_oII = float(len(false_pos_oII))/float((len(true_pos_oII)+len(false_pos_oII)))
#- figure of merit
fom_oII = efficiency_oII*purity_oII
oii8e17 = np.where((np.abs(dz)<0.0033) & (zw==0) & (trfloii>8e-17))[0]
stdzoii = np.std(dz[oii8e17])
stdvoii = np.std(dv[oii8e17])
acczoii = np.mean(dz[oii8e17])
accvoii = np.mean(dv[oii8e17])
dz_quant_oii = np.percentile(dz[oii8e17], (2.5, 16, 50, 84, 97.5))
dv_quant_oii = np.percentile(dv[oii8e17], (2.5, 16, 50, 84, 97.5))
dz_err68_oii = (dz_quant_oii[3] - dz_quant_oii[1])/2.
dz_err95_oii = (dz_quant_oii[4] - dz_quant_oii[0])/2.
dv_err68_oii = (dv_quant_oii[3] - dv_quant_oii[1])/2.
dv_err95_oii = (dv_quant_oii[4] - dv_quant_oii[0])/2.
dz_acc50_oii = dz_quant_oii[2]
dv_acc50_oii = dv_quant_oii[2]
log.info("=====================================")
log.info("%s: For OII > 8e-17 erg/s/cm2"%o)
log.info("=====================================")
log.info('Efficiency_oII: %d/%d=%f'%(len(true_pos_oII),total_oII,efficiency_oII))
if req is not None:
if (efficiency_oII < req['EFFICIENCY']):
log.info("Efficiency_oII does not meet DESI requirements for %s"%o)
log.info('Purity_oII: %d/%d=%f'%(len(true_pos_oII),(len(true_pos_oII)+len(false_pos_oII)),purity_oII))
log.info('Catastrophic failures_oII: %d/%d=%f'%(len(false_pos_oII),len(true_pos_oII)+len(false_pos_oII),cata_fail_oII))
if req is not None:
if (cata_fail_oII>req['CATA_FAIL_MAX']):
log.info("Catastrophic failure rate does not meet DESI requirements for %s"%o)
log.info('sigma_z_oii: %f, mu_z_oii: %f'%(stdzoii, acczoii))
log.info('quantile: dz_err68_oii: %f'%(dz_err68_oii))
log.info('quantile: dz_err95_oii: %f'%(dz_err95_oii))
log.info('quantile: dz_acc50_oii: %f'%(dz_acc50_oii))
log.info('sigma_v_oii: %f, mu_v_oii: %f'%(stdvoii, accvoii))
log.info('quantile: dv_err68_oii: %f'%(dv_err68_oii))
log.info('quantile: dv_err95_oii: %f'%(dv_err95_oii))
log.info('quantile: dv_acc50_oii: %f'%(dv_acc50_oii))
log.info('FOM_oII: %f x %f=%f'%(efficiency_oII,purity_oII,fom_oII))
log.info(" ")
file.write("=====================================\n")
file.write("%s: For OII > 8e-17 erg/s/cm2\n"%o)
file.write("=====================================\n")
file.write('Efficiency_oII: %d/%d=%f\n'%(len(true_pos_oII),total_oII,efficiency_oII))
if req is not None:
if (efficiency_oII < req['EFFICIENCY']):
file.write("Efficiency_oII does not meet DESI requirements for %s\n"%o)
file.write('Purity_oII: %d/%d=%f\n'%(len(true_pos_oII),(len(true_pos_oII)+len(false_pos_oII)),purity_oII))
file.write('Catastrophic failures_oII: %d/%d=%f\n'%(len(false_pos_oII),len(true_pos_oII)+len(false_pos_oII),cata_fail_oII))
if req is not None:
if (cata_fail_oII>req['CATA_FAIL_MAX']):
file.write("Catastrophic failure rate does not meet DESI requirements for %s\n"%o)
file.write('sigma_z_oii: %f, mu_z_oii: %f\n'%(stdzoii, acczoii))
file.write('quantile: dz_err68_oii: %f\n'%(dz_err68_oii))
file.write('quantile: dz_err95_oii: %f\n'%(dz_err95_oii))
file.write('quantile: dz_acc50_oii: %f\n'%(dz_acc50_oii))
file.write('sigma_v_oii: %f, mu_v_oii: %f\n'%(stdvoii, accvoii))
file.write('quantile: dv_err68_oii: %f\n'%(dv_err68_oii))
file.write('quantile: dv_err95_oii: %f\n'%(dv_err95_oii))
file.write('quantile: dv_acc50_oii: %f\n'%(dv_acc50_oii))
file.write('FOM_oII: %f x %f=%f\n'%(efficiency_oII,purity_oII,fom_oII))
file.write("\n")
log.info("=====================================")
log.info("%s: Total sample"%o)
log.info("=====================================")
log.info("zwarn = 0: %d"%zw0)
log.info("zwarn !=0: %d"%zw_non0)
log.info('Efficiency: %d/%d=%f'%(len(true_pos),total,efficiency))
if req is not None:
if (efficiency < req['EFFICIENCY']):
log.info("Efficiency does not meet DESI requirements for %s"%o)
log.info('Purity: %d/%d=%f'%(len(true_pos),(len(true_pos)+len(false_pos)),purity))
log.info('Catastrophic failures: %d/%d=%f'%(len(false_pos),len(true_pos)+len(false_pos),cata_fail))
if req is not None:
if (cata_fail>req['CATA_FAIL_MAX']):
log.info("Catastrophic failure rate does not meet DESI requirements for %s"%o)
log.info('FOM: %f x %f=%f'%(efficiency,purity,fom))
log.info("=====================================")
log.info(" ")
file.write("=====================================\n")
file.write("%s: Total sample\n"%o)
file.write("=====================================\n")
file.write("zwarn = 0: %d\n"%zw0)
file.write("zwarn !=0: %d\n"%zw_non0)
file.write('Efficiency: %d/%d=%f\n'%(len(true_pos),total,efficiency))
if req is not None:
if (efficiency < req['EFFICIENCY']):
file.write("Efficiency does not meet DESI requirements for %s\n"%o)
file.write('Purity: %d/%d=%f\n'%(len(true_pos),(len(true_pos)+len(false_pos)),purity))
file.write('Catastrophic failures: %d/%d=%f\n'%(len(false_pos),len(true_pos)+len(false_pos),cata_fail))
if req is not None:
if (cata_fail>req['CATA_FAIL_MAX']):
file.write("Catastrophic failure rate does not meet DESI requirements for %s\n"%o)
file.write('FOM: %f x %f=%f\n'%(efficiency,purity,fom))
file.write("=====================================\n")
file.write("\n")
# computes spectrum S/N
mean_ston=np.zeros(len(index))
mean_ston_oII=np.zeros(len(index))
for spec,sp in zip(index,range(len(index))):
flux=[b_brick.hdu_list[0].data[spec],r_brick.hdu_list[0].data[spec],z_brick.hdu_list[0].data[spec]]
ivar=[b_brick.hdu_list[1].data[spec],r_brick.hdu_list[1].data[spec],z_brick.hdu_list[1].data[spec]]
wave=[b_brick.hdu_list[2].data,r_brick.hdu_list[2].data,z_brick.hdu_list[2].data]
for i in range(3):
mean_ston[sp] += np.sum(np.abs(flux[i])*np.sqrt(ivar[i]))/len(wave[i])
# computes mean S/N in OII lines for ELG
if (o == 'ELG'):
for i in range(3):
ok = np.where((wave[i]>3722) & (wave[i]< 3734))[0]
if (len(ok) == 0):
break
else:
mean_ston_oII[sp] += np.sum(np.abs(flux[i][ok])*np.sqrt(ivar[i][ok]))/len(ok)
#- plots
if (args.plots):
ok=np.where(zw==0)[0]
cata = np.where((zw == 0) & (np.abs(dz)>0.0033))[0]
not_ok = np.where(zw !=0)[0]
# ok_no_cata = np.where((zw == 0) & (np.abs(dz)<0.0033))[0]
#- histograms
pylab.figure()
n, bins, patches = pylab.hist(dz[ok_no_cata], 30, normed=1, histtype='stepfilled')
pylab.setp(patches, 'facecolor', 'b', 'alpha', 0.75)
if (o != 'QSO'):
muz = np.mean(dz[ok_no_cata])
sigmaz = np.std(dz[ok_no_cata])
gauss = pylab.normpdf(bins, muz, sigmaz)
l = pylab.plot(bins, gauss, 'k--', linewidth=1.5, label="mu=%2.0f *1e-6, sig=%2.0f *1e-6"%(muz/1e-6,sigmaz/1e-6))
pylab.legend()
pylab.xlabel("(zb-zt)/(1+zt) (ZWARN=0 without catastrophic failures)")
pylab.ylabel("Num. of %s targets per bin"%o)
pylab.figure()
n, bins, patches = pylab.hist(dv[ok_no_cata], 30, normed=1, histtype='stepfilled')
pylab.setp(patches, 'facecolor', 'g', 'alpha', 0.75)
if (o != 'QSO'):
muv = np.mean(dv[ok_no_cata])
sigmav = np.std(dv[ok_no_cata])
gauss = pylab.normpdf(bins, muv, sigmav)
l = pylab.plot(bins, gauss, 'k--', linewidth=1.5, label="mu=%2.0f, sig=%2.0f"%(muv,sigmav))
pylab.legend()
pylab.xlabel("Delta v = c(zb-zt)/(1+zt) [km/s] (ZWARN=0 without catastrophic failures)")
pylab.ylabel("Num. of %s targets per bin"%o)
#- pull distribution
pylab.figure()
n, bins, patches = pylab.hist(pull, 30, normed=1, histtype='stepfilled')
pylab.setp(patches, 'facecolor', 'c', 'alpha', 0.75)
# mu_pull = np.mean(pull[ok_no_cata])
# sigma_pull = np.std(pull[ok_no_cata])
gauss = pylab.normpdf(bins, mu_pull, sigma_pull)
l = pylab.plot(bins, gauss, 'k--', linewidth=1.5, label="mu=%2.3f, sig=%2.3f"%(mu_pull,sigma_pull))
pylab.legend()
mu=0.
sig=1.
gauss1 = pylab.normpdf(bins, mu, sig)
l1 = pylab.plot(bins, gauss1, 'k--', linewidth=1.5, color='r', label="mu=0., sig=1.")
pylab.legend()
pylab.xlabel("Pull = (zb-<zt>)/zerr (ZWARN=0 without catastrophic failures)")
pylab.ylabel("Num. of %s targets per bin"%o)
#- other plots
pylab.figure()
nx = 1
if len(cata) !=0:
ny = 3
else:
ny = 2
ai = 1
# catastrophic failures in green
# zwarn != 0 in red
# zw =0 no catastrophic in blue
a=pylab.subplot(ny,nx,ai); ai +=1
a.errorbar(mean_ston[ok],dz[ok],errz[ok],fmt="bo")
a.errorbar(mean_ston[cata],dz[cata],errz[cata],fmt="go")
a.set_xlabel("%s <S/N>"%o)
a.set_ylabel("(zb-zt)/(1+zt) (ZWARN=0)")
a=pylab.subplot(ny,nx,ai); ai +=1
a.errorbar(mean_ston[ok],dz[ok],errz[ok],fmt="bo")
a.errorbar(mean_ston[not_ok],dz[not_ok],errz[not_ok],fmt="ro")
a.errorbar(mean_ston[cata],dz[cata],errz[cata],fmt="go")
a.set_xlabel("%s <S/N> "%o)
a.set_ylabel("(zb-zt)/(1+zt) (all ZWARN)")
if len(cata) !=0:
a=pylab.subplot(ny,nx,ai); ai +=1
a.errorbar(mean_ston[ok_no_cata],dz[ok_no_cata],errz[ok_no_cata],fmt="bo")
a.set_xlabel("%s <S/N> "%o)
a.set_ylabel("(zb-zt)/(1+zt) (ZWARN=0, no cata. fail.)")
if (o == 'ELG'):
pylab.figure()
nx=1
if len(cata) !=0:
ny=3
else:
ny=2
ai=1
a=pylab.subplot(ny,nx,ai); ai +=1
a.errorbar(zb_zt['OIIFLUX'][ok],dz[ok],errz[ok],fmt="bo")
a.errorbar(zb_zt['OIIFLUX'][cata],dz[cata],errz[cata],fmt="ro")
a.set_xlabel("%s True [OII] flux"%o)
a.set_ylabel("(zb-zt)/(1+zt) (all ZWARN)")
a=pylab.subplot(ny,nx,ai); ai +=1
a.errorbar(zb_zt['OIIFLUX'][ok],dz[ok],errz[ok],fmt="bo")
a.errorbar(zb_zt['OIIFLUX'][not_ok],dz[not_ok],errz[not_ok],fmt="ro")
a.errorbar(zb_zt['OIIFLUX'][cata],dz[cata],errz[cata],fmt="go")
a.set_xlabel("%s True [OII] flux"%o)
a.set_ylabel("(zb-zt)/(1+zt) (ZWARN=0)")
if len(cata) != 0:
a=pylab.subplot(ny,nx,ai); ai +=1
a.errorbar(zb_zt['OIIFLUX'][ok_no_cata],dz[ok_no_cata],errz[ok_no_cata],fmt="bo")
a.set_xlabel("%s True [OII] flux"%o)
a.set_ylabel("(zb-zt)/(1+zt) (ZWARN=0, no cata. fail.)")
pylab.figure()
nx=2
if len(cata) !=0:
ny=3
else:
ny=2
ai=1
a=pylab.subplot(ny,nx,ai); ai +=1
a.errorbar(tz[ok],dz[ok],errz[ok],fmt="o",c="b")
a.errorbar(tz[not_ok],dz[not_ok],errz[not_ok],fmt="o",c="r")
a.errorbar(tz[cata],dz[cata],errz[cata],fmt="o",c="g")
a.set_xlabel("%s zt (all ZWARN)"%o)
a.set_ylabel("(zb-zt)/(1+zt) (all ZWARN)")
a=pylab.subplot(ny,nx,ai); ai +=1
a.errorbar(bz[ok],dz[ok],errz[ok],fmt="o",c="b")
a.errorbar(bz[not_ok],dz[not_ok],errz[not_ok],fmt="o",c="r")
a.errorbar(bz[cata],dz[cata],errz[cata],fmt="o",c="g")
a.set_xlabel("%s zb (all ZWARN)"%o)
a.set_ylabel("(zb-zt)/(1+zt)")
a=pylab.subplot(ny,nx,ai); ai +=1
a.errorbar(tz[ok],dz[ok],errz[ok],fmt="o",c="b")
a.errorbar(tz[cata],dz[cata],errz[cata],fmt="o",c="g")
a.set_xlabel("%s zt (ZWARN=0)"%o)
a.set_ylabel("(zb-zt)/(1+zt)")
a=pylab.subplot(ny,nx,ai); ai +=1
a.errorbar(bz[ok],dz[ok],errz[ok],fmt="o",c="b")
a.errorbar(bz[cata],dz[cata],errz[cata],fmt="o",c="g")
a.set_xlabel("%s zb (ZWARN=0)"%o)
a.set_ylabel("(zb-zt)/(1+zt)")
if len(cata) !=0:
a=pylab.subplot(ny,nx,ai); ai +=1
a.errorbar(tz[ok_no_cata],dz[ok_no_cata],errz[ok_no_cata],fmt="o",c="b")
a.set_xlabel("%s zt (ZWARN=0, no cata. fail.)"%o)
a.set_ylabel("(zb-zt)/(1+zt)")
a=pylab.subplot(ny,nx,ai); ai +=1
a.errorbar(bz[ok_no_cata],dz[ok_no_cata],errz[ok_no_cata],fmt="o",c="b")
a.set_xlabel("%s zb (All ZWARN, no cata. fail.)"%o)
a.set_ylabel("(zb-zt)/(1+zt)")
pylab.show()
def main(args):
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
if args.brick is None:
log.critical('Missing required brick argument.')
return -1
# Open the combined coadd file for this brick, for updating.
coadd_all_path = desispec.io.meta.findfile('coadd_all',
brickname=args.brick,
specprod_dir=args.specprod)
coadd_all_file = desispec.io.brick.CoAddedBrick(coadd_all_path,
mode='update')
# Initialize dictionaries of co-added spectra for each object ID.
coadded_spectra = {}
# Keep track of the index we assign to each target.
next_coadd_index = 0
target_index = {}
# The HDU4 table for the global coadd will go here.
coadd_all_info = None
# Loop over bands for this brick.
for band in args.bands:
# Open this band's brick file for reading.
brick_path = desispec.io.meta.findfile('brick',
brickname=args.brick,
band=band,
specprod_dir=args.specprod)
if not os.path.exists(brick_path):
log.info(
'Skipping non-existent brick file {0}.'.format(brick_path))
continue
brick_file = desispec.io.brick.Brick(brick_path, mode='readonly')
flux_in, ivar_in, wlen, resolution_in = (brick_file.hdu_list[0].data,
brick_file.hdu_list[1].data,
brick_file.hdu_list[2].data,
brick_file.hdu_list[3].data)
log.debug('Processing %s with %d exposures of %d targets...' %
(brick_path, brick_file.get_num_spectra(),
brick_file.get_num_targets()))
if resolution_in.shape[1] != desispec.resolution.num_diagonals:
log.error(
'resolution has unexpected shape (ndiag=%d != %d). Skipping this file.'
% (resolution_in.shape[1], desispec.resolution.num_diagonals))
brick_file.close()
continue
# Open this band's coadd file for updating.
coadd_path = desispec.io.meta.findfile('coadd',
brickname=args.brick,
band=band,
specprod_dir=args.specprod)
coadd_file = desispec.io.brick.CoAddedBrick(coadd_path, mode='update')
# Copy the input fibermap info for each exposure into memory.
coadd_info = np.copy(brick_file.hdu_list[4].data)
# Also copy the first band's info to initialize the global coadd info, but remember that this
# band might not have all targets so we could see new targets in other bands.
if coadd_all_info is None:
coadd_all_info = np.copy(brick_file.hdu_list[4].data)
# Loop over objects in the input brick file.
for index, info in enumerate(brick_file.hdu_list[4].data):
assert index == info['INDEX'], 'Index mismatch: %d != %d' % (
index, info['INDEX'])
resolution_matrix = desispec.resolution.Resolution(
resolution_in[index])
spectrum = desispec.coaddition.Spectrum(wlen, flux_in[index],
ivar_in[index],
resolution_matrix)
target_id = info['TARGETID']
# Are we only processing specified targets?
if len(args.target) > 0 and target_id not in args.target:
continue
# Have we seen this target before?
if target_id not in coadded_spectra:
coadded_spectra[target_id] = {}
target_index[target_id] = next_coadd_index
next_coadd_index += 1
# Save the coadd index to our output table.
coadd_info['INDEX'][index] = target_index[target_id]
# Initialize the coadd for this band and target if necessary.
if band not in coadded_spectra[target_id]:
coadded_spectra[target_id][
band] = desispec.coaddition.Spectrum(wlen)
# Do the coaddition.
coadded_spectra[target_id][band] += spectrum
# Is this exposure of this target already in our global coadd table?
exposure = info['EXPID']
seen = (coadd_all_info['EXPID']
== exposure) & (coadd_all_info['TARGETID'] == target_id)
if not np.any(seen):
log.info(
'Adding exposure %d of target %d to global coadd with partial band coverage.'
% (exposure, target_id))
coadd_all_info.append(coadd_info[index])
else:
coadd_all_info['INDEX'][index] = target_index[target_id]
# Allocate arrays for the coadded results for this band. Since we always use the same index
# for the same target in each band, there might be some unused entries in these arrays if
# some bands are missing for some targets.
num_targets = 1 + np.max(coadd_info['INDEX'])
nbins = len(wlen)
flux_out = np.zeros((num_targets, nbins))
ivar_out = np.zeros_like(flux_out)
resolution_out = np.zeros(
(num_targets, desispec.resolution.num_diagonals, nbins))
# Save the coadded spectra for this band.
for target_id in coadded_spectra:
if band not in coadded_spectra[target_id]:
continue
exposures = (coadd_info['TARGETID'] == target_id)
index = target_index[target_id]
log.debug(
'Saving coadd of %d exposures for target ID %d to index %d.' %
(np.count_nonzero(exposures), target_id, index))
coadd = coadded_spectra[target_id][band]
coadd.finalize()
flux_out[index] = coadd.flux
ivar_out[index] = coadd.ivar
resolution_out[index] = coadd.resolution.to_fits_array()
# Save the coadds for this band.
coadd_file.add_objects(flux_out, ivar_out, wlen, resolution_out)
coadd_file.hdu_list[4].data = coadd_info
# Close files for this band.
coadd_file.close()
brick_file.close()
# Allocate space for the global coadded results.
num_targets = next_coadd_index
nbins = len(desispec.coaddition.global_wavelength_grid)
flux_all = np.empty((num_targets, nbins))
ivar_all = np.empty_like(flux_all)
resolution_all = np.empty(
(num_targets, desispec.resolution.num_diagonals, nbins))
# Coadd the bands for each target ID.
all_bands = ','.join(sorted(args.bands))
for target_id in coadded_spectra:
index = target_index[target_id]
bands = ','.join(sorted(coadded_spectra[target_id].keys()))
log.debug('Combining %s bands for target %d at index %d.' %
(bands, target_id, index))
if bands != all_bands:
log.warning('WARNING: target %d has partial band coverage: %s' %
(target_id, bands))
coadd_all = desispec.coaddition.Spectrum(
desispec.coaddition.global_wavelength_grid)
for coadd_band in coadded_spectra[target_id].itervalues():
coadd_all += coadd_band
coadd_all.finalize()
flux_all[index] = coadd_all.flux
ivar_all[index] = coadd_all.ivar
resolution_all[index] = coadd_all.resolution.to_fits_array()
# Save the global coadds.
coadd_all_file.add_objects(flux_all, ivar_all,
desispec.coaddition.global_wavelength_grid,
resolution_all)
coadd_all_file.hdu_list[4].data = coadd_all_info
# Close the combined coadd file.
coadd_all_file.close()
"""
desisim.templates
=================
Functions to simulate spectral templates for DESI.
"""
from __future__ import division, print_function
import os
import sys
import numpy as np
from desispec.log import get_logger
log = get_logger()
class TargetCuts():
"""Select targets from flux cuts. This is a placeholder class that will be
refactored into desitarget. Hence, the documentation here is
intentionally sparse.
"""
def __init__(self):
pass
def BGS(self,rflux=None):
BGS = rflux > 10**((22.5-19.35)/2.5)
return BGS
def ELG(self,gflux=None, rflux=None, zflux=None):
ELG = rflux > 10**((22.5-23.4)/2.5)