def qa_skysub(param, frame, skymodel, quick_look=False):
"""Calculate QA on SkySubtraction
Note: Pixels rejected in generating the SkyModel (as above), are
not rejected in the stats calculated here. Would need to carry
along current_ivar to do so.
Args:
param : dict of QA parameters
frame : desispec.Frame object
skymodel : desispec.SkyModel object
quick_look : bool, optional
If True, do QuickLook specific QA (or avoid some)
Returns:
qadict: dict of QA outputs
Need to record simple Python objects for yaml (str, float, int)
"""
log = get_logger()
# Output dict
qadict = {}
qadict['NREJ'] = int(skymodel.nrej)
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nfibers = len(skyfibers)
qadict['NSKY_FIB'] = int(nfibers)
current_ivar = frame.ivar[skyfibers].copy()
flux = frame.flux[skyfibers]
# Subtract
res = flux - skymodel.flux[skyfibers] # Residuals
res_ivar = util.combine_ivar(current_ivar, skymodel.ivar[skyfibers])
# Chi^2 and Probability
chi2_fiber = np.sum(res_ivar * (res**2), 1)
chi2_prob = np.zeros(nfibers)
for ii in range(nfibers):
# Stats
dof = np.sum(res_ivar[ii, :] > 0.)
chi2_prob[ii] = scipy.stats.chisqprob(chi2_fiber[ii], dof)
# Bad models
qadict['NBAD_PCHI'] = int(np.sum(chi2_prob < param['PCHI_RESID']))
if qadict['NBAD_PCHI'] > 0:
log.warn("Bad Sky Subtraction in {:d} fibers".format(
qadict['NBAD_PCHI']))
# Median residual
qadict['MED_RESID'] = float(np.median(res)) # Median residual (counts)
log.info("Median residual for sky fibers = {:g}".format(
qadict['MED_RESID']))
# Residual percentiles
perc = dustat.perc(res, per=param['PER_RESID'])
qadict['RESID_PER'] = [float(iperc) for iperc in perc]
# Mean Sky Continuum from all skyfibers
# need to limit in wavelength?
if quick_look:
continuum = scipy.ndimage.filters.median_filter(
flux, 200) # taking 200 bins (somewhat arbitrarily)
mean_continuum = np.zeros(flux.shape[1])
for ii in range(flux.shape[1]):
mean_continuum[ii] = np.mean(continuum[:, ii])
qadict['MEAN_CONTIN'] = mean_continuum
# Median Signal to Noise on sky subtracted spectra
# first do the subtraction:
if quick_look:
fframe = frame # make a copy
sskymodel = skymodel # make a copy
subtract_sky(fframe, sskymodel)
medsnr = np.zeros(fframe.flux.shape[0])
totsnr = np.zeros(fframe.flux.shape[0])
for ii in range(fframe.flux.shape[0]):
signalmask = fframe.flux[ii, :] > 0
# total snr considering bin by bin uncorrelated S/N
snr = fframe.flux[ii, signalmask] * np.sqrt(
fframe.ivar[ii, signalmask])
medsnr[ii] = np.median(snr)
totsnr[ii] = np.sqrt(np.sum(snr**2))
qadict['MED_SNR'] = medsnr # for each fiber
qadict['TOT_SNR'] = totsnr # for each fiber
# Return
return qadict
def sky_resid(param, frame, skymodel, quick_look=False):
""" QA Algorithm for sky residual
To be called from desispec.sky.qa_skysub and desispec.qa.qa_quicklook.Sky_residual.run_qa
Args:
param : dict of QA parameters
frame : desispec.Frame object after sky subtraction
skymodel : desispec.SkyModel object
Returns a qa dictionary for sky resid
"""
# Output dict
qadict = {}
qadict['NREJ'] = int(skymodel.nrej)
if quick_look:
qadict['RA'] = frame.fibermap['RA_TARGET']
qadict['DEC'] = frame.fibermap['DEC_TARGET']
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nfibers = len(skyfibers)
qadict['NSKY_FIB'] = int(nfibers)
#current_ivar=frame.ivar[skyfibers].copy()
#flux = frame.flux[skyfibers]
# Record median flux
qadict['MED_SKY'] = np.median(skymodel.flux[skyfibers]) # Counts
#- Residuals
res = frame.flux[skyfibers] #- as this frame is already sky subtracted
res_ivar = frame.ivar[skyfibers]
# Chi^2 and Probability
chi2_fiber = np.sum(res_ivar * (res**2), 1)
chi2_prob = np.zeros(nfibers)
for ii in range(nfibers):
# Stats
dof = np.sum(res_ivar[ii, :] > 0.)
chi2_prob[ii] = scipy.stats.chisqprob(chi2_fiber[ii], dof)
# Bad models
qadict['NBAD_PCHI'] = int(np.sum(chi2_prob < param['PCHI_RESID']))
if qadict['NBAD_PCHI'] > 0:
log.warning("Bad Sky Subtraction in {:d} fibers".format(
qadict['NBAD_PCHI']))
# Median residual
qadict['MED_RESID'] = float(np.median(res)) # Median residual (counts)
log.info("Median residual for sky fibers = {:g}".format(
qadict['MED_RESID']))
# Residual percentiles
perc = dustat.perc(res, per=param['PER_RESID'])
qadict['RESID_PER'] = [float(iperc) for iperc in perc]
qadict['RESID_RMS'] = []
qadict["SKY_FIBERID"] = skyfibers.tolist()
#- Residuals in wave and fiber axes
if quick_look:
qadict["MED_RESID_WAVE"] = np.median(res, axis=0)
qadict["MED_RESID_FIBER"] = np.median(res, axis=1)
#- Weighted average for each bin on all fibers
qadict["WAVG_RES_WAVE"] = np.sum(res * res_ivar, 0) / np.sum(
res_ivar, 0)
#- Histograms for residual/sigma #- inherited from qa_plots.frame_skyres()
if quick_look:
binsz = param['BIN_SZ']
gd_res = res_ivar > 0.
devs = res[gd_res] * np.sqrt(res_ivar[gd_res])
i0, i1 = int(np.min(devs) / binsz) - 1, int(np.max(devs) / binsz) + 1
rng = tuple(binsz * np.array([i0, i1]))
nbin = i1 - i0
hist, edges = np.histogram(devs, range=rng, bins=nbin)
qadict['DEVS_1D'] = hist.tolist() #- histograms for deviates
qadict['DEVS_EDGES'] = edges.tolist() #- Bin edges
#- Add additional metrics for quicklook
if quick_look:
qadict["WAVELENGTH"] = frame.wave
# Return
return qadict
def qa_skysub(param, frame, skymodel, quick_look=False):
"""Calculate QA on SkySubtraction
Note: Pixels rejected in generating the SkyModel (as above), are
not rejected in the stats calculated here. Would need to carry
along current_ivar to do so.
Args:
param : dict of QA parameters
frame : desispec.Frame object
skymodel : desispec.SkyModel object
quick_look : bool, optional
If True, do QuickLook specific QA (or avoid some)
Returns:
qadict: dict of QA outputs
Need to record simple Python objects for yaml (str, float, int)
"""
log=get_logger()
# Output dict
qadict = {}
qadict['NREJ'] = int(skymodel.nrej)
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nfibers=len(skyfibers)
qadict['NSKY_FIB'] = int(nfibers)
current_ivar=frame.ivar[skyfibers].copy()
flux = frame.flux[skyfibers]
# Subtract
res = flux - skymodel.flux[skyfibers] # Residuals
res_ivar = util.combine_ivar(current_ivar, skymodel.ivar[skyfibers])
# Chi^2 and Probability
chi2_fiber = np.sum(res_ivar*(res**2),1)
chi2_prob = np.zeros(nfibers)
for ii in range(nfibers):
# Stats
dof = np.sum(res_ivar[ii,:] > 0.)
chi2_prob[ii] = scipy.stats.chisqprob(chi2_fiber[ii], dof)
# Bad models
qadict['NBAD_PCHI'] = int(np.sum(chi2_prob < param['PCHI_RESID']))
if qadict['NBAD_PCHI'] > 0:
log.warn("Bad Sky Subtraction in {:d} fibers".format(
qadict['NBAD_PCHI']))
# Median residual
qadict['MED_RESID'] = float(np.median(res)) # Median residual (counts)
log.info("Median residual for sky fibers = {:g}".format(
qadict['MED_RESID']))
# Residual percentiles
perc = dustat.perc(res, per=param['PER_RESID'])
qadict['RESID_PER'] = [float(iperc) for iperc in perc]
# Mean Sky Continuum from all skyfibers
# need to limit in wavelength?
if quick_look:
continuum=scipy.ndimage.filters.median_filter(flux,200) # taking 200 bins (somewhat arbitrarily)
mean_continuum=np.zeros(flux.shape[1])
for ii in range(flux.shape[1]):
mean_continuum[ii]=np.mean(continuum[:,ii])
qadict['MEAN_CONTIN'] = mean_continuum
# Median Signal to Noise on sky subtracted spectra
# first do the subtraction:
if quick_look:
fframe=frame # make a copy
sskymodel=skymodel # make a copy
subtract_sky(fframe,sskymodel)
medsnr=np.zeros(fframe.flux.shape[0])
totsnr=np.zeros(fframe.flux.shape[0])
for ii in range(fframe.flux.shape[0]):
signalmask=fframe.flux[ii,:]>0
# total snr considering bin by bin uncorrelated S/N
snr=fframe.flux[ii,signalmask]*np.sqrt(fframe.ivar[ii,signalmask])
medsnr[ii]=np.median(snr)
totsnr[ii]=np.sqrt(np.sum(snr**2))
qadict['MED_SNR']=medsnr # for each fiber
qadict['TOT_SNR']=totsnr # for each fiber
# Return
return qadict
def desi_qso_templates(z_wind=0.2, zmnx=(0.4,4.), outfil=None, N_perz=500,
boss_pca_fil=None, wvmnx=(3500., 10000.),
rebin_wave=None, rstate=None,
sdss_pca_fil=None, no_write=False, redshift=None,
seed=None, old_read=False, ipad=40, cosmo=None):
""" Generate QSO templates for DESI
Rebins to input wavelength array (or log10 in wvmnx)
Parameters
----------
z_wind : float, optional
Window for sampling PCAs
zmnx : tuple, optional
Min/max for generation
N_perz : int, optional
Number of draws per redshift window
old_read : bool, optional
Read the files the old way
seed : int, optional
Seed for the random number state
rebin_wave : ndarray, optional
Input wavelengths for rebinning
wvmnx : tuple, optional
Wavelength limits for rebinning (not used with rebin_wave)
redshift : ndarray, optional
Redshifts desired for the templates
ipad : int, optional
Padding for enabling enough models
cosmo: astropy.cosmology.core, optional
Cosmology inistantiation from astropy.cosmology.code
Returns
-------
wave : ndarray
Wavelengths that the spectra were rebinned to
flux : ndarray (2D; flux vs. model)
z : ndarray
Redshifts
"""
# Cosmology
if cosmo is None:
from astropy import cosmology
cosmo = cosmology.core.FlatLambdaCDM(70., 0.3)
if old_read:
# PCA values
if boss_pca_fil is None:
boss_pca_fil = 'BOSS_DR10Lya_PCA_values_nocut.fits.gz'
hdu = fits.open(boss_pca_fil)
boss_pca_coeff = hdu[1].data
if sdss_pca_fil is None:
sdss_pca_fil = 'SDSS_DR7Lya_PCA_values_nocut.fits.gz'
hdu2 = fits.open(sdss_pca_fil)
sdss_pca_coeff = hdu2[1].data
# Open the BOSS catalog file
boss_cat_fil = os.environ.get('BOSSPATH')+'/DR10/BOSSLyaDR10_cat_v2.1.fits.gz'
bcat_hdu = fits.open(boss_cat_fil)
t_boss = bcat_hdu[1].data
boss_zQSO = t_boss['z_pipe']
# Open the SDSS catalog file
sdss_cat_fil = os.environ.get('SDSSPATH')+'/DR7_QSO/dr7_qso.fits.gz'
scat_hdu = fits.open(sdss_cat_fil)
t_sdss = scat_hdu[1].data
sdss_zQSO = t_sdss['z']
if len(sdss_pca_coeff) != len(sdss_zQSO):
print('Need to finish running the SDSS models!')
sdss_zQSO = sdss_zQSO[0:len(sdss_pca_coeff)]
# Eigenvectors
eigen, eigen_wave = fbq.read_qso_eigen()
else:
infile = desisim.io.find_basis_template('qso')
with fits.open(infile) as hdus:
hdu_names = [hdus[ii].name for ii in range(len(hdus))]
boss_pca_coeff = hdus[hdu_names.index('BOSS_PCA')].data
sdss_pca_coeff = hdus[hdu_names.index('SDSS_PCA')].data
boss_zQSO = hdus[hdu_names.index('BOSS_Z')].data
sdss_zQSO = hdus[hdu_names.index('SDSS_Z')].data
eigen = hdus[hdu_names.index('SDSS_EIGEN')].data
eigen_wave = hdus[hdu_names.index('SDSS_EIGEN_WAVE')].data
# Fiddle with the eigen-vectors
npix = len(eigen_wave)
chkpix = np.where((eigen_wave > 900.) & (eigen_wave < 5000.) )[0]
lambda_912 = 911.76
pix912 = np.argmin( np.abs(eigen_wave-lambda_912) )
# Loop on redshift. If the
if redshift is None:
z0 = np.arange(zmnx[0],zmnx[1],z_wind)
z1 = z0 + z_wind
else:
if np.isscalar(redshift):
z0 = np.array([redshift])
else:
z0 = redshift.copy()
z1 = z0.copy() #+ z_wind
pca_list = ['PCA0', 'PCA1', 'PCA2', 'PCA3']
PCA_mean = np.zeros(4)
PCA_sig = np.zeros(4)
PCA_rand = np.zeros((4,N_perz*ipad))
final_spec = np.zeros((npix, N_perz * len(z0)))
final_wave = np.zeros((npix, N_perz * len(z0)))
final_z = np.zeros(N_perz * len(z0))
# Random state
if rstate is None:
rstate = np.random.RandomState(seed)
for ii in range(len(z0)):
# BOSS or SDSS?
if z0[ii] > 2.15:
zQSO = boss_zQSO
pca_coeff = boss_pca_coeff
else:
zQSO = sdss_zQSO
pca_coeff = sdss_pca_coeff
# Random z values and wavelengths
zrand = rstate.uniform( z0[ii], z1[ii], N_perz*ipad)
wave = np.outer(eigen_wave, 1+zrand)
# MFP (Worseck+14)
mfp = 37. * ( (1+zrand)/5. )**(-5.4) # Physical Mpc
# Grab PCA mean + sigma
if redshift is None:
idx = np.where( (zQSO >= z0[ii]) & (zQSO < z1[ii]) )[0]
else:
# Hack by @moustakas: add a little jitter to get the set of QSOs
# that are *nearest* in redshift to the desired output redshift.
idx = np.where( (zQSO >= z0[ii]-0.01) & (zQSO < z1[ii]+0.01) )[0]
if len(idx) == 0:
idx = np.array([(np.abs(zQSO-zrand[0])).argmin()])
#pdb.set_trace()
log.debug('Making z=({:g},{:g}) with {:d} input quasars'.format(z0[ii],z1[ii],len(idx)))
# Get PCA stats and random values
for jj,ipca in enumerate(pca_list):
if jj == 0: # Use bounds for PCA0 [avoids negative values]
xmnx = perc(pca_coeff[ipca][idx], per=95)
PCA_rand[jj, :] = rstate.uniform(xmnx[0], xmnx[1], N_perz*ipad)
else:
PCA_mean[jj] = np.mean(pca_coeff[ipca][idx])
PCA_sig[jj] = np.std(pca_coeff[ipca][idx])
# Draws
PCA_rand[jj, :] = rstate.uniform( PCA_mean[jj] - 2*PCA_sig[jj],
PCA_mean[jj] + 2*PCA_sig[jj], N_perz*ipad)
# Generate the templates (ipad*N_perz)
spec = np.dot(eigen.T, PCA_rand)
# Take first good N_perz
# Truncate, MFP, Fill
ngd = 0
nbad = 0
for kk in range(ipad*N_perz):
# Any zero values?
mn = np.min(spec[chkpix, kk])
if mn < 0.:
nbad += 1
continue
# MFP
if z0[ii] > 2.39:
z912 = wave[0:pix912,kk]/lambda_912 - 1.
phys_dist = np.fabs( cosmo.lookback_distance(z912) -
cosmo.lookback_distance(zrand[kk]) ) # Mpc
spec[0:pix912, kk] = spec[0:pix912,kk] * np.exp(-phys_dist.value/mfp[kk])
# Write
final_spec[:, ii*N_perz+ngd] = spec[:,kk]
final_wave[:, ii*N_perz+ngd] = wave[:,kk]
final_z[ii*N_perz+ngd] = zrand[kk]
ngd += 1
if ngd == N_perz:
break
if ngd != N_perz:
print('Did not make enough!')
#pdb.set_trace()
log.warning('Did not make enough qso templates. ngd = {}, N_perz = {}'.format(ngd,N_perz))
# Rebin
if rebin_wave is None:
light = C_LIGHT # [km/s]
velpixsize = 10. # [km/s]
pixsize = velpixsize/light/np.log(10) # [pixel size in log-10 A]
minwave = np.log10(wvmnx[0]) # minimum wavelength [log10-A]
maxwave = np.log10(wvmnx[1]) # maximum wavelength [log10-A]
r_npix = np.round((maxwave-minwave)/pixsize+1)
log_wave = minwave+np.arange(r_npix)*pixsize # constant log-10 spacing
else:
log_wave = np.log10(rebin_wave)
r_npix = len(log_wave)
totN = N_perz * len(z0)
rebin_spec = np.zeros((r_npix, totN))
for ii in range(totN):
# Interpolate (in log space)
rebin_spec[:, ii] = resample_flux(log_wave, np.log10(final_wave[:, ii]), final_spec[:, ii])
#f1d = interp1d(np.log10(final_wave[:,ii]), final_spec[:,ii])
#rebin_spec[:,ii] = f1d(log_wave)
if outfil is None:
return 10.**log_wave, rebin_spec, final_z
# Transpose for consistency
out_spec = np.array(rebin_spec.T, dtype='float32')
# Write
hdu = fits.PrimaryHDU(out_spec)
hdu.header.set('PROJECT', 'DESI QSO TEMPLATES')
hdu.header.set('VERSION', '1.1')
hdu.header.set('OBJTYPE', 'QSO')
hdu.header.set('DISPAXIS', 1, 'dispersion axis')
hdu.header.set('CRPIX1', 1, 'reference pixel number')
hdu.header.set('CRVAL1', minwave, 'reference log10(Ang)')
hdu.header.set('CDELT1', pixsize, 'delta log10(Ang)')
hdu.header.set('LOGLAM', 1, 'log10 spaced wavelengths?')
hdu.header.set('AIRORVAC', 'vac', ' wavelengths in vacuum (vac) or air')
hdu.header.set('VELSCALE', velpixsize, ' pixel size in km/s')
hdu.header.set('WAVEUNIT', 'Angstrom', ' wavelength units')
hdu.header.set('BUNIT', '1e-17 erg/s/cm2/A', ' flux unit')
idval = list(range(totN))
col0 = fits.Column(name=str('TEMPLATEID'),format=str('J'), array=idval)
col1 = fits.Column(name=str('Z'),format=str('E'),array=final_z)
cols = fits.ColDefs([col0, col1])
tbhdu = fits.BinTableHDU.from_columns(cols)
tbhdu.header.set('EXTNAME','METADATA')
hdulist = fits.HDUList([hdu, tbhdu])
hdulist.writeto(outfil, overwrite=True)
return final_wave, final_spec, final_z
def qa_skysub(param, frame, skymodel, quick_look=False):
"""Calculate QA on SkySubtraction
Note: Pixels rejected in generating the SkyModel (as above), are
not rejected in the stats calculated here. Would need to carry
along current_ivar to do so.
Args:
param : dict of QA parameters
frame : desispec.Frame object
skymodel : desispec.SkyModel object
quick_look : bool, optional
If True, do QuickLook specific QA (or avoid some)
Returns:
qadict: dict of QA outputs
Need to record simple Python objects for yaml (str, float, int)
"""
log = get_logger()
# Output dict
qadict = {}
qadict['NREJ'] = int(skymodel.nrej)
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nfibers = len(skyfibers)
qadict['NSKY_FIB'] = int(nfibers)
current_ivar = frame.ivar[skyfibers].copy()
flux = frame.flux[skyfibers]
# Subtract
res = flux - skymodel.flux[skyfibers] # Residuals
res_ivar = util.combine_ivar(current_ivar, skymodel.ivar[skyfibers])
# Chi^2 and Probability
chi2_fiber = np.sum(res_ivar * (res**2), 1)
chi2_prob = np.zeros(nfibers)
for ii in range(nfibers):
# Stats
dof = np.sum(res_ivar[ii, :] > 0.)
chi2_prob[ii] = scipy.stats.chisqprob(chi2_fiber[ii], dof)
# Bad models
qadict['NBAD_PCHI'] = int(np.sum(chi2_prob < param['PCHI_RESID']))
if qadict['NBAD_PCHI'] > 0:
log.warning("Bad Sky Subtraction in {:d} fibers".format(
qadict['NBAD_PCHI']))
# Median residual
qadict['MED_RESID'] = float(np.median(res)) # Median residual (counts)
log.info("Median residual for sky fibers = {:g}".format(
qadict['MED_RESID']))
# Residual percentiles
perc = dustat.perc(res, per=param['PER_RESID'])
qadict['RESID_PER'] = [float(iperc) for iperc in perc]
#- Add per fiber median residuals
qadict["MED_RESID_FIBER"] = np.median(res, axis=1)
#- Evaluate residuals in wave axis for quicklook
if quick_look:
qadict["MED_RESID_WAVE"] = np.median(res, axis=0)
qadict["WAVELENGTH"] = frame.wave
# Return
return qadict