def subtract_sky(fframe, skymodel):
"""
skymodel: skymodel object.
fframe: frame object to do the sky subtraction, should be already fiber flat fielded
need same number of fibers and same wavelength grid
"""
#- Check number of specs
assert fframe.nspec == skymodel.nspec
assert fframe.nwave == skymodel.nwave
#- check same wavelength grid, die if not
if not np.allclose(fframe.wave, skymodel.wave):
message = "frame and sky not on same wavelength grid"
raise ValueError(message)
sflux = fframe.flux - skymodel.flux
sivar = util.combine_ivar(fframe.ivar.clip(0), skymodel.ivar.clip(0))
smask = fframe.mask | skymodel.mask
#- create a frame object now
sframe = fr.Frame(fframe.wave,
sflux,
sivar,
smask,
fframe.resolution_data,
meta=fframe.meta,
fibermap=fframe.fibermap)
return sframe
def subtract_sky(frame, skymodel):
"""Subtract skymodel from frame, altering frame.flux, .ivar, and .mask
"""
assert frame.nspec == skymodel.nspec
assert frame.nwave == skymodel.nwave
log = get_logger()
log.info("starting")
# check same wavelength, die if not the case
if not np.allclose(frame.wave, skymodel.wave):
message = "frame and sky not on same wavelength grid"
log.error(message)
raise ValueError(message)
frame.flux -= skymodel.flux
frame.ivar = util.combine_ivar(frame.ivar, skymodel.ivar)
frame.mask |= skymodel.mask
log.info("done")
def subtract_sky(frame, skymodel) :
"""Subtract skymodel from frame, altering frame.flux, .ivar, and .mask
"""
assert frame.nspec == skymodel.nspec
assert frame.nwave == skymodel.nwave
log=get_logger()
log.info("starting")
# check same wavelength, die if not the case
if not np.allclose(frame.wave, skymodel.wave):
message = "frame and sky not on same wavelength grid"
log.error(message)
raise ValueError(message)
frame.flux -= skymodel.flux
frame.ivar = util.combine_ivar(frame.ivar, skymodel.ivar)
frame.mask |= skymodel.mask
log.info("done")
def subtract_sky(fframe,skymodel):
"""
skymodel: skymodel object.
fframe: frame object to do the sky subtraction, should be already fiber flat fielded
need same number of fibers and same wavelength grid
"""
#- Check number of specs
assert fframe.nspec == skymodel.nspec
assert fframe.nwave == skymodel.nwave
#- check same wavelength grid, die if not
if not np.allclose(fframe.wave, skymodel.wave):
message = "frame and sky not on same wavelength grid"
raise ValueError(message)
sflux = fframe.flux-skymodel.flux
sivar = util.combine_ivar(fframe.ivar.clip(0), skymodel.ivar.clip(0))
smask = fframe.mask | skymodel.mask
#- create a frame object now
sframe=fr.Frame(fframe.wave,sflux,sivar,smask,fframe.resolution_data,meta=fframe.meta,fibermap=fframe.fibermap)
return sframe
def subtract_sky(fframe, skymodel):
"""
skymodel: skymodel object.
fframe: frame object to do the sky subtraction, should be already fiber flat fielded
need same number of fibers and same wavelength grid
"""
#- Check number of specs
assert fframe.nspec == skymodel.nspec
assert fframe.nwave == skymodel.nwave
#- check same wavelength grid, die if not
if not np.allclose(fframe.wave, skymodel.wave):
message = "frame and sky not on same wavelength grid"
raise ValueError(message)
#SK. This wouldn't work since not all properties of the input
#frame is modified. Just modify input frame directly instead!
fframe.flux = fframe.flux - skymodel.flux
fframe.ivar = util.combine_ivar(fframe.ivar.clip(0), skymodel.ivar.clip(0))
fframe.mask = fframe.mask | skymodel.mask
#- create a frame object now
#sframe=fr.Frame(fframe.wave,sflux,sivar,smask,fframe.resolution_data,meta=fframe.meta,fibermap=fframe.fibermap)
return fframe
def subtract_sky(fframe,skymodel):
"""
skymodel: skymodel object.
fframe: frame object to do the sky subtraction, should be already fiber flat fielded
need same number of fibers and same wavelength grid
"""
#- Check number of specs
assert fframe.nspec == skymodel.nspec
assert fframe.nwave == skymodel.nwave
#- check same wavelength grid, die if not
if not np.allclose(fframe.wave, skymodel.wave):
message = "frame and sky not on same wavelength grid"
raise ValueError(message)
#SK. This wouldn't work since not all properties of the input
#frame is modified. Just modify input frame directly instead!
fframe.flux= fframe.flux-skymodel.flux
fframe.ivar = util.combine_ivar(fframe.ivar.clip(1e-8), skymodel.ivar.clip(1e-8))
fframe.mask = fframe.mask | skymodel.mask
#- create a frame object now
#sframe=fr.Frame(fframe.wave,sflux,sivar,smask,fframe.resolution_data,meta=fframe.meta,fibermap=fframe.fibermap)
return fframe
def test_combine_ivar(self):
#- input inverse variances with some zeros (1D)
ivar1 = np.random.uniform(-1, 10, size=200).clip(0)
ivar2 = np.random.uniform(-1, 10, size=200).clip(0)
ivar = util.combine_ivar(ivar1, ivar2)
izero = np.where(ivar1 == 0)
self.assertTrue(np.all(ivar[izero] == 0))
izero = np.where(ivar2 == 0)
self.assertTrue(np.all(ivar[izero] == 0))
self.assertTrue(ivar.dtype == np.float64)
#- input inverse variances with some zeros (2D)
np.random.seed(0)
ivar1 = np.random.uniform(-1, 10, size=(10, 20)).clip(0)
ivar2 = np.random.uniform(-1, 10, size=(10, 20)).clip(0)
ivar = util.combine_ivar(ivar1, ivar2)
izero = np.where(ivar1 == 0)
self.assertTrue(np.all(ivar[izero] == 0))
izero = np.where(ivar2 == 0)
self.assertTrue(np.all(ivar[izero] == 0))
self.assertTrue(ivar.dtype == np.float64)
#- Dimensionality
self.assertRaises(AssertionError, util.combine_ivar, ivar1, ivar2[0])
#- ivar must be positive
self.assertRaises(AssertionError, util.combine_ivar, -ivar1, ivar2)
self.assertRaises(AssertionError, util.combine_ivar, ivar1, -ivar2)
#- does it actually combine them correctly?
ivar = util.combine_ivar(1, 2)
self.assertEqual(ivar, 1.0 / (1.0 + 0.5))
#- float -> float, int -> float, 0-dim ndarray -> 0-dim ndarray
ivar = util.combine_ivar(1, 2)
self.assertTrue(isinstance(ivar, float))
ivar = util.combine_ivar(1.0, 2.0)
self.assertTrue(isinstance(ivar, float))
ivar = util.combine_ivar(np.asarray(1.0), np.asarray(2.0))
self.assertTrue(isinstance(ivar, np.ndarray))
self.assertEqual(ivar.ndim, 0)
def test_combine_ivar(self):
#- input inverse variances with some zeros (1D)
ivar1 = np.random.uniform(-1, 10, size=200).clip(0)
ivar2 = np.random.uniform(-1, 10, size=200).clip(0)
ivar = util.combine_ivar(ivar1, ivar2)
izero = np.where(ivar1 == 0)
self.assertTrue(np.all(ivar[izero] == 0))
izero = np.where(ivar2 == 0)
self.assertTrue(np.all(ivar[izero] == 0))
self.assertTrue(ivar.dtype == np.float64)
#- input inverse variances with some zeros (2D)
np.random.seed(0)
ivar1 = np.random.uniform(-1, 10, size=(10,20)).clip(0)
ivar2 = np.random.uniform(-1, 10, size=(10,20)).clip(0)
ivar = util.combine_ivar(ivar1, ivar2)
izero = np.where(ivar1 == 0)
self.assertTrue(np.all(ivar[izero] == 0))
izero = np.where(ivar2 == 0)
self.assertTrue(np.all(ivar[izero] == 0))
self.assertTrue(ivar.dtype == np.float64)
#- Dimensionality
self.assertRaises(AssertionError, util.combine_ivar, ivar1, ivar2[0])
#- ivar must be positive
self.assertRaises(AssertionError, util.combine_ivar, -ivar1, ivar2)
self.assertRaises(AssertionError, util.combine_ivar, ivar1, -ivar2)
#- does it actually combine them correctly?
ivar = util.combine_ivar(1, 2)
self.assertEqual(ivar, 1.0/(1.0 + 0.5))
#- float -> float, int -> float, 0-dim ndarray -> 0-dim ndarray
ivar = util.combine_ivar(1, 2)
self.assertTrue(isinstance(ivar, float))
ivar = util.combine_ivar(1.0, 2.0)
self.assertTrue(isinstance(ivar, float))
ivar = util.combine_ivar(np.asarray(1.0), np.asarray(2.0))
self.assertTrue(isinstance(ivar, np.ndarray))
self.assertEqual(ivar.ndim, 0)
def _model_variance(frame, cskyflux, cskyivar, skyfibers):
"""look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1
"""
log = get_logger()
tivar = util.combine_ivar(frame.ivar[skyfibers], cskyivar[skyfibers])
# the chi2 at a given wavelength can be large because on a cosmic
# and not a psf error or sky non uniformity
# so we need to consider only waves for which
# a reasonable sky model error can be computed
# mean sky
msky = np.mean(cskyflux, axis=0)
dwave = np.mean(np.gradient(frame.wave))
dskydw = np.zeros(msky.shape)
dskydw[1:-1] = (msky[2:] - msky[:-2]) / (frame.wave[2:] - frame.wave[:-2])
dskydw = np.abs(dskydw)
# now we consider a worst possible sky model error (20% error on flat, 0.5A )
max_possible_var = 1. / (tivar +
(tivar == 0)) + (0.2 * msky)**2 + (0.5 *
dskydw)**2
# exclude residuals inconsistent with this max possible variance (at 3 sigma)
bad = (frame.flux[skyfibers] -
cskyflux[skyfibers])**2 > 3**2 * max_possible_var
tivar[bad] = 0
ndata = np.sum(tivar > 0, axis=0)
ok = np.where(ndata > 1)[0]
chi2 = np.zeros(frame.wave.size)
chi2[ok] = np.sum(tivar * (frame.flux[skyfibers] - cskyflux[skyfibers])**2,
axis=0)[ok] / (ndata[ok] - 1)
chi2[ndata <= 1] = 1. # default
# now we are going to evaluate a sky model error based on this chi2,
# but only around sky flux peaks (>0.1*max)
tmp = np.zeros(frame.wave.size)
tmp = (msky[1:-1] > msky[2:]) * (msky[1:-1] > msky[:-2]) * (
msky[1:-1] > 0.1 * np.max(msky))
peaks = np.where(tmp)[0] + 1
dpix = int(np.ceil(3 / dwave)) # +- n Angstrom around each peak
skyvar = 1. / (cskyivar + (cskyivar == 0))
# loop on peaks
for peak in peaks:
b = peak - dpix
e = peak + dpix + 1
mchi2 = np.mean(chi2[b:e]) # mean reduced chi2 around peak
mndata = np.mean(ndata[b:e]) # mean number of fibers contributing
# sky model variance = sigma_flat * msky + sigma_wave * dmskydw
sigma_flat = 0.000 # the fiber flat error is already included in the flux ivar
sigma_wave = 0.005 # A, minimum value
res2 = (frame.flux[skyfibers, b:e] - cskyflux[skyfibers, b:e])**2
var = 1. / (tivar[:, b:e] + (tivar[:, b:e] == 0))
nd = np.sum(tivar[:, b:e] > 0)
while (sigma_wave < 2):
pivar = 1. / (var + (sigma_flat * msky[b:e])**2 +
(sigma_wave * dskydw[b:e])**2)
pchi2 = np.sum(pivar * res2) / nd
if pchi2 <= 1:
log.info("peak at {}A : sigma_wave={}".format(
int(frame.wave[peak]), sigma_wave))
skyvar[:, b:e] += ((sigma_flat * msky[b:e])**2 +
(sigma_wave * dskydw[b:e])**2)
break
sigma_wave += 0.005
return (cskyivar > 0) / (skyvar + (skyvar == 0))
def subtract_sky(frame,
skymodel,
throughput_correction=False,
default_throughput_correction=1.):
"""Subtract skymodel from frame, altering frame.flux, .ivar, and .mask
Args:
frame : desispec.Frame object
skymodel : desispec.SkyModel object
Option:
throughput_correction : if True, fit for an achromatic throughput correction. This is to absorb variations of Focal Ratio Degradation with fiber flexure.
default_throughput_correction : float, default value of correction if the fit on sky lines failed.
"""
assert frame.nspec == skymodel.nspec
assert frame.nwave == skymodel.nwave
log = get_logger()
log.info("starting")
# check same wavelength, die if not the case
if not np.allclose(frame.wave, skymodel.wave):
message = "frame and sky not on same wavelength grid"
log.error(message)
raise ValueError(message)
if throughput_correction:
# need to fit for a multiplicative factor of the sky model
# before subtraction
# we are going to use a set of bright sky lines,
# and fit a multiplicative factor + background around
# each of them individually, and then combine the results
# with outlier rejection in case a source emission line
# coincides with one of the sky lines.
# it's more robust to have a hardcoded set of sky lines here
# these are all the sky lines with a flux >5% of the max flux
# except in b where we add an extra weaker line at 5199.4A
skyline = np.array([
5199.4, 5578.4, 5656.4, 5891.4, 5897.4, 6302.4, 6308.4, 6365.4,
6500.4, 6546.4, 6555.4, 6618.4, 6663.4, 6679.4, 6690.4, 6765.4,
6831.4, 6836.4, 6865.4, 6925.4, 6951.4, 6980.4, 7242.4, 7247.4,
7278.4, 7286.4, 7305.4, 7318.4, 7331.4, 7343.4, 7360.4, 7371.4,
7394.4, 7404.4, 7440.4, 7526.4, 7714.4, 7719.4, 7752.4, 7762.4,
7782.4, 7796.4, 7810.4, 7823.4, 7843.4, 7855.4, 7862.4, 7873.4,
7881.4, 7892.4, 7915.4, 7923.4, 7933.4, 7951.4, 7966.4, 7982.4,
7995.4, 8016.4, 8028.4, 8064.4, 8280.4, 8284.4, 8290.4, 8298.4,
8301.4, 8313.4, 8346.4, 8355.4, 8367.4, 8384.4, 8401.4, 8417.4,
8432.4, 8454.4, 8467.4, 8495.4, 8507.4, 8627.4, 8630.4, 8634.4,
8638.4, 8652.4, 8657.4, 8662.4, 8667.4, 8672.4, 8677.4, 8683.4,
8763.4, 8770.4, 8780.4, 8793.4, 8829.4, 8835.4, 8838.4, 8852.4,
8870.4, 8888.4, 8905.4, 8922.4, 8945.4, 8960.4, 8990.4, 9003.4,
9040.4, 9052.4, 9105.4, 9227.4, 9309.4, 9315.4, 9320.4, 9326.4,
9340.4, 9378.4, 9389.4, 9404.4, 9422.4, 9442.4, 9461.4, 9479.4,
9505.4, 9521.4, 9555.4, 9570.4, 9610.4, 9623.4, 9671.4, 9684.4,
9693.4, 9702.4, 9714.4, 9722.4, 9740.4, 9748.4, 9793.4, 9802.4,
9814.4, 9820.4
])
sw = []
swf = []
sws = []
sws2 = []
swsf = []
# half width of wavelength region around each sky line
# larger values give a better statistical precision
# but also a larger sensitivity to source features
# best solution on one dark night exposure obtained with
# a half width of 4A.
hw = 4 #A
tivar = frame.ivar
if frame.mask is not None:
tivar *= (frame.mask == 0)
tivar *= (skymodel.ivar > 0)
# we precompute the quantities needed to fit each sky line + continuum
# the sky "line profile" is the actual sky model
# and we consider an additive constant
for line in skyline:
if line <= frame.wave[0] or line >= frame.wave[-1]: continue
ii = np.where((frame.wave >= line - hw)
& (frame.wave <= line + hw))[0]
if ii.size < 2: continue
sw.append(np.sum(tivar[:, ii], axis=1))
swf.append(np.sum(tivar[:, ii] * frame.flux[:, ii], axis=1))
swsf.append(
np.sum(tivar[:, ii] * frame.flux[:, ii] * skymodel.flux[:, ii],
axis=1))
sws.append(np.sum(tivar[:, ii] * skymodel.flux[:, ii], axis=1))
sws2.append(np.sum(tivar[:, ii] * skymodel.flux[:, ii]**2, axis=1))
nlines = len(sw)
for fiber in range(frame.flux.shape[0]):
# we solve the 2x2 linear system for each fiber and sky line
# and save the results for each fiber
coef = [] # list of scale values
var = [] # list of variance on scale values
for line in range(nlines):
if sw[line][fiber] <= 0: continue
A = np.array([[sw[line][fiber], sws[line][fiber]],
[sws[line][fiber], sws2[line][fiber]]])
B = np.array([swf[line][fiber], swsf[line][fiber]])
try:
Ai = np.linalg.inv(A)
X = Ai.dot(B)
coef.append(
X[1]
) # the scale coef (marginalized over cst background)
var.append(Ai[1, 1])
except:
pass
if len(coef) == 0:
log.warning("cannot corr. throughput. for fiber %d" % fiber)
continue
coef = np.array(coef)
var = np.array(var)
ivar = (var > 0) / (var + (var == 0) + 0.005**2)
ivar_for_outliers = (var > 0) / (var + (var == 0) + 0.02**2)
# loop for outlier rejection
failed = False
for loop in range(50):
a = np.sum(ivar)
if a <= 0:
log.warning(
"cannot corr. throughput. ivar=0 everywhere on sky lines for fiber %d"
% fiber)
failed = True
break
mcoef = np.sum(ivar * coef) / a
mcoeferr = 1 / np.sqrt(a)
nsig = 3.
chi2 = ivar_for_outliers * (coef - mcoef)**2
worst = np.argmax(chi2)
if chi2[worst] > nsig**2 * np.median(
chi2[chi2 > 0]): # with rough scaling of errors
#log.debug("discard a bad measurement for fiber %d"%(fiber))
ivar[worst] = 0
ivar_for_outliers[worst] = 0
else:
break
if failed:
continue
log.info(
"fiber #%03d throughput corr = %5.4f +- %5.4f (mean fiber flux=%f)"
% (fiber, mcoef, mcoeferr, np.median(frame.flux[fiber])))
'''
if np.abs(mcoef)>0.01 :
print(fiber,"mean coef=",mcoef,"all coef=",coef)
print(fiber,"all err=",np.sqrt(var))
print(fiber,"mean coef=",mcoef,"selected coef=",coef[ivar>0])
print(fiber,"select err=",np.sqrt(var[ivar>0]))
import matplotlib.pyplot as plt
x=np.arange(coef.size)
plt.errorbar(x,coef,np.sqrt(var),fmt="o")
plt.errorbar(x[ivar>0],coef[ivar>0],np.sqrt(var[ivar>0]),fmt="o")
plt.axhline(0.)
plt.axhline(mcoef)
plt.ylim(-0.11,0.11)
plt.grid()
plt.show()
'''
if mcoeferr > 0.01:
log.warning(
"throughput corr error = %5.4f is too large for fiber #%03d, do not apply correction"
% (mcoeferr, fiber))
throughput_correction_value = default_throughput_correction
else:
throughput_correction_value = mcoef
# apply this correction to the sky model even if we have not fit it (default can be 1 or 0)
skymodel.flux[fiber] *= throughput_correction_value
frame.flux -= skymodel.flux
frame.ivar = util.combine_ivar(frame.ivar, skymodel.ivar)
frame.mask |= skymodel.mask
log.info("done")
def qa_skysub(param, frame, skymodel, quick_look=False):
"""Calculate QA on SkySubtraction
Note: Pixels rejected in generating the SkyModel (as above), are
not rejected in the stats calculated here. Would need to carry
along current_ivar to do so.
Args:
param : dict of QA parameters
frame : desispec.Frame object
skymodel : desispec.SkyModel object
quick_look : bool, optional
If True, do QuickLook specific QA (or avoid some)
Returns:
qadict: dict of QA outputs
Need to record simple Python objects for yaml (str, float, int)
"""
log = get_logger()
# Output dict
qadict = {}
qadict['NREJ'] = int(skymodel.nrej)
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nfibers = len(skyfibers)
qadict['NSKY_FIB'] = int(nfibers)
current_ivar = frame.ivar[skyfibers].copy()
flux = frame.flux[skyfibers]
# Subtract
res = flux - skymodel.flux[skyfibers] # Residuals
res_ivar = util.combine_ivar(current_ivar, skymodel.ivar[skyfibers])
# Chi^2 and Probability
chi2_fiber = np.sum(res_ivar * (res**2), 1)
chi2_prob = np.zeros(nfibers)
for ii in range(nfibers):
# Stats
dof = np.sum(res_ivar[ii, :] > 0.)
chi2_prob[ii] = scipy.stats.chisqprob(chi2_fiber[ii], dof)
# Bad models
qadict['NBAD_PCHI'] = int(np.sum(chi2_prob < param['PCHI_RESID']))
if qadict['NBAD_PCHI'] > 0:
log.warn("Bad Sky Subtraction in {:d} fibers".format(
qadict['NBAD_PCHI']))
# Median residual
qadict['MED_RESID'] = float(np.median(res)) # Median residual (counts)
log.info("Median residual for sky fibers = {:g}".format(
qadict['MED_RESID']))
# Residual percentiles
perc = dustat.perc(res, per=param['PER_RESID'])
qadict['RESID_PER'] = [float(iperc) for iperc in perc]
# Mean Sky Continuum from all skyfibers
# need to limit in wavelength?
if quick_look:
continuum = scipy.ndimage.filters.median_filter(
flux, 200) # taking 200 bins (somewhat arbitrarily)
mean_continuum = np.zeros(flux.shape[1])
for ii in range(flux.shape[1]):
mean_continuum[ii] = np.mean(continuum[:, ii])
qadict['MEAN_CONTIN'] = mean_continuum
# Median Signal to Noise on sky subtracted spectra
# first do the subtraction:
if quick_look:
fframe = frame # make a copy
sskymodel = skymodel # make a copy
subtract_sky(fframe, sskymodel)
medsnr = np.zeros(fframe.flux.shape[0])
totsnr = np.zeros(fframe.flux.shape[0])
for ii in range(fframe.flux.shape[0]):
signalmask = fframe.flux[ii, :] > 0
# total snr considering bin by bin uncorrelated S/N
snr = fframe.flux[ii, signalmask] * np.sqrt(
fframe.ivar[ii, signalmask])
medsnr[ii] = np.median(snr)
totsnr[ii] = np.sqrt(np.sum(snr**2))
qadict['MED_SNR'] = medsnr # for each fiber
qadict['TOT_SNR'] = totsnr # for each fiber
# Return
return qadict
def frame_skyres(outfil, frame, skymodel, qaframe):
"""
Generate QA plots and files for sky residuals of a given frame
Parameters
----------
outfil: str
Name of output file
frame: Frame object
skymodel: SkyModel object
qaframe: QAFrame object
"""
# Sky fibers
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
# Residuals
res = frame.flux[skyfibers] - skymodel.flux[skyfibers] # Residuals
res_ivar = util.combine_ivar(frame.ivar[skyfibers], skymodel.ivar[skyfibers])
med_res = np.median(res,0)
# Deviates
gd_res = res_ivar > 0.
devs = res[gd_res] * np.sqrt(res_ivar[gd_res])
# Calculations
wavg_res = np.sum(res*res_ivar,0) / np.sum(res_ivar,0)
'''
wavg_ivar = np.sum(res_ivar,0)
chi2_wavg = np.sum(wavg_res**2 * wavg_ivar)
dof_wavg = np.sum(wavg_ivar > 0.)
pchi2_wavg = scipy.stats.chisqprob(chi2_wavg, dof_wavg)
chi2_med = np.sum(med_res**2 * wavg_ivar)
pchi2_med = scipy.stats.chisqprob(chi2_med, dof_wavg)
'''
# Plot
fig = plt.figure(figsize=(8, 5.0))
gs = gridspec.GridSpec(2,2)
xmin,xmax = np.min(frame.wave), np.max(frame.wave)
# Simple residual plot
ax0 = plt.subplot(gs[0,:])
ax0.plot(frame.wave, med_res, label='Median Res')
ax0.plot(frame.wave, signal.medfilt(med_res,51), color='black', label='Median**2 Res')
ax0.plot(frame.wave, signal.medfilt(wavg_res,51), color='red', label='Med WAvgRes')
#ax_flux.plot(wave, sky_sig, label='Model Error')
#ax_flux.plot(wave,true_flux*scl, label='Truth')
#ax_flux.get_xaxis().set_ticks([]) # Suppress labeling
#
ax0.plot([xmin,xmax], [0., 0], '--', color='gray')
ax0.plot([xmin,xmax], [0., 0], '--', color='gray')
ax0.set_xlabel('Wavelength')
ax0.set_ylabel('Sky Residuals (Counts)')
ax0.set_xlim(xmin,xmax)
ax0.set_xlabel('Wavelength')
ax0.set_ylabel('Sky Residuals (Counts)')
ax0.set_xlim(xmin,xmax)
med0 = np.maximum(np.abs(np.median(med_res)), 1.)
ax0.set_ylim(-5.*med0, 5.*med0)
#ax0.text(0.5, 0.85, 'Sky Meanspec',
# transform=ax_flux.transAxes, ha='center')
# Legend
legend = ax0.legend(loc='upper right', borderpad=0.3,
handletextpad=0.3, fontsize='small')
# Histogram of all residuals
ax1 = plt.subplot(gs[1,0])
binsz = 0.1
xmin,xmax = -5., 5.
i0, i1 = int( np.min(devs) / binsz) - 1, int( np.max(devs) / binsz) + 1
rng = tuple( binsz*np.array([i0,i1]) )
nbin = i1-i0
# Histogram
hist, edges = np.histogram(devs, range=rng, bins=nbin)
xhist = (edges[1:] + edges[:-1])/2.
#ax.hist(xhist, color='black', bins=edges, weights=hist)#, histtype='step')
ax1.hist(xhist, color='blue', bins=edges, weights=hist)#, histtype='step')
# PDF for Gaussian
area = len(devs) * binsz
xppf = np.linspace(scipy.stats.norm.ppf(0.0001), scipy.stats.norm.ppf(0.9999), 100)
ax1.plot(xppf, area*scipy.stats.norm.pdf(xppf), 'r-', alpha=1.0)
ax1.set_xlabel(r'Res/$\sigma$')
ax1.set_ylabel('N')
ax1.set_xlim(xmin,xmax)
# Meta text
ax2 = plt.subplot(gs[1,1])
ax2.set_axis_off()
show_meta(ax2, qaframe, 'SKYSUB', outfil)
"""
# Meta
xlbl = 0.1
ylbl = 0.85
i0 = outfil.rfind('/')
ax2.text(xlbl, ylbl, outfil[i0+1:], color='black', transform=ax2.transAxes, ha='left')
yoff=0.15
for key in sorted(qaframe.data['SKYSUB']['QA'].keys()):
if key in ['QA_FIG']:
continue
# Show
ylbl -= yoff
ax2.text(xlbl+0.1, ylbl, key+': '+str(qaframe.data['SKYSUB']['QA'][key]),
transform=ax2.transAxes, ha='left', fontsize='small')
"""
'''
# Residuals
scatt_sz = 0.5
ax_res = plt.subplot(gs[1])
ax_res.get_xaxis().set_ticks([]) # Suppress labeling
res = (sky_model - (true_flux*scl))/(true_flux*scl)
rms = np.sqrt(np.sum(res**2)/len(res))
#ax_res.set_ylim(-3.*rms, 3.*rms)
ax_res.set_ylim(-2, 2)
ax_res.set_ylabel('Frac Res')
# Error
#ax_res.plot(true_wave, 2.*ms_sig/sky_model, color='red')
ax_res.scatter(wave,res, marker='o',s=scatt_sz)
ax_res.plot([xmin,xmax], [0.,0], 'g-')
ax_res.set_xlim(xmin,xmax)
# Relative to error
ax_sig = plt.subplot(gs[2])
ax_sig.set_xlabel('Wavelength')
sig_res = (sky_model - (true_flux*scl))/sky_sig
ax_sig.scatter(wave, sig_res, marker='o',s=scatt_sz)
ax_sig.set_ylabel(r'Res $\delta/\sigma$')
ax_sig.set_ylim(-5., 5.)
ax_sig.plot([xmin,xmax], [0.,0], 'g-')
ax_sig.set_xlim(xmin,xmax)
'''
# Finish
plt.tight_layout(pad=0.1,h_pad=0.0,w_pad=0.0)
plt.savefig(outfil)
plt.close()
print('Wrote QA SkyRes file: {:s}'.format(outfil))
def subtract_sky(frame, skymodel, throughput_correction = False, default_throughput_correction = 1.) :
"""Subtract skymodel from frame, altering frame.flux, .ivar, and .mask
Args:
frame : desispec.Frame object
skymodel : desispec.SkyModel object
Option:
throughput_correction : if True, fit for an achromatic throughput correction. This is to absorb variations of Focal Ratio Degradation with fiber flexure.
default_throughput_correction : float, default value of correction if the fit on sky lines failed.
"""
assert frame.nspec == skymodel.nspec
assert frame.nwave == skymodel.nwave
log=get_logger()
log.info("starting")
# check same wavelength, die if not the case
if not np.allclose(frame.wave, skymodel.wave):
message = "frame and sky not on same wavelength grid"
log.error(message)
raise ValueError(message)
if throughput_correction :
# need to fit for a multiplicative factor of the sky model
# before subtraction
# we are going to use a set of bright sky lines,
# and fit a multiplicative factor + background around
# each of them individually, and then combine the results
# with outlier rejection in case a source emission line
# coincides with one of the sky lines.
# it's more robust to have a hardcoded set of sky lines here
# these are all the sky lines with a flux >5% of the max flux
# except in b where we add an extra weaker line at 5199.4A
skyline=np.array([5199.4,5578.4,5656.4,5891.4,5897.4,6302.4,6308.4,6365.4,6500.4,6546.4,6555.4,6618.4,6663.4,6679.4,6690.4,6765.4,6831.4,6836.4,6865.4,6925.4,6951.4,6980.4,7242.4,7247.4,7278.4,7286.4,7305.4,7318.4,7331.4,7343.4,7360.4,7371.4,7394.4,7404.4,7440.4,7526.4,7714.4,7719.4,7752.4,7762.4,7782.4,7796.4,7810.4,7823.4,7843.4,7855.4,7862.4,7873.4,7881.4,7892.4,7915.4,7923.4,7933.4,7951.4,7966.4,7982.4,7995.4,8016.4,8028.4,8064.4,8280.4,8284.4,8290.4,8298.4,8301.4,8313.4,8346.4,8355.4,8367.4,8384.4,8401.4,8417.4,8432.4,8454.4,8467.4,8495.4,8507.4,8627.4,8630.4,8634.4,8638.4,8652.4,8657.4,8662.4,8667.4,8672.4,8677.4,8683.4,8763.4,8770.4,8780.4,8793.4,8829.4,8835.4,8838.4,8852.4,8870.4,8888.4,8905.4,8922.4,8945.4,8960.4,8990.4,9003.4,9040.4,9052.4,9105.4,9227.4,9309.4,9315.4,9320.4,9326.4,9340.4,9378.4,9389.4,9404.4,9422.4,9442.4,9461.4,9479.4,9505.4,9521.4,9555.4,9570.4,9610.4,9623.4,9671.4,9684.4,9693.4,9702.4,9714.4,9722.4,9740.4,9748.4,9793.4,9802.4,9814.4,9820.4])
sw=[]
swf=[]
sws=[]
sws2=[]
swsf=[]
# half width of wavelength region around each sky line
# larger values give a better statistical precision
# but also a larger sensitivity to source features
# best solution on one dark night exposure obtained with
# a half width of 4A.
hw=4#A
tivar=frame.ivar
if frame.mask is not None :
tivar *= (frame.mask==0)
tivar *= (skymodel.ivar>0)
# we precompute the quantities needed to fit each sky line + continuum
# the sky "line profile" is the actual sky model
# and we consider an additive constant
for line in skyline :
if line<=frame.wave[0] or line>=frame.wave[-1] : continue
ii=np.where((frame.wave>=line-hw)&(frame.wave<=line+hw))[0]
if ii.size<2 : continue
sw.append(np.sum(tivar[:,ii],axis=1))
swf.append(np.sum(tivar[:,ii]*frame.flux[:,ii],axis=1))
swsf.append(np.sum(tivar[:,ii]*frame.flux[:,ii]*skymodel.flux[:,ii],axis=1))
sws.append(np.sum(tivar[:,ii]*skymodel.flux[:,ii],axis=1))
sws2.append(np.sum(tivar[:,ii]*skymodel.flux[:,ii]**2,axis=1))
nlines=len(sw)
for fiber in range(frame.flux.shape[0]) :
# we solve the 2x2 linear system for each fiber and sky line
# and save the results for each fiber
coef=[] # list of scale values
var=[] # list of variance on scale values
for line in range(nlines) :
if sw[line][fiber]<=0 : continue
A=np.array([[sw[line][fiber],sws[line][fiber]],[sws[line][fiber],sws2[line][fiber]]])
B=np.array([swf[line][fiber],swsf[line][fiber]])
try :
Ai=np.linalg.inv(A)
X=Ai.dot(B)
coef.append(X[1]) # the scale coef (marginalized over cst background)
var.append(Ai[1,1])
except :
pass
if len(coef)==0 :
log.warning("cannot corr. throughput. for fiber %d"%fiber)
continue
coef=np.array(coef)
var=np.array(var)
ivar=(var>0)/(var+(var==0)+0.005**2)
ivar_for_outliers=(var>0)/(var+(var==0)+0.02**2)
# loop for outlier rejection
failed=False
for loop in range(50) :
a=np.sum(ivar)
if a <= 0 :
log.warning("cannot corr. throughput. ivar=0 everywhere on sky lines for fiber %d"%fiber)
failed=True
break
mcoef=np.sum(ivar*coef)/a
mcoeferr=1/np.sqrt(a)
nsig=3.
chi2=ivar_for_outliers*(coef-mcoef)**2
worst=np.argmax(chi2)
if chi2[worst]>nsig**2*np.median(chi2[chi2>0]) : # with rough scaling of errors
#log.debug("discard a bad measurement for fiber %d"%(fiber))
ivar[worst]=0
ivar_for_outliers[worst]=0
else :
break
if failed :
continue
log.info("fiber #%03d throughput corr = %5.4f +- %5.4f (mean fiber flux=%f)"%(fiber,mcoef,mcoeferr,np.median(frame.flux[fiber])))
'''
if np.abs(mcoef)>0.01 :
print(fiber,"mean coef=",mcoef,"all coef=",coef)
print(fiber,"all err=",np.sqrt(var))
print(fiber,"mean coef=",mcoef,"selected coef=",coef[ivar>0])
print(fiber,"select err=",np.sqrt(var[ivar>0]))
import matplotlib.pyplot as plt
x=np.arange(coef.size)
plt.errorbar(x,coef,np.sqrt(var),fmt="o")
plt.errorbar(x[ivar>0],coef[ivar>0],np.sqrt(var[ivar>0]),fmt="o")
plt.axhline(0.)
plt.axhline(mcoef)
plt.ylim(-0.11,0.11)
plt.grid()
plt.show()
'''
if mcoeferr>0.01 :
log.warning("throughput corr error = %5.4f is too large for fiber #%03d, do not apply correction"%(mcoeferr,fiber))
throughput_correction_value = default_throughput_correction
else :
throughput_correction_value = mcoef
# apply this correction to the sky model even if we have not fit it (default can be 1 or 0)
skymodel.flux[fiber] *= throughput_correction_value
frame.flux -= skymodel.flux
frame.ivar = util.combine_ivar(frame.ivar, skymodel.ivar)
frame.mask |= skymodel.mask
log.info("done")
def _model_variance(frame,cskyflux,cskyivar,skyfibers) :
"""look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1
"""
log = get_logger()
tivar = util.combine_ivar(frame.ivar[skyfibers], cskyivar[skyfibers])
# the chi2 at a given wavelength can be large because on a cosmic
# and not a psf error or sky non uniformity
# so we need to consider only waves for which
# a reasonable sky model error can be computed
# mean sky
msky = np.mean(cskyflux,axis=0)
dwave = np.mean(np.gradient(frame.wave))
dskydw = np.zeros(msky.shape)
dskydw[1:-1]=(msky[2:]-msky[:-2])/(frame.wave[2:]-frame.wave[:-2])
dskydw = np.abs(dskydw)
# now we consider a worst possible sky model error (20% error on flat, 0.5A )
max_possible_var = 1./(tivar+(tivar==0)) + (0.2*msky)**2 + (0.5*dskydw)**2
# exclude residuals inconsistent with this max possible variance (at 3 sigma)
bad = (frame.flux[skyfibers]-cskyflux[skyfibers])**2 > 3**2*max_possible_var
tivar[bad]=0
ndata = np.sum(tivar>0,axis=0)
ok=np.where(ndata>1)[0]
chi2 = np.zeros(frame.wave.size)
chi2[ok] = np.sum(tivar*(frame.flux[skyfibers]-cskyflux[skyfibers])**2,axis=0)[ok]/(ndata[ok]-1)
chi2[ndata<=1] = 1. # default
# now we are going to evaluate a sky model error based on this chi2,
# but only around sky flux peaks (>0.1*max)
tmp = np.zeros(frame.wave.size)
tmp = (msky[1:-1]>msky[2:])*(msky[1:-1]>msky[:-2])*(msky[1:-1]>0.1*np.max(msky))
peaks = np.where(tmp)[0]+1
dpix = int(np.ceil(3/dwave)) # +- n Angstrom around each peak
skyvar = 1./(cskyivar+(cskyivar==0))
# loop on peaks
for peak in peaks :
b=peak-dpix
e=peak+dpix+1
mchi2 = np.mean(chi2[b:e]) # mean reduced chi2 around peak
mndata = np.mean(ndata[b:e]) # mean number of fibers contributing
# sky model variance = sigma_flat * msky + sigma_wave * dmskydw
sigma_flat=0.000 # the fiber flat error is already included in the flux ivar
sigma_wave=0.005 # A, minimum value
res2=(frame.flux[skyfibers,b:e]-cskyflux[skyfibers,b:e])**2
var=1./(tivar[:,b:e]+(tivar[:,b:e]==0))
nd=np.sum(tivar[:,b:e]>0)
while(sigma_wave<2) :
pivar=1./(var+(sigma_flat*msky[b:e])**2+(sigma_wave*dskydw[b:e])**2)
pchi2=np.sum(pivar*res2)/nd
if pchi2<=1 :
log.info("peak at {}A : sigma_wave={}".format(int(frame.wave[peak]),sigma_wave))
skyvar[:,b:e] += ( (sigma_flat*msky[b:e])**2 + (sigma_wave*dskydw[b:e])**2 )
break
sigma_wave += 0.005
return (cskyivar>0)/(skyvar+(skyvar==0))
def qa_skysub(param, frame, skymodel, quick_look=False):
"""Calculate QA on SkySubtraction
Note: Pixels rejected in generating the SkyModel (as above), are
not rejected in the stats calculated here. Would need to carry
along current_ivar to do so.
Args:
param : dict of QA parameters
frame : desispec.Frame object
skymodel : desispec.SkyModel object
quick_look : bool, optional
If True, do QuickLook specific QA (or avoid some)
Returns:
qadict: dict of QA outputs
Need to record simple Python objects for yaml (str, float, int)
"""
log=get_logger()
# Output dict
qadict = {}
qadict['NREJ'] = int(skymodel.nrej)
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nfibers=len(skyfibers)
qadict['NSKY_FIB'] = int(nfibers)
current_ivar=frame.ivar[skyfibers].copy()
flux = frame.flux[skyfibers]
# Subtract
res = flux - skymodel.flux[skyfibers] # Residuals
res_ivar = util.combine_ivar(current_ivar, skymodel.ivar[skyfibers])
# Chi^2 and Probability
chi2_fiber = np.sum(res_ivar*(res**2),1)
chi2_prob = np.zeros(nfibers)
for ii in range(nfibers):
# Stats
dof = np.sum(res_ivar[ii,:] > 0.)
chi2_prob[ii] = scipy.stats.chisqprob(chi2_fiber[ii], dof)
# Bad models
qadict['NBAD_PCHI'] = int(np.sum(chi2_prob < param['PCHI_RESID']))
if qadict['NBAD_PCHI'] > 0:
log.warn("Bad Sky Subtraction in {:d} fibers".format(
qadict['NBAD_PCHI']))
# Median residual
qadict['MED_RESID'] = float(np.median(res)) # Median residual (counts)
log.info("Median residual for sky fibers = {:g}".format(
qadict['MED_RESID']))
# Residual percentiles
perc = dustat.perc(res, per=param['PER_RESID'])
qadict['RESID_PER'] = [float(iperc) for iperc in perc]
# Mean Sky Continuum from all skyfibers
# need to limit in wavelength?
if quick_look:
continuum=scipy.ndimage.filters.median_filter(flux,200) # taking 200 bins (somewhat arbitrarily)
mean_continuum=np.zeros(flux.shape[1])
for ii in range(flux.shape[1]):
mean_continuum[ii]=np.mean(continuum[:,ii])
qadict['MEAN_CONTIN'] = mean_continuum
# Median Signal to Noise on sky subtracted spectra
# first do the subtraction:
if quick_look:
fframe=frame # make a copy
sskymodel=skymodel # make a copy
subtract_sky(fframe,sskymodel)
medsnr=np.zeros(fframe.flux.shape[0])
totsnr=np.zeros(fframe.flux.shape[0])
for ii in range(fframe.flux.shape[0]):
signalmask=fframe.flux[ii,:]>0
# total snr considering bin by bin uncorrelated S/N
snr=fframe.flux[ii,signalmask]*np.sqrt(fframe.ivar[ii,signalmask])
medsnr[ii]=np.median(snr)
totsnr[ii]=np.sqrt(np.sum(snr**2))
qadict['MED_SNR']=medsnr # for each fiber
qadict['TOT_SNR']=totsnr # for each fiber
# Return
return qadict
def qa_skysub(param, frame, skymodel, quick_look=False):
"""Calculate QA on SkySubtraction
Note: Pixels rejected in generating the SkyModel (as above), are
not rejected in the stats calculated here. Would need to carry
along current_ivar to do so.
Args:
param : dict of QA parameters
frame : desispec.Frame object
skymodel : desispec.SkyModel object
quick_look : bool, optional
If True, do QuickLook specific QA (or avoid some)
Returns:
qadict: dict of QA outputs
Need to record simple Python objects for yaml (str, float, int)
"""
log = get_logger()
# Output dict
qadict = {}
qadict['NREJ'] = int(skymodel.nrej)
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nfibers = len(skyfibers)
qadict['NSKY_FIB'] = int(nfibers)
current_ivar = frame.ivar[skyfibers].copy()
flux = frame.flux[skyfibers]
# Subtract
res = flux - skymodel.flux[skyfibers] # Residuals
res_ivar = util.combine_ivar(current_ivar, skymodel.ivar[skyfibers])
# Chi^2 and Probability
chi2_fiber = np.sum(res_ivar * (res**2), 1)
chi2_prob = np.zeros(nfibers)
for ii in range(nfibers):
# Stats
dof = np.sum(res_ivar[ii, :] > 0.)
chi2_prob[ii] = scipy.stats.chisqprob(chi2_fiber[ii], dof)
# Bad models
qadict['NBAD_PCHI'] = int(np.sum(chi2_prob < param['PCHI_RESID']))
if qadict['NBAD_PCHI'] > 0:
log.warning("Bad Sky Subtraction in {:d} fibers".format(
qadict['NBAD_PCHI']))
# Median residual
qadict['MED_RESID'] = float(np.median(res)) # Median residual (counts)
log.info("Median residual for sky fibers = {:g}".format(
qadict['MED_RESID']))
# Residual percentiles
perc = dustat.perc(res, per=param['PER_RESID'])
qadict['RESID_PER'] = [float(iperc) for iperc in perc]
#- Add per fiber median residuals
qadict["MED_RESID_FIBER"] = np.median(res, axis=1)
#- Evaluate residuals in wave axis for quicklook
if quick_look:
qadict["MED_RESID_WAVE"] = np.median(res, axis=0)
qadict["WAVELENGTH"] = frame.wave
# Return
return qadict
