def compute_fiberflat(frame, nsig_clipping=4.) :
"""Compute fiber flat by deriving an average spectrum and dividing all fiber data by this average.
Input data are expected to be on the same wavelenght grid, with uncorrelated noise.
They however do not have exactly the same resolution.
args:
frame (desispec.Frame): input Frame object with attributes
wave, flux, ivar, resolution_data
nsig_clipping : [optional] sigma clipping value for outlier rejection
returns tuple (fiberflat, ivar, mask, meanspec):
fiberflat : 2D[nwave, nflux] fiberflat (data have to be divided by this to be flatfielded)
ivar : inverse variance of that fiberflat
mask : 0=ok >0 if problems
meanspec : deconvolved mean spectrum
- we first iteratively :
- compute a deconvolved mean spectrum
- compute a fiber flat using the resolution convolved mean spectrum for each fiber
- smooth the fiber flat along wavelength
- clip outliers
- then we compute a fiberflat at the native fiber resolution (not smoothed)
- the routine returns the fiberflat, its inverse variance , mask, and the deconvolved mean spectrum
- the fiberflat is the ratio data/mean , so this flat should be divided to the data
NOTE THAT THIS CODE HAS NOT BEEN TESTED WITH ACTUAL FIBER TRANSMISSION VARIATIONS,
OUTLIER PIXELS, DEAD COLUMNS ...
"""
log=get_logger()
log.info("starting")
#
# chi2 = sum_(fiber f) sum_(wavelenght i) w_fi ( D_fi - F_fi (R_f M)_i )
#
# where
# w = inverse variance
# D = flux data (at the resolution of the fiber)
# F = smooth fiber flat
# R = resolution data
# M = mean deconvolved spectrum
#
# M = A^{-1} B
# with
# A_kl = sum_(fiber f) sum_(wavelenght i) w_fi F_fi^2 (R_fki R_fli)
# B_k = sum_(fiber f) sum_(wavelenght i) w_fi D_fi F_fi R_fki
#
# defining R'_fi = sqrt(w_fi) F_fi R_fi
# and D'_fi = sqrt(w_fi) D_fi
#
# A = sum_(fiber f) R'_f R'_f^T
# B = sum_(fiber f) R'_f D'_f
# (it's faster that way, and we try to use sparse matrices as much as possible)
#
#- Shortcuts
nwave=frame.nwave
nfibers=frame.nspec
wave = frame.wave.copy() #- this will become part of output too
flux = frame.flux
ivar = frame.ivar
# iterative fitting and clipping to get precise mean spectrum
current_ivar=ivar.copy()
smooth_fiberflat=np.ones((frame.flux.shape))
chi2=np.zeros((flux.shape))
sqrtwflat=np.sqrt(current_ivar)*smooth_fiberflat
sqrtwflux=np.sqrt(current_ivar)*flux
# test
#nfibers=20
nout_tot=0
for iteration in range(20) :
# fit mean spectrum
A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr()
B=np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD=scipy.sparse.lil_matrix((nwave,nwave))
# loop on fiber to handle resolution
for fiber in range(nfibers) :
if fiber%10==0 :
log.info("iter %d fiber %d"%(iteration,fiber))
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
# diagonal sparse matrix with content = sqrt(ivar)*flat
SD.setdiag(sqrtwflat[fiber])
sqrtwflatR = SD*R # each row r of R is multiplied by sqrtwflat[r]
A = A+(sqrtwflatR.T*sqrtwflatR).tocsr()
B += sqrtwflatR.T*sqrtwflux[fiber]
log.info("iter %d solving"%iteration)
mean_spectrum=cholesky_solve(A.todense(),B)
log.info("iter %d smoothing"%iteration)
# fit smooth fiberflat and compute chi2
smoothing_res=100. #A
for fiber in range(nfibers) :
#if fiber%10==0 :
# log.info("iter %d fiber %d (smoothing)"%(iteration,fiber))
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
#M = np.array(np.dot(R.todense(),mean_spectrum)).flatten()
M = R.dot(mean_spectrum)
F = flux[fiber]/(M+(M==0))
smooth_fiberflat[fiber]=spline_fit(wave,wave,F,smoothing_res,current_ivar[fiber]*(M!=0))
chi2[fiber]=current_ivar[fiber]*(flux[fiber]-smooth_fiberflat[fiber]*M)**2
log.info("rejecting")
nout_iter=0
if iteration<1 :
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave=np.sum(chi2>nsig_clipping**2,axis=0)
selection=np.where(nout_per_wave>0)[0]
for i in selection :
worst_entry=np.argmax(chi2[:,i])
current_ivar[worst_entry,i]=0
sqrtwflat[worst_entry,i]=0
sqrtwflux[worst_entry,i]=0
nout_iter += 1
else :
# remove all of them at once
bad=(chi2>nsig_clipping**2)
current_ivar *= (bad==0)
sqrtwflat *= (bad==0)
sqrtwflux *= (bad==0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2=float(np.sum(chi2))
ndf=int(np.sum(chi2>0)-nwave-nfibers*(nwave/smoothing_res))
chi2pdf=0.
if ndf>0 :
chi2pdf=sum_chi2/ndf
log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d"%(iteration,sum_chi2,ndf,chi2pdf,nout_iter))
# normalize to get a mean fiberflat=1
mean=np.mean(smooth_fiberflat,axis=0)
smooth_fiberflat = smooth_fiberflat/mean
mean_spectrum = mean_spectrum*mean
if nout_iter == 0 :
break
log.info("nout tot=%d"%nout_tot)
# now use mean spectrum to compute flat field correction without any smoothing
# because sharp feature can arise if dead columns
fiberflat=np.ones((flux.shape))
fiberflat_ivar=np.zeros((flux.shape))
mask=np.zeros((flux.shape)).astype(long) # SOMEONE CHECK THIS !
fiberflat_mask=12 # place holder for actual mask bit when defined
nsig_for_mask=4 # only mask out 4 sigma outliers
for fiber in range(nfibers) :
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = np.array(np.dot(R.todense(),mean_spectrum)).flatten()
fiberflat[fiber] = (M!=0)*flux[fiber]/(M+(M==0)) + (M==0)
fiberflat_ivar[fiber] = ivar[fiber]*M**2
smooth_fiberflat=spline_fit(wave,wave,fiberflat[fiber],smoothing_res,current_ivar[fiber]*M**2*(M!=0))
bad=np.where(fiberflat_ivar[fiber]*(fiberflat[fiber]-smooth_fiberflat)**2>nsig_for_mask**2)[0]
if bad.size>0 :
mask[fiber,bad] += fiberflat_mask
return FiberFlat(wave, fiberflat, fiberflat_ivar, mask, mean_spectrum)
def compute_fiberflat(frame,
nsig_clipping=10.,
accuracy=5.e-4,
minval=0.1,
maxval=10.,
max_iterations=100,
smoothing_res=5.,
max_bad=100,
max_rej_it=5,
min_sn=0,
diag_epsilon=1e-3):
"""Compute fiber flat by deriving an average spectrum and dividing all fiber data by this average.
Input data are expected to be on the same wavelength grid, with uncorrelated noise.
They however do not have exactly the same resolution.
Args:
frame (desispec.Frame): input Frame object with attributes
wave, flux, ivar, resolution_data
nsig_clipping : [optional] sigma clipping value for outlier rejection
accuracy : [optional] accuracy of fiberflat (end test for the iterative loop)
minval: [optional] mask pixels with flux < minval * median fiberflat.
maxval: [optional] mask pixels with flux > maxval * median fiberflat.
max_iterations: [optional] maximum number of iterations
smoothing_res: [optional] spacing between spline fit nodes for smoothing the fiberflat
max_bad: [optional] mask entire fiber if more than max_bad-1 initially unmasked pixels are masked during the iterations
max_rej_it: [optional] reject at most the max_rej_it worst pixels in each iteration
min_sn: [optional] mask portions with signal to noise less than min_sn
diag_epsilon: [optional] size of the regularization term in the deconvolution
Returns:
desispec.FiberFlat object with attributes
wave, fiberflat, ivar, mask, meanspec
Notes:
- we first iteratively :
- compute a deconvolved mean spectrum
- compute a fiber flat using the resolution convolved mean spectrum for each fiber
- smooth the fiber flat along wavelength
- clip outliers
- then we compute a fiberflat at the native fiber resolution (not smoothed)
- the routine returns the fiberflat, its inverse variance , mask, and the deconvolved mean spectrum
- the fiberflat is the ratio data/mean , so this flat should be divided to the data
NOTE THAT THIS CODE HAS NOT BEEN TESTED WITH ACTUAL FIBER TRANSMISSION VARIATIONS,
OUTLIER PIXELS, DEAD COLUMNS ...
"""
log = get_logger()
log.info("starting")
#
# chi2 = sum_(fiber f) sum_(wavelenght i) w_fi ( D_fi - F_fi (R_f M)_i )
#
# where
# w = inverse variance
# D = flux data (at the resolution of the fiber)
# F = smooth fiber flat
# R = resolution data
# M = mean deconvolved spectrum
#
# M = A^{-1} B
# with
# A_kl = sum_(fiber f) sum_(wavelenght i) w_fi F_fi^2 (R_fki R_fli)
# B_k = sum_(fiber f) sum_(wavelenght i) w_fi D_fi F_fi R_fki
#
# defining R'_fi = sqrt(w_fi) F_fi R_fi
# and D'_fi = sqrt(w_fi) D_fi
#
# A = sum_(fiber f) R'_f R'_f^T
# B = sum_(fiber f) R'_f D'_f
# (it's faster that way, and we try to use sparse matrices as much as possible)
#
#- Shortcuts
nwave = frame.nwave
nfibers = frame.nspec
wave = frame.wave.copy() #- this will become part of output too
flux = frame.flux.copy()
ivar = frame.ivar * (frame.mask == 0)
# iterative fitting and clipping to get precise mean spectrum
# we first need to iterate to converge on a solution of mean spectrum
# and smooth fiber flat. several interations are needed when
# throughput AND resolution vary from fiber to fiber.
# the end test is that the fiber flat has varied by less than accuracy
# of previous iteration for all wavelength
# we also have a max. number of iterations for this code
nout_tot = 0
chi2pdf = 0.
smooth_fiberflat = np.ones((flux.shape))
chi2 = np.zeros((flux.shape))
## mask low sn portions
w = flux * np.sqrt(ivar) < min_sn
ivar[w] = 0
## 0th pass: reject pixels according to minval and maxval
mean_spectrum = np.zeros(flux.shape[1])
nbad = np.zeros(nfibers, dtype=int)
for iteration in range(max_iterations):
for i in range(flux.shape[1]):
w = ivar[:, i] > 0
if w.sum() > 0:
mean_spectrum[i] = np.median(flux[w, i])
nbad_it = 0
for fib in range(nfibers):
w = ((flux[fib, :] < minval * mean_spectrum) |
(flux[fib, :] > maxval * mean_spectrum)) & (ivar[fib, :] > 0)
nbad_it += w.sum()
nbad[fib] += w.sum()
if w.sum() > 0:
ivar[fib, w] = 0
log.warning("0th pass: masking {} pixels in fiber {}".format(
w.sum(), fib))
if nbad[fib] >= max_bad:
ivar[fib, :] = 0
log.warning(
"0th pass: masking entire fiber {} (nbad={})".format(
fib, nbad[fib]))
if nbad_it == 0:
break
# 1st pass is median for spectrum, flat field without resolution
# outlier rejection
for iteration in range(max_iterations):
# use median for spectrum
mean_spectrum = np.zeros((flux.shape[1]))
for i in range(flux.shape[1]):
w = ivar[:, i] > 0
if w.sum() > 0:
mean_spectrum[i] = np.median(flux[w, i])
nbad_it = 0
sum_chi2 = 0
# not more than max_rej_it pixels per fiber at a time
for fib in range(nfibers):
w = ivar[fib, :] > 0
if w.sum() == 0:
continue
F = flux[fib, :] * 0
w = (mean_spectrum != 0) & (ivar[fib, :] > 0)
F[w] = flux[fib, w] / mean_spectrum[w]
smooth_fiberflat[fib, :] = spline_fit(
wave, wave[w], F[w], smoothing_res,
ivar[fib, w] * mean_spectrum[w]**2)
chi2 = ivar[fib, :] * (flux[fib, :] -
mean_spectrum * smooth_fiberflat[fib, :])**2
w = np.isnan(chi2)
bad = np.where(chi2 > nsig_clipping**2)[0]
if bad.size > 0:
if bad.size > max_rej_it: # not more than 5 pixels at a time
ii = np.argsort(chi2[bad])
bad = bad[ii[-max_rej_it:]]
ivar[fib, bad] = 0
log.warning(
"1st pass: rejecting {} pixels from fiber {}".format(
len(bad), fib))
nbad[fib] += len(bad)
if nbad[fib] >= max_bad:
ivar[fib, :] = 0
log.warning(
"1st pass: rejecting fiber {} due to too many (new) bad pixels"
.format(fib))
nbad_it += len(bad)
sum_chi2 += chi2.sum()
ndf = int((ivar > 0).sum() - nwave - nfibers * (nwave / smoothing_res))
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info(
"1st pass iter #{} chi2={}/{} chi2pdf={} nout={} (nsig={})".format(
iteration, sum_chi2, ndf, chi2pdf, nbad_it, nsig_clipping))
if nbad_it == 0:
break
## flatten fiberflat
## normalize smooth_fiberflat:
mean = np.ones(smooth_fiberflat.shape[1])
for i in range(smooth_fiberflat.shape[1]):
w = ivar[:, i] > 0
if w.sum() > 0:
mean[i] = np.median(smooth_fiberflat[w, i])
smooth_fiberflat = smooth_fiberflat / mean
median_spectrum = mean_spectrum * 1.
previous_smooth_fiberflat = smooth_fiberflat * 0
log.info("after 1st pass : nout = %d/%d" %
(np.sum(ivar == 0), np.size(ivar.flatten())))
# 2nd pass is full solution including deconvolved spectrum, no outlier rejection
for iteration in range(max_iterations):
## reset sum_chi2
sum_chi2 = 0
log.info("2nd pass, iter %d : mean deconvolved spectrum" % iteration)
# fit mean spectrum
A = scipy.sparse.lil_matrix((nwave, nwave)).tocsr()
B = np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD = scipy.sparse.lil_matrix((nwave, nwave))
# this is to go a bit faster
sqrtwflat = np.sqrt(ivar) * smooth_fiberflat
# loop on fiber to handle resolution (this is long)
for fiber in range(nfibers):
if fiber % 10 == 0:
log.info("2nd pass, filling matrix, iter %d fiber %d" %
(iteration, fiber))
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
SD.setdiag(sqrtwflat[fiber])
sqrtwflatR = SD * R # each row r of R is multiplied by sqrtwflat[r]
A = A + (sqrtwflatR.T * sqrtwflatR).tocsr()
B += sqrtwflatR.T.dot(np.sqrt(ivar[fiber]) * flux[fiber])
A_pos_def = A.todense()
log.info("deconvolving")
w = A.diagonal() > 0
A_pos_def = A_pos_def[w, :]
A_pos_def = A_pos_def[:, w]
mean_spectrum = np.zeros(nwave)
try:
mean_spectrum[w] = cholesky_solve(A_pos_def, B[w])
except:
mean_spectrum[w] = np.linalg.lstsq(A_pos_def, B[w])[0]
log.info("cholesky failes, trying svd inverse in iter {}".format(
iteration))
for fiber in range(nfibers):
if np.sum(ivar[fiber] > 0) == 0:
continue
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = R.dot(mean_spectrum)
ok = (M != 0) & (ivar[fiber, :] > 0)
if ok.sum() == 0:
continue
smooth_fiberflat[fiber] = spline_fit(
wave, wave[ok], flux[fiber, ok] / M[ok], smoothing_res,
ivar[fiber, ok] * M[ok]**2) * (ivar[fiber, :] * M**2 > 0)
chi2 = ivar[fiber] * (flux[fiber] - smooth_fiberflat[fiber] * M)**2
sum_chi2 += chi2.sum()
w = np.isnan(smooth_fiberflat[fiber])
if w.sum() > 0:
ivar[fiber] = 0
smooth_fiberflat[fiber] = 1
# normalize to get a mean fiberflat=1
mean = np.ones(smooth_fiberflat.shape[1])
for i in range(nwave):
w = ivar[:, i] > 0
if w.sum() > 0:
mean[i] = np.median(smooth_fiberflat[w, i])
ok = np.where(mean != 0)[0]
smooth_fiberflat[:, ok] /= mean[ok]
# this is the max difference between two iterations
max_diff = np.max(
np.abs(smooth_fiberflat - previous_smooth_fiberflat) * (ivar > 0.))
previous_smooth_fiberflat = smooth_fiberflat.copy()
ndf = int(np.sum(ivar > 0) - nwave - nfibers * (nwave / smoothing_res))
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info("2nd pass, iter %d, chi2=%f ndf=%d chi2pdf=%f" %
(iteration, sum_chi2, ndf, chi2pdf))
if max_diff < accuracy:
break
log.info(
"2nd pass, iter %d, max diff. = %g > requirement = %g, continue iterating"
% (iteration, max_diff, accuracy))
log.info("Total number of masked pixels=%d" % nout_tot)
log.info("3rd pass, final computation of fiber flat")
# now use mean spectrum to compute flat field correction without any smoothing
# because sharp feature can arise if dead columns
fiberflat = np.ones((flux.shape))
fiberflat_ivar = np.zeros((flux.shape))
mask = np.zeros((flux.shape), dtype='uint32')
# reset ivar
ivar = frame.ivar
fiberflat_mask = 12 # place holder for actual mask bit when defined
nsig_for_mask = nsig_clipping # only mask out N sigma outliers
for fiber in range(nfibers):
if np.sum(ivar[fiber] > 0) == 0:
continue
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = np.array(np.dot(R.todense(), mean_spectrum)).flatten()
fiberflat[fiber] = (M != 0) * flux[fiber] / (M + (M == 0)) + (M == 0)
fiberflat_ivar[fiber] = ivar[fiber] * M**2
nbad_tot = 0
iteration = 0
while iteration < 500:
w = fiberflat_ivar[fiber, :] > 0
if w.sum() < 100:
break
smooth_fiberflat = spline_fit(wave, wave[w], fiberflat[fiber, w],
smoothing_res, fiberflat_ivar[fiber,
w])
chi2 = fiberflat_ivar[fiber] * (fiberflat[fiber] -
smooth_fiberflat)**2
bad = np.where(chi2 > nsig_for_mask**2)[0]
if bad.size > 0:
nbadmax = 1
if bad.size > nbadmax: # not more than nbadmax pixels at a time
ii = np.argsort(chi2[bad])
bad = bad[ii[-nbadmax:]]
mask[fiber, bad] += fiberflat_mask
fiberflat_ivar[fiber, bad] = 0.
nbad_tot += bad.size
else:
break
iteration += 1
log.info("3rd pass : fiber #%d , number of iterations %d" %
(fiber, iteration))
# set median flat to 1
log.info("3rd pass : set median fiberflat to 1")
mean = np.ones((flux.shape[1]))
for i in range(flux.shape[1]):
ok = np.where((mask[:, i] == 0) & (ivar[:, i] > 0))[0]
if ok.size > 0:
mean[i] = np.median(fiberflat[ok, i])
ok = np.where(mean != 0)[0]
for fiber in range(nfibers):
fiberflat[fiber, ok] /= mean[ok]
log.info("3rd pass : interpolating over masked pixels")
for fiber in range(nfibers):
if np.sum(ivar[fiber] > 0) == 0:
continue
# replace bad by smooth fiber flat
bad = np.where((mask[fiber] > 0) | (fiberflat_ivar[fiber] == 0)
| (fiberflat[fiber] < minval)
| (fiberflat[fiber] > maxval))[0]
if bad.size > 0:
fiberflat_ivar[fiber, bad] = 0
# find max length of segment with bad pix
length = 0
for i in range(bad.size):
ib = bad[i]
ilength = 1
tmp = ib
for jb in bad[i + 1:]:
if jb == tmp + 1:
ilength += 1
tmp = jb
else:
break
length = max(length, ilength)
if length > 10:
log.info(
"3rd pass : fiber #%d has a max length of bad pixels=%d" %
(fiber, length))
smoothing_res = float(max(100, length))
x = np.arange(wave.size)
ok = fiberflat_ivar[fiber] > 0
if ok.sum() == 0:
continue
try:
smooth_fiberflat = spline_fit(x, x[ok], fiberflat[fiber, ok],
smoothing_res,
fiberflat_ivar[fiber, ok])
fiberflat[fiber, bad] = smooth_fiberflat[bad]
except:
fiberflat[fiber, bad] = 1
fiberflat_ivar[fiber, bad] = 0
if nbad_tot > 0:
log.info(
"3rd pass : fiber #%d masked pixels = %d (%d iterations)" %
(fiber, nbad_tot, iteration))
# set median flat to 1
log.info("set median fiberflat to 1")
mean = np.ones((flux.shape[1]))
for i in range(flux.shape[1]):
ok = np.where((mask[:, i] == 0) & (ivar[:, i] > 0))[0]
if ok.size > 0:
mean[i] = np.median(fiberflat[ok, i])
ok = np.where(mean != 0)[0]
for fiber in range(nfibers):
fiberflat[fiber, ok] /= mean[ok]
log.info("done fiberflat")
return FiberFlat(wave,
fiberflat,
fiberflat_ivar,
mask,
mean_spectrum,
chi2pdf=chi2pdf)
def compute_fiberflat(frame, nsig_clipping=4.):
"""Compute fiber flat by deriving an average spectrum and dividing all fiber data by this average.
Input data are expected to be on the same wavelenght grid, with uncorrelated noise.
They however do not have exactly the same resolution.
args:
frame (desispec.Frame): input Frame object with attributes
wave, flux, ivar, resolution_data
nsig_clipping : [optional] sigma clipping value for outlier rejection
returns tuple (fiberflat, ivar, mask, meanspec):
fiberflat : 2D[nwave, nflux] fiberflat (data have to be divided by this to be flatfielded)
ivar : inverse variance of that fiberflat
mask : 0=ok >0 if problems
meanspec : deconvolved mean spectrum
- we first iteratively :
- compute a deconvolved mean spectrum
- compute a fiber flat using the resolution convolved mean spectrum for each fiber
- smooth the fiber flat along wavelength
- clip outliers
- then we compute a fiberflat at the native fiber resolution (not smoothed)
- the routine returns the fiberflat, its inverse variance , mask, and the deconvolved mean spectrum
- the fiberflat is the ratio data/mean , so this flat should be divided to the data
NOTE THAT THIS CODE HAS NOT BEEN TESTED WITH ACTUAL FIBER TRANSMISSION VARIATIONS,
OUTLIER PIXELS, DEAD COLUMNS ...
"""
log = get_logger()
log.info("starting")
#
# chi2 = sum_(fiber f) sum_(wavelenght i) w_fi ( D_fi - F_fi (R_f M)_i )
#
# where
# w = inverse variance
# D = flux data (at the resolution of the fiber)
# F = smooth fiber flat
# R = resolution data
# M = mean deconvolved spectrum
#
# M = A^{-1} B
# with
# A_kl = sum_(fiber f) sum_(wavelenght i) w_fi F_fi^2 (R_fki R_fli)
# B_k = sum_(fiber f) sum_(wavelenght i) w_fi D_fi F_fi R_fki
#
# defining R'_fi = sqrt(w_fi) F_fi R_fi
# and D'_fi = sqrt(w_fi) D_fi
#
# A = sum_(fiber f) R'_f R'_f^T
# B = sum_(fiber f) R'_f D'_f
# (it's faster that way, and we try to use sparse matrices as much as possible)
#
#- Shortcuts
nwave = frame.nwave
nfibers = frame.nspec
wave = frame.wave.copy() #- this will become part of output too
flux = frame.flux
ivar = frame.ivar
# iterative fitting and clipping to get precise mean spectrum
current_ivar = ivar.copy()
smooth_fiberflat = np.ones((frame.flux.shape))
chi2 = np.zeros((flux.shape))
sqrtwflat = np.sqrt(current_ivar) * smooth_fiberflat
sqrtwflux = np.sqrt(current_ivar) * flux
# test
#nfibers=20
nout_tot = 0
for iteration in range(20):
# fit mean spectrum
A = scipy.sparse.lil_matrix((nwave, nwave)).tocsr()
B = np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD = scipy.sparse.lil_matrix((nwave, nwave))
# loop on fiber to handle resolution
for fiber in range(nfibers):
if fiber % 10 == 0:
log.info("iter %d fiber %d" % (iteration, fiber))
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
# diagonal sparse matrix with content = sqrt(ivar)*flat
SD.setdiag(sqrtwflat[fiber])
sqrtwflatR = SD * R # each row r of R is multiplied by sqrtwflat[r]
A = A + (sqrtwflatR.T * sqrtwflatR).tocsr()
B += sqrtwflatR.T * sqrtwflux[fiber]
log.info("iter %d solving" % iteration)
mean_spectrum = cholesky_solve(A.todense(), B)
log.info("iter %d smoothing" % iteration)
# fit smooth fiberflat and compute chi2
smoothing_res = 100. #A
for fiber in range(nfibers):
#if fiber%10==0 :
# log.info("iter %d fiber %d (smoothing)"%(iteration,fiber))
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
#M = np.array(np.dot(R.todense(),mean_spectrum)).flatten()
M = R.dot(mean_spectrum)
F = flux[fiber] / (M + (M == 0))
smooth_fiberflat[fiber] = spline_fit(
wave, wave, F, smoothing_res, current_ivar[fiber] * (M != 0))
chi2[fiber] = current_ivar[fiber] * (
flux[fiber] - smooth_fiberflat[fiber] * M)**2
log.info("rejecting")
nout_iter = 0
if iteration < 1:
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave = np.sum(chi2 > nsig_clipping**2, axis=0)
selection = np.where(nout_per_wave > 0)[0]
for i in selection:
worst_entry = np.argmax(chi2[:, i])
current_ivar[worst_entry, i] = 0
sqrtwflat[worst_entry, i] = 0
sqrtwflux[worst_entry, i] = 0
nout_iter += 1
else:
# remove all of them at once
bad = (chi2 > nsig_clipping**2)
current_ivar *= (bad == 0)
sqrtwflat *= (bad == 0)
sqrtwflux *= (bad == 0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2 = float(np.sum(chi2))
ndf = int(np.sum(chi2 > 0) - nwave - nfibers * (nwave / smoothing_res))
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d" %
(iteration, sum_chi2, ndf, chi2pdf, nout_iter))
# normalize to get a mean fiberflat=1
mean = np.mean(smooth_fiberflat, axis=0)
smooth_fiberflat = smooth_fiberflat / mean
mean_spectrum = mean_spectrum * mean
if nout_iter == 0:
break
log.info("nout tot=%d" % nout_tot)
# now use mean spectrum to compute flat field correction without any smoothing
# because sharp feature can arise if dead columns
fiberflat = np.ones((flux.shape))
fiberflat_ivar = np.zeros((flux.shape))
mask = np.zeros((flux.shape)).astype(long) # SOMEONE CHECK THIS !
fiberflat_mask = 12 # place holder for actual mask bit when defined
nsig_for_mask = 4 # only mask out 4 sigma outliers
for fiber in range(nfibers):
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = np.array(np.dot(R.todense(), mean_spectrum)).flatten()
fiberflat[fiber] = (M != 0) * flux[fiber] / (M + (M == 0)) + (M == 0)
fiberflat_ivar[fiber] = ivar[fiber] * M**2
smooth_fiberflat = spline_fit(wave, wave, fiberflat[fiber],
smoothing_res,
current_ivar[fiber] * M**2 * (M != 0))
bad = np.where(
fiberflat_ivar[fiber] *
(fiberflat[fiber] - smooth_fiberflat)**2 > nsig_for_mask**2)[0]
if bad.size > 0:
mask[fiber, bad] += fiberflat_mask
return FiberFlat(wave, fiberflat, fiberflat_ivar, mask, mean_spectrum)
def compute_fiberflat(frame, nsig_clipping=10., accuracy=5.e-4, minval=0.1, maxval=10.,max_iterations=100,smoothing_res=5.,max_bad=100,max_rej_it=5,min_sn=0,diag_epsilon=1e-3) :
"""Compute fiber flat by deriving an average spectrum and dividing all fiber data by this average.
Input data are expected to be on the same wavelength grid, with uncorrelated noise.
They however do not have exactly the same resolution.
Args:
frame (desispec.Frame): input Frame object with attributes
wave, flux, ivar, resolution_data
nsig_clipping : [optional] sigma clipping value for outlier rejection
accuracy : [optional] accuracy of fiberflat (end test for the iterative loop)
minval: [optional] mask pixels with flux < minval * median fiberflat.
maxval: [optional] mask pixels with flux > maxval * median fiberflat.
max_iterations: [optional] maximum number of iterations
smoothing_res: [optional] spacing between spline fit nodes for smoothing the fiberflat
max_bad: [optional] mask entire fiber if more than max_bad-1 initially unmasked pixels are masked during the iterations
max_rej_it: [optional] reject at most the max_rej_it worst pixels in each iteration
min_sn: [optional] mask portions with signal to noise less than min_sn
diag_epsilon: [optional] size of the regularization term in the deconvolution
Returns:
desispec.FiberFlat object with attributes
wave, fiberflat, ivar, mask, meanspec
Notes:
- we first iteratively :
- compute a deconvolved mean spectrum
- compute a fiber flat using the resolution convolved mean spectrum for each fiber
- smooth the fiber flat along wavelength
- clip outliers
- then we compute a fiberflat at the native fiber resolution (not smoothed)
- the routine returns the fiberflat, its inverse variance , mask, and the deconvolved mean spectrum
- the fiberflat is the ratio data/mean , so this flat should be divided to the data
NOTE THAT THIS CODE HAS NOT BEEN TESTED WITH ACTUAL FIBER TRANSMISSION VARIATIONS,
OUTLIER PIXELS, DEAD COLUMNS ...
"""
log=get_logger()
log.info("starting")
#
# chi2 = sum_(fiber f) sum_(wavelenght i) w_fi ( D_fi - F_fi (R_f M)_i )
#
# where
# w = inverse variance
# D = flux data (at the resolution of the fiber)
# F = smooth fiber flat
# R = resolution data
# M = mean deconvolved spectrum
#
# M = A^{-1} B
# with
# A_kl = sum_(fiber f) sum_(wavelenght i) w_fi F_fi^2 (R_fki R_fli)
# B_k = sum_(fiber f) sum_(wavelenght i) w_fi D_fi F_fi R_fki
#
# defining R'_fi = sqrt(w_fi) F_fi R_fi
# and D'_fi = sqrt(w_fi) D_fi
#
# A = sum_(fiber f) R'_f R'_f^T
# B = sum_(fiber f) R'_f D'_f
# (it's faster that way, and we try to use sparse matrices as much as possible)
#
#- Shortcuts
nwave=frame.nwave
nfibers=frame.nspec
wave = frame.wave.copy() #- this will become part of output too
flux = frame.flux.copy()
ivar = frame.ivar*(frame.mask==0)
# iterative fitting and clipping to get precise mean spectrum
# we first need to iterate to converge on a solution of mean spectrum
# and smooth fiber flat. several interations are needed when
# throughput AND resolution vary from fiber to fiber.
# the end test is that the fiber flat has varied by less than accuracy
# of previous iteration for all wavelength
# we also have a max. number of iterations for this code
nout_tot=0
chi2pdf = 0.
smooth_fiberflat=np.ones((flux.shape))
chi2=np.zeros((flux.shape))
## mask low sn portions
w = flux*np.sqrt(ivar)<min_sn
ivar[w]=0
## 0th pass: reject pixels according to minval and maxval
mean_spectrum = np.zeros(flux.shape[1])
nbad=np.zeros(nfibers,dtype=int)
for iteration in range(max_iterations):
for i in range(flux.shape[1]):
w = ivar[:,i]>0
if w.sum()>0:
mean_spectrum[i] = np.median(flux[w,i])
nbad_it=0
for fib in range(nfibers):
w = ((flux[fib,:]<minval*mean_spectrum) | (flux[fib,:]>maxval*mean_spectrum)) & (ivar[fib,:]>0)
nbad_it+=w.sum()
nbad[fib]+=w.sum()
if w.sum()>0:
ivar[fib,w]=0
log.warning("0th pass: masking {} pixels in fiber {}".format(w.sum(),fib))
if nbad[fib]>=max_bad:
ivar[fib,:]=0
log.warning("0th pass: masking entire fiber {} (nbad={})".format(fib,nbad[fib]))
if nbad_it == 0:
break
# 1st pass is median for spectrum, flat field without resolution
# outlier rejection
for iteration in range(max_iterations) :
# use median for spectrum
mean_spectrum=np.zeros((flux.shape[1]))
for i in range(flux.shape[1]) :
w=ivar[:,i]>0
if w.sum() > 0 :
mean_spectrum[i]=np.median(flux[w,i])
nbad_it=0
sum_chi2 = 0
# not more than max_rej_it pixels per fiber at a time
for fib in range(nfibers) :
w=ivar[fib,:]>0
if w.sum()==0:
continue
F = flux[fib,:]*0
w=(mean_spectrum!=0) & (ivar[fib,:]>0)
F[w]= flux[fib,w]/mean_spectrum[w]
try :
smooth_fiberflat[fib,:] = spline_fit(wave,wave[w],F[w],smoothing_res,ivar[fib,w]*mean_spectrum[w]**2,max_resolution=1.5*smoothing_res)
except ValueError as err :
log.error("Error when smoothing the flat")
log.error("Setting ivar=0 for fiber {} because spline fit failed".format(fib))
ivar[fib,:] *= 0
chi2 = ivar[fib,:]*(flux[fib,:]-mean_spectrum*smooth_fiberflat[fib,:])**2
w=np.isnan(chi2)
bad=np.where(chi2>nsig_clipping**2)[0]
if bad.size>0 :
if bad.size>max_rej_it : # not more than 5 pixels at a time
ii=np.argsort(chi2[bad])
bad=bad[ii[-max_rej_it:]]
ivar[fib,bad] = 0
log.warning("1st pass: rejecting {} pixels from fiber {}".format(len(bad),fib))
nbad[fib]+=len(bad)
if nbad[fib]>=max_bad:
ivar[fib,:]=0
log.warning("1st pass: rejecting fiber {} due to too many (new) bad pixels".format(fib))
nbad_it+=len(bad)
sum_chi2+=chi2.sum()
ndf=int((ivar>0).sum()-nwave-nfibers*(nwave/smoothing_res))
chi2pdf=0.
if ndf>0 :
chi2pdf=sum_chi2/ndf
log.info("1st pass iter #{} chi2={}/{} chi2pdf={} nout={} (nsig={})".format(iteration,sum_chi2,ndf,chi2pdf,nbad_it,nsig_clipping))
if nbad_it == 0 :
break
## flatten fiberflat
## normalize smooth_fiberflat:
mean=np.ones(smooth_fiberflat.shape[1])
for i in range(smooth_fiberflat.shape[1]):
w=ivar[:,i]>0
if w.sum()>0:
mean[i]=np.median(smooth_fiberflat[w,i])
smooth_fiberflat = smooth_fiberflat/mean
median_spectrum = mean_spectrum*1.
previous_smooth_fiberflat = smooth_fiberflat*0
previous_max_diff = 0.
log.info("after 1st pass : nout = %d/%d"%(np.sum(ivar==0),np.size(ivar.flatten())))
# 2nd pass is full solution including deconvolved spectrum, no outlier rejection
for iteration in range(max_iterations) :
## reset sum_chi2
sum_chi2=0
log.info("2nd pass, iter %d : mean deconvolved spectrum"%iteration)
# fit mean spectrum
A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr()
B=np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD=scipy.sparse.lil_matrix((nwave,nwave))
# this is to go a bit faster
sqrtwflat=np.sqrt(ivar)*smooth_fiberflat
# loop on fiber to handle resolution (this is long)
for fiber in range(nfibers) :
if fiber%10==0 :
log.info("2nd pass, filling matrix, iter %d fiber %d"%(iteration,fiber))
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
SD.setdiag(sqrtwflat[fiber])
sqrtwflatR = SD*R # each row r of R is multiplied by sqrtwflat[r]
A = A+(sqrtwflatR.T*sqrtwflatR).tocsr()
B += sqrtwflatR.T.dot(np.sqrt(ivar[fiber])*flux[fiber])
A_pos_def = A.todense()
log.info("deconvolving")
w = A.diagonal() > 0
A_pos_def = A_pos_def[w,:]
A_pos_def = A_pos_def[:,w]
mean_spectrum = np.zeros(nwave)
try:
mean_spectrum[w]=cholesky_solve(A_pos_def,B[w])
except:
mean_spectrum[w]=np.linalg.lstsq(A_pos_def,B[w])[0]
log.info("cholesky failes, trying svd inverse in iter {}".format(iteration))
for fiber in range(nfibers) :
if np.sum(ivar[fiber]>0)==0 :
continue
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = R.dot(mean_spectrum)
ok=(M!=0) & (ivar[fiber,:]>0)
if ok.sum()==0:
continue
try :
smooth_fiberflat[fiber] = spline_fit(wave,wave[ok],flux[fiber,ok]/M[ok],smoothing_res,ivar[fiber,ok]*M[ok]**2,max_resolution=1.5*smoothing_res)*(ivar[fiber,:]*M**2>0)
except ValueError as err :
log.error("Error when smoothing the flat")
log.error("Setting ivar=0 for fiber {} because spline fit failed".format(fiber))
ivar[fiber,:] *= 0
chi2 = ivar[fiber]*(flux[fiber]-smooth_fiberflat[fiber]*M)**2
sum_chi2 += chi2.sum()
w=np.isnan(smooth_fiberflat[fiber])
if w.sum()>0:
ivar[fiber]=0
smooth_fiberflat[fiber]=1
# normalize to get a mean fiberflat=1
mean = np.ones(smooth_fiberflat.shape[1])
for i in range(nwave):
w = ivar[:,i]>0
if w.sum()>0:
mean[i]=np.median(smooth_fiberflat[w,i])
ok=np.where(mean!=0)[0]
smooth_fiberflat[:,ok] /= mean[ok]
# this is the max difference between two iterations
max_diff=np.max(np.abs(smooth_fiberflat-previous_smooth_fiberflat)*(ivar>0.))
previous_smooth_fiberflat=smooth_fiberflat.copy()
ndf=int(np.sum(ivar>0)-nwave-nfibers*(nwave/smoothing_res))
chi2pdf=0.
if ndf>0 :
chi2pdf=sum_chi2/ndf
log.info("2nd pass, iter %d, chi2=%f ndf=%d chi2pdf=%f"%(iteration,sum_chi2,ndf,chi2pdf))
if max_diff<accuracy :
break
if np.abs(max_diff-previous_max_diff)<accuracy*0.1 :
log.warning("no significant improvement on max diff, quit loop")
break
previous_max_diff=max_diff
log.info("2nd pass, iter %d, max diff. = %g > requirement = %g, continue iterating"%(iteration,max_diff,accuracy))
log.info("Total number of masked pixels=%d"%nout_tot)
log.info("3rd pass, final computation of fiber flat")
# now use mean spectrum to compute flat field correction without any smoothing
# because sharp feature can arise if dead columns
fiberflat=np.ones((flux.shape))
fiberflat_ivar=np.zeros((flux.shape))
mask=np.zeros((flux.shape), dtype='uint32')
# reset ivar
ivar=frame.ivar
fiberflat_mask=12 # place holder for actual mask bit when defined
nsig_for_mask=nsig_clipping # only mask out N sigma outliers
for fiber in range(nfibers) :
if np.sum(ivar[fiber]>0)==0 :
continue
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = np.array(np.dot(R.todense(),mean_spectrum)).flatten()
fiberflat[fiber] = (M!=0)*flux[fiber]/(M+(M==0)) + (M==0)
fiberflat_ivar[fiber] = ivar[fiber]*M**2
nbad_tot=0
iteration=0
while iteration<500 :
w=fiberflat_ivar[fiber,:]>0
if w.sum()<100:
break
smooth_fiberflat=spline_fit(wave,wave[w],fiberflat[fiber,w],smoothing_res,fiberflat_ivar[fiber,w])
chi2=fiberflat_ivar[fiber]*(fiberflat[fiber]-smooth_fiberflat)**2
bad=np.where(chi2>nsig_for_mask**2)[0]
if bad.size>0 :
nbadmax=1
if bad.size>nbadmax : # not more than nbadmax pixels at a time
ii=np.argsort(chi2[bad])
bad=bad[ii[-nbadmax:]]
mask[fiber,bad] += fiberflat_mask
fiberflat_ivar[fiber,bad] = 0.
nbad_tot += bad.size
else :
break
iteration += 1
log.info("3rd pass : fiber #%d , number of iterations %d"%(fiber,iteration))
# set median flat to 1
log.info("3rd pass : set median fiberflat to 1")
mean=np.ones((flux.shape[1]))
for i in range(flux.shape[1]) :
ok=np.where((mask[:,i]==0)&(ivar[:,i]>0))[0]
if ok.size > 0 :
mean[i] = np.median(fiberflat[ok,i])
ok=np.where(mean!=0)[0]
for fiber in range(nfibers) :
fiberflat[fiber,ok] /= mean[ok]
log.info("3rd pass : interpolating over masked pixels")
for fiber in range(nfibers) :
if np.sum(ivar[fiber]>0)==0 :
continue
# replace bad by smooth fiber flat
bad=np.where((mask[fiber]>0)|(fiberflat_ivar[fiber]==0)|(fiberflat[fiber]<minval)|(fiberflat[fiber]>maxval))[0]
if bad.size>0 :
fiberflat_ivar[fiber,bad] = 0
# find max length of segment with bad pix
length=0
for i in range(bad.size) :
ib=bad[i]
ilength=1
tmp=ib
for jb in bad[i+1:] :
if jb==tmp+1 :
ilength +=1
tmp=jb
else :
break
length=max(length,ilength)
if length>10 :
log.info("3rd pass : fiber #%d has a max length of bad pixels=%d"%(fiber,length))
smoothing_res=float(max(100,length))
x=np.arange(wave.size)
ok=fiberflat_ivar[fiber]>0
if ok.sum()==0:
continue
try:
smooth_fiberflat=spline_fit(x,x[ok],fiberflat[fiber,ok],smoothing_res,fiberflat_ivar[fiber,ok])
fiberflat[fiber,bad] = smooth_fiberflat[bad]
except:
fiberflat[fiber,bad] = 1
fiberflat_ivar[fiber,bad]=0
if nbad_tot>0 :
log.info("3rd pass : fiber #%d masked pixels = %d (%d iterations)"%(fiber,nbad_tot,iteration))
# set median flat to 1
log.info("set median fiberflat to 1")
mean=np.ones((flux.shape[1]))
for i in range(flux.shape[1]) :
ok=np.where((mask[:,i]==0)&(ivar[:,i]>0))[0]
if ok.size > 0 :
mean[i] = np.median(fiberflat[ok,i])
ok=np.where(mean!=0)[0]
for fiber in range(nfibers) :
fiberflat[fiber,ok] /= mean[ok]
log.info("done fiberflat")
log.info("add a systematic error of 0.0035 to fiberflat variance (calibrated on sims)")
fiberflat_ivar = (fiberflat_ivar>0)/( 1./ (fiberflat_ivar+(fiberflat_ivar==0) ) + 0.0035**2)
return FiberFlat(wave, fiberflat, fiberflat_ivar, mask, mean_spectrum,
chi2pdf=chi2pdf)
def qproc_compute_fiberflat(qframe,niter_meanspec=4,nsig_clipping=3.,spline_res_clipping=20.,spline_res_flat=5.) :
"""
Fast estimation of fiberflat
"""
log = get_logger()
t0=time.time()
log.info("Starting...")
twave=np.mean(qframe.wave,axis=0)
tflux=np.zeros(qframe.flux.shape)
tivar=np.zeros(qframe.flux.shape)
if qframe.mask is not None :
qframe.ivar *= (qframe.mask==0)
for i in range(qframe.flux.shape[0]) :
jj=(qframe.ivar[i]>0)
tflux[i]=np.interp(twave,qframe.wave[i,jj],qframe.flux[i,jj])
tivar[i]=np.interp(twave,qframe.wave[i,jj],qframe.ivar[i,jj],left=0,right=0)
# iterative loop to a absorb constant term in fiber (should have more parameters)
a=np.ones(tflux.shape[0])
for iter in range(niter_meanspec) :
mflux=np.median(a[:,np.newaxis]*tflux,axis=0)
for i in range(qframe.flux.shape[0]) :
a[i] = np.median(tflux[i,mflux>0]/mflux[mflux>0])
# trivial fiberflat
fflat=tflux/(mflux+(mflux==0))
fivar=tivar*mflux**2
mask=np.zeros((fflat.shape), dtype='uint32')
chi2=0
# spline fit to reject outliers and smooth the flat
for fiber in range(fflat.shape[0]) :
# iterative spline fit
max_rej_it=5# not more than 5 pixels at a time
max_bad=1000
nbad_tot=0
for loop in range(20) :
good=(fivar[fiber]>0)
splineflat = spline_fit(twave,twave[good],fflat[fiber,good],required_resolution=spline_res_clipping,input_ivar=fivar[fiber,good],max_resolution=3*spline_res_clipping)
fchi2 = fivar[fiber]*(fflat[fiber]-splineflat)**2
bad=np.where(fchi2>nsig_clipping**2)[0]
if bad.size>0 :
if bad.size>max_rej_it : # not more than 5 pixels at a time
ii=np.argsort(fchi2[bad])
bad=bad[ii[-max_rej_it:]]
fivar[fiber,bad] = 0
nbad_tot += len(bad)
#log.warning("iteration {} rejecting {} pixels (tot={}) from fiber {}".format(loop,len(bad),nbad_tot,fiber))
if nbad_tot>=max_bad:
fivar[fiber,:]=0
log.warning("1st pass: rejecting fiber {} due to too many (new) bad pixels".format(fiber))
else :
break
chi2 += np.sum(fchi2)
min_ivar = 0.1*np.median(fivar[fiber])
med_flat = np.median(fflat[fiber])
good=(fivar[fiber]>0)
splineflat = spline_fit(twave,twave[good],fflat[fiber,good],required_resolution=spline_res_flat,input_ivar=fivar[fiber,good],max_resolution=3*spline_res_flat)
fflat[fiber] = splineflat # replace by spline
ii=np.where(fivar[fiber]>min_ivar)[0]
if ii.size<2 :
fflat[fiber] = 1
fivar[fiber] = 0
# set flat in unkown edges to median value of fiber (and ivar to 0)
b=ii[0]
e=ii[-1]+1
fflat[fiber,:b]=med_flat # default
fivar[fiber,:b]=0
mask[fiber,:b]=1 # need to change this
fflat[fiber,e:]=med_flat # default
fivar[fiber,e:]=0
mask[fiber,e:]=1 # need to change this
# internal interpolation
bad=(fivar[fiber][b:e]<=min_ivar)
good=(fivar[fiber][b:e]>min_ivar)
fflat[fiber][b:e][bad]=np.interp(twave[b:e][bad],twave[b:e][good],fflat[fiber][b:e][good])
ndata=np.sum(fivar>0)
if ndata>0 :
chi2pdf = chi2/ndata
else :
chi2pdf = 0
t1=time.time()
log.info(" done in {:3.1f} sec".format(t1-t0))
# return a fiberflat object ...
return FiberFlat(twave, fflat, fivar, mask, mflux,chi2pdf=chi2pdf)
def compute_fiberflat(frame, nsig_clipping=4., accuracy=5.e-4, minval=0.1, maxval=10.) :
"""Compute fiber flat by deriving an average spectrum and dividing all fiber data by this average.
Input data are expected to be on the same wavelength grid, with uncorrelated noise.
They however do not have exactly the same resolution.
Args:
frame (desispec.Frame): input Frame object with attributes
wave, flux, ivar, resolution_data
nsig_clipping : [optional] sigma clipping value for outlier rejection
accuracy : [optional] accuracy of fiberflat (end test for the iterative loop)
Returns:
desispec.FiberFlat object with attributes
wave, fiberflat, ivar, mask, meanspec
Notes:
- we first iteratively :
- compute a deconvolved mean spectrum
- compute a fiber flat using the resolution convolved mean spectrum for each fiber
- smooth the fiber flat along wavelength
- clip outliers
- then we compute a fiberflat at the native fiber resolution (not smoothed)
- the routine returns the fiberflat, its inverse variance , mask, and the deconvolved mean spectrum
- the fiberflat is the ratio data/mean , so this flat should be divided to the data
NOTE THAT THIS CODE HAS NOT BEEN TESTED WITH ACTUAL FIBER TRANSMISSION VARIATIONS,
OUTLIER PIXELS, DEAD COLUMNS ...
"""
log=get_logger()
log.info("starting")
#
# chi2 = sum_(fiber f) sum_(wavelenght i) w_fi ( D_fi - F_fi (R_f M)_i )
#
# where
# w = inverse variance
# D = flux data (at the resolution of the fiber)
# F = smooth fiber flat
# R = resolution data
# M = mean deconvolved spectrum
#
# M = A^{-1} B
# with
# A_kl = sum_(fiber f) sum_(wavelenght i) w_fi F_fi^2 (R_fki R_fli)
# B_k = sum_(fiber f) sum_(wavelenght i) w_fi D_fi F_fi R_fki
#
# defining R'_fi = sqrt(w_fi) F_fi R_fi
# and D'_fi = sqrt(w_fi) D_fi
#
# A = sum_(fiber f) R'_f R'_f^T
# B = sum_(fiber f) R'_f D'_f
# (it's faster that way, and we try to use sparse matrices as much as possible)
#
#- Shortcuts
nwave=frame.nwave
nfibers=frame.nspec
wave = frame.wave.copy() #- this will become part of output too
flux = frame.flux
ivar = frame.ivar*(frame.mask==0)
# iterative fitting and clipping to get precise mean spectrum
# we first need to iterate to converge on a solution of mean spectrum
# and smooth fiber flat. several interations are needed when
# throughput AND resolution vary from fiber to fiber.
# the end test is that the fiber flat has varied by less than accuracy
# of previous iteration for all wavelength
# we also have a max. number of iterations for this code
max_iterations = 100
nout_tot=0
chi2pdf = 0.
smooth_fiberflat=np.ones((frame.flux.shape))
previous_smooth_fiberflat=smooth_fiberflat.copy()
chi2=np.zeros((flux.shape))
# 1st pass is median for spectrum, flat field without resolution
# outlier rejection
for iteration in range(max_iterations) :
# use median for spectrum
mean_spectrum=np.zeros((flux.shape[1]))
for i in range(flux.shape[1]) :
ok=np.where(ivar[:,i]>0)[0]
if ok.size > 0 :
mean_spectrum[i]=np.median(flux[ok,i])
# max pixels far from mean spectrum.
#log.info("mask pixels with difference smaller than %f or larger than %f of mean")
nout_iter=0
for fiber in range(nfibers) :
bad=np.where((ivar[fiber]>0)&((flux[fiber]>maxval*mean_spectrum)|(flux[fiber]<minval*mean_spectrum)))[0]
if bad.size>100 :
log.warning("masking fiber %d because of bad flat field with %d bad pixels"%(fiber,bad.size))
ivar[fiber]=0.
if bad.size>0 :
log.warning("masking %d bad pixels for fiber %d"%(bad.size,fiber))
ivar[fiber,bad]=0.
nout_iter += bad.size
# fit smooth fiberflat and compute chi2
smoothing_res=100. #A
for fiber in range(nfibers) :
if np.sum(ivar[fiber]>0)==0 :
continue
F = np.ones((flux.shape[1]))
ok=np.where((mean_spectrum!=0)&(ivar[fiber]>0))[0]
F[ok] = flux[fiber,ok]/mean_spectrum[ok]
smooth_fiberflat[fiber]=spline_fit(wave,wave[ok],F[ok],smoothing_res,ivar[fiber,ok])
# normalize to get a mean fiberflat=1
mean=np.mean(smooth_fiberflat,axis=0)
ok=np.where(mean!=0)[0]
for fiber in range(nfibers) :
smooth_fiberflat[fiber,ok] = smooth_fiberflat[fiber,ok]/mean[ok]
mean_spectrum *= mean
# this is the max difference between two iterations
max_diff=np.max(np.abs(smooth_fiberflat-previous_smooth_fiberflat)*(ivar>0.))
previous_smooth_fiberflat=smooth_fiberflat.copy()
# we don't start the rejection tests until we have converged on this
if max_diff>0.01 :
log.info("1st pass, max diff. = %g > 0.01 , continue iterating before outlier rejection"%(max_diff))
continue
chi2=ivar*(flux-smooth_fiberflat*mean_spectrum)**2
if True :
nsig_clipping_for_this_pass = nsig_clipping
# not more than 5 pixels per fiber at a time
for fiber in range(nfibers) :
for loop in range(max_iterations) :
bad=np.where(chi2[fiber]>nsig_clipping_for_this_pass**2)[0]
if bad.size>0 :
if bad.size>5 : # not more than 5 pixels at a time
ii=np.argsort(chi2[fiber,bad])
bad=bad[ii[-5:]]
ivar[fiber,bad] = 0
nout_iter += bad.size
ok=np.where((mean_spectrum!=0)&(ivar[fiber]>0))[0]
F[ok] = flux[fiber,ok]/mean_spectrum[ok]
smooth_fiberflat[fiber]=spline_fit(wave,wave[ok],F[ok],smoothing_res,ivar[fiber,ok])
chi2[fiber]=ivar[fiber]*(flux[fiber]-smooth_fiberflat[fiber]*mean_spectrum)**2
else :
break
nout_tot += nout_iter
sum_chi2=float(np.sum(chi2))
ndf=int(np.sum(chi2>0)-nwave-nfibers*(nwave/smoothing_res))
chi2pdf=0.
if ndf>0 :
chi2pdf=sum_chi2/ndf
log.info("1st pass iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d (nsig=%f)"%(iteration,sum_chi2,ndf,chi2pdf,nout_iter,nsig_clipping_for_this_pass))
if max_diff>accuracy :
log.info("1st pass iter #%d max diff. = %g > requirement = %g , continue iterating"%(iteration,max_diff,accuracy))
continue
if nout_iter == 0 :
break
log.info("after 1st pass : nout = %d/%d"%(np.sum(ivar==0),np.size(ivar.flatten())))
# 2nd pass is full solution including deconvolved spectrum, no outlier rejection
for iteration in range(max_iterations) :
log.info("2nd pass, iter %d : mean deconvolved spectrum"%iteration)
# fit mean spectrum
A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr()
B=np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD=scipy.sparse.lil_matrix((nwave,nwave))
# this is to go a bit faster
sqrtwflat=np.sqrt(ivar)*smooth_fiberflat
# loop on fiber to handle resolution (this is long)
for fiber in range(nfibers) :
if fiber%10==0 :
log.info("2nd pass, filling matrix, iter %d fiber %d"%(iteration,fiber))
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
SD.setdiag(sqrtwflat[fiber])
sqrtwflatR = SD*R # each row r of R is multiplied by sqrtwflat[r]
A = A+(sqrtwflatR.T*sqrtwflatR).tocsr()
B += sqrtwflatR.T.dot(np.sqrt(ivar[fiber])*flux[fiber])
mean_spectrum=cholesky_solve(A.todense(),B)
# fit smooth fiberflat
smoothing_res=100. #A
for fiber in range(nfibers) :
if np.sum(ivar[fiber]>0)==0 :
continue
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = R.dot(mean_spectrum)
ok=np.where(M!=0)[0]
smooth_fiberflat[fiber]=spline_fit(wave,wave[ok],flux[fiber,ok]/M[ok],smoothing_res,ivar[fiber,ok])
# normalize to get a mean fiberflat=1
mean=np.mean(smooth_fiberflat,axis=0)
ok=np.where(mean!=0)[0]
smooth_fiberflat[:,ok] /= mean[ok]
mean_spectrum *= mean
chi2=ivar*(flux-smooth_fiberflat*mean_spectrum)**2
# this is the max difference between two iterations
max_diff=np.max(np.abs(smooth_fiberflat-previous_smooth_fiberflat)*(ivar>0.))
previous_smooth_fiberflat=smooth_fiberflat.copy()
sum_chi2=float(np.sum(chi2))
ndf=int(np.sum(chi2>0)-nwave-nfibers*(nwave/smoothing_res))
chi2pdf=0.
if ndf>0 :
chi2pdf=sum_chi2/ndf
log.info("2nd pass, iter %d, chi2=%f ndf=%d chi2pdf=%f"%(iteration,sum_chi2,ndf,chi2pdf))
if max_diff<accuracy :
break
log.info("2nd pass, iter %d, max diff. = %g > requirement = %g, continue iterating"%(iteration,max_diff,accuracy))
log.info("Total number of masked pixels=%d"%nout_tot)
log.info("3rd pass, final computation of fiber flat")
# now use mean spectrum to compute flat field correction without any smoothing
# because sharp feature can arise if dead columns
fiberflat=np.ones((flux.shape))
fiberflat_ivar=np.zeros((flux.shape))
mask=np.zeros((flux.shape)).astype(long) # SOMEONE CHECK THIS !
# reset ivar
ivar=frame.ivar
fiberflat_mask=12 # place holder for actual mask bit when defined
nsig_for_mask=nsig_clipping # only mask out N sigma outliers
for fiber in range(nfibers) :
if np.sum(ivar[fiber]>0)==0 :
continue
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = np.array(np.dot(R.todense(),mean_spectrum)).flatten()
fiberflat[fiber] = (M!=0)*flux[fiber]/(M+(M==0)) + (M==0)
fiberflat_ivar[fiber] = ivar[fiber]*M**2
nbad_tot=0
iteration=0
while iteration<500 :
smooth_fiberflat=spline_fit(wave,wave,fiberflat[fiber],smoothing_res,fiberflat_ivar[fiber])
chi2=fiberflat_ivar[fiber]*(fiberflat[fiber]-smooth_fiberflat)**2
bad=np.where(chi2>nsig_for_mask**2)[0]
if bad.size>0 :
if bad.size>5 : # not more than 5 pixels at a time
ii=np.argsort(chi2[bad])
bad=bad[ii[-5:]]
mask[fiber,bad] += fiberflat_mask
fiberflat_ivar[fiber,bad] = 0.
nbad_tot += bad.size
else :
break
iteration += 1
# replace bad by smooth fiber flat
bad=np.where((mask[fiber]>0)|(fiberflat_ivar[fiber]==0)|(fiberflat[fiber]<minval)|(fiberflat[fiber]>maxval))[0]
if bad.size>0 :
fiberflat_ivar[fiber,bad] = 0
# find max length of segment with bad pix
length=0
for i in range(bad.size) :
ib=bad[i]
ilength=1
tmp=ib
for jb in bad[i+1:] :
if jb==tmp+1 :
ilength +=1
tmp=jb
else :
break
length=max(length,ilength)
if length>10 :
log.info("3rd pass : fiber #%d has a max length of bad pixels=%d"%(fiber,length))
smoothing_res=float(max(100,2*length))
x=np.arange(wave.size)
ok=np.where(fiberflat_ivar[fiber]>0)[0]
smooth_fiberflat=spline_fit(x,x[ok],fiberflat[fiber,ok],smoothing_res,fiberflat_ivar[fiber,ok])
fiberflat[fiber,bad] = smooth_fiberflat[bad]
if nbad_tot>0 :
log.info("3rd pass : fiber #%d masked pixels = %d (%d iterations)"%(fiber,nbad_tot,iteration))
# set median flat to 1
log.info("set median fiberflat to 1")
mean=np.ones((flux.shape[1]))
for i in range(flux.shape[1]) :
ok=np.where((mask[:,i]==0)&(ivar[:,i]>0))[0]
if ok.size > 0 :
mean[i] = np.median(fiberflat[ok,i])
ok=np.where(mean!=0)[0]
for fiber in range(nfibers) :
fiberflat[fiber,ok] /= mean[ok]
log.info("done fiberflat")
return FiberFlat(wave, fiberflat, fiberflat_ivar, mask, mean_spectrum,
chi2pdf=chi2pdf)
def qproc_compute_fiberflat(qframe,
niter_meanspec=4,
nsig_clipping=3.,
spline_res_clipping=20.,
spline_res_flat=5.):
"""
Fast estimation of fiberflat
"""
log = get_logger()
t0 = time.time()
log.info("Starting...")
twave = np.mean(qframe.wave, axis=0)
tflux = np.zeros(qframe.flux.shape)
tivar = np.zeros(qframe.flux.shape)
if qframe.mask is not None:
qframe.ivar *= (qframe.mask == 0)
for i in range(qframe.flux.shape[0]):
jj = (qframe.ivar[i] > 0)
tflux[i] = np.interp(twave, qframe.wave[i, jj], qframe.flux[i, jj])
tivar[i] = np.interp(twave,
qframe.wave[i, jj],
qframe.ivar[i, jj],
left=0,
right=0)
# iterative loop to a absorb constant term in fiber (should have more parameters)
a = np.ones(tflux.shape[0])
for iter in range(niter_meanspec):
mflux = np.median(a[:, np.newaxis] * tflux, axis=0)
for i in range(qframe.flux.shape[0]):
a[i] = np.median(tflux[i, mflux > 0] / mflux[mflux > 0])
# trivial fiberflat
fflat = tflux / (mflux + (mflux == 0))
fivar = tivar * mflux**2
mask = np.zeros((fflat.shape), dtype='uint32')
chi2 = 0
# spline fit to reject outliers and smooth the flat
for fiber in range(fflat.shape[0]):
# iterative spline fit
max_rej_it = 5 # not more than 5 pixels at a time
max_bad = 1000
nbad_tot = 0
for loop in range(20):
good = (fivar[fiber] > 0)
splineflat = spline_fit(twave,
twave[good],
fflat[fiber, good],
required_resolution=spline_res_clipping,
input_ivar=fivar[fiber, good],
max_resolution=3 * spline_res_clipping)
fchi2 = fivar[fiber] * (fflat[fiber] - splineflat)**2
bad = np.where(fchi2 > nsig_clipping**2)[0]
if bad.size > 0:
if bad.size > max_rej_it: # not more than 5 pixels at a time
ii = np.argsort(fchi2[bad])
bad = bad[ii[-max_rej_it:]]
fivar[fiber, bad] = 0
nbad_tot += len(bad)
#log.warning("iteration {} rejecting {} pixels (tot={}) from fiber {}".format(loop,len(bad),nbad_tot,fiber))
if nbad_tot >= max_bad:
fivar[fiber, :] = 0
log.warning(
"1st pass: rejecting fiber {} due to too many (new) bad pixels"
.format(fiber))
else:
break
chi2 += np.sum(fchi2)
min_ivar = 0.1 * np.median(fivar[fiber])
med_flat = np.median(fflat[fiber])
good = (fivar[fiber] > 0)
splineflat = spline_fit(twave,
twave[good],
fflat[fiber, good],
required_resolution=spline_res_flat,
input_ivar=fivar[fiber, good],
max_resolution=3 * spline_res_flat)
fflat[fiber] = splineflat # replace by spline
ii = np.where(fivar[fiber] > min_ivar)[0]
if ii.size < 2:
fflat[fiber] = 1
fivar[fiber] = 0
# set flat in unkown edges to median value of fiber (and ivar to 0)
b = ii[0]
e = ii[-1] + 1
fflat[fiber, :b] = med_flat # default
fivar[fiber, :b] = 0
mask[fiber, :b] = 1 # need to change this
fflat[fiber, e:] = med_flat # default
fivar[fiber, e:] = 0
mask[fiber, e:] = 1 # need to change this
# internal interpolation
bad = (fivar[fiber][b:e] <= min_ivar)
good = (fivar[fiber][b:e] > min_ivar)
fflat[fiber][b:e][bad] = np.interp(twave[b:e][bad], twave[b:e][good],
fflat[fiber][b:e][good])
ndata = np.sum(fivar > 0)
if ndata > 0:
chi2pdf = chi2 / ndata
else:
chi2pdf = 0
t1 = time.time()
log.info(" done in {:3.1f} sec".format(t1 - t0))
# return a fiberflat object ...
return FiberFlat(twave, fflat, fivar, mask, mflux, chi2pdf=chi2pdf)
def compute_fiberflat(frame,
nsig_clipping=4.,
accuracy=5.e-4,
minval=0.1,
maxval=10.):
"""Compute fiber flat by deriving an average spectrum and dividing all fiber data by this average.
Input data are expected to be on the same wavelength grid, with uncorrelated noise.
They however do not have exactly the same resolution.
Args:
frame (desispec.Frame): input Frame object with attributes
wave, flux, ivar, resolution_data
nsig_clipping : [optional] sigma clipping value for outlier rejection
accuracy : [optional] accuracy of fiberflat (end test for the iterative loop)
Returns:
desispec.FiberFlat object with attributes
wave, fiberflat, ivar, mask, meanspec
Notes:
- we first iteratively :
- compute a deconvolved mean spectrum
- compute a fiber flat using the resolution convolved mean spectrum for each fiber
- smooth the fiber flat along wavelength
- clip outliers
- then we compute a fiberflat at the native fiber resolution (not smoothed)
- the routine returns the fiberflat, its inverse variance , mask, and the deconvolved mean spectrum
- the fiberflat is the ratio data/mean , so this flat should be divided to the data
NOTE THAT THIS CODE HAS NOT BEEN TESTED WITH ACTUAL FIBER TRANSMISSION VARIATIONS,
OUTLIER PIXELS, DEAD COLUMNS ...
"""
log = get_logger()
log.info("starting")
#
# chi2 = sum_(fiber f) sum_(wavelenght i) w_fi ( D_fi - F_fi (R_f M)_i )
#
# where
# w = inverse variance
# D = flux data (at the resolution of the fiber)
# F = smooth fiber flat
# R = resolution data
# M = mean deconvolved spectrum
#
# M = A^{-1} B
# with
# A_kl = sum_(fiber f) sum_(wavelenght i) w_fi F_fi^2 (R_fki R_fli)
# B_k = sum_(fiber f) sum_(wavelenght i) w_fi D_fi F_fi R_fki
#
# defining R'_fi = sqrt(w_fi) F_fi R_fi
# and D'_fi = sqrt(w_fi) D_fi
#
# A = sum_(fiber f) R'_f R'_f^T
# B = sum_(fiber f) R'_f D'_f
# (it's faster that way, and we try to use sparse matrices as much as possible)
#
#- Shortcuts
nwave = frame.nwave
nfibers = frame.nspec
wave = frame.wave.copy() #- this will become part of output too
flux = frame.flux
ivar = frame.ivar * (frame.mask == 0)
# iterative fitting and clipping to get precise mean spectrum
# we first need to iterate to converge on a solution of mean spectrum
# and smooth fiber flat. several interations are needed when
# throughput AND resolution vary from fiber to fiber.
# the end test is that the fiber flat has varied by less than accuracy
# of previous iteration for all wavelength
# we also have a max. number of iterations for this code
max_iterations = 100
nout_tot = 0
chi2pdf = 0.
smooth_fiberflat = np.ones((frame.flux.shape))
previous_smooth_fiberflat = smooth_fiberflat.copy()
chi2 = np.zeros((flux.shape))
# 1st pass is median for spectrum, flat field without resolution
# outlier rejection
for iteration in range(max_iterations):
# use median for spectrum
mean_spectrum = np.zeros((flux.shape[1]))
for i in range(flux.shape[1]):
ok = np.where(ivar[:, i] > 0)[0]
if ok.size > 0:
mean_spectrum[i] = np.median(flux[ok, i])
# max pixels far from mean spectrum.
#log.info("mask pixels with difference smaller than %f or larger than %f of mean")
nout_iter = 0
for fiber in range(nfibers):
bad = np.where((ivar[fiber] > 0)
& ((flux[fiber] > maxval * mean_spectrum)
| (flux[fiber] < minval * mean_spectrum)))[0]
if bad.size > 100:
log.warning(
"masking fiber %d because of bad flat field with %d bad pixels"
% (fiber, bad.size))
ivar[fiber] = 0.
if bad.size > 0:
log.warning("masking %d bad pixels for fiber %d" %
(bad.size, fiber))
ivar[fiber, bad] = 0.
nout_iter += bad.size
# fit smooth fiberflat and compute chi2
smoothing_res = 100. #A
for fiber in range(nfibers):
if np.sum(ivar[fiber] > 0) == 0:
continue
F = np.ones((flux.shape[1]))
ok = np.where((mean_spectrum != 0) & (ivar[fiber] > 0))[0]
F[ok] = flux[fiber, ok] / mean_spectrum[ok]
smooth_fiberflat[fiber] = spline_fit(wave, wave[ok], F[ok],
smoothing_res, ivar[fiber,
ok])
# normalize to get a mean fiberflat=1
mean = np.mean(smooth_fiberflat, axis=0)
ok = np.where(mean != 0)[0]
for fiber in range(nfibers):
smooth_fiberflat[fiber,
ok] = smooth_fiberflat[fiber, ok] / mean[ok]
mean_spectrum *= mean
# this is the max difference between two iterations
max_diff = np.max(
np.abs(smooth_fiberflat - previous_smooth_fiberflat) * (ivar > 0.))
previous_smooth_fiberflat = smooth_fiberflat.copy()
# we don't start the rejection tests until we have converged on this
if max_diff > 0.01:
log.info(
"1st pass, max diff. = %g > 0.01 , continue iterating before outlier rejection"
% (max_diff))
continue
chi2 = ivar * (flux - smooth_fiberflat * mean_spectrum)**2
if True:
nsig_clipping_for_this_pass = nsig_clipping
# not more than 5 pixels per fiber at a time
for fiber in range(nfibers):
for loop in range(max_iterations):
bad = np.where(
chi2[fiber] > nsig_clipping_for_this_pass**2)[0]
if bad.size > 0:
if bad.size > 5: # not more than 5 pixels at a time
ii = np.argsort(chi2[fiber, bad])
bad = bad[ii[-5:]]
ivar[fiber, bad] = 0
nout_iter += bad.size
ok = np.where((mean_spectrum != 0)
& (ivar[fiber] > 0))[0]
F[ok] = flux[fiber, ok] / mean_spectrum[ok]
smooth_fiberflat[fiber] = spline_fit(
wave, wave[ok], F[ok], smoothing_res, ivar[fiber,
ok])
chi2[fiber] = ivar[fiber] * (
flux[fiber] -
smooth_fiberflat[fiber] * mean_spectrum)**2
else:
break
nout_tot += nout_iter
sum_chi2 = float(np.sum(chi2))
ndf = int(
np.sum(chi2 > 0) - nwave - nfibers * (nwave / smoothing_res))
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info(
"1st pass iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d (nsig=%f)"
% (iteration, sum_chi2, ndf, chi2pdf, nout_iter,
nsig_clipping_for_this_pass))
if max_diff > accuracy:
log.info(
"1st pass iter #%d max diff. = %g > requirement = %g , continue iterating"
% (iteration, max_diff, accuracy))
continue
if nout_iter == 0:
break
log.info("after 1st pass : nout = %d/%d" %
(np.sum(ivar == 0), np.size(ivar.flatten())))
# 2nd pass is full solution including deconvolved spectrum, no outlier rejection
for iteration in range(max_iterations):
log.info("2nd pass, iter %d : mean deconvolved spectrum" % iteration)
# fit mean spectrum
A = scipy.sparse.lil_matrix((nwave, nwave)).tocsr()
B = np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD = scipy.sparse.lil_matrix((nwave, nwave))
# this is to go a bit faster
sqrtwflat = np.sqrt(ivar) * smooth_fiberflat
# loop on fiber to handle resolution (this is long)
for fiber in range(nfibers):
if fiber % 10 == 0:
log.info("2nd pass, filling matrix, iter %d fiber %d" %
(iteration, fiber))
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
SD.setdiag(sqrtwflat[fiber])
sqrtwflatR = SD * R # each row r of R is multiplied by sqrtwflat[r]
A = A + (sqrtwflatR.T * sqrtwflatR).tocsr()
B += sqrtwflatR.T.dot(np.sqrt(ivar[fiber]) * flux[fiber])
mean_spectrum = cholesky_solve(A.todense(), B)
# fit smooth fiberflat
smoothing_res = 100. #A
for fiber in range(nfibers):
if np.sum(ivar[fiber] > 0) == 0:
continue
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = R.dot(mean_spectrum)
ok = np.where(M != 0)[0]
smooth_fiberflat[fiber] = spline_fit(wave, wave[ok],
flux[fiber, ok] / M[ok],
smoothing_res, ivar[fiber,
ok])
# normalize to get a mean fiberflat=1
mean = np.mean(smooth_fiberflat, axis=0)
ok = np.where(mean != 0)[0]
smooth_fiberflat[:, ok] /= mean[ok]
mean_spectrum *= mean
chi2 = ivar * (flux - smooth_fiberflat * mean_spectrum)**2
# this is the max difference between two iterations
max_diff = np.max(
np.abs(smooth_fiberflat - previous_smooth_fiberflat) * (ivar > 0.))
previous_smooth_fiberflat = smooth_fiberflat.copy()
sum_chi2 = float(np.sum(chi2))
ndf = int(np.sum(chi2 > 0) - nwave - nfibers * (nwave / smoothing_res))
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info("2nd pass, iter %d, chi2=%f ndf=%d chi2pdf=%f" %
(iteration, sum_chi2, ndf, chi2pdf))
if max_diff < accuracy:
break
log.info(
"2nd pass, iter %d, max diff. = %g > requirement = %g, continue iterating"
% (iteration, max_diff, accuracy))
log.info("Total number of masked pixels=%d" % nout_tot)
log.info("3rd pass, final computation of fiber flat")
# now use mean spectrum to compute flat field correction without any smoothing
# because sharp feature can arise if dead columns
fiberflat = np.ones((flux.shape))
fiberflat_ivar = np.zeros((flux.shape))
mask = np.zeros((flux.shape), dtype='uint32')
# reset ivar
ivar = frame.ivar
fiberflat_mask = 12 # place holder for actual mask bit when defined
nsig_for_mask = nsig_clipping # only mask out N sigma outliers
for fiber in range(nfibers):
if np.sum(ivar[fiber] > 0) == 0:
continue
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = np.array(np.dot(R.todense(), mean_spectrum)).flatten()
fiberflat[fiber] = (M != 0) * flux[fiber] / (M + (M == 0)) + (M == 0)
fiberflat_ivar[fiber] = ivar[fiber] * M**2
nbad_tot = 0
iteration = 0
while iteration < 500:
smooth_fiberflat = spline_fit(wave, wave, fiberflat[fiber],
smoothing_res, fiberflat_ivar[fiber])
chi2 = fiberflat_ivar[fiber] * (fiberflat[fiber] -
smooth_fiberflat)**2
bad = np.where(chi2 > nsig_for_mask**2)[0]
if bad.size > 0:
if bad.size > 5: # not more than 5 pixels at a time
ii = np.argsort(chi2[bad])
bad = bad[ii[-5:]]
mask[fiber, bad] += fiberflat_mask
fiberflat_ivar[fiber, bad] = 0.
nbad_tot += bad.size
else:
break
iteration += 1
# replace bad by smooth fiber flat
bad = np.where((mask[fiber] > 0) | (fiberflat_ivar[fiber] == 0)
| (fiberflat[fiber] < minval)
| (fiberflat[fiber] > maxval))[0]
if bad.size > 0:
fiberflat_ivar[fiber, bad] = 0
# find max length of segment with bad pix
length = 0
for i in range(bad.size):
ib = bad[i]
ilength = 1
tmp = ib
for jb in bad[i + 1:]:
if jb == tmp + 1:
ilength += 1
tmp = jb
else:
break
length = max(length, ilength)
if length > 10:
log.info(
"3rd pass : fiber #%d has a max length of bad pixels=%d" %
(fiber, length))
smoothing_res = float(max(100, 2 * length))
x = np.arange(wave.size)
ok = np.where(fiberflat_ivar[fiber] > 0)[0]
smooth_fiberflat = spline_fit(x, x[ok], fiberflat[fiber, ok],
smoothing_res, fiberflat_ivar[fiber,
ok])
fiberflat[fiber, bad] = smooth_fiberflat[bad]
if nbad_tot > 0:
log.info(
"3rd pass : fiber #%d masked pixels = %d (%d iterations)" %
(fiber, nbad_tot, iteration))
# set median flat to 1
log.info("set median fiberflat to 1")
mean = np.ones((flux.shape[1]))
for i in range(flux.shape[1]):
ok = np.where((mask[:, i] == 0) & (ivar[:, i] > 0))[0]
if ok.size > 0:
mean[i] = np.median(fiberflat[ok, i])
ok = np.where(mean != 0)[0]
for fiber in range(nfibers):
fiberflat[fiber, ok] /= mean[ok]
log.info("done fiberflat")
return FiberFlat(wave,
fiberflat,
fiberflat_ivar,
mask,
mean_spectrum,
chi2pdf=chi2pdf)
def qproc_compute_fiberflat(qframe,niter_meanspec=4,nsig_clipping=3.,spline_res_clipping=20.,spline_res_flat=5.) :
"""
Fast estimation of fiberflat
"""
log = get_logger()
t0=time.time()
log.info("Starting...")
twave=np.mean(qframe.wave,axis=0)
tflux=np.zeros(qframe.flux.shape)
tivar=np.zeros(qframe.flux.shape)
if qframe.mask is not None :
qframe.ivar *= (qframe.mask==0)
for i in range(qframe.flux.shape[0]) :
jj=(qframe.ivar[i]>0)
tflux[i]=np.interp(twave,qframe.wave[i,jj],qframe.flux[i,jj])
tivar[i]=np.interp(twave,qframe.wave[i,jj],qframe.ivar[i,jj],left=0,right=0)
# iterative loop to a absorb constant term in fiber
if 1 : # simple scaling per fiber
a=np.ones(tflux.shape[0])
for iter in range(niter_meanspec) :
mflux=np.median(a[:,np.newaxis]*tflux,axis=0)
for i in range(qframe.flux.shape[0]) :
a[i] = np.median(tflux[i,mflux>0]/mflux[mflux>0])
else : # polynomial fit does not improve much and s more fragile
x=np.linspace(-1,1,tflux.shape[1])
pol=np.ones(tflux.shape)
for iteration in range(niter_meanspec) :
if iteration>0 :
for i in range(tflux.shape[0]) :
jj=(mflux>0)&(tivar[i]>0)
c = np.polyfit(x[jj],tflux[i,jj]/mflux[jj],1,w=mflux[jj]**2*tivar[i,jj])
pol[i] = np.poly1d(c)(x)
mflux=np.median(pol*tflux,axis=0)
# trivial fiberflat
fflat=tflux/(mflux+(mflux==0))
fivar=tivar*mflux**2
mask=np.zeros((fflat.shape), dtype='uint32')
chi2=0
# special case with test slit
mask_lines = ( qframe.flux.shape[0]<50 )
if mask_lines :
log.warning("Will interpolate over absorption lines in input continuum spectrum from illumination bench")
# spline fit to reject outliers and smooth the flat
for fiber in range(fflat.shape[0]) :
# iterative spline fit
max_rej_it=5# not more than 5 pixels at a time
max_bad=1000
nbad_tot=0
for loop in range(20) :
good=(fivar[fiber]>0)
splineflat = spline_fit(twave,twave[good],fflat[fiber,good],required_resolution=spline_res_clipping,input_ivar=fivar[fiber,good],max_resolution=3*spline_res_clipping)
fchi2 = fivar[fiber]*(fflat[fiber]-splineflat)**2
bad=np.where(fchi2>nsig_clipping**2)[0]
if bad.size>0 :
if bad.size>max_rej_it : # not more than 5 pixels at a time
ii=np.argsort(fchi2[bad])
bad=bad[ii[-max_rej_it:]]
fivar[fiber,bad] = 0
nbad_tot += len(bad)
#log.warning("iteration {} rejecting {} pixels (tot={}) from fiber {}".format(loop,len(bad),nbad_tot,fiber))
if nbad_tot>=max_bad:
fivar[fiber,:]=0
log.warning("1st pass: rejecting fiber {} due to too many (new) bad pixels".format(fiber))
else :
break
chi2 += np.sum(fchi2)
min_ivar = 0.1*np.median(fivar[fiber])
med_flat = np.median(fflat[fiber])
good=(fivar[fiber]>0)
splineflat = spline_fit(twave,twave[good],fflat[fiber,good],required_resolution=spline_res_flat,input_ivar=fivar[fiber,good],max_resolution=3*spline_res_flat)
fflat[fiber] = splineflat # replace by spline
ii=np.where(fivar[fiber]>min_ivar)[0]
if ii.size<2 :
fflat[fiber] = 1
fivar[fiber] = 0
# set flat in unknown edges to median value of fiber (and ivar to 0)
b=ii[0]
e=ii[-1]+1
fflat[fiber,:b]=med_flat # default
fivar[fiber,:b]=0
mask[fiber,:b]=1 # need to change this
fflat[fiber,e:]=med_flat # default
fivar[fiber,e:]=0
mask[fiber,e:]=1 # need to change this
# internal interpolation
bad=(fivar[fiber][b:e]<=min_ivar)
good=(fivar[fiber][b:e]>min_ivar)
fflat[fiber][b:e][bad]=np.interp(twave[b:e][bad],twave[b:e][good],fflat[fiber][b:e][good])
# special case with test slit
if mask_lines :
if qframe.meta["camera"].upper()[0] == "B" :
jj=((twave>3900)&(twave<3960))|((twave>4350)&(twave<4440))|(twave>5800)
elif qframe.meta["camera"].upper()[0] == "R" :
jj=(twave<5750)
else :
jj=(twave<7550)|(twave>9800)
if np.sum(jj)>0 :
njj=np.logical_not(jj)
fflat[fiber,jj] = np.interp(twave[jj],twave[njj],fflat[fiber,njj])
ndata=np.sum(fivar>0)
if ndata>0 :
chi2pdf = chi2/ndata
else :
chi2pdf = 0
t1=time.time()
log.info(" done in {:3.1f} sec".format(t1-t0))
# return a fiberflat object ...
return FiberFlat(twave, fflat, fivar, mask, mflux,chi2pdf=chi2pdf)
