def test_errors(self):
#- Bad shaped input
data = np.random.uniform(size=(10, 5))
with self.assertRaises(ValueError):
R = Resolution(data)
#- Meaningless type for input
with self.assertRaises(ValueError):
R = Resolution('blat')
#- Non-uniform x spacing
with self.assertRaises(ValueError):
R = desispec.resolution._gauss_pix([-1, 0, 2])
#- missing offsets
with self.assertRaises(ValueError):
desispec.resolution._sort_and_symmeterize(data, [-2, -1, 0, 1, 3])
#- length of offsets too large or small
with self.assertRaises(ValueError):
Resolution(data, offsets=[1, 2])
with self.assertRaises(ValueError):
Resolution(data,
offsets=np.arange(10 *
desispec.resolution.default_ndiag))
def _get_spectra(self):
#- Setup data for a Resolution matrix
sigma = 4.0
ndiag = 21
xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag)
Rdata = np.zeros((self.nspec, ndiag, self.nwave))
for i in range(self.nspec):
for j in range(self.nwave):
kernel = np.exp(-xx**2 / (2 * sigma))
kernel /= sum(kernel)
Rdata[i, :, j] = kernel
flux = np.zeros((self.nspec, self.nwave))
ivar = np.ones((self.nspec, self.nwave))
mask = np.zeros((self.nspec, self.nwave), dtype=int)
for i in range(self.nspec):
R = Resolution(Rdata[i])
flux[i] = R.dot(self.flux)
fibermap = desispec.io.empty_fibermap(self.nspec, 1500)
fibermap['OBJTYPE'][0::2] = 'SKY'
return Frame(self.wave,
flux,
ivar,
mask,
Rdata,
spectrograph=2,
fibermap=fibermap)
def test_resolution_sparsedia(self):
data = np.random.uniform(size=(5,10))
offsets = np.arange(-2,3)
#- Original case: symetric and odd number of diagonals
Rdia = scipy.sparse.dia_matrix((data, offsets), shape=(10,10))
R = Resolution(Rdia)
self.assertTrue(np.all(R.diagonal() == Rdia.diagonal()))
#- Non symetric but still odd number of diagonals
Rdia = scipy.sparse.dia_matrix((data, offsets+1), shape=(10,10))
R = Resolution(Rdia)
self.assertTrue(np.all(R.diagonal() == Rdia.diagonal()))
#- Even number of diagonals
Rdia = scipy.sparse.dia_matrix((data[1:,:], offsets[1:]), shape=(10,10))
R = Resolution(Rdia)
self.assertTrue(np.all(R.diagonal() == Rdia.diagonal()))
#- Unordered diagonals
data = np.random.uniform(size=(5,10))
offsets = [0,1,-1,2,-2]
Rdia = scipy.sparse.dia_matrix((data, offsets), shape=(10,10))
R1 = Resolution(Rdia)
R2 = Resolution(data, offsets)
self.assertTrue(np.all(R1.diagonal() == Rdia.diagonal()))
self.assertTrue(np.all(R2.diagonal() == Rdia.diagonal()))
self.assertTrue(np.all(R1.data == R2.data))
def test_resolution_dense(self):
#- dense with no offsets specified
data = np.random.uniform(size=(10,10))
R = Resolution(data)
Rdense = R.todense()
self.assertTrue(np.all(Rdense == data))
#- with offsets
offsets = np.arange(-2,4)
R = Resolution(data, offsets)
Rdense = R.todense()
for i in offsets:
self.assertTrue(np.all(Rdense.diagonal(i) == data.diagonal(i)), \
"diagonal {} doesn't match".format(i))
#- dense without offsets but larger than default_ndiag
ndiag = desispec.resolution.default_ndiag + 5
data = np.random.uniform(size=(ndiag, ndiag))
Rdense = Resolution(data).todense()
for i in range(ndiag):
if i <= desispec.resolution.default_ndiag//2:
self.assertTrue(np.all(Rdense.diagonal(i) == data.diagonal(i)), \
"diagonal {} doesn't match".format(i))
self.assertTrue(np.all(Rdense.diagonal(-i) == data.diagonal(-i)), \
"diagonal {} doesn't match".format(-i))
else:
self.assertTrue(np.all(Rdense.diagonal(i) == 0.0), \
"diagonal {} not 0s".format(i))
self.assertTrue(np.all(Rdense.diagonal(-i) == 0.0), \
"diagonal {} not 0s".format(-i))
def _get_spectra(self, with_gradient=False):
#- Setup data for a Resolution matrix
sigma2 = 4.0
ndiag = 21
xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag)
Rdata = np.zeros((self.nspec, ndiag, self.nwave))
for i in range(self.nspec):
kernel = np.exp(-(xx + float(i) / self.nspec * 0.3)**2 /
(2 * sigma2))
#kernel = np.exp(-xx**2/(2*sigma2))
kernel /= sum(kernel)
for j in range(self.nwave):
Rdata[i, :, j] = kernel
flux = np.zeros((self.nspec, self.nwave), dtype=float)
ivar = np.ones((self.nspec, self.nwave), dtype=float)
# Add a random component
for i in range(self.nspec):
ivar[i] += 0.4 * np.random.uniform(size=self.nwave)
mask = np.zeros((self.nspec, self.nwave), dtype=int)
fibermap = empty_fibermap(self.nspec, 1500)
fibermap['OBJTYPE'][0::2] = 'SKY'
x = fibermap["FIBERASSIGN_X"]
y = fibermap["FIBERASSIGN_Y"]
x = x - np.mean(x)
y = y - np.mean(y)
if np.std(x) > 0: x /= np.std(x)
if np.std(y) > 0: y /= np.std(y)
w = (self.wave - self.wave[0]) / (self.wave[-1] -
self.wave[0]) * 2. - 1
for i in range(self.nspec):
R = Resolution(Rdata[i])
if with_gradient:
scale = 1. + (0.1 * x[i] + 0.2 * y[i]) * (1 + 0.4 * w)
flux[i] = R.dot(scale * self.flux)
else:
flux[i] = R.dot(self.flux)
meta = {"camera": "r2"}
return Frame(self.wave,
flux,
ivar,
mask,
Rdata,
spectrograph=2,
fibermap=fibermap,
meta=meta)
def _X2(x):
z = x[0]
v = x[1]
lnA = x[2]
line_flux = np.exp(lnA)
result = 0.0
for cam in cams:
wave = spectra.wave[cam]
res = Resolution(spectra.resolution_data[cam][fiber, :])
flux = spectra.flux[cam][fiber, :]
ivar = spectra.ivar[cam][fiber, :]
mask = spectra.mask[cam][fiber, :]
tw = twave(wave.min(), wave.max())
_, _, X2, _ = doublet_chi2(z,
tw,
wave,
res,
flux,
ivar,
mask,
continuum=0.0,
sigmav=v,
r=0.0,
line_flux=line_flux,
linea=0.0,
lineb=lineb)
result += X2
return result
def get_frame_data(nspec=10, objtype=None):
"""
Return basic test data for desispec.frame object:
"""
nwave = 100
wavemin, wavemax = 4000, 4100
wave, model_flux = get_models(nspec,
nwave,
wavemin=wavemin,
wavemax=wavemax)
resol_data = set_resolmatrix(nspec, nwave)
calib = np.sin((wave - wavemin) * np.pi / np.max(wave))
flux = np.zeros((nspec, nwave))
for i in range(nspec):
flux[i] = Resolution(resol_data[i]).dot(model_flux[i] * calib)
sigma = 0.01
# flux += np.random.normal(scale=sigma, size=flux.shape)
ivar = np.ones(flux.shape) / sigma**2
mask = np.zeros(flux.shape, dtype=int)
fibermap = empty_fibermap(nspec, 1500)
if objtype is None:
fibermap['OBJTYPE'] = 'QSO'
fibermap['OBJTYPE'][0:3] = 'STD' # For flux tests
else:
fibermap['OBJTYPE'] = objtype
frame = Frame(wave, flux, ivar, mask, resol_data, fibermap=fibermap)
frame.meta = {}
frame.meta['EXPTIME'] = 1. # For flux tests
return frame
class TestResolution(unittest.TestCase):
#- TODO: in principle these should be converted to self.assertXYZ
#- In practice, if they fail then the test won't pass so the end result is the same
def test_resolution(self, n=100):
dense = np.arange(n * n).reshape(n, n)
R1 = Resolution(dense)
assert scipy.sparse.isspmatrix_dia(
R1), 'Resolution is not recognized as a scipy.sparse.dia_matrix.'
assert len(
R1.offsets
) == desispec.resolution.default_ndiag, 'Resolution.offsets has wrong size'
R2 = Resolution(R1)
assert np.array_equal(
R1.toarray(),
R2.toarray()), 'Constructor broken for dia_matrix input.'
R3 = Resolution(R1.data)
assert np.array_equal(
R1.toarray(),
R3.toarray()), 'Constructor broken for array data input.'
sparse = scipy.sparse.dia_matrix((R1.data[::-1], R1.offsets[::-1]),
(n, n))
R4 = Resolution(sparse)
assert np.array_equal(
R1.toarray(),
R4.toarray()), 'Constructor broken for permuted offsets input.'
R5 = Resolution(R1.to_fits_array())
assert np.array_equal(R1.toarray(),
R5.toarray()), 'to_fits_array() is broken.'
#- test different sizes of input diagonals
for ndiag in [3, 5, 11]:
R6 = Resolution(np.ones((ndiag, n)))
assert len(
R6.offsets) == ndiag, 'Constructor broken for ndiag={}'.format(
ndiag)
#- An even number if diagonals is not allowed
try:
ndiag = 10
R7 = Resolution(np.ones((ndiag, n)))
raise RuntimeError(
'Incorrectly created Resolution with even number of diagonals')
except ValueError, err:
#- it correctly raised an error, so pass
pass
#- Test creation with asymetric diagonals (should fail)
R1.offsets += 1
try:
R8 = Resolution(R1)
raise RuntimeError(
'Incorrectly created Resolution with non-symmetric input')
except ValueError:
#- correctly raised an error, so pass
pass
def test_resolution(self):
"""
Test that identical spectra convolved with different resolutions
results in identical fiberflats
"""
wave, flux, ivar, mask = _get_data()
nspec, nwave = flux.shape
#- Setup a Resolution matrix that varies with fiber and wavelength
#- Note: this is actually the transpose of the resolution matrix
#- I wish I was creating, but as long as we self-consistently
#- use it for convolving and solving, that shouldn't matter.
sigma = np.linspace(2, 10, nwave * nspec)
ndiag = 21
xx = np.linspace(-ndiag / 2.0, +ndiag / 2.0, ndiag)
Rdata = np.zeros((nspec, len(xx), nwave))
for i in range(nspec):
for j in range(nwave):
kernel = np.exp(-xx**2 / (2 * sigma[i * nwave + j]**2))
kernel /= sum(kernel)
Rdata[i, :, j] = kernel
#- Convolve the data with the resolution matrix
convflux = np.empty_like(flux)
for i in range(nspec):
convflux[i] = Resolution(Rdata[i]).dot(flux[i])
#- Run the code
frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0)
ff = compute_fiberflat(frame)
#- These fiber flats should all be ~1
self.assertTrue(np.all(np.abs(ff.fiberflat - 1) < 0.001))
def _fdgrad(x):
z = x[0]
v = x[1]
r = x[2]
lnA = x[3]
line_flux = np.exp(lnA)
result = 0.0
chisq = 0.0
grad = np.zeros(4)
eps = np.sqrt(np.finfo(float).eps)
for cam in cams:
wave = postage[cam].wave
res = Resolution(postage[cam].R)
flux = postage[cam].flux
ivar = postage[cam].ivar
mask = postage[cam].mask
for i in np.arange(4):
hi = np.array(x, copy=True)
lo = np.array(x, copy=True)
lo[i] -= eps
hi[i] += eps
_, _, hiX2, _ = doublet_chi2(hi[0],
wave,
res,
flux,
ivar,
mask,
continuum=0.0,
sigmav=hi[1],
r=hi[2],
line_flux=np.exp(hi[3]),
linea=linea,
lineb=lineb)
_, _, loX2, _ = doublet_chi2(lo[0],
wave,
res,
flux,
ivar,
mask,
continuum=0.0,
sigmav=lo[1],
r=lo[2],
line_flux=np.exp(lo[3]),
linea=linea,
lineb=lineb)
grad[i] += (hiX2 - loX2) / 2. / eps
break
return grad, -99.
def get_resolution(wave, nspec, psf, usesigma=False):
"""
Calculates approximate resolution values at given wavelengths in the format that can directly
feed resolution data of desispec.frame.Frame object.
wave: wavelength array
nsepc: no of spectra (int)
psf: desispec.psf.PSF like object
usesigma: allows to use sigma from psf file for resolution computation.
If psf file is psfboot, uses per fiber xsigma.
If psf file is from QL arcs processing, uses wsigma
returns : resolution data (nspec,nband,nwave); nband = 1 for usesigma = False, otherwise nband=21
"""
from desispec.resolution import Resolution
nwave = len(wave)
if usesigma:
nband = 21
else:
nband = 1 # only for dimensionality purpose of data model.
resolution_data = np.zeros((nspec, nband, nwave))
if usesigma: #- use sigmas for resolution based on psffile type
if hasattr(psf, 'wcoeff'): #- use if have wsigmas
log.info("Getting resolution from wsigmas from arc lines PSF")
for ispec in range(nspec):
thissigma = psf.wdisp(ispec, wave) / psf.angstroms_per_pixel(
ispec, wave) #- in pixel units
Rsig = Resolution(thissigma)
resolution_data[ispec] = Rsig.data
else:
if hasattr(
psf,
'xsigma_boot'): #- only use if xsigma comes from psfboot
log.info(
"Getting resolution matrix band diagonal elements from constant Gaussing Xsigma"
)
for ispec in range(nspec):
thissigma = psf.xsigma(ispec, wave)
Rsig = Resolution(thissigma)
resolution_data[ispec] = Rsig.data
return resolution_data
def _get_spectra(self,with_gradient=False):
#- Setup data for a Resolution matrix
sigma2 = 4.0
ndiag = 21
xx = np.linspace(-(ndiag-1)/2.0, +(ndiag-1)/2.0, ndiag)
Rdata = np.zeros( (self.nspec, ndiag, self.nwave) )
for i in range(self.nspec):
kernel = np.exp(-(xx+float(i)/self.nspec*0.3)**2/(2*sigma2))
#kernel = np.exp(-xx**2/(2*sigma2))
kernel /= sum(kernel)
for j in range(self.nwave):
Rdata[i,:,j] = kernel
flux = np.zeros((self.nspec, self.nwave),dtype=float)
ivar = np.ones((self.nspec, self.nwave),dtype=float)
# Add a random component
for i in range(self.nspec) :
ivar[i] += 0.4*np.random.uniform(size=self.nwave)
mask = np.zeros((self.nspec, self.nwave), dtype=int)
fibermap = desispec.io.empty_fibermap(self.nspec, 1500)
fibermap['OBJTYPE'][0::2] = 'SKY'
x=fibermap["DESIGN_X"]
y=fibermap["DESIGN_Y"]
x = x-np.mean(x)
y = y-np.mean(y)
if np.std(x)>0 : x /= np.std(x)
if np.std(y)>0 : y /= np.std(y)
w = (self.wave-self.wave[0])/(self.wave[-1]-self.wave[0])*2.-1
for i in range(self.nspec):
R = Resolution(Rdata[i])
if with_gradient :
scale = 1.+(0.1*x[i]+0.2*y[i])*(1+0.4*w)
flux[i] = R.dot(scale*self.flux)
else :
flux[i] = R.dot(self.flux)
return Frame(self.wave, flux, ivar, mask, Rdata, spectrograph=2, fibermap=fibermap)
def test_throughput_resolution(self):
"""
Test that spectra with different throughputs and different resolutions
result in fiberflat variations that are only due to throughput.
"""
wave, flux, ivar, mask = _get_data()
nspec, nwave = flux.shape
#- Setup a Resolution matrix that varies with fiber and wavelength
#- Note: this is actually the transpose of the resolution matrix
#- I wish I was creating, but as long as we self-consistently
#- use it for convolving and solving, that shouldn't matter.
sigma = np.linspace(2, 10, nwave * nspec)
ndiag = 21
xx = np.linspace(-ndiag / 2.0, +ndiag / 2.0, ndiag)
Rdata = np.zeros((nspec, len(xx), nwave))
for i in range(nspec):
for j in range(nwave):
kernel = np.exp(-xx**2 / (2 * sigma[i * nwave + j]**2))
kernel /= sum(kernel)
Rdata[i, :, j] = kernel
#- Vary the input flux prior to calculating the fiber flat
flux[1] *= 1.1
flux[2] *= 1.2
flux[3] /= 1.1
flux[4] /= 1.2
#- Convolve the data with the varying resolution matrix
convflux = np.empty_like(flux)
for i in range(nspec):
convflux[i] = Resolution(Rdata[i]).dot(flux[i])
#- Run the code
frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0)
#- Set an accuracy for this
accuracy = 1.e-9
ff = compute_fiberflat(frame, accuracy=accuracy)
#- Compare variation with middle fiber
mid = ff.fiberflat.shape[0] // 2
diff = (ff.fiberflat[1] / 1.1 - ff.fiberflat[mid])
self.assertLess(np.max(np.abs(diff)), accuracy)
diff = (ff.fiberflat[2] / 1.2 - ff.fiberflat[mid])
self.assertLess(np.max(np.abs(diff)), accuracy)
diff = (ff.fiberflat[3] * 1.1 - ff.fiberflat[mid])
self.assertLess(np.max(np.abs(diff)), accuracy)
diff = (ff.fiberflat[4] * 1.2 - ff.fiberflat[mid])
self.assertLess(np.max(np.abs(diff)), accuracy)
def _getdata(self, n=10):
wave = np.linspace(5000, 5100, n)
flux = np.random.uniform(0, 1, size=n)
ivar = np.random.uniform(0, 1, size=n)
### mask = np.random.randint(0, 256, size=n)
rdat = np.ones((3, n))
rdat[0] *= 0.25
rdat[1] *= 0.5
rdat[2] *= 0.25
R = Resolution(rdat)
### return wave, flux, ivar, mask, R
return wave, flux, ivar, None, R
def _res(x):
z = x[0]
v = x[1]
lnA = x[2]
line_flux = np.exp(lnA)
residuals = []
for cam in cams:
wave = spectra.wave[cam]
res = Resolution(spectra.resolution_data[cam][fiber, :])
flux = spectra.flux[cam][fiber, :]
ivar = spectra.ivar[cam][fiber, :]
mask = spectra.mask[cam][fiber, :]
tw = twave(wave.min(), wave.max())
rflux = doublet_obs(z,
tw,
wave,
res,
continuum=0.0,
sigmav=v,
r=0.0,
linea=0.0,
lineb=lineb)
rflux *= line_flux
res = np.sqrt(ivar[mask]) * np.abs(flux[mask] - rflux[mask])
residuals += res.tolist()
residuals = np.array(residuals)
print(z, v, lnA, residuals.sum())
return residuals
def _get_spectra(self):
#- Setup data for a Resolution matrix
sigma = 4.0
ndiag = 21
xx = np.linspace(-(ndiag-1)/2.0, +(ndiag-1)/2.0, ndiag)
Rdata = np.zeros( (self.nspec, ndiag, self.nwave) )
for i in range(self.nspec):
for j in range(self.nwave):
kernel = np.exp(-xx**2/(2*sigma))
kernel /= sum(kernel)
Rdata[i,:,j] = kernel
flux = np.zeros((self.nspec, self.nwave))
ivar = np.ones((self.nspec, self.nwave))
mask = np.zeros((self.nspec, self.nwave), dtype=int)
for i in range(self.nspec):
R = Resolution(Rdata[i])
flux[i] = R.dot(self.flux)
fibermap = desispec.io.empty_fibermap(self.nspec)
fibermap['OBJTYPE'][0::2] = 'SKY'
return Frame(self.wave, flux, ivar, mask, Rdata), fibermap
def test_main(self):
"""
Test the main program.
"""
# generate the frame data
wave, flux, ivar, mask = _get_data()
nspec, nwave = flux.shape
#- Setup data for a Resolution matrix
sigma = 4.0
ndiag = 11
xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag)
Rdata = np.zeros((nspec, ndiag, nwave))
kernel = np.exp(-xx**2 / (2 * sigma))
kernel /= sum(kernel)
for i in range(nspec):
for j in range(nwave):
Rdata[i, :, j] = kernel
#- Convolve the data with the resolution matrix
convflux = np.empty_like(flux)
for i in range(nspec):
convflux[i] = Resolution(Rdata[i]).dot(flux[i])
# create a fake fibermap
fibermap = io.empty_fibermap(nspec, nwave)
for i in range(0, nspec):
fibermap['OBJTYPE'][i] = 'FAKE'
io.write_fibermap(self.testfibermap, fibermap)
#- write out the frame
frame = Frame(wave,
convflux,
ivar,
mask,
Rdata,
spectrograph=0,
fibermap=fibermap,
meta=dict(FLAVOR='flat'))
write_frame(self.testframe, frame, fibermap=fibermap)
# set program arguments
argstr = ['--infile', self.testframe, '--outfile', self.testflat]
# run it
args = ffscript.parse(options=argstr)
ffscript.main(args)
def get_frame_data(nspec=10, wavemin=4000, wavemax=4100, nwave=100, meta={}):
"""
Return basic test data for desispec.frame object:
"""
wave, model_flux = get_models(nspec,
nwave,
wavemin=wavemin,
wavemax=wavemax)
resol_data = set_resolmatrix(nspec, nwave)
calib = np.sin((wave - wavemin) * np.pi / np.max(wave))
flux = np.zeros((nspec, nwave))
for i in range(nspec):
flux[i] = Resolution(resol_data[i]).dot(model_flux[i] * calib)
sigma = 0.01
# flux += np.random.normal(scale=sigma, size=flux.shape)
ivar = np.ones(flux.shape) / sigma**2
mask = np.zeros(flux.shape, dtype=int)
fibermap = empty_fibermap(nspec, 1500)
fibermap['OBJTYPE'] = 'TGT'
fibermap['DESI_TARGET'] = desi_mask.QSO
fibermap['DESI_TARGET'][0:3] = desi_mask.STD_FAINT # For flux tests
fibermap['FIBER_X'] = np.arange(nspec) * 400. / nspec #mm
fibermap['FIBER_Y'] = np.arange(nspec) * 400. / nspec #mm
fibermap['DELTA_X'] = 0.005 * np.ones(nspec) #mm
fibermap['DELTA_Y'] = 0.003 * np.ones(nspec) #mm
if "EXPTIME" not in meta.keys():
meta['EXPTIME'] = 1.0
frame = Frame(wave,
flux,
ivar,
mask,
resol_data,
fibermap=fibermap,
meta=meta)
return frame
def _agrad(x):
z = x[0]
v = x[1]
r = x[2]
lnA = x[3]
line_flux = np.exp(lnA)
result = 0.0
chisq = 0.0
grad = np.zeros(4)
for cam in cams:
wave = postage[cam].wave
res = Resolution(postage[cam].R)
flux = postage[cam].flux
ivar = postage[cam].ivar
mask = postage[cam].mask
_grad, X2 = doublet_chi2_grad(z,
wave,
res,
flux,
ivar,
mask,
continuum=0.0,
v=v,
r=r,
line_flux=line_flux,
linea=linea,
lineb=lineb)
grad += _grad
chisq += X2
break
return grad, chisq
def test_throughput(self):
"""
Test that spectra with different throughputs but the same resolution
produce a fiberflat mirroring the variations in throughput
"""
wave, flux, ivar, mask = _get_data()
nspec, nwave = flux.shape
#- Setup data for a Resolution matrix
sigma = 4.0
ndiag = 21
xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag)
Rdata = np.zeros((nspec, ndiag, nwave))
kernel = np.exp(-xx**2 / (2 * sigma))
kernel /= sum(kernel)
for i in range(nspec):
for j in range(nwave):
Rdata[i, :, j] = kernel
#- Vary the input flux prior to calculating the fiber flat
flux[1] *= 1.1
flux[2] *= 1.2
flux[3] *= 0.8
#- Convolve with the (common) resolution matrix
convflux = np.empty_like(flux)
for i in range(nspec):
convflux[i] = Resolution(Rdata[i]).dot(flux[i])
frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0)
ff = compute_fiberflat(frame)
#- flux[1] is brighter, so should fiberflat[1]. etc.
self.assertTrue(np.allclose(ff.fiberflat[0], ff.fiberflat[1] / 1.1))
self.assertTrue(np.allclose(ff.fiberflat[0], ff.fiberflat[2] / 1.2))
self.assertTrue(np.allclose(ff.fiberflat[0], ff.fiberflat[3] / 0.8))
def spectroperf_resample_spectrum_singleproc(spectra, target_index, wave,
wavebin, resampling_matrix, ndiag,
flux, ivar, rdata):
cinv = None
for b in spectra._bands:
twave = spectra.wave[b]
jj = (twave >= wave[0]) & (twave <= wave[-1])
twave = twave[jj]
tivar = spectra.ivar[b][target_index][jj]
diag_ivar = scipy.sparse.dia_matrix((tivar, [0]),
(twave.size, twave.size))
RR = Resolution(spectra.resolution_data[b][target_index][:, jj]).dot(
resampling_matrix[b])
tcinv = RR.T.dot(diag_ivar.dot(RR))
tcinvf = RR.T.dot(tivar * spectra.flux[b][target_index][jj])
if cinv is None:
cinv = tcinv
cinvf = tcinvf
else:
cinv += tcinv
cinvf += tcinvf
cinv = cinv.todense()
decorrelate_divide_and_conquer(cinv, cinvf, wavebin, flux[target_index],
ivar[target_index], rdata[target_index])
def _X2(x):
z = x[0]
v = x[1]
r = x[2]
lnA = x[3]
line_flux = np.exp(lnA)
result = 0.0
for cam in cams:
wave = postage[cam].wave
res = Resolution(postage[cam].R)
flux = postage[cam].flux
ivar = postage[cam].ivar
mask = postage[cam].mask
_, _, X2, _ = doublet_chi2(z,
wave,
res,
flux,
ivar,
mask,
continuum=0.0,
sigmav=v,
r=r,
line_flux=line_flux,
linea=linea,
lineb=lineb)
# X2 = doublet(z, twave, sigmav=v, r=r, linea=linea, lineb=lineb, index=5514)
result += X2
break
return result
def compute_uniform_sky(frame,
nsig_clipping=4.,
max_iterations=100,
model_ivar=False,
add_variance=True):
"""Compute a sky model.
Sky[fiber,i] = R[fiber,i,j] Flux[j]
Input flux are expected to be flatfielded!
We don't check this in this routine.
Args:
frame : Frame object, which includes attributes
- wave : 1D wavelength grid in Angstroms
- flux : 2D flux[nspec, nwave] density
- ivar : 2D inverse variance of flux
- mask : 2D inverse mask flux (0=good)
- resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers)
nsig_clipping : [optional] sigma clipping value for outlier rejection
Optional:
max_iterations : int , number of iterations
model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims.
add_variance : evaluate calibration error and add this to the sky model variance
returns SkyModel object with attributes wave, flux, ivar, mask
"""
log = get_logger()
log.info("starting")
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nwave = frame.nwave
nfibers = len(skyfibers)
current_ivar = frame.ivar[skyfibers].copy() * (frame.mask[skyfibers] == 0)
flux = frame.flux[skyfibers]
Rsky = frame.R[skyfibers]
input_ivar = None
if model_ivar:
log.info(
"use a model of the inverse variance to remove bias due to correlated ivar and flux"
)
input_ivar = current_ivar.copy()
median_ivar_vs_wave = np.median(current_ivar, axis=0)
median_ivar_vs_fiber = np.median(current_ivar, axis=1)
median_median_ivar = np.median(median_ivar_vs_fiber)
for f in range(current_ivar.shape[0]):
threshold = 0.01
current_ivar[f] = median_ivar_vs_fiber[
f] / median_median_ivar * median_ivar_vs_wave
# keep input ivar for very low weights
ii = (input_ivar[f] <= (threshold * median_ivar_vs_wave))
#log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1]))
current_ivar[f][ii] = input_ivar[f][ii]
sqrtw = np.sqrt(current_ivar)
sqrtwflux = sqrtw * flux
chi2 = np.zeros(flux.shape)
nout_tot = 0
for iteration in range(max_iterations):
# the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters
# A_ij = 1/2 d2(chi2)/di/dj
# A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w]
# the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters
# B_i = 1/2 d(chi2)/di
# B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w])
# the model is model[fiber]=R[fiber]*sky
# and the parameters are the unconvolved sky flux at the wavelength i
# so, d(model)/di[fiber,w] = R[fiber][w,i]
# this gives
# A_ij = sum_fiber sum_wave_w ivar[fiber,w] R[fiber][w,i] R[fiber][w,j]
# A = sum_fiber ( diag(sqrt(ivar))*R[fiber] ) ( diag(sqrt(ivar))* R[fiber] )^t
# A = sum_fiber sqrtwR[fiber] sqrtwR[fiber]^t
# and
# B = sum_fiber sum_wave_w ivar[fiber,w] R[fiber][w] * flux[fiber,w]
# B = sum_fiber sum_wave_w sqrt(ivar)[fiber,w]*flux[fiber,w] sqrtwR[fiber,wave]
#A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr()
A = np.zeros((nwave, nwave))
B = np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD = scipy.sparse.lil_matrix((nwave, nwave))
# loop on fiber to handle resolution
for fiber in range(nfibers):
if fiber % 10 == 0:
log.info("iter %d sky fiber %d/%d" %
(iteration, fiber, nfibers))
R = Rsky[fiber]
# diagonal sparse matrix with content = sqrt(ivar)
SD.setdiag(sqrtw[fiber])
sqrtwR = SD * R # each row r of R is multiplied by sqrtw[r]
A += (sqrtwR.T * sqrtwR).todense()
B += sqrtwR.T * sqrtwflux[fiber]
log.info("iter %d solving" % iteration)
w = A.diagonal() > 0
A_pos_def = A[w, :]
A_pos_def = A_pos_def[:, w]
parameters = B * 0
try:
parameters[w] = cholesky_solve(A_pos_def, B[w])
except:
log.info("cholesky failed, trying svd in iteration {}".format(
iteration))
parameters[w] = np.linalg.lstsq(A_pos_def, B[w])[0]
log.info("iter %d compute chi2" % iteration)
for fiber in range(nfibers):
# the parameters are directly the unconvolve sky flux
# so we simply have to reconvolve it
fiber_convolved_sky_flux = Rsky[fiber].dot(parameters)
chi2[fiber] = current_ivar[fiber] * (flux[fiber] -
fiber_convolved_sky_flux)**2
log.info("rejecting")
nout_iter = 0
if iteration < 1:
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave = np.sum(chi2 > nsig_clipping**2, axis=0)
selection = np.where(nout_per_wave > 0)[0]
for i in selection:
worst_entry = np.argmax(chi2[:, i])
current_ivar[worst_entry, i] = 0
sqrtw[worst_entry, i] = 0
sqrtwflux[worst_entry, i] = 0
nout_iter += 1
else:
# remove all of them at once
bad = (chi2 > nsig_clipping**2)
current_ivar *= (bad == 0)
sqrtw *= (bad == 0)
sqrtwflux *= (bad == 0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2 = float(np.sum(chi2))
ndf = int(np.sum(chi2 > 0) - nwave)
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d" %
(iteration, sum_chi2, ndf, chi2pdf, nout_iter))
if nout_iter == 0:
break
log.info("nout tot=%d" % nout_tot)
# we know have to compute the sky model for all fibers
# and propagate the uncertainties
# no need to restore the original ivar to compute the model errors when modeling ivar
# the sky inverse variances are very similar
log.info("compute the parameter covariance")
# we may have to use a different method to compute this
# covariance
try:
parameter_covar = cholesky_invert(A)
# the above is too slow
# maybe invert per block, sandwich by R
except np.linalg.linalg.LinAlgError:
log.warning(
"cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv"
)
parameter_covar = np.linalg.pinv(A)
log.info("compute mean resolution")
# we make an approximation for the variance to save CPU time
# we use the average resolution of all fibers in the frame:
mean_res_data = np.mean(frame.resolution_data, axis=0)
Rmean = Resolution(mean_res_data)
log.info("compute convolved sky and ivar")
# The parameters are directly the unconvolved sky
# First convolve with average resolution :
convolved_sky_covar = Rmean.dot(parameter_covar).dot(Rmean.T.todense())
# and keep only the diagonal
convolved_sky_var = np.diagonal(convolved_sky_covar)
# inverse
convolved_sky_ivar = (convolved_sky_var > 0) / (convolved_sky_var +
(convolved_sky_var == 0))
# and simply consider it's the same for all spectra
cskyivar = np.tile(convolved_sky_ivar,
frame.nspec).reshape(frame.nspec, nwave)
# The sky model for each fiber (simple convolution with resolution of each fiber)
cskyflux = np.zeros(frame.flux.shape)
for i in range(frame.nspec):
cskyflux[i] = frame.R[i].dot(parameters)
# look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1
if skyfibers.size > 1 and add_variance:
modified_cskyivar = _model_variance(frame, cskyflux, cskyivar,
skyfibers)
else:
modified_cskyivar = cskyivar.copy()
# need to do better here
mask = (cskyivar == 0).astype(np.uint32)
return SkyModel(
frame.wave.copy(),
cskyflux,
modified_cskyivar,
mask,
nrej=nout_tot,
stat_ivar=cskyivar) # keep a record of the statistical ivar for QA
def __init__(self,
wave,
flux,
ivar,
mask=None,
resolution_data=None,
fibers=None,
spectrograph=None,
meta=None,
fibermap=None,
chi2pix=None,
scores=None,
scores_comments=None,
wsigma=None,
ndiag=21,
suppress_res_warning=False):
"""
Lightweight wrapper for multiple spectra on a common wavelength grid
x.wave, x.flux, x.ivar, x.mask, x.resolution_data, x.header, sp.R
Args:
wave: 1D[nwave] wavelength in Angstroms
flux: 2D[nspec, nwave] flux
ivar: 2D[nspec, nwave] inverse variance of flux
Optional:
mask: 2D[nspec, nwave] integer bitmask of flux. 0=good.
resolution_data: 3D[nspec, ndiag, nwave]
diagonals of resolution matrix data
fibers: ndarray of which fibers these spectra are
spectrograph: integer, which spectrograph [0-9]
meta: dict-like object (e.g. FITS header)
fibermap: fibermap table
chi2pix: 2D[nspec, nwave] chi2 of 2D model to pixel-level data
for pixels that contributed to each flux bin
scores: dictionnary of 1D arrays of size nspec
scores_comments: dictionnary of string (explaining the scores)
suppress_res_warning: bool to suppress Warning message when the Resolution image is not read
Parameters below allow on-the-fly resolution calculation
wsigma: 2D[nspec,nwave] sigma widths for each wavelength bin for all fibers
Notes:
spectrograph input is used only if fibers is None. In this case,
it assumes nspec_per_spectrograph = flux.shape[0] and calculates
the fibers array for this spectrograph, i.e.
fibers = spectrograph * flux.shape[0] + np.arange(flux.shape[0])
Attributes:
All input args become object attributes.
nspec : number of spectra, flux.shape[0]
nwave : number of wavelengths, flux.shape[1]
specmin : minimum fiber number
R: array of sparse Resolution matrix objects converted
from resolution_data
fibermap: fibermap table if provided
"""
assert wave.ndim == 1
assert flux.ndim == 2
assert wave.shape[0] == flux.shape[1]
assert ivar.shape == flux.shape
assert (mask is None) or mask.shape == flux.shape
assert (mask is None) or mask.dtype in \
(int, np.int64, np.int32, np.uint64, np.uint32), "Bad mask type "+str(mask.dtype)
self.wave = wave
self.flux = flux
self.ivar = ivar
self.meta = meta
self.fibermap = fibermap
self.nspec, self.nwave = self.flux.shape
self.chi2pix = chi2pix
self.scores = scores
self.scores_comments = scores_comments
self.ndiag = ndiag
fibers_per_spectrograph = 500 #- hardcode; could get from desimodel
if mask is None:
self.mask = np.zeros(flux.shape, dtype=np.uint32)
else:
self.mask = util.mask32(mask)
if resolution_data is not None:
if resolution_data.ndim != 3 or \
resolution_data.shape[0] != self.nspec or \
resolution_data.shape[2] != self.nwave:
raise ValueError(
"Wrong dimensions for resolution_data[nspec, ndiag, nwave]"
)
#- Maybe setup non-None identity matrix resolution matrix instead?
self.wsigma = wsigma
self.resolution_data = resolution_data
if resolution_data is not None:
self.wsigma = None #ignore width coefficients if resolution data is given explicitly
self.ndiag = None
self.R = np.array([Resolution(r) for r in resolution_data])
elif wsigma is not None:
from desispec.quicklook.qlresolution import QuickResolution
assert ndiag is not None
r = []
for sigma in wsigma:
r.append(QuickResolution(sigma=sigma, ndiag=self.ndiag))
self.R = np.array(r)
else:
#SK I believe this should be error, but looking at the
#tests frame objects are allowed to not to have resolution data
# thus I changed value error to a simple warning message.
if not suppress_res_warning:
log = get_logger()
log.warning("Frame object is constructed without resolution data or respective "\
"sigma widths. Resolution will not be available")
# raise ValueError("Need either resolution_data or coefficients to generate it")
self.spectrograph = spectrograph
# Deal with Fibers (these must be set!)
if fibers is not None:
fibers = np.asarray(fibers)
if len(fibers) != self.nspec:
raise ValueError("len(fibers) != nspec ({} != {})".format(
len(fibers), self.nspec))
if fibermap is not None and np.any(fibers != fibermap['FIBER']):
raise ValueError("fibermap doesn't match fibers")
if (spectrograph is not None):
minfiber = spectrograph * fibers_per_spectrograph
maxfiber = (spectrograph + 1) * fibers_per_spectrograph
if np.any(fibers < minfiber) or np.any(maxfiber <= fibers):
raise ValueError('fibers inconsistent with spectrograph')
self.fibers = fibers
else:
if fibermap is not None:
self.fibers = fibermap['FIBER']
elif spectrograph is not None:
self.fibers = spectrograph * fibers_per_spectrograph + np.arange(
self.nspec, dtype=int)
elif (self.meta is not None) and ('FIBERMIN' in self.meta):
self.fibers = self.meta['FIBERMIN'] + np.arange(self.nspec,
dtype=int)
else:
raise ValueError("Must set fibers by one of the methods!")
if self.meta is not None:
self.meta['FIBERMIN'] = np.min(self.fibers)
def compute_sky(frame, nsig_clipping=4.):
"""Compute a sky model.
Input has to correspond to sky fibers only.
Input flux are expected to be flatfielded!
We don't check this in this routine.
Args:
frame : Frame object, which includes attributes
- wave : 1D wavelength grid in Angstroms
- flux : 2D flux[nspec, nwave] density
- ivar : 2D inverse variance of flux
- mask : 2D inverse mask flux (0=good)
- resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers)
nsig_clipping : [optional] sigma clipping value for outlier rejection
returns SkyModel object with attributes wave, flux, ivar, mask
"""
log = get_logger()
log.info("starting")
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nwave = frame.nwave
nfibers = len(skyfibers)
current_ivar = frame.ivar[skyfibers].copy()
flux = frame.flux[skyfibers]
Rsky = frame.R[skyfibers]
sqrtw = np.sqrt(current_ivar)
sqrtwflux = sqrtw * flux
chi2 = np.zeros(flux.shape)
#debug
#nfibers=min(nfibers,2)
nout_tot = 0
for iteration in range(20):
A = scipy.sparse.lil_matrix((nwave, nwave)).tocsr()
B = np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD = scipy.sparse.lil_matrix((nwave, nwave))
# loop on fiber to handle resolution
for fiber in range(nfibers):
if fiber % 10 == 0:
log.info("iter %d fiber %d" % (iteration, fiber))
R = Rsky[fiber]
# diagonal sparse matrix with content = sqrt(ivar)
SD.setdiag(sqrtw[fiber])
sqrtwR = SD * R # each row r of R is multiplied by sqrtw[r]
A = A + (sqrtwR.T * sqrtwR).tocsr()
B += sqrtwR.T * sqrtwflux[fiber]
log.info("iter %d solving" % iteration)
skyflux = cholesky_solve(A.todense(), B)
log.info("iter %d compute chi2" % iteration)
for fiber in range(nfibers):
S = Rsky[fiber].dot(skyflux)
chi2[fiber] = current_ivar[fiber] * (flux[fiber] - S)**2
log.info("rejecting")
nout_iter = 0
if iteration < 1:
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave = np.sum(chi2 > nsig_clipping**2, axis=0)
selection = np.where(nout_per_wave > 0)[0]
for i in selection:
worst_entry = np.argmax(chi2[:, i])
current_ivar[worst_entry, i] = 0
sqrtw[worst_entry, i] = 0
sqrtwflux[worst_entry, i] = 0
nout_iter += 1
else:
# remove all of them at once
bad = (chi2 > nsig_clipping**2)
current_ivar *= (bad == 0)
sqrtw *= (bad == 0)
sqrtwflux *= (bad == 0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2 = float(np.sum(chi2))
ndf = int(np.sum(chi2 > 0) - nwave)
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d" %
(iteration, sum_chi2, ndf, chi2pdf, nout_iter))
if nout_iter == 0:
break
log.info("nout tot=%d" % nout_tot)
# solve once again to get deconvolved sky variance
skyflux, skycovar = cholesky_solve_and_invert(A.todense(), B)
#- sky inverse variance, but incomplete and not needed anyway
# skyvar=np.diagonal(skycovar)
# skyivar=(skyvar>0)/(skyvar+(skyvar==0))
# Use diagonal of skycovar convolved with mean resolution of all fibers
# first compute average resolution
mean_res_data = np.mean(frame.resolution_data, axis=0)
R = Resolution(mean_res_data)
# compute convolved sky and ivar
cskycovar = R.dot(skycovar).dot(R.T.todense())
cskyvar = np.diagonal(cskycovar)
cskyivar = (cskyvar > 0) / (cskyvar + (cskyvar == 0))
# convert cskyivar to 2D; today it is the same for all spectra,
# but that may not be the case in the future
cskyivar = np.tile(cskyivar, frame.nspec).reshape(frame.nspec, nwave)
# Convolved sky
cskyflux = np.zeros(frame.flux.shape)
for i in range(frame.nspec):
cskyflux[i] = frame.R[i].dot(skyflux)
# need to do better here
mask = (cskyivar == 0).astype(np.uint32)
return SkyModel(frame.wave.copy(), cskyflux, cskyivar, mask, nrej=nout_tot)
def __init__(self,
wave,
flux,
ivar,
mask=None,
resolution_data=None,
fibers=None,
spectrograph=None,
meta=None,
fibermap=None,
chi2pix=None):
"""
Lightweight wrapper for multiple spectra on a common wavelength grid
x.wave, x.flux, x.ivar, x.mask, x.resolution_data, x.header, sp.R
Args:
wave: 1D[nwave] wavelength in Angstroms
flux: 2D[nspec, nwave] flux
ivar: 2D[nspec, nwave] inverse variance of flux
Optional:
mask: 2D[nspec, nwave] integer bitmask of flux. 0=good.
resolution_data: 3D[nspec, ndiag, nwave]
diagonals of resolution matrix data
fibers: ndarray of which fibers these spectra are
spectrograph: integer, which spectrograph [0-9]
meta: dict-like object (e.g. FITS header)
fibermap: fibermap table
chi2pix: 2D[nspec, nwave] chi2 of 2D model to pixel-level data
for pixels that contributed to each flux bin
Notes:
spectrograph input is used only if fibers is None. In this case,
it assumes nspec_per_spectrograph = flux.shape[0] and calculates
the fibers array for this spectrograph, i.e.
fibers = spectrograph * flux.shape[0] + np.arange(flux.shape[0])
Attributes:
All input args become object attributes.
nspec : number of spectra, flux.shape[0]
nwave : number of wavelengths, flux.shape[1]
specmin : minimum fiber number
R: array of sparse Resolution matrix objects converted
from resolution_data
fibermap: fibermap table if provided
"""
assert wave.ndim == 1
assert flux.ndim == 2
assert wave.shape[0] == flux.shape[1]
assert ivar.shape == flux.shape
assert (mask is None) or mask.shape == flux.shape
assert (mask is None) or mask.dtype in \
(int, np.int64, np.int32, np.uint64, np.uint32), "Bad mask type "+str(mask.dtype)
self.wave = wave
self.flux = flux
self.ivar = ivar
self.meta = meta
self.fibermap = fibermap
self.nspec, self.nwave = self.flux.shape
self.chi2pix = chi2pix
fibers_per_spectrograph = 500 #- hardcode; could get from desimodel
if mask is None:
self.mask = np.zeros(flux.shape, dtype=np.uint32)
else:
self.mask = util.mask32(mask)
if resolution_data is not None:
if resolution_data.ndim != 3 or \
resolution_data.shape[0] != self.nspec or \
resolution_data.shape[2] != self.nwave:
raise ValueError(
"Wrong dimensions for resolution_data[nspec, ndiag, nwave]"
)
#- Maybe setup non-None identity matrix resolution matrix instead?
self.resolution_data = resolution_data
if resolution_data is not None:
self.R = np.array([Resolution(r) for r in resolution_data])
self.spectrograph = spectrograph
# Deal with Fibers (these must be set!)
if fibers is not None:
fibers = np.asarray(fibers)
if len(fibers) != self.nspec:
raise ValueError("len(fibers) != nspec ({} != {})".format(
len(fibers), self.nspec))
if fibermap is not None and np.any(fibers != fibermap['FIBER']):
raise ValueError("fibermap doesn't match fibers")
if (spectrograph is not None):
minfiber = spectrograph * fibers_per_spectrograph
maxfiber = (spectrograph + 1) * fibers_per_spectrograph
if np.any(fibers < minfiber) or np.any(maxfiber <= fibers):
raise ValueError('fibers inconsistent with spectrograph')
self.fibers = fibers
else:
if fibermap is not None:
self.fibers = fibermap['FIBER']
elif spectrograph is not None:
self.fibers = spectrograph * fibers_per_spectrograph + np.arange(
self.nspec, dtype=int)
elif (self.meta is not None) and ('FIBERMIN' in self.meta):
self.fibers = self.meta['FIBERMIN'] + np.arange(self.nspec,
dtype=int)
else:
raise ValueError("Must set fibers by one of the methods!")
if self.meta is not None:
self.meta['FIBERMIN'] = np.min(self.fibers)
def sim_source_spectra(allinfo, allzbest, infofile='source-truth.fits', debug=False):
"""Build the residual (source) spectra. No redshift-fitting.
"""
from desispec.io import read_spectra, write_spectra
from desispec.spectra import Spectra
from desispec.interpolation import resample_flux
from desispec.resolution import Resolution
from redrock.external.desi import DistTargetsDESI
from redrock.templates import find_templates, Template
assert(np.all(allinfo['TARGETID'] == allzbest['TARGETID']))
nsim = len(allinfo)
# Select the subset of objects for which we got the correct lens (BGS)
# redshift.
these = np.where((allzbest['SPECTYPE'] == 'GALAXY') *
(np.abs(allzbest['Z'] - allinfo['LENS_Z']) < 0.003))[0]
print('Selecting {}/{} lenses with the correct redshift'.format(len(these), nsim))
if len(these) == 0:
raise ValueError('No spectra passed the cuts!')
allinfo = allinfo[these]
allzbest = allzbest[these]
print('Writing {}'.format(infofile))
allinfo.write(infofile, overwrite=True)
tempfile = find_templates()[0]
rrtemp = Template(tempfile)
# loop on each chunk of lens+source spectra
nchunk = len(set(allinfo['CHUNK']))
for ichunk in set(allinfo['CHUNK']):
I = np.where(allinfo['CHUNK'] == ichunk)[0]
info = allinfo[I]
zbest = allzbest[I]
specfile = 'lenssource-spectra-chunk{:03d}.fits'.format(ichunk)
sourcefile = 'source-spectra-chunk{:03d}.fits'.format(ichunk)
spectra = read_spectra(specfile).select(targets=info['TARGETID'])
for igal, zz in enumerate(zbest):
zwave = rrtemp.wave * (1 + zz['Z'])
zflux = rrtemp.flux.T.dot(zz['COEFF']).T #/ (1 + zz['Z'])
if debug:
fig, ax = plt.subplots()
for band in spectra.bands:
R = Resolution(spectra.resolution_data[band][igal])
# use fastspecfit here
modelflux = R.dot(resample_flux(spectra.wave[band], zwave, zflux))
if debug:
ax.plot(spectra.wave[band], spectra.flux[band][igal, :])
ax.plot(spectra.wave[band], modelflux)
ax.set_ylim(np.median(spectra.flux['r'][igal, :]) + np.std(spectra.flux['r'][igal, :]) * np.array([-1.5, 3]))
#ax.set_xlim(4500, 5500)
spectra.flux[band][igal, :] -= modelflux # subtract
if debug:
qafile = 'source-spectra-chunk{:03d}-{}.png'.format(ichunk, igal)
fig.savefig(qafile)
plt.close()
print('Writing {} spectra to {}'.format(len(zbest), sourcefile))
write_spectra(outfile=sourcefile, spec=spectra)
return allinfo
def sim_spectra(wave, flux, program, spectra_filename, obsconditions=None,
sourcetype=None, targetid=None, redshift=None, expid=0, seed=0, skyerr=0.0, ra=None, dec=None, meta=None, fibermap_columns=None, fullsim=False,use_poisson=True):
"""
Simulate spectra from an input set of wavelength and flux and writes a FITS file in the Spectra format that can
be used as input to the redshift fitter.
Args:
wave : 1D np.array of wavelength in Angstrom (in vacuum) in observer frame (i.e. redshifted)
flux : 1D or 2D np.array. 1D array must have same size as wave, 2D array must have shape[1]=wave.size
flux has to be in units of 10^-17 ergs/s/cm2/A
spectra_filename : path to output FITS file in the Spectra format
program : dark, lrg, qso, gray, grey, elg, bright, mws, bgs
ignored if obsconditions is not None
Optional:
obsconditions : dictionnary of observation conditions with SEEING EXPTIME AIRMASS MOONFRAC MOONALT MOONSEP
sourcetype : list of string, allowed values are (sky,elg,lrg,qso,bgs,star), type of sources, used for fiber aperture loss , default is star
targetid : list of targetids for each target. default of None has them generated as str(range(nspec))
redshift : list/array with each index being the redshifts for that target
expid : this expid number will be saved in the Spectra fibermap
seed : random seed
skyerr : fractional sky subtraction error
ra : numpy array with targets RA (deg)
dec : numpy array with targets Dec (deg)
meta : dictionnary, saved in primary fits header of the spectra file
fibermap_columns : add these columns to the fibermap
fullsim : if True, write full simulation data in extra file per camera
use_poisson : if False, do not use numpy.random.poisson to simulate the Poisson noise. This is useful to get reproducible random realizations.
"""
log = get_logger()
if len(flux.shape)==1 :
flux=flux.reshape((1,flux.size))
nspec=flux.shape[0]
log.info("Starting simulation of {} spectra".format(nspec))
if sourcetype is None :
sourcetype = np.array(["star" for i in range(nspec)])
log.debug("sourcetype = {}".format(sourcetype))
tileid = 0
telera = 0
teledec = 0
dateobs = time.gmtime()
night = desisim.obs.get_night(utc=dateobs)
program = program.lower()
frame_fibermap = desispec.io.fibermap.empty_fibermap(nspec)
frame_fibermap.meta["FLAVOR"]="custom"
frame_fibermap.meta["NIGHT"]=night
frame_fibermap.meta["EXPID"]=expid
# add DESI_TARGET
tm = desitarget.targetmask.desi_mask
frame_fibermap['DESI_TARGET'][sourcetype=="star"]=tm.STD_FAINT
frame_fibermap['DESI_TARGET'][sourcetype=="lrg"]=tm.LRG
frame_fibermap['DESI_TARGET'][sourcetype=="elg"]=tm.ELG
frame_fibermap['DESI_TARGET'][sourcetype=="qso"]=tm.QSO
frame_fibermap['DESI_TARGET'][sourcetype=="sky"]=tm.SKY
frame_fibermap['DESI_TARGET'][sourcetype=="bgs"]=tm.BGS_ANY
if fibermap_columns is not None :
for k in fibermap_columns.keys() :
frame_fibermap[k] = fibermap_columns[k]
if targetid is None:
targetid = np.arange(nspec).astype(int)
# add TARGETID
frame_fibermap['TARGETID'] = targetid
# spectra fibermap has two extra fields : night and expid
# This would be cleaner if desispec would provide the spectra equivalent
# of desispec.io.empty_fibermap()
spectra_fibermap = desispec.io.empty_fibermap(nspec)
spectra_fibermap = desispec.io.util.add_columns(spectra_fibermap,
['NIGHT', 'EXPID', 'TILEID'],
[np.int32(night), np.int32(expid), np.int32(tileid)],
)
for s in range(nspec):
for tp in frame_fibermap.dtype.fields:
spectra_fibermap[s][tp] = frame_fibermap[s][tp]
if ra is not None :
spectra_fibermap["TARGET_RA"] = ra
spectra_fibermap["FIBER_RA"] = ra
if dec is not None :
spectra_fibermap["TARGET_DEC"] = dec
spectra_fibermap["FIBER_DEC"] = dec
if obsconditions is None:
if program in ['dark', 'lrg', 'qso']:
obsconditions = desisim.simexp.reference_conditions['DARK']
elif program in ['elg', 'gray', 'grey']:
obsconditions = desisim.simexp.reference_conditions['GRAY']
elif program in ['mws', 'bgs', 'bright']:
obsconditions = desisim.simexp.reference_conditions['BRIGHT']
else:
raise ValueError('unknown program {}'.format(program))
elif isinstance(obsconditions, str):
try:
obsconditions = desisim.simexp.reference_conditions[obsconditions.upper()]
except KeyError:
raise ValueError('obsconditions {} not in {}'.format(
obsconditions.upper(),
list(desisim.simexp.reference_conditions.keys())))
try:
params = desimodel.io.load_desiparams()
wavemin = params['ccd']['b']['wavemin']
wavemax = params['ccd']['z']['wavemax']
except KeyError:
wavemin = desimodel.io.load_throughput('b').wavemin
wavemax = desimodel.io.load_throughput('z').wavemax
if wave[0] > wavemin:
log.warning('Minimum input wavelength {}>{}; padding with zeros'.format(
wave[0], wavemin))
dwave = wave[1] - wave[0]
npad = int((wave[0] - wavemin)/dwave + 1)
wavepad = np.arange(npad) * dwave
wavepad += wave[0] - dwave - wavepad[-1]
fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype)
wave = np.concatenate([wavepad, wave])
flux = np.hstack([fluxpad, flux])
assert flux.shape[1] == len(wave)
assert np.allclose(dwave, np.diff(wave))
assert wave[0] <= wavemin
if wave[-1] < wavemax:
log.warning('Maximum input wavelength {}<{}; padding with zeros'.format(
wave[-1], wavemax))
dwave = wave[-1] - wave[-2]
npad = int( (wavemax - wave[-1])/dwave + 1 )
wavepad = wave[-1] + dwave + np.arange(npad)*dwave
fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype)
wave = np.concatenate([wave, wavepad])
flux = np.hstack([flux, fluxpad])
assert flux.shape[1] == len(wave)
assert np.allclose(dwave, np.diff(wave))
assert wavemax <= wave[-1]
ii = (wavemin <= wave) & (wave <= wavemax)
flux_unit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm ** 2 )
wave = wave[ii]*u.Angstrom
flux = flux[:,ii]*flux_unit
sim = desisim.simexp.simulate_spectra(wave, flux, fibermap=frame_fibermap,
obsconditions=obsconditions, redshift=redshift, seed=seed,
psfconvolve=True)
random_state = np.random.RandomState(seed)
sim.generate_random_noise(random_state,use_poisson=use_poisson)
scale=1e17
specdata = None
resolution={}
for camera in sim.instrument.cameras:
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1])
skyscale = skyerr * random_state.normal(size=sim.num_fibers)
if fullsim :
for table in sim.camera_output :
band = table.meta['name'].strip()[0]
table_filename=spectra_filename.replace(".fits","-fullsim-{}.fits".format(band))
table.write(table_filename,format="fits",overwrite=True)
print("wrote",table_filename)
for table in sim.camera_output :
wave = table['wavelength'].astype(float)
flux = (table['observed_flux']+table['random_noise_electrons']*table['flux_calibration']).T.astype(float)
if np.any(skyscale):
flux += ((table['num_sky_electrons']*skyscale)*table['flux_calibration']).T.astype(float)
ivar = table['flux_inverse_variance'].T.astype(float)
band = table.meta['name'].strip()[0]
flux = flux * scale
ivar = ivar / scale**2
mask = np.zeros(flux.shape).astype(int)
spec = Spectra([band], {band : wave}, {band : flux}, {band : ivar},
resolution_data={band : resolution[band]},
mask={band : mask},
fibermap=spectra_fibermap,
meta=meta,
single=True)
if specdata is None :
specdata = spec
else :
specdata.update(spec)
desispec.io.write_spectra(spectra_filename, specdata)
log.info('Wrote '+spectra_filename)
# need to clear the simulation buffers that keeps growing otherwise
# because of a different number of fibers each time ...
desisim.specsim._simulators.clear()
desisim.specsim._simdefaults.clear()
def main(args):
# Set up the logger
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
# Make sure all necessary environment variables are set
DESI_SPECTRO_REDUX_DIR = "./quickGen"
if 'DESI_SPECTRO_REDUX' not in os.environ:
log.info('DESI_SPECTRO_REDUX environment is not set.')
else:
DESI_SPECTRO_REDUX_DIR = os.environ['DESI_SPECTRO_REDUX']
if os.path.exists(DESI_SPECTRO_REDUX_DIR):
if not os.path.isdir(DESI_SPECTRO_REDUX_DIR):
raise RuntimeError("Path %s Not a directory" %
DESI_SPECTRO_REDUX_DIR)
else:
try:
os.makedirs(DESI_SPECTRO_REDUX_DIR)
except:
raise
SPECPROD_DIR = 'specprod'
if 'SPECPROD' not in os.environ:
log.info('SPECPROD environment is not set.')
else:
SPECPROD_DIR = os.environ['SPECPROD']
prod_Dir = specprod_root()
if os.path.exists(prod_Dir):
if not os.path.isdir(prod_Dir):
raise RuntimeError("Path %s Not a directory" % prod_Dir)
else:
try:
os.makedirs(prod_Dir)
except:
raise
# Initialize random number generator to use.
np.random.seed(args.seed)
random_state = np.random.RandomState(args.seed)
# Derive spectrograph number from nstart if needed
if args.spectrograph is None:
args.spectrograph = args.nstart / 500
# Read fibermapfile to get object type, night and expid
if args.fibermap:
log.info("Reading fibermap file {}".format(args.fibermap))
fibermap = read_fibermap(args.fibermap)
objtype = get_source_types(fibermap)
stdindx = np.where(objtype == 'STD') # match STD with STAR
mwsindx = np.where(objtype == 'MWS_STAR') # match MWS_STAR with STAR
bgsindx = np.where(objtype == 'BGS') # match BGS with LRG
objtype[stdindx] = 'STAR'
objtype[mwsindx] = 'STAR'
objtype[bgsindx] = 'LRG'
NIGHT = fibermap.meta['NIGHT']
EXPID = fibermap.meta['EXPID']
else:
# Create a blank fake fibermap
fibermap = empty_fibermap(args.nspec)
targetids = random_state.randint(2**62, size=args.nspec)
fibermap['TARGETID'] = targetids
night = get_night()
expid = 0
log.info("Initializing SpecSim with config {}".format(args.config))
desiparams = load_desiparams()
qsim = get_simulator(args.config, num_fibers=1)
if args.simspec:
# Read the input file
log.info('Reading input file {}'.format(args.simspec))
simspec = desisim.io.read_simspec(args.simspec)
nspec = simspec.nspec
if simspec.flavor == 'arc':
log.warning("quickgen doesn't generate flavor=arc outputs")
return
else:
wavelengths = simspec.wave
spectra = simspec.flux
if nspec < args.nspec:
log.info("Only {} spectra in input file".format(nspec))
args.nspec = nspec
else:
# Initialize the output truth table.
spectra = []
wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value
npix = len(wavelengths)
truth = dict()
meta = Table()
truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10))
truth['FLUX'] = np.zeros((args.nspec, npix))
truth['WAVE'] = wavelengths
jj = list()
for thisobj in set(true_objtype):
ii = np.where(true_objtype == thisobj)[0]
nobj = len(ii)
truth['OBJTYPE'][ii] = thisobj
log.info('Generating {} template'.format(thisobj))
# Generate the templates
if thisobj == 'ELG':
elg = desisim.templates.ELG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = elg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_elg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'LRG':
lrg = desisim.templates.LRG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = lrg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_lrg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'QSO':
qso = desisim.templates.QSO(wave=wavelengths)
flux, tmpwave, meta1 = qso.make_templates(
nmodel=nobj, seed=args.seed, zrange=args.zrange_qso)
elif thisobj == 'BGS':
bgs = desisim.templates.BGS(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = bgs.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_bgs,
rmagrange=args.rmagrange_bgs,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'STD':
std = desisim.templates.STD(wave=wavelengths)
flux, tmpwave, meta1 = std.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'QSO_BAD': # use STAR template no color cuts
star = desisim.templates.STAR(wave=wavelengths)
flux, tmpwave, meta1 = star.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'MWS_STAR' or thisobj == 'MWS':
mwsstar = desisim.templates.MWS_STAR(wave=wavelengths)
flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'WD':
wd = desisim.templates.WD(wave=wavelengths)
flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'SKY':
flux = np.zeros((nobj, npix))
meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32)))
elif thisobj == 'TEST':
flux = np.zeros((args.nspec, npix))
indx = np.where(wave > 5800.0 - 1E-6)[0][0]
ref_integrated_flux = 1E-10
ref_cst_flux_density = 1E-17
single_line = (np.arange(args.nspec) % 2 == 0).astype(
np.float32)
continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32)
for spec in range(args.nspec):
flux[spec, indx] = single_line[
spec] * ref_integrated_flux / np.gradient(wavelengths)[
indx] # single line
flux[spec] += continuum[
spec] * ref_cst_flux_density # flat continuum
meta1 = Table(
dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32),
LINE=wave[indx] *
np.ones(args.nspec, dtype=np.float32),
LINEFLUX=single_line * ref_integrated_flux,
CONSTFLUXDENSITY=continuum * ref_cst_flux_density))
else:
log.fatal('Unknown object type {}'.format(thisobj))
sys.exit(1)
# Pack it in.
truth['FLUX'][ii] = flux
meta = vstack([meta, meta1])
jj.append(ii.tolist())
# Sanity check on units; templates currently return ergs, not 1e-17 ergs...
# assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6)
# Sort the metadata table.
jj = sum(jj, [])
meta_new = Table()
for k in range(args.nspec):
index = int(np.where(np.array(jj) == k)[0])
meta_new = vstack([meta_new, meta[index]])
meta = meta_new
# Add TARGETID and the true OBJTYPE to the metadata table.
meta.add_column(
Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE'))
meta.add_column(Column(targetids, name='TARGETID'))
# Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z
meta.rename_column('REDSHIFT', 'TRUEZ')
# explicitly set location on focal plane if needed to support airmass
# variations when using specsim v0.5
if qsim.source.focal_xy is None:
qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm'))
# Set simulation parameters from the simspec header or desiparams
bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX']
gray_objects = ['gray', 'grey']
if args.simspec is None:
object_type = objtype
flavor = None
elif simspec.flavor == 'science':
object_type = None
flavor = simspec.header['PROGRAM']
else:
object_type = None
flavor = simspec.flavor
log.warning(
'Maybe using an outdated simspec file with flavor={}'.format(
flavor))
# Set airmass
if args.airmass is not None:
qsim.atmosphere.airmass = args.airmass
elif args.simspec and 'AIRMASS' in simspec.header:
qsim.atmosphere.airmass = simspec.header['AIRMASS']
else:
qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2
# Set exptime
if args.exptime is not None:
qsim.observation.exposure_time = args.exptime * u.s
elif args.simspec and 'EXPTIME' in simspec.header:
qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s
elif objtype in bright_objects:
qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s
else:
qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s
# Set Moon Phase
if args.moon_phase is not None:
qsim.atmosphere.moon.moon_phase = args.moon_phase
elif args.simspec and 'MOONFRAC' in simspec.header:
qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC']
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_phase = 0.7
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_phase = 0.1
else:
qsim.atmosphere.moon.moon_phase = 0.5
# Set Moon Zenith
if args.moon_zenith is not None:
qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg
elif args.simspec and 'MOONALT' in simspec.header:
qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_zenith = 30 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_zenith = 80 * u.deg
else:
qsim.atmosphere.moon.moon_zenith = 100 * u.deg
# Set Moon - Object Angle
if args.moon_angle is not None:
qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg
elif args.simspec and 'MOONSEP' in simspec.header:
qsim.atmosphere.moon.separation_angle = simspec.header[
'MOONSEP'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.separation_angle = 50 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
else:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
# Initialize per-camera output arrays that will be saved
waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {}
maxbin = 0
nmax = args.nspec
for camera in qsim.instrument.cameras:
# Lookup this camera's resolution matrix and convert to the sparse
# format used in desispec.
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(),
[args.nspec, 1, 1])
waves[camera.name] = (camera.output_wavelength.to(
u.Angstrom).value.astype(np.float32))
nwave = len(waves[camera.name])
maxbin = max(maxbin, len(waves[camera.name]))
nobj = np.zeros((nmax, 3, maxbin)) # object photons
nsky = np.zeros((nmax, 3, maxbin)) # sky photons
nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky)
cframe_observedflux = np.zeros(
(nmax, 3, maxbin)) # calibrated object flux
cframe_ivar = np.zeros(
(nmax, 3, maxbin)) # inverse variance of calibrated object flux
cframe_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to calibrated flux
sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky
sky_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to sky only
frame_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky
trueflux[camera.name] = np.empty(
(args.nspec, nwave)) # calibrated flux
noisyflux[camera.name] = np.empty(
(args.nspec, nwave)) # observed flux with noise
obsivar[camera.name] = np.empty(
(args.nspec, nwave)) # inverse variance of flux
if args.simspec:
for i in range(10):
cn = camera.name + str(i)
if cn in simspec.cameras:
dw = np.gradient(simspec.cameras[cn].wave)
break
else:
raise RuntimeError(
'Unable to find a {} camera in input simspec'.format(
camera))
else:
sflux = np.empty((args.nspec, npix))
#- Check if input simspec is for a continuum flat lamp instead of science
#- This does not convolve to per-fiber resolution
if args.simspec:
if simspec.flavor == 'flat':
log.info("Simulating flat lamp exposure")
for i, camera in enumerate(qsim.instrument.cameras):
channel = camera.name #- from simspec, b/r/z not b0/r1/z9
assert camera.output_wavelength.unit == u.Angstrom
num_pixels = len(waves[channel])
phot = list()
for j in range(10):
cn = camera.name + str(j)
if cn in simspec.cameras:
camwave = simspec.cameras[cn].wave
dw = np.gradient(camwave)
phot.append(simspec.cameras[cn].phot)
if len(phot) == 0:
raise RuntimeError(
'Unable to find a {} camera in input simspec'.format(
camera))
else:
phot = np.vstack(phot)
meanspec = resample_flux(waves[channel], camwave,
np.average(phot / dw, axis=0))
fiberflat = random_state.normal(loc=1.0,
scale=1.0 / np.sqrt(meanspec),
size=(nspec, num_pixels))
ivar = np.tile(meanspec, [nspec, 1])
mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32)
for kk in range((args.nspec + args.nstart - 1) // 500 + 1):
camera = channel + str(kk)
outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID,
camera)
start = max(500 * kk, args.nstart)
end = min(500 * (kk + 1), nmax)
if (args.spectrograph <= kk):
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, kk, start, end))
ff = FiberFlat(waves[channel],
fiberflat[start:end, :],
ivar[start:end, :],
mask[start:end, :],
meanspec,
header=dict(CAMERA=camera))
write_fiberflat(outfile, ff)
filePath = desispec.io.findfile("fiberflat", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(filePath))
sys.exit(0)
# Repeat the simulation for all spectra
fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom)
for j in range(args.nspec):
thisobjtype = objtype[j]
sys.stdout.flush()
if flavor == 'arc':
qsim.source.update_in('Quickgen source {0}'.format, 'perfect',
wavelengths * u.Angstrom,
spectra * fluxunits)
else:
qsim.source.update_in('Quickgen source {0}'.format(j),
thisobjtype.lower(),
wavelengths * u.Angstrom,
spectra[j, :] * fluxunits)
qsim.source.update_out()
qsim.simulate()
qsim.generate_random_noise(random_state)
for i, output in enumerate(qsim.camera_output):
assert output['observed_flux'].unit == 1e17 * fluxunits
# Extract the simulation results needed to create our uncalibrated
# frame output file.
num_pixels = len(output)
nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0]
nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0]
nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0]
# Get results for our flux-calibrated output file.
cframe_observedflux[
j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0]
cframe_ivar[
j,
i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0]
# Fill brick arrays from the results.
camera = output.meta['name']
trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0]
noisyflux[camera][j][:] = 1e17 * (
output['observed_flux'][:, 0] +
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,
0]
# Use the same noise realization in the cframe and frame, without any
# additional noise from sky subtraction for now.
frame_rand_noise[
j, i, :num_pixels] = output['random_noise_electrons'][:, 0]
cframe_rand_noise[j, i, :num_pixels] = 1e17 * (
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
# The sky output file represents a model fit to ~40 sky fibers.
# We reduce the variance by a factor of 25 to account for this and
# give the sky an independent (Gaussian) noise realization.
sky_ivar[
j,
i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] -
output['num_source_electrons'][:, 0])
sky_rand_noise[j, i, :num_pixels] = random_state.normal(
scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]),
size=num_pixels)
armName = {"b": 0, "r": 1, "z": 2}
for channel in 'brz':
#Before writing, convert from counts/bin to counts/A (as in Pixsim output)
#Quicksim Default:
#FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang]
#COUNTS_OBJ - object counts in 0.5 Ang bin
#COUNTS_SKY - sky counts in 0.5 Ang bin
num_pixels = len(waves[channel])
dwave = np.gradient(waves[channel])
nobj[:, armName[channel], :num_pixels] /= dwave
frame_rand_noise[:, armName[channel], :num_pixels] /= dwave
nivar[:, armName[channel], :num_pixels] *= dwave**2
nsky[:, armName[channel], :num_pixels] /= dwave
sky_rand_noise[:, armName[channel], :num_pixels] /= dwave
sky_ivar[:, armName[channel], :num_pixels] /= dwave**2
# Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra
# Looping over spectrograph
for ii in range((args.nspec + args.nstart - 1) // 500 + 1):
start = max(500 * ii,
args.nstart) # first spectrum for a given spectrograph
end = min(500 * (ii + 1),
nmax) # last spectrum for the spectrograph
if (args.spectrograph <= ii):
camera = "{}{}".format(channel, ii)
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, ii, start, end))
num_pixels = len(waves[channel])
# Write frame file
framefileName = desispec.io.findfile("frame", NIGHT, EXPID,
camera)
frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \
nsky[start:end,armName[channel],:num_pixels] + \
frame_rand_noise[start:end,armName[channel],:num_pixels]
frame_ivar = nivar[start:end, armName[channel], :num_pixels]
sh1 = frame_flux.shape[
0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave)
# for example if nstart =400, nspec=150: two spectrographs:
# 400-499=> 0 spectrograph, 500-549 => 1
if (args.nstart == start):
resol = resolution[channel][:sh1, :, :]
else:
resol = resolution[channel][-sh1:, :, :]
# must create desispec.Frame object
frame=Frame(waves[channel], frame_flux, frame_ivar,\
resolution_data=resol, spectrograph=ii, \
fibermap=fibermap[start:end], \
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.write_frame(framefileName, frame)
framefilePath = desispec.io.findfile("frame", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(framefilePath))
if args.frameonly or simspec.flavor == 'arc':
continue
# Write cframe file
cframeFileName = desispec.io.findfile("cframe", NIGHT, EXPID,
camera)
cframeFlux = cframe_observedflux[
start:end,
armName[channel], :num_pixels] + cframe_rand_noise[
start:end, armName[channel], :num_pixels]
cframeIvar = cframe_ivar[start:end,
armName[channel], :num_pixels]
# must create desispec.Frame object
cframe = Frame(waves[channel], cframeFlux, cframeIvar, \
resolution_data=resol, spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.frame.write_frame(cframeFileName, cframe)
cframefilePath = desispec.io.findfile("cframe", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(cframefilePath))
# Write sky file
skyfileName = desispec.io.findfile("sky", NIGHT, EXPID, camera)
skyflux=nsky[start:end,armName[channel],:num_pixels] + \
sky_rand_noise[start:end,armName[channel],:num_pixels]
skyivar = sky_ivar[start:end, armName[channel], :num_pixels]
skymask = np.zeros(skyflux.shape, dtype=np.uint32)
# must create desispec.Sky object
skymodel = SkyModel(waves[channel],
skyflux,
skyivar,
skymask,
header=dict(CAMERA=camera))
desispec.io.sky.write_sky(skyfileName, skymodel)
skyfilePath = desispec.io.findfile("sky", NIGHT, EXPID, camera)
log.info("Wrote file {}".format(skyfilePath))
# Write calib file
calibVectorFile = desispec.io.findfile("calib", NIGHT, EXPID,
camera)
flux = cframe_observedflux[start:end,
armName[channel], :num_pixels]
phot = nobj[start:end, armName[channel], :num_pixels]
calibration = np.zeros_like(phot)
jj = (flux > 0)
calibration[jj] = phot[jj] / flux[jj]
#- TODO: what should calibivar be?
#- For now, model it as the noise of combining ~10 spectra
calibivar = 10 / cframe_ivar[start:end,
armName[channel], :num_pixels]
#mask=(1/calibivar>0).astype(int)??
mask = np.zeros(calibration.shape, dtype=np.uint32)
# write flux calibration
fluxcalib = FluxCalib(waves[channel], calibration, calibivar,
mask)
write_flux_calibration(calibVectorFile, fluxcalib)
calibfilePath = desispec.io.findfile("calib", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(calibfilePath))
def compute_polynomial_times_sky(frame, nsig_clipping=4.,max_iterations=30,model_ivar=False,add_variance=True,angular_variation_deg=1,chromatic_variation_deg=1) :
"""Compute a sky model.
Sky[fiber,i] = R[fiber,i,j] Polynomial(x[fiber],y[fiber],wavelength[j]) Flux[j]
Input flux are expected to be flatfielded!
We don't check this in this routine.
Args:
frame : Frame object, which includes attributes
- wave : 1D wavelength grid in Angstroms
- flux : 2D flux[nspec, nwave] density
- ivar : 2D inverse variance of flux
- mask : 2D inverse mask flux (0=good)
- resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers)
nsig_clipping : [optional] sigma clipping value for outlier rejection
Optional:
max_iterations : int , number of iterations
model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims.
add_variance : evaluate calibration error and add this to the sky model variance
returns SkyModel object with attributes wave, flux, ivar, mask
"""
log=get_logger()
log.info("starting")
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nwave=frame.nwave
nfibers=len(skyfibers)
current_ivar=frame.ivar[skyfibers].copy()*(frame.mask[skyfibers]==0)
flux = frame.flux[skyfibers]
Rsky = frame.R[skyfibers]
input_ivar=None
if model_ivar :
log.info("use a model of the inverse variance to remove bias due to correlated ivar and flux")
input_ivar=current_ivar.copy()
median_ivar_vs_wave = np.median(current_ivar,axis=0)
median_ivar_vs_fiber = np.median(current_ivar,axis=1)
median_median_ivar = np.median(median_ivar_vs_fiber)
for f in range(current_ivar.shape[0]) :
threshold=0.01
current_ivar[f] = median_ivar_vs_fiber[f]/median_median_ivar * median_ivar_vs_wave
# keep input ivar for very low weights
ii=(input_ivar[f]<=(threshold*median_ivar_vs_wave))
#log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1]))
current_ivar[f][ii] = input_ivar[f][ii]
# need focal plane coordinates
x = frame.fibermap["DESIGN_X"]
y = frame.fibermap["DESIGN_Y"]
# normalize for numerical stability
xm = np.mean(x)
ym = np.mean(y)
xs = np.std(x)
ys = np.std(y)
if xs==0 : xs = 1
if ys==0 : ys = 1
x = (x-xm)/xs
y = (y-ym)/ys
w = (frame.wave-frame.wave[0])/(frame.wave[-1]-frame.wave[0])*2.-1
# precompute the monomials for the sky fibers
log.debug("compute monomials for deg={} and {}".format(angular_variation_deg,chromatic_variation_deg))
monomials=[]
for dx in range(angular_variation_deg+1) :
for dy in range(angular_variation_deg+1-dx) :
xypol = (x**dx)*(y**dy)
for dw in range(chromatic_variation_deg+1) :
wpol=w**dw
monomials.append(np.outer(xypol,wpol))
ncoef=len(monomials)
coef=np.zeros((ncoef))
allfibers_monomials=np.array(monomials)
log.debug("shape of allfibers_monomials = {}".format(allfibers_monomials.shape))
skyfibers_monomials = allfibers_monomials[:,skyfibers,:]
log.debug("shape of skyfibers_monomials = {}".format(skyfibers_monomials.shape))
sqrtw=np.sqrt(current_ivar)
sqrtwflux=sqrtw*flux
chi2=np.zeros(flux.shape)
Pol = np.ones(flux.shape,dtype=float)
coef[0] = 1.
nout_tot=0
previous_chi2=-10.
for iteration in range(max_iterations) :
# the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters
# A_ij = 1/2 d2(chi2)/di/dj
# A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w]
# the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters
# B_i = 1/2 d(chi2)/di
# B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w])
# the model is model[fiber]=R[fiber]*Pol(x,y,wave)*sky
# the parameters are the unconvolved sky flux at the wavelength i
# and the polynomial coefficients
A=np.zeros((nwave,nwave),dtype=float)
B=np.zeros((nwave),dtype=float)
D=scipy.sparse.lil_matrix((nwave,nwave))
D2=scipy.sparse.lil_matrix((nwave,nwave))
Pol /= coef[0] # force constant term to 1.
# solving for the deconvolved mean sky spectrum
# loop on fiber to handle resolution
for fiber in range(nfibers) :
if fiber%10==0 :
log.info("iter %d sky fiber (1st fit) %d/%d"%(iteration,fiber,nfibers))
D.setdiag(sqrtw[fiber])
D2.setdiag(Pol[fiber])
sqrtwRP = D.dot(Rsky[fiber]).dot(D2) # each row r of R is multiplied by sqrtw[r]
A += (sqrtwRP.T*sqrtwRP).todense()
B += sqrtwRP.T*sqrtwflux[fiber]
log.info("iter %d solving"%iteration)
w = A.diagonal()>0
A_pos_def = A[w,:]
A_pos_def = A_pos_def[:,w]
parameters = B*0
try:
parameters[w]=cholesky_solve(A_pos_def,B[w])
except:
log.info("cholesky failed, trying svd in iteration {}".format(iteration))
parameters[w]=np.linalg.lstsq(A_pos_def,B[w])[0]
# parameters = the deconvolved mean sky spectrum
# now evaluate the polynomial coefficients
Ap=np.zeros((ncoef,ncoef),dtype=float)
Bp=np.zeros((ncoef),dtype=float)
D2.setdiag(parameters)
for fiber in range(nfibers) :
if fiber%10==0 :
log.info("iter %d sky fiber (2nd fit) %d/%d"%(iteration,fiber,nfibers))
D.setdiag(sqrtw[fiber])
sqrtwRSM = D.dot(Rsky[fiber]).dot(D2).dot(skyfibers_monomials[:,fiber,:].T)
Ap += sqrtwRSM.T.dot(sqrtwRSM)
Bp += sqrtwRSM.T.dot(sqrtwflux[fiber])
# Add huge prior on zeroth angular order terms to converge faster
# (because those terms are degenerate with the mean deconvolved spectrum)
weight=1e24
Ap[0,0] += weight
Bp[0] += weight # force 0th term to 1
for i in range(1,chromatic_variation_deg+1) :
Ap[i,i] += weight # force other wavelength terms to 0
coef=cholesky_solve(Ap,Bp)
log.info("pol coef = {}".format(coef))
# recompute the polynomial values
Pol = skyfibers_monomials.T.dot(coef).T
# chi2 and outlier rejection
log.info("iter %d compute chi2"%iteration)
for fiber in range(nfibers) :
chi2[fiber]=current_ivar[fiber]*(flux[fiber]-Rsky[fiber].dot(Pol[fiber]*parameters))**2
log.info("rejecting")
nout_iter=0
if iteration<1 :
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave=np.sum(chi2>nsig_clipping**2,axis=0)
selection=np.where(nout_per_wave>0)[0]
for i in selection :
worst_entry=np.argmax(chi2[:,i])
current_ivar[worst_entry,i]=0
sqrtw[worst_entry,i]=0
sqrtwflux[worst_entry,i]=0
nout_iter += 1
else :
# remove all of them at once
bad=(chi2>nsig_clipping**2)
current_ivar *= (bad==0)
sqrtw *= (bad==0)
sqrtwflux *= (bad==0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2=float(np.sum(chi2))
ndf=int(np.sum(chi2>0)-nwave)
chi2pdf=0.
if ndf>0 :
chi2pdf=sum_chi2/ndf
log.info("iter #%d chi2=%g ndf=%d chi2pdf=%f delta=%f nout=%d"%(iteration,sum_chi2,ndf,chi2pdf,abs(sum_chi2-previous_chi2),nout_iter))
if nout_iter == 0 and abs(sum_chi2-previous_chi2)<0.2 :
break
previous_chi2 = sum_chi2+0.
log.info("nout tot=%d"%nout_tot)
# we know have to compute the sky model for all fibers
# and propagate the uncertainties
# no need to restore the original ivar to compute the model errors when modeling ivar
# the sky inverse variances are very similar
# we ignore here the fact that we have fit a angular variation,
# so the sky model uncertainties are inaccurate
log.info("compute the parameter covariance")
try :
parameter_covar=cholesky_invert(A)
except np.linalg.linalg.LinAlgError :
log.warning("cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv")
parameter_covar = np.linalg.pinv(A)
log.info("compute mean resolution")
# we make an approximation for the variance to save CPU time
# we use the average resolution of all fibers in the frame:
mean_res_data=np.mean(frame.resolution_data,axis=0)
Rmean = Resolution(mean_res_data)
log.info("compute convolved sky and ivar")
# The parameters are directly the unconvolved sky
# First convolve with average resolution :
convolved_sky_covar=Rmean.dot(parameter_covar).dot(Rmean.T.todense())
# and keep only the diagonal
convolved_sky_var=np.diagonal(convolved_sky_covar)
# inverse
convolved_sky_ivar=(convolved_sky_var>0)/(convolved_sky_var+(convolved_sky_var==0))
# and simply consider it's the same for all spectra
cskyivar = np.tile(convolved_sky_ivar, frame.nspec).reshape(frame.nspec, nwave)
# The sky model for each fiber (simple convolution with resolution of each fiber)
cskyflux = np.zeros(frame.flux.shape)
Pol = allfibers_monomials.T.dot(coef).T
for fiber in range(frame.nspec):
cskyflux[fiber] = frame.R[fiber].dot(Pol[fiber]*parameters)
# look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1
if skyfibers.size > 1 and add_variance :
modified_cskyivar = _model_variance(frame,cskyflux,cskyivar,skyfibers)
else :
modified_cskyivar = cskyivar.copy()
# need to do better here
mask = (cskyivar==0).astype(np.uint32)
return SkyModel(frame.wave.copy(), cskyflux, modified_cskyivar, mask,
nrej=nout_tot, stat_ivar = cskyivar) # keep a record of the statistical ivar for QA
def spectroperf_resample_spectrum_multiproc(shm_in_wave, shm_in_flux,
shm_in_ivar, shm_in_rdata,
in_nwave, in_ndiag, in_bands,
target_indices, wave, wavebin,
resampling_matrix, ndiag, ntarget,
shm_flux, shm_ivar, shm_rdata):
nwave = wave.size
# manipulate shared memory as np arrays
# input shared memory
in_wave = list()
in_flux = list()
in_ivar = list()
in_rdata = list()
nbands = len(shm_in_wave)
for b in range(nbands):
in_wave.append(
np.array(shm_in_wave[b], copy=False).reshape(in_nwave[b]))
in_flux.append(
np.array(shm_in_flux[b], copy=False).reshape(
(ntarget, in_nwave[b])))
in_ivar.append(
np.array(shm_in_ivar[b], copy=False).reshape(
(ntarget, in_nwave[b])))
in_rdata.append(
np.array(shm_in_rdata[b], copy=False).reshape(
(ntarget, in_ndiag[b], in_nwave[b])))
# output shared memory
flux = np.array(shm_flux, copy=False).reshape(ntarget, nwave)
ivar = np.array(shm_ivar, copy=False).reshape(ntarget, nwave)
rdata = np.array(shm_rdata, copy=False).reshape(ntarget, ndiag, nwave)
for target_index in target_indices:
cinv = None
for b in range(nbands):
twave = in_wave[b]
jj = (twave >= wave[0]) & (twave <= wave[-1])
twave = twave[jj]
tivar = in_ivar[b][target_index][jj]
diag_ivar = scipy.sparse.dia_matrix((tivar, [0]),
(twave.size, twave.size))
RR = Resolution(in_rdata[b][target_index][:, jj]).dot(
resampling_matrix[in_bands[b]])
tcinv = RR.T.dot(diag_ivar.dot(RR))
tcinvf = RR.T.dot(tivar * in_flux[b][target_index][jj])
if cinv is None:
cinv = tcinv
cinvf = tcinvf
else:
cinv += tcinv
cinvf += tcinvf
cinv = cinv.todense()
decorrelate_divide_and_conquer(cinv, cinvf, wavebin,
flux[target_index], ivar[target_index],
rdata[target_index])
def compute_uniform_sky(frame, nsig_clipping=4.,max_iterations=100,model_ivar=False,add_variance=True) :
"""Compute a sky model.
Sky[fiber,i] = R[fiber,i,j] Flux[j]
Input flux are expected to be flatfielded!
We don't check this in this routine.
Args:
frame : Frame object, which includes attributes
- wave : 1D wavelength grid in Angstroms
- flux : 2D flux[nspec, nwave] density
- ivar : 2D inverse variance of flux
- mask : 2D inverse mask flux (0=good)
- resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers)
nsig_clipping : [optional] sigma clipping value for outlier rejection
Optional:
max_iterations : int , number of iterations
model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims.
add_variance : evaluate calibration error and add this to the sky model variance
returns SkyModel object with attributes wave, flux, ivar, mask
"""
log=get_logger()
log.info("starting")
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nwave=frame.nwave
nfibers=len(skyfibers)
current_ivar=frame.ivar[skyfibers].copy()*(frame.mask[skyfibers]==0)
flux = frame.flux[skyfibers]
Rsky = frame.R[skyfibers]
input_ivar=None
if model_ivar :
log.info("use a model of the inverse variance to remove bias due to correlated ivar and flux")
input_ivar=current_ivar.copy()
median_ivar_vs_wave = np.median(current_ivar,axis=0)
median_ivar_vs_fiber = np.median(current_ivar,axis=1)
median_median_ivar = np.median(median_ivar_vs_fiber)
for f in range(current_ivar.shape[0]) :
threshold=0.01
current_ivar[f] = median_ivar_vs_fiber[f]/median_median_ivar * median_ivar_vs_wave
# keep input ivar for very low weights
ii=(input_ivar[f]<=(threshold*median_ivar_vs_wave))
#log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1]))
current_ivar[f][ii] = input_ivar[f][ii]
sqrtw=np.sqrt(current_ivar)
sqrtwflux=sqrtw*flux
chi2=np.zeros(flux.shape)
nout_tot=0
for iteration in range(max_iterations) :
# the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters
# A_ij = 1/2 d2(chi2)/di/dj
# A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w]
# the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters
# B_i = 1/2 d(chi2)/di
# B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w])
# the model is model[fiber]=R[fiber]*sky
# and the parameters are the unconvolved sky flux at the wavelength i
# so, d(model)/di[fiber,w] = R[fiber][w,i]
# this gives
# A_ij = sum_fiber sum_wave_w ivar[fiber,w] R[fiber][w,i] R[fiber][w,j]
# A = sum_fiber ( diag(sqrt(ivar))*R[fiber] ) ( diag(sqrt(ivar))* R[fiber] )^t
# A = sum_fiber sqrtwR[fiber] sqrtwR[fiber]^t
# and
# B = sum_fiber sum_wave_w ivar[fiber,w] R[fiber][w] * flux[fiber,w]
# B = sum_fiber sum_wave_w sqrt(ivar)[fiber,w]*flux[fiber,w] sqrtwR[fiber,wave]
#A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr()
A=np.zeros((nwave,nwave))
B=np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD=scipy.sparse.lil_matrix((nwave,nwave))
# loop on fiber to handle resolution
for fiber in range(nfibers) :
if fiber%10==0 :
log.info("iter %d sky fiber %d/%d"%(iteration,fiber,nfibers))
R = Rsky[fiber]
# diagonal sparse matrix with content = sqrt(ivar)
SD.setdiag(sqrtw[fiber])
sqrtwR = SD*R # each row r of R is multiplied by sqrtw[r]
A += (sqrtwR.T*sqrtwR).todense()
B += sqrtwR.T*sqrtwflux[fiber]
log.info("iter %d solving"%iteration)
w = A.diagonal()>0
A_pos_def = A[w,:]
A_pos_def = A_pos_def[:,w]
parameters = B*0
try:
parameters[w]=cholesky_solve(A_pos_def,B[w])
except:
log.info("cholesky failed, trying svd in iteration {}".format(iteration))
parameters[w]=np.linalg.lstsq(A_pos_def,B[w])[0]
log.info("iter %d compute chi2"%iteration)
for fiber in range(nfibers) :
# the parameters are directly the unconvolve sky flux
# so we simply have to reconvolve it
fiber_convolved_sky_flux = Rsky[fiber].dot(parameters)
chi2[fiber]=current_ivar[fiber]*(flux[fiber]-fiber_convolved_sky_flux)**2
log.info("rejecting")
nout_iter=0
if iteration<1 :
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave=np.sum(chi2>nsig_clipping**2,axis=0)
selection=np.where(nout_per_wave>0)[0]
for i in selection :
worst_entry=np.argmax(chi2[:,i])
current_ivar[worst_entry,i]=0
sqrtw[worst_entry,i]=0
sqrtwflux[worst_entry,i]=0
nout_iter += 1
else :
# remove all of them at once
bad=(chi2>nsig_clipping**2)
current_ivar *= (bad==0)
sqrtw *= (bad==0)
sqrtwflux *= (bad==0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2=float(np.sum(chi2))
ndf=int(np.sum(chi2>0)-nwave)
chi2pdf=0.
if ndf>0 :
chi2pdf=sum_chi2/ndf
log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d"%(iteration,sum_chi2,ndf,chi2pdf,nout_iter))
if nout_iter == 0 :
break
log.info("nout tot=%d"%nout_tot)
# we know have to compute the sky model for all fibers
# and propagate the uncertainties
# no need to restore the original ivar to compute the model errors when modeling ivar
# the sky inverse variances are very similar
log.info("compute the parameter covariance")
# we may have to use a different method to compute this
# covariance
try :
parameter_covar=cholesky_invert(A)
# the above is too slow
# maybe invert per block, sandwich by R
except np.linalg.linalg.LinAlgError :
log.warning("cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv")
parameter_covar = np.linalg.pinv(A)
log.info("compute mean resolution")
# we make an approximation for the variance to save CPU time
# we use the average resolution of all fibers in the frame:
mean_res_data=np.mean(frame.resolution_data,axis=0)
Rmean = Resolution(mean_res_data)
log.info("compute convolved sky and ivar")
# The parameters are directly the unconvolved sky
# First convolve with average resolution :
convolved_sky_covar=Rmean.dot(parameter_covar).dot(Rmean.T.todense())
# and keep only the diagonal
convolved_sky_var=np.diagonal(convolved_sky_covar)
# inverse
convolved_sky_ivar=(convolved_sky_var>0)/(convolved_sky_var+(convolved_sky_var==0))
# and simply consider it's the same for all spectra
cskyivar = np.tile(convolved_sky_ivar, frame.nspec).reshape(frame.nspec, nwave)
# The sky model for each fiber (simple convolution with resolution of each fiber)
cskyflux = np.zeros(frame.flux.shape)
for i in range(frame.nspec):
cskyflux[i] = frame.R[i].dot(parameters)
# look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1
if skyfibers.size > 1 and add_variance :
modified_cskyivar = _model_variance(frame,cskyflux,cskyivar,skyfibers)
else :
modified_cskyivar = cskyivar.copy()
# need to do better here
mask = (cskyivar==0).astype(np.uint32)
return SkyModel(frame.wave.copy(), cskyflux, modified_cskyivar, mask,
nrej=nout_tot, stat_ivar = cskyivar) # keep a record of the statistical ivar for QA
def compute_sky(frame, nsig_clipping=4.) :
"""Compute a sky model.
Input has to correspond to sky fibers only.
Input flux are expected to be flatfielded!
We don't check this in this routine.
Args:
frame : Frame object, which includes attributes
- wave : 1D wavelength grid in Angstroms
- flux : 2D flux[nspec, nwave] density
- ivar : 2D inverse variance of flux
- mask : 2D inverse mask flux (0=good)
- resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers)
nsig_clipping : [optional] sigma clipping value for outlier rejection
returns SkyModel object with attributes wave, flux, ivar, mask
"""
log=get_logger()
log.info("starting")
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nwave=frame.nwave
nfibers=len(skyfibers)
current_ivar=frame.ivar[skyfibers].copy()
flux = frame.flux[skyfibers]
Rsky = frame.R[skyfibers]
sqrtw=np.sqrt(current_ivar)
sqrtwflux=sqrtw*flux
chi2=np.zeros(flux.shape)
#debug
#nfibers=min(nfibers,2)
nout_tot=0
for iteration in range(20) :
A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr()
B=np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD=scipy.sparse.lil_matrix((nwave,nwave))
# loop on fiber to handle resolution
for fiber in range(nfibers) :
if fiber%10==0 :
log.info("iter %d fiber %d"%(iteration,fiber))
R = Rsky[fiber]
# diagonal sparse matrix with content = sqrt(ivar)
SD.setdiag(sqrtw[fiber])
sqrtwR = SD*R # each row r of R is multiplied by sqrtw[r]
A = A+(sqrtwR.T*sqrtwR).tocsr()
B += sqrtwR.T*sqrtwflux[fiber]
log.info("iter %d solving"%iteration)
skyflux=cholesky_solve(A.todense(),B)
log.info("iter %d compute chi2"%iteration)
for fiber in range(nfibers) :
S = Rsky[fiber].dot(skyflux)
chi2[fiber]=current_ivar[fiber]*(flux[fiber]-S)**2
log.info("rejecting")
nout_iter=0
if iteration<1 :
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave=np.sum(chi2>nsig_clipping**2,axis=0)
selection=np.where(nout_per_wave>0)[0]
for i in selection :
worst_entry=np.argmax(chi2[:,i])
current_ivar[worst_entry,i]=0
sqrtw[worst_entry,i]=0
sqrtwflux[worst_entry,i]=0
nout_iter += 1
else :
# remove all of them at once
bad=(chi2>nsig_clipping**2)
current_ivar *= (bad==0)
sqrtw *= (bad==0)
sqrtwflux *= (bad==0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2=float(np.sum(chi2))
ndf=int(np.sum(chi2>0)-nwave)
chi2pdf=0.
if ndf>0 :
chi2pdf=sum_chi2/ndf
log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d"%(iteration,sum_chi2,ndf,chi2pdf,nout_iter))
if nout_iter == 0 :
break
log.info("nout tot=%d"%nout_tot)
# solve once again to get deconvolved sky variance
skyflux,skycovar=cholesky_solve_and_invert(A.todense(),B)
#- sky inverse variance, but incomplete and not needed anyway
# skyvar=np.diagonal(skycovar)
# skyivar=(skyvar>0)/(skyvar+(skyvar==0))
# Use diagonal of skycovar convolved with mean resolution of all fibers
# first compute average resolution
mean_res_data=np.mean(frame.resolution_data,axis=0)
R = Resolution(mean_res_data)
# compute convolved sky and ivar
cskycovar=R.dot(skycovar).dot(R.T.todense())
cskyvar=np.diagonal(cskycovar)
cskyivar=(cskyvar>0)/(cskyvar+(cskyvar==0))
# convert cskyivar to 2D; today it is the same for all spectra,
# but that may not be the case in the future
cskyivar = np.tile(cskyivar, frame.nspec).reshape(frame.nspec, nwave)
# Convolved sky
cskyflux = np.zeros(frame.flux.shape)
for i in range(frame.nspec):
cskyflux[i] = frame.R[i].dot(skyflux)
# need to do better here
mask = (cskyivar==0).astype(np.uint32)
return SkyModel(frame.wave.copy(), cskyflux, cskyivar, mask,
nrej=nout_tot)
def compute_non_uniform_sky(frame, nsig_clipping=4.,max_iterations=10,model_ivar=False,add_variance=True,angular_variation_deg=1) :
"""Compute a sky model.
Sky[fiber,i] = R[fiber,i,j] ( Flux_0[j] + x[fiber]*Flux_x[j] + y[fiber]*Flux_y[j] + ... )
Input flux are expected to be flatfielded!
We don't check this in this routine.
Args:
frame : Frame object, which includes attributes
- wave : 1D wavelength grid in Angstroms
- flux : 2D flux[nspec, nwave] density
- ivar : 2D inverse variance of flux
- mask : 2D inverse mask flux (0=good)
- resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers)
nsig_clipping : [optional] sigma clipping value for outlier rejection
Optional:
max_iterations : int , number of iterations
model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims.
add_variance : evaluate calibration error and add this to the sky model variance
angular_variation_deg : degree of 2D polynomial correction as a function of fiber focal plane coordinates (default=1). One set of coefficients per wavelength
returns SkyModel object with attributes wave, flux, ivar, mask
"""
log=get_logger()
log.info("starting")
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nwave=frame.nwave
nfibers=len(skyfibers)
current_ivar=frame.ivar[skyfibers].copy()*(frame.mask[skyfibers]==0)
flux = frame.flux[skyfibers]
Rsky = frame.R[skyfibers]
# need focal plane coordinates of fibers
x = frame.fibermap["DESIGN_X"][skyfibers]
y = frame.fibermap["DESIGN_Y"][skyfibers]
# normalize for numerical stability
xm = np.mean(frame.fibermap["DESIGN_X"])
ym = np.mean(frame.fibermap["DESIGN_Y"])
xs = np.std(frame.fibermap["DESIGN_X"])
ys = np.std(frame.fibermap["DESIGN_Y"])
if xs==0 : xs = 1
if ys==0 : ys = 1
x = (x-xm)/xs
y = (y-ym)/ys
# precompute the monomials for the sky fibers
log.debug("compute monomials for deg={}".format(angular_variation_deg))
monomials=[]
for dx in range(angular_variation_deg+1) :
for dy in range(angular_variation_deg+1-dx) :
monomials.append((x**dx)*(y**dy))
ncoef=len(monomials)
monomials=np.array(monomials)
input_ivar=None
if model_ivar :
log.info("use a model of the inverse variance to remove bias due to correlated ivar and flux")
input_ivar=current_ivar.copy()
median_ivar_vs_wave = np.median(current_ivar,axis=0)
median_ivar_vs_fiber = np.median(current_ivar,axis=1)
median_median_ivar = np.median(median_ivar_vs_fiber)
for f in range(current_ivar.shape[0]) :
threshold=0.01
current_ivar[f] = median_ivar_vs_fiber[f]/median_median_ivar * median_ivar_vs_wave
# keep input ivar for very low weights
ii=(input_ivar[f]<=(threshold*median_ivar_vs_wave))
#log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1]))
current_ivar[f][ii] = input_ivar[f][ii]
sqrtw=np.sqrt(current_ivar)
sqrtwflux=sqrtw*flux
chi2=np.zeros(flux.shape)
nout_tot=0
for iteration in range(max_iterations) :
# the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters
# A_ij = 1/2 d2(chi2)/di/dj
# A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w]
# the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters
# B_i = 1/2 d(chi2)/di
# B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w])
# with x_fiber,y_fiber the fiber coordinates in the focal plane (or sky)
# the unconvolved sky flux at wavelength i is a polynomial of x_fiber,y_fiber
# sky(fiber,i) = pol(x_fiber,y_fiber,p) = sum_p a_ip * x_fiber**degx(p) y_fiber**degy(p)
# sky(fiber,i) = sum_p monom[fiber,p] * a_ip
# the convolved sky flux at wavelength w is
# model[fiber,w] = sum_i R[fiber][w,i] sum_p monom[fiber,p] * a_ip
# model[fiber,w] = sum_p monom[fiber,p] R[fiber][w,i] a_ip
#
# so, the matrix A is composed of blocks (p,k) corresponding to polynomial coefficient indices where
# A[pk] = sum_fiber monom[fiber,p]*monom[fiber,k] sqrtwR[fiber] sqrtwR[fiber]^t
# similarily
# B[p] = sum_fiber monom[fiber,p] * sum_wave_w (sqrt(ivar)[fiber,w]*flux[fiber,w]) sqrtwR[fiber,wave]
A=np.zeros((nwave*ncoef,nwave*ncoef))
B=np.zeros((nwave*ncoef))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD=scipy.sparse.lil_matrix((nwave,nwave))
# loop on fiber to handle resolution
for fiber in range(nfibers) :
if fiber%10==0 :
log.info("iter %d sky fiber %d/%d"%(iteration,fiber,nfibers))
R = Rsky[fiber]
# diagonal sparse matrix with content = sqrt(ivar)
SD.setdiag(sqrtw[fiber])
sqrtwR = SD*R # each row r of R is multiplied by sqrtw[r]
#wRtR=(sqrtwR.T*sqrtwR).tocsr()
wRtR=(sqrtwR.T*sqrtwR).todense()
wRtF=sqrtwR.T*sqrtwflux[fiber]
# loop on polynomial coefficients (double loop for A)
# fill only blocks of A and B
for p in range(ncoef) :
for k in range(ncoef) :
A[p*nwave:(p+1)*nwave,k*nwave:(k+1)*nwave] += monomials[p,fiber]*monomials[k,fiber]*wRtR
B[p*nwave:(p+1)*nwave] += monomials[p,fiber]*wRtF
log.info("iter %d solving"%iteration)
w = A.diagonal()>0
A_pos_def = A[w,:]
A_pos_def = A_pos_def[:,w]
parameters = B*0
try:
parameters[w]=cholesky_solve(A_pos_def,B[w])
except:
log.info("cholesky failed, trying svd in iteration {}".format(iteration))
parameters[w]=np.linalg.lstsq(A_pos_def,B[w])[0]
log.info("iter %d compute chi2"%iteration)
for fiber in range(nfibers) :
# loop on polynomial indices
unconvolved_fiber_sky_flux = np.zeros(nwave)
for p in range(ncoef) :
unconvolved_fiber_sky_flux += monomials[p,fiber]*parameters[p*nwave:(p+1)*nwave]
# then convolve
fiber_convolved_sky_flux = Rsky[fiber].dot(unconvolved_fiber_sky_flux)
chi2[fiber]=current_ivar[fiber]*(flux[fiber]-fiber_convolved_sky_flux)**2
log.info("rejecting")
nout_iter=0
if iteration<1 :
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave=np.sum(chi2>nsig_clipping**2,axis=0)
selection=np.where(nout_per_wave>0)[0]
for i in selection :
worst_entry=np.argmax(chi2[:,i])
current_ivar[worst_entry,i]=0
sqrtw[worst_entry,i]=0
sqrtwflux[worst_entry,i]=0
nout_iter += 1
else :
# remove all of them at once
bad=(chi2>nsig_clipping**2)
current_ivar *= (bad==0)
sqrtw *= (bad==0)
sqrtwflux *= (bad==0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2=float(np.sum(chi2))
ndf=int(np.sum(chi2>0)-nwave)
chi2pdf=0.
if ndf>0 :
chi2pdf=sum_chi2/ndf
log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d"%(iteration,sum_chi2,ndf,chi2pdf,nout_iter))
if nout_iter == 0 :
break
log.info("nout tot=%d"%nout_tot)
# we know have to compute the sky model for all fibers
# and propagate the uncertainties
# no need to restore the original ivar to compute the model errors when modeling ivar
# the sky inverse variances are very similar
# is there a different method to compute this ?
log.info("compute covariance")
try :
parameter_covar=cholesky_invert(A)
except np.linalg.linalg.LinAlgError :
log.warning("cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv")
parameter_covar = np.linalg.pinv(A)
log.info("compute mean resolution")
# we make an approximation for the variance to save CPU time
# we use the average resolution of all fibers in the frame:
mean_res_data=np.mean(frame.resolution_data,axis=0)
Rmean = Resolution(mean_res_data)
log.info("compute convolved sky and ivar")
cskyflux = np.zeros(frame.flux.shape)
cskyivar = np.zeros(frame.flux.shape)
log.info("compute convolved parameter covariance")
# The covariance of the parameters is composed of ncoef*ncoef blocks each of size nwave*nwave
# A block (p,k) is the covariance of the unconvolved spectra p and k , corresponding to the polynomial indices p and k
# We first sandwich each block with the average resolution.
convolved_parameter_covar=np.zeros((ncoef,ncoef,nwave))
for p in range(ncoef) :
for k in range(ncoef) :
convolved_parameter_covar[p,k] = np.diagonal(Rmean.dot(parameter_covar[p*nwave:(p+1)*nwave,k*nwave:(k+1)*nwave]).dot(Rmean.T.todense()))
'''
import astropy.io.fits as pyfits
pyfits.writeto("convolved_parameter_covar.fits",convolved_parameter_covar,overwrite=True)
# other approach
log.info("dense Rmean...")
Rmean=Rmean.todense()
log.info("invert Rinv...")
Rinv=np.linalg.inv(Rmean)
# check this
print("0?",np.max(np.abs(Rinv.dot(Rmean)-np.eye(Rmean.shape[0])))/np.max(np.abs(Rmean)))
convolved_parameter_ivar=np.zeros((ncoef,ncoef,nwave))
for p in range(ncoef) :
for k in range(ncoef) :
convolved_parameter_ivar[p,k] = np.diagonal(Rinv.T.dot(A[p*nwave:(p+1)*nwave,k*nwave:(k+1)*nwave]).dot(Rinv))
# solve for each wave separately
convolved_parameter_covar=np.zeros((ncoef,ncoef,nwave))
for i in range(nwave) :
print("inverting ivar of wave %d/%d"%(i,nwave))
convolved_parameter_covar[:,:,i] = cholesky_invert(convolved_parameter_ivar[:,:,i])
pyfits.writeto("convolved_parameter_covar_bis.fits",convolved_parameter_covar,overwrite=True)
import sys
sys.exit(12)
'''
# Now we compute the sky model variance for each fiber individually
# accounting for its focal plane coordinates
# so that a target fiber distant for a sky fiber will naturally have a larger
# sky model variance
log.info("compute sky and variance per fiber")
for i in range(frame.nspec):
# compute monomials
M = []
xi=(frame.fibermap["DESIGN_X"][i]-xm)/xs
yi=(frame.fibermap["DESIGN_Y"][i]-ym)/ys
for dx in range(angular_variation_deg+1) :
for dy in range(angular_variation_deg+1-dx) :
M.append((xi**dx)*(yi**dy))
M = np.array(M)
unconvolved_fiber_sky_flux=np.zeros(nwave)
convolved_fiber_skyvar=np.zeros(nwave)
for p in range(ncoef) :
unconvolved_fiber_sky_flux += M[p]*parameters[p*nwave:(p+1)*nwave]
for k in range(ncoef) :
convolved_fiber_skyvar += M[p]*M[k]*convolved_parameter_covar[p,k]
# convolve sky model with this fiber's resolution
cskyflux[i] = frame.R[i].dot(unconvolved_fiber_sky_flux)
# save inverse of variance
cskyivar[i] = (convolved_fiber_skyvar>0)/(convolved_fiber_skyvar+(convolved_fiber_skyvar==0))
# look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1
if skyfibers.size > 1 and add_variance :
modified_cskyivar = _model_variance(frame,cskyflux,cskyivar,skyfibers)
else :
modified_cskyivar = cskyivar.copy()
# need to do better here
mask = (cskyivar==0).astype(np.uint32)
return SkyModel(frame.wave.copy(), cskyflux, modified_cskyivar, mask,
nrej=nout_tot, stat_ivar = cskyivar) # keep a record of the statistical ivar for QA
def test_resolution(self, n = 100):
dense = np.arange(n*n).reshape(n,n)
R1 = Resolution(dense)
assert scipy.sparse.isspmatrix_dia(R1),'Resolution is not recognized as a scipy.sparse.dia_matrix.'
assert len(R1.offsets) == desispec.resolution.default_ndiag, 'Resolution.offsets has wrong size'
R2 = Resolution(R1)
assert np.array_equal(R1.toarray(),R2.toarray()),'Constructor broken for dia_matrix input.'
R3 = Resolution(R1.data)
assert np.array_equal(R1.toarray(),R3.toarray()),'Constructor broken for array data input.'
sparse = scipy.sparse.dia_matrix((R1.data[::-1],R1.offsets[::-1]),(n,n))
R4 = Resolution(sparse)
assert np.array_equal(R1.toarray(),R4.toarray()),'Constructor broken for permuted offsets input.'
R5 = Resolution(R1.to_fits_array())
assert np.array_equal(R1.toarray(),R5.toarray()),'to_fits_array() is broken.'
#- test different sizes of input diagonals
for ndiag in [3,5,11]:
R6 = Resolution(np.ones((ndiag, n)))
assert len(R6.offsets) == ndiag, 'Constructor broken for ndiag={}'.format(ndiag)
#- An even number if diagonals is not allowed
try:
ndiag = 10
R7 = Resolution(np.ones((ndiag, n)))
raise RuntimeError('Incorrectly created Resolution with even number of diagonals')
except ValueError, err:
#- it correctly raised an error, so pass
pass
def compute_polynomial_times_sky(frame,
nsig_clipping=4.,
max_iterations=30,
model_ivar=False,
add_variance=True,
angular_variation_deg=1,
chromatic_variation_deg=1):
"""Compute a sky model.
Sky[fiber,i] = R[fiber,i,j] Polynomial(x[fiber],y[fiber],wavelength[j]) Flux[j]
Input flux are expected to be flatfielded!
We don't check this in this routine.
Args:
frame : Frame object, which includes attributes
- wave : 1D wavelength grid in Angstroms
- flux : 2D flux[nspec, nwave] density
- ivar : 2D inverse variance of flux
- mask : 2D inverse mask flux (0=good)
- resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers)
nsig_clipping : [optional] sigma clipping value for outlier rejection
Optional:
max_iterations : int , number of iterations
model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims.
add_variance : evaluate calibration error and add this to the sky model variance
returns SkyModel object with attributes wave, flux, ivar, mask
"""
log = get_logger()
log.info("starting")
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nwave = frame.nwave
nfibers = len(skyfibers)
current_ivar = frame.ivar[skyfibers].copy() * (frame.mask[skyfibers] == 0)
flux = frame.flux[skyfibers]
Rsky = frame.R[skyfibers]
input_ivar = None
if model_ivar:
log.info(
"use a model of the inverse variance to remove bias due to correlated ivar and flux"
)
input_ivar = current_ivar.copy()
median_ivar_vs_wave = np.median(current_ivar, axis=0)
median_ivar_vs_fiber = np.median(current_ivar, axis=1)
median_median_ivar = np.median(median_ivar_vs_fiber)
for f in range(current_ivar.shape[0]):
threshold = 0.01
current_ivar[f] = median_ivar_vs_fiber[
f] / median_median_ivar * median_ivar_vs_wave
# keep input ivar for very low weights
ii = (input_ivar[f] <= (threshold * median_ivar_vs_wave))
#log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1]))
current_ivar[f][ii] = input_ivar[f][ii]
# need focal plane coordinates
x = frame.fibermap["FIBERASSIGN_X"]
y = frame.fibermap["FIBERASSIGN_Y"]
# normalize for numerical stability
xm = np.mean(x)
ym = np.mean(y)
xs = np.std(x)
ys = np.std(y)
if xs == 0: xs = 1
if ys == 0: ys = 1
x = (x - xm) / xs
y = (y - ym) / ys
w = (frame.wave - frame.wave[0]) / (frame.wave[-1] -
frame.wave[0]) * 2. - 1
# precompute the monomials for the sky fibers
log.debug("compute monomials for deg={} and {}".format(
angular_variation_deg, chromatic_variation_deg))
monomials = []
for dx in range(angular_variation_deg + 1):
for dy in range(angular_variation_deg + 1 - dx):
xypol = (x**dx) * (y**dy)
for dw in range(chromatic_variation_deg + 1):
wpol = w**dw
monomials.append(np.outer(xypol, wpol))
ncoef = len(monomials)
coef = np.zeros((ncoef))
allfibers_monomials = np.array(monomials)
log.debug("shape of allfibers_monomials = {}".format(
allfibers_monomials.shape))
skyfibers_monomials = allfibers_monomials[:, skyfibers, :]
log.debug("shape of skyfibers_monomials = {}".format(
skyfibers_monomials.shape))
sqrtw = np.sqrt(current_ivar)
sqrtwflux = sqrtw * flux
chi2 = np.zeros(flux.shape)
Pol = np.ones(flux.shape, dtype=float)
coef[0] = 1.
nout_tot = 0
previous_chi2 = -10.
for iteration in range(max_iterations):
# the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters
# A_ij = 1/2 d2(chi2)/di/dj
# A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w]
# the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters
# B_i = 1/2 d(chi2)/di
# B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w])
# the model is model[fiber]=R[fiber]*Pol(x,y,wave)*sky
# the parameters are the unconvolved sky flux at the wavelength i
# and the polynomial coefficients
A = np.zeros((nwave, nwave), dtype=float)
B = np.zeros((nwave), dtype=float)
D = scipy.sparse.lil_matrix((nwave, nwave))
D2 = scipy.sparse.lil_matrix((nwave, nwave))
Pol /= coef[0] # force constant term to 1.
# solving for the deconvolved mean sky spectrum
# loop on fiber to handle resolution
for fiber in range(nfibers):
if fiber % 10 == 0:
log.info("iter %d sky fiber (1st fit) %d/%d" %
(iteration, fiber, nfibers))
D.setdiag(sqrtw[fiber])
D2.setdiag(Pol[fiber])
sqrtwRP = D.dot(Rsky[fiber]).dot(
D2) # each row r of R is multiplied by sqrtw[r]
A += (sqrtwRP.T * sqrtwRP).todense()
B += sqrtwRP.T * sqrtwflux[fiber]
log.info("iter %d solving" % iteration)
w = A.diagonal() > 0
A_pos_def = A[w, :]
A_pos_def = A_pos_def[:, w]
parameters = B * 0
try:
parameters[w] = cholesky_solve(A_pos_def, B[w])
except:
log.info("cholesky failed, trying svd in iteration {}".format(
iteration))
parameters[w] = np.linalg.lstsq(A_pos_def, B[w])[0]
# parameters = the deconvolved mean sky spectrum
# now evaluate the polynomial coefficients
Ap = np.zeros((ncoef, ncoef), dtype=float)
Bp = np.zeros((ncoef), dtype=float)
D2.setdiag(parameters)
for fiber in range(nfibers):
if fiber % 10 == 0:
log.info("iter %d sky fiber (2nd fit) %d/%d" %
(iteration, fiber, nfibers))
D.setdiag(sqrtw[fiber])
sqrtwRSM = D.dot(Rsky[fiber]).dot(D2).dot(
skyfibers_monomials[:, fiber, :].T)
Ap += sqrtwRSM.T.dot(sqrtwRSM)
Bp += sqrtwRSM.T.dot(sqrtwflux[fiber])
# Add huge prior on zeroth angular order terms to converge faster
# (because those terms are degenerate with the mean deconvolved spectrum)
weight = 1e24
Ap[0, 0] += weight
Bp[0] += weight # force 0th term to 1
for i in range(1, chromatic_variation_deg + 1):
Ap[i, i] += weight # force other wavelength terms to 0
coef = cholesky_solve(Ap, Bp)
log.info("pol coef = {}".format(coef))
# recompute the polynomial values
Pol = skyfibers_monomials.T.dot(coef).T
# chi2 and outlier rejection
log.info("iter %d compute chi2" % iteration)
for fiber in range(nfibers):
chi2[fiber] = current_ivar[fiber] * (
flux[fiber] - Rsky[fiber].dot(Pol[fiber] * parameters))**2
log.info("rejecting")
nout_iter = 0
if iteration < 1:
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave = np.sum(chi2 > nsig_clipping**2, axis=0)
selection = np.where(nout_per_wave > 0)[0]
for i in selection:
worst_entry = np.argmax(chi2[:, i])
current_ivar[worst_entry, i] = 0
sqrtw[worst_entry, i] = 0
sqrtwflux[worst_entry, i] = 0
nout_iter += 1
else:
# remove all of them at once
bad = (chi2 > nsig_clipping**2)
current_ivar *= (bad == 0)
sqrtw *= (bad == 0)
sqrtwflux *= (bad == 0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2 = float(np.sum(chi2))
ndf = int(np.sum(chi2 > 0) - nwave)
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info("iter #%d chi2=%g ndf=%d chi2pdf=%f delta=%f nout=%d" %
(iteration, sum_chi2, ndf, chi2pdf,
abs(sum_chi2 - previous_chi2), nout_iter))
if nout_iter == 0 and abs(sum_chi2 - previous_chi2) < 0.2:
break
previous_chi2 = sum_chi2 + 0.
log.info("nout tot=%d" % nout_tot)
# we know have to compute the sky model for all fibers
# and propagate the uncertainties
# no need to restore the original ivar to compute the model errors when modeling ivar
# the sky inverse variances are very similar
# we ignore here the fact that we have fit a angular variation,
# so the sky model uncertainties are inaccurate
log.info("compute the parameter covariance")
try:
parameter_covar = cholesky_invert(A)
except np.linalg.linalg.LinAlgError:
log.warning(
"cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv"
)
parameter_covar = np.linalg.pinv(A)
log.info("compute mean resolution")
# we make an approximation for the variance to save CPU time
# we use the average resolution of all fibers in the frame:
mean_res_data = np.mean(frame.resolution_data, axis=0)
Rmean = Resolution(mean_res_data)
log.info("compute convolved sky and ivar")
# The parameters are directly the unconvolved sky
# First convolve with average resolution :
convolved_sky_covar = Rmean.dot(parameter_covar).dot(Rmean.T.todense())
# and keep only the diagonal
convolved_sky_var = np.diagonal(convolved_sky_covar)
# inverse
convolved_sky_ivar = (convolved_sky_var > 0) / (convolved_sky_var +
(convolved_sky_var == 0))
# and simply consider it's the same for all spectra
cskyivar = np.tile(convolved_sky_ivar,
frame.nspec).reshape(frame.nspec, nwave)
# The sky model for each fiber (simple convolution with resolution of each fiber)
cskyflux = np.zeros(frame.flux.shape)
Pol = allfibers_monomials.T.dot(coef).T
for fiber in range(frame.nspec):
cskyflux[fiber] = frame.R[fiber].dot(Pol[fiber] * parameters)
# look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1
if skyfibers.size > 1 and add_variance:
modified_cskyivar = _model_variance(frame, cskyflux, cskyivar,
skyfibers)
else:
modified_cskyivar = cskyivar.copy()
# need to do better here
mask = (cskyivar == 0).astype(np.uint32)
return SkyModel(
frame.wave.copy(),
cskyflux,
modified_cskyivar,
mask,
nrej=nout_tot,
stat_ivar=cskyivar) # keep a record of the statistical ivar for QA
def myspecupdate(spectra_in, other) :
# Does the other Spectra object have any data?
if other.num_spectra() == 0:
return
# Do we have new bands to add?
newbands = []
for b in other.bands:
if b not in spectra_in.bands:
newbands.append(b)
else:
if not np.allclose(spectra_in.wave[b], other.wave[b]):
raise RuntimeError("band {} has an incompatible wavelength grid".format(b))
bands = list(spectra_in.bands)
bands.extend(newbands)
# Are we adding mask data in this update?
add_mask = False
if other.mask is None:
if spectra_in.mask is not None:
raise RuntimeError("existing spectra has a mask, cannot "
"update it to a spectra with no mask")
else:
if spectra_in.mask is None:
add_mask = True
# Are we adding resolution data in this update?
ndiag = {}
add_res = False
if other.resolution_data is None:
if spectra_in.resolution_data is not None:
raise RuntimeError("existing spectra has resolution data, cannot "
"update it to a spectra with none")
else:
if spectra_in.resolution_data is not None:
for b in spectra_in.bands:
ndiag[b] = spectra_in.resolution_data[b].shape[1]
for b in other.bands:
odiag = other.resolution_data[b].shape[1]
if b not in spectra_in.bands:
ndiag[b] = odiag
else:
if odiag != ndiag[b]:
raise RuntimeError("Resolution matrices for a"
" given band must have the same dimensoins")
else:
add_res = True
for b in other.bands:
ndiag[b] = other.resolution_data[b].shape[1]
# Are we adding extra data in this update?
add_extra = False
if other.extra is None:
if spectra_in.extra is not None:
raise RuntimeError("existing spectra has extra data, cannot "
"update it to a spectra with none")
else:
if spectra_in.extra is None:
add_extra = True
# Compute which targets / exposures are new
nother = len(other.fibermap)
exists = np.zeros(nother, dtype=np.int)
indx_original = []
# EA modif :
check_exists = True
if ( (spectra_in.fibermap is None)
or ("EXPID" not in spectra_in.fibermap.keys())
or ("EXPID" not in other.fibermap.keys())
or ("FIBER" not in spectra_in.fibermap.keys())
or ("FIBER" not in other.fibermap.keys()) ) :
check_exists = False
if check_exists :
for r in range(nother):
expid = other.fibermap[r]["EXPID"]
fiber = other.fibermap[r]["FIBER"]
for i, row in enumerate(spectra_in.fibermap):
if (expid == row["EXPID"]) and (fiber == row["FIBER"]):
indx_original.append(i)
exists[r] += 1
if len(np.where(exists > 1)[0]) > 0:
raise RuntimeError("found duplicate spectra (same EXPID and FIBER) in the fibermap")
indx_exists = np.where(exists == 1)[0]
indx_new = np.where(exists == 0)[0]
# Make new data arrays of the correct size to hold both the old and
# new data
nupdate = len(indx_exists)
nnew = len(indx_new)
if spectra_in.fibermap is None:
nold = 0
newfmap = other.fibermap.copy()
else:
nold = len(spectra_in.fibermap)
newfmap = encode_table(np.zeros( (nold + nnew, ),
dtype=spectra_in.fibermap.dtype))
newwave = {}
newflux = {}
newivar = {}
newmask = None
if add_mask or spectra_in.mask is not None:
newmask = {}
newres = None
newR = None
if add_res or spectra_in.resolution_data is not None:
newres = {}
newR = {}
newextra = None
if add_extra or spectra_in.extra is not None:
newextra = {}
for b in bands:
nwave = None
if b in spectra_in.bands:
nwave = spectra_in.wave[b].shape[0]
newwave[b] = spectra_in.wave[b]
else:
nwave = other.wave[b].shape[0]
newwave[b] = other.wave[b].astype(spectra_in._ftype)
newflux[b] = np.zeros( (nold + nnew, nwave), dtype=spectra_in._ftype)
newivar[b] = np.zeros( (nold + nnew, nwave), dtype=spectra_in._ftype)
if newmask is not None:
newmask[b] = np.zeros( (nold + nnew, nwave), dtype=np.uint32)
newmask[b][:,:] = specmask["NODATA"]
if newres is not None:
newres[b] = np.zeros( (nold + nnew, ndiag[b], nwave), dtype=spectra_in._ftype)
if newextra is not None:
newextra[b] = {}
# Copy the old data
if nold > 0:
# We have some data (i.e. we are not starting with an empty Spectra)
newfmap[:nold] = spectra_in.fibermap
for b in spectra_in.bands:
newflux[b][:nold,:] = spectra_in.flux[b]
newivar[b][:nold,:] = spectra_in.ivar[b]
if spectra_in.mask is not None:
newmask[b][:nold,:] = spectra_in.mask[b]
elif add_mask:
newmask[b][:nold,:] = 0
if spectra_in.resolution_data is not None:
newres[b][:nold,:,:] = spectra_in.resolution_data[b]
if spectra_in.extra is not None:
for ex in spectra_in.extra[b].items():
newextra[b][ex[0]] = np.zeros( newflux[b].shape,
dtype=spectra_in._ftype)
newextra[b][ex[0]][:nold,:] = ex[1]
# Update existing spectra
for i, s in enumerate(indx_exists):
row = indx_original[i]
for b in other.bands:
newflux[b][row,:] = other.flux[b][s,:].astype(spectra_in._ftype)
newivar[b][row,:] = other.ivar[b][s,:].astype(spectra_in._ftype)
if other.mask is not None:
newmask[b][row,:] = other.mask[b][s,:]
else:
newmask[b][row,:] = 0
if other.resolution_data is not None:
newres[b][row,:,:] = other.resolution_data[b][s,:,:].astype(spectra_in._ftype)
if other.extra is not None:
for ex in other.extra[b].items():
if ex[0] not in newextra[b]:
newextra[b][ex[0]] = np.zeros(newflux[b].shape,
dtype=spectra_in._ftype)
newextra[b][ex[0]][row,:] = ex[1][s,:].astype(spectra_in._ftype)
# Append new spectra
if nnew > 0:
newfmap[nold:] = other.fibermap[indx_new]
for b in other.bands:
newflux[b][nold:,:] = other.flux[b][indx_new].astype(spectra_in._ftype)
newivar[b][nold:,:] = other.ivar[b][indx_new].astype(spectra_in._ftype)
if other.mask is not None:
newmask[b][nold:,:] = other.mask[b][indx_new]
else:
newmask[b][nold:,:] = 0
if other.resolution_data is not None:
newres[b][nold:,:,:] = other.resolution_data[b][indx_new].astype(spectra_in._ftype)
if other.extra is not None:
for ex in other.extra[b].items():
if ex[0] not in newextra[b]:
newextra[b][ex[0]] = np.zeros(newflux[b].shape,
dtype=spectra_in._ftype)
newextra[b][ex[0]][nold:,:] = ex[1][indx_new].astype(spectra_in._ftype)
# Update all sparse resolution matrices
for b in bands:
if newres is not None:
newR[b] = np.array( [ Resolution(r) for r in newres[b] ] )
# Swap data into place
spectra_in._bands = bands
spectra_in.wave = newwave
spectra_in.fibermap = newfmap
spectra_in.flux = newflux
spectra_in.ivar = newivar
spectra_in.mask = newmask
spectra_in.resolution_data = newres
spectra_in.R = newR
spectra_in.extra = newextra
return spectra_in
def compute_non_uniform_sky(frame,
nsig_clipping=4.,
max_iterations=10,
model_ivar=False,
add_variance=True,
angular_variation_deg=1):
"""Compute a sky model.
Sky[fiber,i] = R[fiber,i,j] ( Flux_0[j] + x[fiber]*Flux_x[j] + y[fiber]*Flux_y[j] + ... )
Input flux are expected to be flatfielded!
We don't check this in this routine.
Args:
frame : Frame object, which includes attributes
- wave : 1D wavelength grid in Angstroms
- flux : 2D flux[nspec, nwave] density
- ivar : 2D inverse variance of flux
- mask : 2D inverse mask flux (0=good)
- resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers)
nsig_clipping : [optional] sigma clipping value for outlier rejection
Optional:
max_iterations : int , number of iterations
model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims.
add_variance : evaluate calibration error and add this to the sky model variance
angular_variation_deg : degree of 2D polynomial correction as a function of fiber focal plane coordinates (default=1). One set of coefficients per wavelength
returns SkyModel object with attributes wave, flux, ivar, mask
"""
log = get_logger()
log.info("starting")
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nwave = frame.nwave
nfibers = len(skyfibers)
current_ivar = frame.ivar[skyfibers].copy() * (frame.mask[skyfibers] == 0)
flux = frame.flux[skyfibers]
Rsky = frame.R[skyfibers]
# need focal plane coordinates of fibers
x = frame.fibermap["FIBERASSIGN_X"][skyfibers]
y = frame.fibermap["FIBERASSIGN_Y"][skyfibers]
# normalize for numerical stability
xm = np.mean(frame.fibermap["FIBERASSIGN_X"])
ym = np.mean(frame.fibermap["FIBERASSIGN_Y"])
xs = np.std(frame.fibermap["FIBERASSIGN_X"])
ys = np.std(frame.fibermap["FIBERASSIGN_Y"])
if xs == 0: xs = 1
if ys == 0: ys = 1
x = (x - xm) / xs
y = (y - ym) / ys
# precompute the monomials for the sky fibers
log.debug("compute monomials for deg={}".format(angular_variation_deg))
monomials = []
for dx in range(angular_variation_deg + 1):
for dy in range(angular_variation_deg + 1 - dx):
monomials.append((x**dx) * (y**dy))
ncoef = len(monomials)
monomials = np.array(monomials)
input_ivar = None
if model_ivar:
log.info(
"use a model of the inverse variance to remove bias due to correlated ivar and flux"
)
input_ivar = current_ivar.copy()
median_ivar_vs_wave = np.median(current_ivar, axis=0)
median_ivar_vs_fiber = np.median(current_ivar, axis=1)
median_median_ivar = np.median(median_ivar_vs_fiber)
for f in range(current_ivar.shape[0]):
threshold = 0.01
current_ivar[f] = median_ivar_vs_fiber[
f] / median_median_ivar * median_ivar_vs_wave
# keep input ivar for very low weights
ii = (input_ivar[f] <= (threshold * median_ivar_vs_wave))
#log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1]))
current_ivar[f][ii] = input_ivar[f][ii]
sqrtw = np.sqrt(current_ivar)
sqrtwflux = sqrtw * flux
chi2 = np.zeros(flux.shape)
nout_tot = 0
for iteration in range(max_iterations):
# the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters
# A_ij = 1/2 d2(chi2)/di/dj
# A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w]
# the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters
# B_i = 1/2 d(chi2)/di
# B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w])
# with x_fiber,y_fiber the fiber coordinates in the focal plane (or sky)
# the unconvolved sky flux at wavelength i is a polynomial of x_fiber,y_fiber
# sky(fiber,i) = pol(x_fiber,y_fiber,p) = sum_p a_ip * x_fiber**degx(p) y_fiber**degy(p)
# sky(fiber,i) = sum_p monom[fiber,p] * a_ip
# the convolved sky flux at wavelength w is
# model[fiber,w] = sum_i R[fiber][w,i] sum_p monom[fiber,p] * a_ip
# model[fiber,w] = sum_p monom[fiber,p] R[fiber][w,i] a_ip
#
# so, the matrix A is composed of blocks (p,k) corresponding to polynomial coefficient indices where
# A[pk] = sum_fiber monom[fiber,p]*monom[fiber,k] sqrtwR[fiber] sqrtwR[fiber]^t
# similarily
# B[p] = sum_fiber monom[fiber,p] * sum_wave_w (sqrt(ivar)[fiber,w]*flux[fiber,w]) sqrtwR[fiber,wave]
A = np.zeros((nwave * ncoef, nwave * ncoef))
B = np.zeros((nwave * ncoef))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD = scipy.sparse.lil_matrix((nwave, nwave))
# loop on fiber to handle resolution
for fiber in range(nfibers):
if fiber % 10 == 0:
log.info("iter %d sky fiber %d/%d" %
(iteration, fiber, nfibers))
R = Rsky[fiber]
# diagonal sparse matrix with content = sqrt(ivar)
SD.setdiag(sqrtw[fiber])
sqrtwR = SD * R # each row r of R is multiplied by sqrtw[r]
#wRtR=(sqrtwR.T*sqrtwR).tocsr()
wRtR = (sqrtwR.T * sqrtwR).todense()
wRtF = sqrtwR.T * sqrtwflux[fiber]
# loop on polynomial coefficients (double loop for A)
# fill only blocks of A and B
for p in range(ncoef):
for k in range(ncoef):
A[p * nwave:(p + 1) * nwave, k * nwave:(k + 1) *
nwave] += monomials[p, fiber] * monomials[k,
fiber] * wRtR
B[p * nwave:(p + 1) * nwave] += monomials[p, fiber] * wRtF
log.info("iter %d solving" % iteration)
w = A.diagonal() > 0
A_pos_def = A[w, :]
A_pos_def = A_pos_def[:, w]
parameters = B * 0
try:
parameters[w] = cholesky_solve(A_pos_def, B[w])
except:
log.info("cholesky failed, trying svd in iteration {}".format(
iteration))
parameters[w] = np.linalg.lstsq(A_pos_def, B[w])[0]
log.info("iter %d compute chi2" % iteration)
for fiber in range(nfibers):
# loop on polynomial indices
unconvolved_fiber_sky_flux = np.zeros(nwave)
for p in range(ncoef):
unconvolved_fiber_sky_flux += monomials[
p, fiber] * parameters[p * nwave:(p + 1) * nwave]
# then convolve
fiber_convolved_sky_flux = Rsky[fiber].dot(
unconvolved_fiber_sky_flux)
chi2[fiber] = current_ivar[fiber] * (flux[fiber] -
fiber_convolved_sky_flux)**2
log.info("rejecting")
nout_iter = 0
if iteration < 1:
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave = np.sum(chi2 > nsig_clipping**2, axis=0)
selection = np.where(nout_per_wave > 0)[0]
for i in selection:
worst_entry = np.argmax(chi2[:, i])
current_ivar[worst_entry, i] = 0
sqrtw[worst_entry, i] = 0
sqrtwflux[worst_entry, i] = 0
nout_iter += 1
else:
# remove all of them at once
bad = (chi2 > nsig_clipping**2)
current_ivar *= (bad == 0)
sqrtw *= (bad == 0)
sqrtwflux *= (bad == 0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2 = float(np.sum(chi2))
ndf = int(np.sum(chi2 > 0) - nwave)
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d" %
(iteration, sum_chi2, ndf, chi2pdf, nout_iter))
if nout_iter == 0:
break
log.info("nout tot=%d" % nout_tot)
# we know have to compute the sky model for all fibers
# and propagate the uncertainties
# no need to restore the original ivar to compute the model errors when modeling ivar
# the sky inverse variances are very similar
# is there a different method to compute this ?
log.info("compute covariance")
try:
parameter_covar = cholesky_invert(A)
except np.linalg.linalg.LinAlgError:
log.warning(
"cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv"
)
parameter_covar = np.linalg.pinv(A)
log.info("compute mean resolution")
# we make an approximation for the variance to save CPU time
# we use the average resolution of all fibers in the frame:
mean_res_data = np.mean(frame.resolution_data, axis=0)
Rmean = Resolution(mean_res_data)
log.info("compute convolved sky and ivar")
cskyflux = np.zeros(frame.flux.shape)
cskyivar = np.zeros(frame.flux.shape)
log.info("compute convolved parameter covariance")
# The covariance of the parameters is composed of ncoef*ncoef blocks each of size nwave*nwave
# A block (p,k) is the covariance of the unconvolved spectra p and k , corresponding to the polynomial indices p and k
# We first sandwich each block with the average resolution.
convolved_parameter_covar = np.zeros((ncoef, ncoef, nwave))
for p in range(ncoef):
for k in range(ncoef):
convolved_parameter_covar[p, k] = np.diagonal(
Rmean.dot(parameter_covar[p * nwave:(p + 1) * nwave,
k * nwave:(k + 1) * nwave]).dot(
Rmean.T.todense()))
'''
import astropy.io.fits as pyfits
pyfits.writeto("convolved_parameter_covar.fits",convolved_parameter_covar,overwrite=True)
# other approach
log.info("dense Rmean...")
Rmean=Rmean.todense()
log.info("invert Rinv...")
Rinv=np.linalg.inv(Rmean)
# check this
print("0?",np.max(np.abs(Rinv.dot(Rmean)-np.eye(Rmean.shape[0])))/np.max(np.abs(Rmean)))
convolved_parameter_ivar=np.zeros((ncoef,ncoef,nwave))
for p in range(ncoef) :
for k in range(ncoef) :
convolved_parameter_ivar[p,k] = np.diagonal(Rinv.T.dot(A[p*nwave:(p+1)*nwave,k*nwave:(k+1)*nwave]).dot(Rinv))
# solve for each wave separately
convolved_parameter_covar=np.zeros((ncoef,ncoef,nwave))
for i in range(nwave) :
print("inverting ivar of wave %d/%d"%(i,nwave))
convolved_parameter_covar[:,:,i] = cholesky_invert(convolved_parameter_ivar[:,:,i])
pyfits.writeto("convolved_parameter_covar_bis.fits",convolved_parameter_covar,overwrite=True)
import sys
sys.exit(12)
'''
# Now we compute the sky model variance for each fiber individually
# accounting for its focal plane coordinates
# so that a target fiber distant for a sky fiber will naturally have a larger
# sky model variance
log.info("compute sky and variance per fiber")
for i in range(frame.nspec):
# compute monomials
M = []
xi = (frame.fibermap["FIBERASSIGN_X"][i] - xm) / xs
yi = (frame.fibermap["FIBERASSIGN_Y"][i] - ym) / ys
for dx in range(angular_variation_deg + 1):
for dy in range(angular_variation_deg + 1 - dx):
M.append((xi**dx) * (yi**dy))
M = np.array(M)
unconvolved_fiber_sky_flux = np.zeros(nwave)
convolved_fiber_skyvar = np.zeros(nwave)
for p in range(ncoef):
unconvolved_fiber_sky_flux += M[p] * parameters[p * nwave:(p + 1) *
nwave]
for k in range(ncoef):
convolved_fiber_skyvar += M[p] * M[
k] * convolved_parameter_covar[p, k]
# convolve sky model with this fiber's resolution
cskyflux[i] = frame.R[i].dot(unconvolved_fiber_sky_flux)
# save inverse of variance
cskyivar[i] = (convolved_fiber_skyvar > 0) / (
convolved_fiber_skyvar + (convolved_fiber_skyvar == 0))
# look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1
if skyfibers.size > 1 and add_variance:
modified_cskyivar = _model_variance(frame, cskyflux, cskyivar,
skyfibers)
else:
modified_cskyivar = cskyivar.copy()
# need to do better here
mask = (cskyivar == 0).astype(np.uint32)
return SkyModel(
frame.wave.copy(),
cskyflux,
modified_cskyivar,
mask,
nrej=nout_tot,
stat_ivar=cskyivar) # keep a record of the statistical ivar for QA
def main(args=None):
'''
Converts simspec -> frame files; see fastframe --help for usage options
'''
#- TODO: use desiutil.log
if isinstance(args, (list, tuple, type(None))):
args = parse(args)
print('Reading files')
simspec = desisim.io.read_simspec(args.simspec)
if simspec.flavor == 'arc':
print('arc exposure; no frames to output')
return
fibermap = simspec.fibermap
obs = simspec.obs
night = simspec.header['NIGHT']
expid = simspec.header['EXPID']
firstspec = args.firstspec
nspec = min(args.nspec, len(fibermap) - firstspec)
print('Simulating spectra {}-{}'.format(firstspec, firstspec + nspec))
wave = simspec.wave['brz']
flux = simspec.flux
ii = slice(firstspec, firstspec + nspec)
if simspec.flavor == 'science':
sim = desisim.simexp.simulate_spectra(wave,
flux[ii],
fibermap=fibermap[ii],
obsconditions=obs,
dwave_out=1.0)
elif simspec.flavor in ['arc', 'flat', 'calib']:
x = fibermap['X_TARGET']
y = fibermap['Y_TARGET']
fiber_area = desisim.simexp.fiber_area_arcsec2(fibermap['X_TARGET'],
fibermap['Y_TARGET'])
surface_brightness = (flux.T / fiber_area).T
config = desisim.simexp._specsim_config_for_wave(wave, dwave_out=1.0)
# sim = specsim.simulator.Simulator(config, num_fibers=nspec)
sim = desisim.specsim.get_simulator(config, num_fibers=nspec)
sim.observation.exposure_time = simspec.header['EXPTIME'] * u.s
sbunit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm**2 * u.arcsec**2)
xy = np.vstack([x, y]).T * u.mm
sim.simulate(calibration_surface_brightness=surface_brightness[ii] *
sbunit,
focal_positions=xy[ii])
else:
raise ValueError('Unknown simspec flavor {}'.format(simspec.flavor))
sim.generate_random_noise()
for i, results in enumerate(sim.camera_output):
results = sim.camera_output[i]
wave = results['wavelength']
phot = (results['num_source_electrons'] + \
results['num_sky_electrons'] + \
results['num_dark_electrons'] + \
results['random_noise_electrons']).T
ivar = 1.0 / results['variance_electrons'].T
R = Resolution(
sim.instrument.cameras[i].get_output_resolution_matrix())
Rdata = np.tile(R.data.T, nspec).T.reshape(nspec, R.data.shape[0],
R.data.shape[1])
assert np.all(Rdata[0] == R.data)
assert phot.shape == (nspec, len(wave))
for spectro in range(10):
imin = max(firstspec, spectro * 500) - firstspec
imax = min(firstspec + nspec, (spectro + 1) * 500) - firstspec
if imax <= imin:
continue
xphot = phot[imin:imax]
xivar = ivar[imin:imax]
xfibermap = fibermap[ii][imin:imax]
camera = '{}{}'.format(sim.camera_names[i], spectro)
meta = simspec.header.copy()
meta['CAMERA'] = camera
frame = Frame(wave,
xphot,
xivar,
resolution_data=Rdata[0:imax - imin],
spectrograph=spectro,
fibermap=xfibermap,
meta=meta)
outfile = desispec.io.findfile('frame',
night,
expid,
camera,
outdir=args.outdir)
print('writing {}'.format(outfile))
desispec.io.write_frame(outfile, frame)
# be in the next spectrograph file ('b0' has 500 total, 'b1' has 500 total, etc.)
for n in range(nadd_start, nadd_stop):
# First, get the overall wavelength range of the spectrum
# Individual cameras have 0.5 A resolutions
# We make the final coadded spectrum uniformly spaced with 1.0 A bins
spectrograph = int(n/500)
ncameras = cameras[spectrograph::int(np.ceil(nspec/500.))]
global_wavelength_grid = np.arange(cframes['b{:d}'.format(spectrograph)].wave[0],
cframes['z{:d}'.format(spectrograph)].wave[-1] + 0.5,
1.0, dtype=np.float32)
coadd_all_bands = Spectrum(global_wavelength_grid)
for band in ncameras:
nn = n - 500*int(n / 500)
band_specobj = Spectrum(cframes[band].wave,
flux=cframes[band].flux[nn],
ivar=cframes[band].ivar[nn],
resolution=Resolution(cframes[band].resolution_data[nn]))
coadd_all_bands += band_specobj
print('Spectra %d: Added band %s' % (n, band))
# Finalize the overall spectrum
coadd_all_bands.finalize()
print('{}/{} coadded'.format(n+1, nspec))
# And write the output
outfile = args.outdir + "spectra-%s-expid%03d-%05d.fits" % (args.night, args.expid, n)
write_coadd_spectra(outfile, coadd_all_bands, meta[n], simspecfile) # fibermap=fiberap[n]
print('written to {}'.format(outfile))
def main(args):
# Set up the logger
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
# Make sure all necessary environment variables are set
DESI_SPECTRO_REDUX_DIR="./quickGen"
if 'DESI_SPECTRO_REDUX' not in os.environ:
log.info('DESI_SPECTRO_REDUX environment is not set.')
else:
DESI_SPECTRO_REDUX_DIR=os.environ['DESI_SPECTRO_REDUX']
if os.path.exists(DESI_SPECTRO_REDUX_DIR):
if not os.path.isdir(DESI_SPECTRO_REDUX_DIR):
raise RuntimeError("Path %s Not a directory"%DESI_SPECTRO_REDUX_DIR)
else:
try:
os.makedirs(DESI_SPECTRO_REDUX_DIR)
except:
raise
SPECPROD_DIR='specprod'
if 'SPECPROD' not in os.environ:
log.info('SPECPROD environment is not set.')
else:
SPECPROD_DIR=os.environ['SPECPROD']
prod_Dir=specprod_root()
if os.path.exists(prod_Dir):
if not os.path.isdir(prod_Dir):
raise RuntimeError("Path %s Not a directory"%prod_Dir)
else:
try:
os.makedirs(prod_Dir)
except:
raise
# Initialize random number generator to use.
np.random.seed(args.seed)
random_state = np.random.RandomState(args.seed)
# Derive spectrograph number from nstart if needed
if args.spectrograph is None:
args.spectrograph = args.nstart / 500
# Read fibermapfile to get object type, night and expid
if args.fibermap:
log.info("Reading fibermap file {}".format(args.fibermap))
fibermap=read_fibermap(args.fibermap)
objtype = get_source_types(fibermap)
stdindx=np.where(objtype=='STD') # match STD with STAR
mwsindx=np.where(objtype=='MWS_STAR') # match MWS_STAR with STAR
bgsindx=np.where(objtype=='BGS') # match BGS with LRG
objtype[stdindx]='STAR'
objtype[mwsindx]='STAR'
objtype[bgsindx]='LRG'
NIGHT=fibermap.meta['NIGHT']
EXPID=fibermap.meta['EXPID']
else:
# Create a blank fake fibermap
fibermap = empty_fibermap(args.nspec)
targetids = random_state.randint(2**62, size=args.nspec)
fibermap['TARGETID'] = targetids
night = get_night()
expid = 0
log.info("Initializing SpecSim with config {}".format(args.config))
desiparams = load_desiparams()
qsim = get_simulator(args.config, num_fibers=1)
if args.simspec:
# Read the input file
log.info('Reading input file {}'.format(args.simspec))
simspec = desisim.io.read_simspec(args.simspec)
nspec = simspec.nspec
if simspec.flavor == 'arc':
log.warning("quickgen doesn't generate flavor=arc outputs")
return
else:
wavelengths = simspec.wave
spectra = simspec.flux
if nspec < args.nspec:
log.info("Only {} spectra in input file".format(nspec))
args.nspec = nspec
else:
# Initialize the output truth table.
spectra = []
wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value
npix = len(wavelengths)
truth = dict()
meta = Table()
truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10))
truth['FLUX'] = np.zeros((args.nspec, npix))
truth['WAVE'] = wavelengths
jj = list()
for thisobj in set(true_objtype):
ii = np.where(true_objtype == thisobj)[0]
nobj = len(ii)
truth['OBJTYPE'][ii] = thisobj
log.info('Generating {} template'.format(thisobj))
# Generate the templates
if thisobj == 'ELG':
elg = desisim.templates.ELG(wave=wavelengths, add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = elg.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_elg,sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'LRG':
lrg = desisim.templates.LRG(wave=wavelengths, add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = lrg.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_lrg,sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'QSO':
qso = desisim.templates.QSO(wave=wavelengths)
flux, tmpwave, meta1 = qso.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_qso)
elif thisobj == 'BGS':
bgs = desisim.templates.BGS(wave=wavelengths, add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = bgs.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_bgs,rmagrange=args.rmagrange_bgs,sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj =='STD':
std = desisim.templates.STD(wave=wavelengths)
flux, tmpwave, meta1 = std.make_templates(nmodel=nobj, seed=args.seed)
elif thisobj == 'QSO_BAD': # use STAR template no color cuts
star = desisim.templates.STAR(wave=wavelengths)
flux, tmpwave, meta1 = star.make_templates(nmodel=nobj, seed=args.seed)
elif thisobj == 'MWS_STAR' or thisobj == 'MWS':
mwsstar = desisim.templates.MWS_STAR(wave=wavelengths)
flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj, seed=args.seed)
elif thisobj == 'WD':
wd = desisim.templates.WD(wave=wavelengths)
flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj, seed=args.seed)
elif thisobj == 'SKY':
flux = np.zeros((nobj, npix))
meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32)))
elif thisobj == 'TEST':
flux = np.zeros((args.nspec, npix))
indx = np.where(wave>5800.0-1E-6)[0][0]
ref_integrated_flux = 1E-10
ref_cst_flux_density = 1E-17
single_line = (np.arange(args.nspec)%2 == 0).astype(np.float32)
continuum = (np.arange(args.nspec)%2 == 1).astype(np.float32)
for spec in range(args.nspec) :
flux[spec,indx] = single_line[spec]*ref_integrated_flux/np.gradient(wavelengths)[indx] # single line
flux[spec] += continuum[spec]*ref_cst_flux_density # flat continuum
meta1 = Table(dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32),
LINE=wave[indx]*np.ones(args.nspec, dtype=np.float32),
LINEFLUX=single_line*ref_integrated_flux,
CONSTFLUXDENSITY=continuum*ref_cst_flux_density))
else:
log.fatal('Unknown object type {}'.format(thisobj))
sys.exit(1)
# Pack it in.
truth['FLUX'][ii] = flux
meta = vstack([meta, meta1])
jj.append(ii.tolist())
# Sanity check on units; templates currently return ergs, not 1e-17 ergs...
# assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6)
# Sort the metadata table.
jj = sum(jj,[])
meta_new = Table()
for k in range(args.nspec):
index = int(np.where(np.array(jj) == k)[0])
meta_new = vstack([meta_new, meta[index]])
meta = meta_new
# Add TARGETID and the true OBJTYPE to the metadata table.
meta.add_column(Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE'))
meta.add_column(Column(targetids, name='TARGETID'))
# Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z
meta.rename_column('REDSHIFT', 'TRUEZ')
# explicitly set location on focal plane if needed to support airmass
# variations when using specsim v0.5
if qsim.source.focal_xy is None:
qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm'))
# Set simulation parameters from the simspec header or desiparams
bright_objects = ['bgs','mws','bright','BGS','MWS','BRIGHT_MIX']
gray_objects = ['gray','grey']
if args.simspec is None:
object_type = objtype
flavor = None
elif simspec.flavor == 'science':
object_type = None
flavor = simspec.header['PROGRAM']
else:
object_type = None
flavor = simspec.flavor
log.warning('Maybe using an outdated simspec file with flavor={}'.format(flavor))
# Set airmass
if args.airmass is not None:
qsim.atmosphere.airmass = args.airmass
elif args.simspec and 'AIRMASS' in simspec.header:
qsim.atmosphere.airmass = simspec.header['AIRMASS']
else:
qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2
# Set exptime
if args.exptime is not None:
qsim.observation.exposure_time = args.exptime * u.s
elif args.simspec and 'EXPTIME' in simspec.header:
qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s
elif objtype in bright_objects:
qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s
else:
qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s
# Set Moon Phase
if args.moon_phase is not None:
qsim.atmosphere.moon.moon_phase = args.moon_phase
elif args.simspec and 'MOONFRAC' in simspec.header:
qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC']
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_phase = 0.7
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_phase = 0.1
else:
qsim.atmosphere.moon.moon_phase = 0.5
# Set Moon Zenith
if args.moon_zenith is not None:
qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg
elif args.simspec and 'MOONALT' in simspec.header:
qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_zenith = 30 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_zenith = 80 * u.deg
else:
qsim.atmosphere.moon.moon_zenith = 100 * u.deg
# Set Moon - Object Angle
if args.moon_angle is not None:
qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg
elif args.simspec and 'MOONSEP' in simspec.header:
qsim.atmosphere.moon.separation_angle = simspec.header['MOONSEP'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.separation_angle = 50 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
else:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
# Initialize per-camera output arrays that will be saved
waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {}
maxbin = 0
nmax= args.nspec
for camera in qsim.instrument.cameras:
# Lookup this camera's resolution matrix and convert to the sparse
# format used in desispec.
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(), [args.nspec, 1, 1])
waves[camera.name] = (camera.output_wavelength.to(u.Angstrom).value.astype(np.float32))
nwave = len(waves[camera.name])
maxbin = max(maxbin, len(waves[camera.name]))
nobj = np.zeros((nmax,3,maxbin)) # object photons
nsky = np.zeros((nmax,3,maxbin)) # sky photons
nivar = np.zeros((nmax,3,maxbin)) # inverse variance (object+sky)
cframe_observedflux = np.zeros((nmax,3,maxbin)) # calibrated object flux
cframe_ivar = np.zeros((nmax,3,maxbin)) # inverse variance of calibrated object flux
cframe_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to calibrated flux
sky_ivar = np.zeros((nmax,3,maxbin)) # inverse variance of sky
sky_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to sky only
frame_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to nobj+nsky
trueflux[camera.name] = np.empty((args.nspec, nwave)) # calibrated flux
noisyflux[camera.name] = np.empty((args.nspec, nwave)) # observed flux with noise
obsivar[camera.name] = np.empty((args.nspec, nwave)) # inverse variance of flux
if args.simspec:
for i in range(10):
cn = camera.name + str(i)
if cn in simspec.cameras:
dw = np.gradient(simspec.cameras[cn].wave)
break
else:
raise RuntimeError('Unable to find a {} camera in input simspec'.format(camera))
else:
sflux = np.empty((args.nspec, npix))
#- Check if input simspec is for a continuum flat lamp instead of science
#- This does not convolve to per-fiber resolution
if args.simspec:
if simspec.flavor == 'flat':
log.info("Simulating flat lamp exposure")
for i,camera in enumerate(qsim.instrument.cameras):
channel = camera.name #- from simspec, b/r/z not b0/r1/z9
assert camera.output_wavelength.unit == u.Angstrom
num_pixels = len(waves[channel])
phot = list()
for j in range(10):
cn = camera.name + str(j)
if cn in simspec.cameras:
camwave = simspec.cameras[cn].wave
dw = np.gradient(camwave)
phot.append(simspec.cameras[cn].phot)
if len(phot) == 0:
raise RuntimeError('Unable to find a {} camera in input simspec'.format(camera))
else:
phot = np.vstack(phot)
meanspec = resample_flux(
waves[channel], camwave, np.average(phot/dw, axis=0))
fiberflat = random_state.normal(loc=1.0,
scale=1.0 / np.sqrt(meanspec), size=(nspec, num_pixels))
ivar = np.tile(meanspec, [nspec, 1])
mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32)
for kk in range((args.nspec+args.nstart-1)//500+1):
camera = channel+str(kk)
outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID, camera)
start=max(500*kk,args.nstart)
end=min(500*(kk+1),nmax)
if (args.spectrograph <= kk):
log.info("Writing files for channel:{}, spectrograph:{}, spectra:{} to {}".format(channel,kk,start,end))
ff = FiberFlat(
waves[channel], fiberflat[start:end,:],
ivar[start:end,:], mask[start:end,:], meanspec,
header=dict(CAMERA=camera))
write_fiberflat(outfile, ff)
filePath=desispec.io.findfile("fiberflat",NIGHT,EXPID,camera)
log.info("Wrote file {}".format(filePath))
sys.exit(0)
# Repeat the simulation for all spectra
fluxunits = 1e-17 * u.erg / (u.s * u.cm ** 2 * u.Angstrom)
for j in range(args.nspec):
thisobjtype = objtype[j]
sys.stdout.flush()
if flavor == 'arc':
qsim.source.update_in(
'Quickgen source {0}'.format, 'perfect',
wavelengths * u.Angstrom, spectra * fluxunits)
else:
qsim.source.update_in(
'Quickgen source {0}'.format(j), thisobjtype.lower(),
wavelengths * u.Angstrom, spectra[j, :] * fluxunits)
qsim.source.update_out()
qsim.simulate()
qsim.generate_random_noise(random_state)
for i, output in enumerate(qsim.camera_output):
assert output['observed_flux'].unit == 1e17 * fluxunits
# Extract the simulation results needed to create our uncalibrated
# frame output file.
num_pixels = len(output)
nobj[j, i, :num_pixels] = output['num_source_electrons'][:,0]
nsky[j, i, :num_pixels] = output['num_sky_electrons'][:,0]
nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:,0]
# Get results for our flux-calibrated output file.
cframe_observedflux[j, i, :num_pixels] = 1e17 * output['observed_flux'][:,0]
cframe_ivar[j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:,0]
# Fill brick arrays from the results.
camera = output.meta['name']
trueflux[camera][j][:] = 1e17 * output['observed_flux'][:,0]
noisyflux[camera][j][:] = 1e17 * (output['observed_flux'][:,0] +
output['flux_calibration'][:,0] * output['random_noise_electrons'][:,0])
obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,0]
# Use the same noise realization in the cframe and frame, without any
# additional noise from sky subtraction for now.
frame_rand_noise[j, i, :num_pixels] = output['random_noise_electrons'][:,0]
cframe_rand_noise[j, i, :num_pixels] = 1e17 * (
output['flux_calibration'][:,0] * output['random_noise_electrons'][:,0])
# The sky output file represents a model fit to ~40 sky fibers.
# We reduce the variance by a factor of 25 to account for this and
# give the sky an independent (Gaussian) noise realization.
sky_ivar[j, i, :num_pixels] = 25.0 / (
output['variance_electrons'][:,0] - output['num_source_electrons'][:,0])
sky_rand_noise[j, i, :num_pixels] = random_state.normal(
scale=1.0 / np.sqrt(sky_ivar[j,i,:num_pixels]),size=num_pixels)
armName={"b":0,"r":1,"z":2}
for channel in 'brz':
#Before writing, convert from counts/bin to counts/A (as in Pixsim output)
#Quicksim Default:
#FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang]
#COUNTS_OBJ - object counts in 0.5 Ang bin
#COUNTS_SKY - sky counts in 0.5 Ang bin
num_pixels = len(waves[channel])
dwave=np.gradient(waves[channel])
nobj[:,armName[channel],:num_pixels]/=dwave
frame_rand_noise[:,armName[channel],:num_pixels]/=dwave
nivar[:,armName[channel],:num_pixels]*=dwave**2
nsky[:,armName[channel],:num_pixels]/=dwave
sky_rand_noise[:,armName[channel],:num_pixels]/=dwave
sky_ivar[:,armName[channel],:num_pixels]/=dwave**2
# Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra
# Looping over spectrograph
for ii in range((args.nspec+args.nstart-1)//500+1):
start=max(500*ii,args.nstart) # first spectrum for a given spectrograph
end=min(500*(ii+1),nmax) # last spectrum for the spectrograph
if (args.spectrograph <= ii):
camera = "{}{}".format(channel, ii)
log.info("Writing files for channel:{}, spectrograph:{}, spectra:{} to {}".format(channel,ii,start,end))
num_pixels = len(waves[channel])
# Write frame file
framefileName=desispec.io.findfile("frame",NIGHT,EXPID,camera)
frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \
nsky[start:end,armName[channel],:num_pixels] + \
frame_rand_noise[start:end,armName[channel],:num_pixels]
frame_ivar=nivar[start:end,armName[channel],:num_pixels]
sh1=frame_flux.shape[0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave)
# for example if nstart =400, nspec=150: two spectrographs:
# 400-499=> 0 spectrograph, 500-549 => 1
if (args.nstart==start):
resol=resolution[channel][:sh1,:,:]
else:
resol=resolution[channel][-sh1:,:,:]
# must create desispec.Frame object
frame=Frame(waves[channel], frame_flux, frame_ivar,\
resolution_data=resol, spectrograph=ii, \
fibermap=fibermap[start:end], \
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.write_frame(framefileName, frame)
framefilePath=desispec.io.findfile("frame",NIGHT,EXPID,camera)
log.info("Wrote file {}".format(framefilePath))
if args.frameonly or simspec.flavor == 'arc':
continue
# Write cframe file
cframeFileName=desispec.io.findfile("cframe",NIGHT,EXPID,camera)
cframeFlux=cframe_observedflux[start:end,armName[channel],:num_pixels]+cframe_rand_noise[start:end,armName[channel],:num_pixels]
cframeIvar=cframe_ivar[start:end,armName[channel],:num_pixels]
# must create desispec.Frame object
cframe = Frame(waves[channel], cframeFlux, cframeIvar, \
resolution_data=resol, spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.frame.write_frame(cframeFileName,cframe)
cframefilePath=desispec.io.findfile("cframe",NIGHT,EXPID,camera)
log.info("Wrote file {}".format(cframefilePath))
# Write sky file
skyfileName=desispec.io.findfile("sky",NIGHT,EXPID,camera)
skyflux=nsky[start:end,armName[channel],:num_pixels] + \
sky_rand_noise[start:end,armName[channel],:num_pixels]
skyivar=sky_ivar[start:end,armName[channel],:num_pixels]
skymask=np.zeros(skyflux.shape, dtype=np.uint32)
# must create desispec.Sky object
skymodel = SkyModel(waves[channel], skyflux, skyivar, skymask,
header=dict(CAMERA=camera))
desispec.io.sky.write_sky(skyfileName, skymodel)
skyfilePath=desispec.io.findfile("sky",NIGHT,EXPID,camera)
log.info("Wrote file {}".format(skyfilePath))
# Write calib file
calibVectorFile=desispec.io.findfile("calib",NIGHT,EXPID,camera)
flux = cframe_observedflux[start:end,armName[channel],:num_pixels]
phot = nobj[start:end,armName[channel],:num_pixels]
calibration = np.zeros_like(phot)
jj = (flux>0)
calibration[jj] = phot[jj] / flux[jj]
#- TODO: what should calibivar be?
#- For now, model it as the noise of combining ~10 spectra
calibivar=10/cframe_ivar[start:end,armName[channel],:num_pixels]
#mask=(1/calibivar>0).astype(int)??
mask=np.zeros(calibration.shape, dtype=np.uint32)
# write flux calibration
fluxcalib = FluxCalib(waves[channel], calibration, calibivar, mask)
write_flux_calibration(calibVectorFile, fluxcalib)
calibfilePath=desispec.io.findfile("calib",NIGHT,EXPID,camera)
log.info("Wrote file {}".format(calibfilePath))