def test_errors(self): #- Bad shaped input data = np.random.uniform(size=(10, 5)) with self.assertRaises(ValueError): R = Resolution(data) #- Meaningless type for input with self.assertRaises(ValueError): R = Resolution('blat') #- Non-uniform x spacing with self.assertRaises(ValueError): R = desispec.resolution._gauss_pix([-1, 0, 2]) #- missing offsets with self.assertRaises(ValueError): desispec.resolution._sort_and_symmeterize(data, [-2, -1, 0, 1, 3]) #- length of offsets too large or small with self.assertRaises(ValueError): Resolution(data, offsets=[1, 2]) with self.assertRaises(ValueError): Resolution(data, offsets=np.arange(10 * desispec.resolution.default_ndiag))
def _get_spectra(self): #- Setup data for a Resolution matrix sigma = 4.0 ndiag = 21 xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag) Rdata = np.zeros((self.nspec, ndiag, self.nwave)) for i in range(self.nspec): for j in range(self.nwave): kernel = np.exp(-xx**2 / (2 * sigma)) kernel /= sum(kernel) Rdata[i, :, j] = kernel flux = np.zeros((self.nspec, self.nwave)) ivar = np.ones((self.nspec, self.nwave)) mask = np.zeros((self.nspec, self.nwave), dtype=int) for i in range(self.nspec): R = Resolution(Rdata[i]) flux[i] = R.dot(self.flux) fibermap = desispec.io.empty_fibermap(self.nspec, 1500) fibermap['OBJTYPE'][0::2] = 'SKY' return Frame(self.wave, flux, ivar, mask, Rdata, spectrograph=2, fibermap=fibermap)
def test_resolution_sparsedia(self): data = np.random.uniform(size=(5,10)) offsets = np.arange(-2,3) #- Original case: symetric and odd number of diagonals Rdia = scipy.sparse.dia_matrix((data, offsets), shape=(10,10)) R = Resolution(Rdia) self.assertTrue(np.all(R.diagonal() == Rdia.diagonal())) #- Non symetric but still odd number of diagonals Rdia = scipy.sparse.dia_matrix((data, offsets+1), shape=(10,10)) R = Resolution(Rdia) self.assertTrue(np.all(R.diagonal() == Rdia.diagonal())) #- Even number of diagonals Rdia = scipy.sparse.dia_matrix((data[1:,:], offsets[1:]), shape=(10,10)) R = Resolution(Rdia) self.assertTrue(np.all(R.diagonal() == Rdia.diagonal())) #- Unordered diagonals data = np.random.uniform(size=(5,10)) offsets = [0,1,-1,2,-2] Rdia = scipy.sparse.dia_matrix((data, offsets), shape=(10,10)) R1 = Resolution(Rdia) R2 = Resolution(data, offsets) self.assertTrue(np.all(R1.diagonal() == Rdia.diagonal())) self.assertTrue(np.all(R2.diagonal() == Rdia.diagonal())) self.assertTrue(np.all(R1.data == R2.data))
def test_resolution_dense(self): #- dense with no offsets specified data = np.random.uniform(size=(10,10)) R = Resolution(data) Rdense = R.todense() self.assertTrue(np.all(Rdense == data)) #- with offsets offsets = np.arange(-2,4) R = Resolution(data, offsets) Rdense = R.todense() for i in offsets: self.assertTrue(np.all(Rdense.diagonal(i) == data.diagonal(i)), \ "diagonal {} doesn't match".format(i)) #- dense without offsets but larger than default_ndiag ndiag = desispec.resolution.default_ndiag + 5 data = np.random.uniform(size=(ndiag, ndiag)) Rdense = Resolution(data).todense() for i in range(ndiag): if i <= desispec.resolution.default_ndiag//2: self.assertTrue(np.all(Rdense.diagonal(i) == data.diagonal(i)), \ "diagonal {} doesn't match".format(i)) self.assertTrue(np.all(Rdense.diagonal(-i) == data.diagonal(-i)), \ "diagonal {} doesn't match".format(-i)) else: self.assertTrue(np.all(Rdense.diagonal(i) == 0.0), \ "diagonal {} not 0s".format(i)) self.assertTrue(np.all(Rdense.diagonal(-i) == 0.0), \ "diagonal {} not 0s".format(-i))
def _get_spectra(self, with_gradient=False): #- Setup data for a Resolution matrix sigma2 = 4.0 ndiag = 21 xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag) Rdata = np.zeros((self.nspec, ndiag, self.nwave)) for i in range(self.nspec): kernel = np.exp(-(xx + float(i) / self.nspec * 0.3)**2 / (2 * sigma2)) #kernel = np.exp(-xx**2/(2*sigma2)) kernel /= sum(kernel) for j in range(self.nwave): Rdata[i, :, j] = kernel flux = np.zeros((self.nspec, self.nwave), dtype=float) ivar = np.ones((self.nspec, self.nwave), dtype=float) # Add a random component for i in range(self.nspec): ivar[i] += 0.4 * np.random.uniform(size=self.nwave) mask = np.zeros((self.nspec, self.nwave), dtype=int) fibermap = empty_fibermap(self.nspec, 1500) fibermap['OBJTYPE'][0::2] = 'SKY' x = fibermap["FIBERASSIGN_X"] y = fibermap["FIBERASSIGN_Y"] x = x - np.mean(x) y = y - np.mean(y) if np.std(x) > 0: x /= np.std(x) if np.std(y) > 0: y /= np.std(y) w = (self.wave - self.wave[0]) / (self.wave[-1] - self.wave[0]) * 2. - 1 for i in range(self.nspec): R = Resolution(Rdata[i]) if with_gradient: scale = 1. + (0.1 * x[i] + 0.2 * y[i]) * (1 + 0.4 * w) flux[i] = R.dot(scale * self.flux) else: flux[i] = R.dot(self.flux) meta = {"camera": "r2"} return Frame(self.wave, flux, ivar, mask, Rdata, spectrograph=2, fibermap=fibermap, meta=meta)
def _X2(x): z = x[0] v = x[1] lnA = x[2] line_flux = np.exp(lnA) result = 0.0 for cam in cams: wave = spectra.wave[cam] res = Resolution(spectra.resolution_data[cam][fiber, :]) flux = spectra.flux[cam][fiber, :] ivar = spectra.ivar[cam][fiber, :] mask = spectra.mask[cam][fiber, :] tw = twave(wave.min(), wave.max()) _, _, X2, _ = doublet_chi2(z, tw, wave, res, flux, ivar, mask, continuum=0.0, sigmav=v, r=0.0, line_flux=line_flux, linea=0.0, lineb=lineb) result += X2 return result
def get_frame_data(nspec=10, objtype=None): """ Return basic test data for desispec.frame object: """ nwave = 100 wavemin, wavemax = 4000, 4100 wave, model_flux = get_models(nspec, nwave, wavemin=wavemin, wavemax=wavemax) resol_data = set_resolmatrix(nspec, nwave) calib = np.sin((wave - wavemin) * np.pi / np.max(wave)) flux = np.zeros((nspec, nwave)) for i in range(nspec): flux[i] = Resolution(resol_data[i]).dot(model_flux[i] * calib) sigma = 0.01 # flux += np.random.normal(scale=sigma, size=flux.shape) ivar = np.ones(flux.shape) / sigma**2 mask = np.zeros(flux.shape, dtype=int) fibermap = empty_fibermap(nspec, 1500) if objtype is None: fibermap['OBJTYPE'] = 'QSO' fibermap['OBJTYPE'][0:3] = 'STD' # For flux tests else: fibermap['OBJTYPE'] = objtype frame = Frame(wave, flux, ivar, mask, resol_data, fibermap=fibermap) frame.meta = {} frame.meta['EXPTIME'] = 1. # For flux tests return frame
class TestResolution(unittest.TestCase): #- TODO: in principle these should be converted to self.assertXYZ #- In practice, if they fail then the test won't pass so the end result is the same def test_resolution(self, n=100): dense = np.arange(n * n).reshape(n, n) R1 = Resolution(dense) assert scipy.sparse.isspmatrix_dia( R1), 'Resolution is not recognized as a scipy.sparse.dia_matrix.' assert len( R1.offsets ) == desispec.resolution.default_ndiag, 'Resolution.offsets has wrong size' R2 = Resolution(R1) assert np.array_equal( R1.toarray(), R2.toarray()), 'Constructor broken for dia_matrix input.' R3 = Resolution(R1.data) assert np.array_equal( R1.toarray(), R3.toarray()), 'Constructor broken for array data input.' sparse = scipy.sparse.dia_matrix((R1.data[::-1], R1.offsets[::-1]), (n, n)) R4 = Resolution(sparse) assert np.array_equal( R1.toarray(), R4.toarray()), 'Constructor broken for permuted offsets input.' R5 = Resolution(R1.to_fits_array()) assert np.array_equal(R1.toarray(), R5.toarray()), 'to_fits_array() is broken.' #- test different sizes of input diagonals for ndiag in [3, 5, 11]: R6 = Resolution(np.ones((ndiag, n))) assert len( R6.offsets) == ndiag, 'Constructor broken for ndiag={}'.format( ndiag) #- An even number if diagonals is not allowed try: ndiag = 10 R7 = Resolution(np.ones((ndiag, n))) raise RuntimeError( 'Incorrectly created Resolution with even number of diagonals') except ValueError, err: #- it correctly raised an error, so pass pass #- Test creation with asymetric diagonals (should fail) R1.offsets += 1 try: R8 = Resolution(R1) raise RuntimeError( 'Incorrectly created Resolution with non-symmetric input') except ValueError: #- correctly raised an error, so pass pass
def test_resolution(self): """ Test that identical spectra convolved with different resolutions results in identical fiberflats """ wave, flux, ivar, mask = _get_data() nspec, nwave = flux.shape #- Setup a Resolution matrix that varies with fiber and wavelength #- Note: this is actually the transpose of the resolution matrix #- I wish I was creating, but as long as we self-consistently #- use it for convolving and solving, that shouldn't matter. sigma = np.linspace(2, 10, nwave * nspec) ndiag = 21 xx = np.linspace(-ndiag / 2.0, +ndiag / 2.0, ndiag) Rdata = np.zeros((nspec, len(xx), nwave)) for i in range(nspec): for j in range(nwave): kernel = np.exp(-xx**2 / (2 * sigma[i * nwave + j]**2)) kernel /= sum(kernel) Rdata[i, :, j] = kernel #- Convolve the data with the resolution matrix convflux = np.empty_like(flux) for i in range(nspec): convflux[i] = Resolution(Rdata[i]).dot(flux[i]) #- Run the code frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0) ff = compute_fiberflat(frame) #- These fiber flats should all be ~1 self.assertTrue(np.all(np.abs(ff.fiberflat - 1) < 0.001))
def _fdgrad(x): z = x[0] v = x[1] r = x[2] lnA = x[3] line_flux = np.exp(lnA) result = 0.0 chisq = 0.0 grad = np.zeros(4) eps = np.sqrt(np.finfo(float).eps) for cam in cams: wave = postage[cam].wave res = Resolution(postage[cam].R) flux = postage[cam].flux ivar = postage[cam].ivar mask = postage[cam].mask for i in np.arange(4): hi = np.array(x, copy=True) lo = np.array(x, copy=True) lo[i] -= eps hi[i] += eps _, _, hiX2, _ = doublet_chi2(hi[0], wave, res, flux, ivar, mask, continuum=0.0, sigmav=hi[1], r=hi[2], line_flux=np.exp(hi[3]), linea=linea, lineb=lineb) _, _, loX2, _ = doublet_chi2(lo[0], wave, res, flux, ivar, mask, continuum=0.0, sigmav=lo[1], r=lo[2], line_flux=np.exp(lo[3]), linea=linea, lineb=lineb) grad[i] += (hiX2 - loX2) / 2. / eps break return grad, -99.
def get_resolution(wave, nspec, psf, usesigma=False): """ Calculates approximate resolution values at given wavelengths in the format that can directly feed resolution data of desispec.frame.Frame object. wave: wavelength array nsepc: no of spectra (int) psf: desispec.psf.PSF like object usesigma: allows to use sigma from psf file for resolution computation. If psf file is psfboot, uses per fiber xsigma. If psf file is from QL arcs processing, uses wsigma returns : resolution data (nspec,nband,nwave); nband = 1 for usesigma = False, otherwise nband=21 """ from desispec.resolution import Resolution nwave = len(wave) if usesigma: nband = 21 else: nband = 1 # only for dimensionality purpose of data model. resolution_data = np.zeros((nspec, nband, nwave)) if usesigma: #- use sigmas for resolution based on psffile type if hasattr(psf, 'wcoeff'): #- use if have wsigmas log.info("Getting resolution from wsigmas from arc lines PSF") for ispec in range(nspec): thissigma = psf.wdisp(ispec, wave) / psf.angstroms_per_pixel( ispec, wave) #- in pixel units Rsig = Resolution(thissigma) resolution_data[ispec] = Rsig.data else: if hasattr( psf, 'xsigma_boot'): #- only use if xsigma comes from psfboot log.info( "Getting resolution matrix band diagonal elements from constant Gaussing Xsigma" ) for ispec in range(nspec): thissigma = psf.xsigma(ispec, wave) Rsig = Resolution(thissigma) resolution_data[ispec] = Rsig.data return resolution_data
def _get_spectra(self,with_gradient=False): #- Setup data for a Resolution matrix sigma2 = 4.0 ndiag = 21 xx = np.linspace(-(ndiag-1)/2.0, +(ndiag-1)/2.0, ndiag) Rdata = np.zeros( (self.nspec, ndiag, self.nwave) ) for i in range(self.nspec): kernel = np.exp(-(xx+float(i)/self.nspec*0.3)**2/(2*sigma2)) #kernel = np.exp(-xx**2/(2*sigma2)) kernel /= sum(kernel) for j in range(self.nwave): Rdata[i,:,j] = kernel flux = np.zeros((self.nspec, self.nwave),dtype=float) ivar = np.ones((self.nspec, self.nwave),dtype=float) # Add a random component for i in range(self.nspec) : ivar[i] += 0.4*np.random.uniform(size=self.nwave) mask = np.zeros((self.nspec, self.nwave), dtype=int) fibermap = desispec.io.empty_fibermap(self.nspec, 1500) fibermap['OBJTYPE'][0::2] = 'SKY' x=fibermap["DESIGN_X"] y=fibermap["DESIGN_Y"] x = x-np.mean(x) y = y-np.mean(y) if np.std(x)>0 : x /= np.std(x) if np.std(y)>0 : y /= np.std(y) w = (self.wave-self.wave[0])/(self.wave[-1]-self.wave[0])*2.-1 for i in range(self.nspec): R = Resolution(Rdata[i]) if with_gradient : scale = 1.+(0.1*x[i]+0.2*y[i])*(1+0.4*w) flux[i] = R.dot(scale*self.flux) else : flux[i] = R.dot(self.flux) return Frame(self.wave, flux, ivar, mask, Rdata, spectrograph=2, fibermap=fibermap)
def test_throughput_resolution(self): """ Test that spectra with different throughputs and different resolutions result in fiberflat variations that are only due to throughput. """ wave, flux, ivar, mask = _get_data() nspec, nwave = flux.shape #- Setup a Resolution matrix that varies with fiber and wavelength #- Note: this is actually the transpose of the resolution matrix #- I wish I was creating, but as long as we self-consistently #- use it for convolving and solving, that shouldn't matter. sigma = np.linspace(2, 10, nwave * nspec) ndiag = 21 xx = np.linspace(-ndiag / 2.0, +ndiag / 2.0, ndiag) Rdata = np.zeros((nspec, len(xx), nwave)) for i in range(nspec): for j in range(nwave): kernel = np.exp(-xx**2 / (2 * sigma[i * nwave + j]**2)) kernel /= sum(kernel) Rdata[i, :, j] = kernel #- Vary the input flux prior to calculating the fiber flat flux[1] *= 1.1 flux[2] *= 1.2 flux[3] /= 1.1 flux[4] /= 1.2 #- Convolve the data with the varying resolution matrix convflux = np.empty_like(flux) for i in range(nspec): convflux[i] = Resolution(Rdata[i]).dot(flux[i]) #- Run the code frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0) #- Set an accuracy for this accuracy = 1.e-9 ff = compute_fiberflat(frame, accuracy=accuracy) #- Compare variation with middle fiber mid = ff.fiberflat.shape[0] // 2 diff = (ff.fiberflat[1] / 1.1 - ff.fiberflat[mid]) self.assertLess(np.max(np.abs(diff)), accuracy) diff = (ff.fiberflat[2] / 1.2 - ff.fiberflat[mid]) self.assertLess(np.max(np.abs(diff)), accuracy) diff = (ff.fiberflat[3] * 1.1 - ff.fiberflat[mid]) self.assertLess(np.max(np.abs(diff)), accuracy) diff = (ff.fiberflat[4] * 1.2 - ff.fiberflat[mid]) self.assertLess(np.max(np.abs(diff)), accuracy)
def _getdata(self, n=10): wave = np.linspace(5000, 5100, n) flux = np.random.uniform(0, 1, size=n) ivar = np.random.uniform(0, 1, size=n) ### mask = np.random.randint(0, 256, size=n) rdat = np.ones((3, n)) rdat[0] *= 0.25 rdat[1] *= 0.5 rdat[2] *= 0.25 R = Resolution(rdat) ### return wave, flux, ivar, mask, R return wave, flux, ivar, None, R
def _res(x): z = x[0] v = x[1] lnA = x[2] line_flux = np.exp(lnA) residuals = [] for cam in cams: wave = spectra.wave[cam] res = Resolution(spectra.resolution_data[cam][fiber, :]) flux = spectra.flux[cam][fiber, :] ivar = spectra.ivar[cam][fiber, :] mask = spectra.mask[cam][fiber, :] tw = twave(wave.min(), wave.max()) rflux = doublet_obs(z, tw, wave, res, continuum=0.0, sigmav=v, r=0.0, linea=0.0, lineb=lineb) rflux *= line_flux res = np.sqrt(ivar[mask]) * np.abs(flux[mask] - rflux[mask]) residuals += res.tolist() residuals = np.array(residuals) print(z, v, lnA, residuals.sum()) return residuals
def _get_spectra(self): #- Setup data for a Resolution matrix sigma = 4.0 ndiag = 21 xx = np.linspace(-(ndiag-1)/2.0, +(ndiag-1)/2.0, ndiag) Rdata = np.zeros( (self.nspec, ndiag, self.nwave) ) for i in range(self.nspec): for j in range(self.nwave): kernel = np.exp(-xx**2/(2*sigma)) kernel /= sum(kernel) Rdata[i,:,j] = kernel flux = np.zeros((self.nspec, self.nwave)) ivar = np.ones((self.nspec, self.nwave)) mask = np.zeros((self.nspec, self.nwave), dtype=int) for i in range(self.nspec): R = Resolution(Rdata[i]) flux[i] = R.dot(self.flux) fibermap = desispec.io.empty_fibermap(self.nspec) fibermap['OBJTYPE'][0::2] = 'SKY' return Frame(self.wave, flux, ivar, mask, Rdata), fibermap
def test_main(self): """ Test the main program. """ # generate the frame data wave, flux, ivar, mask = _get_data() nspec, nwave = flux.shape #- Setup data for a Resolution matrix sigma = 4.0 ndiag = 11 xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag) Rdata = np.zeros((nspec, ndiag, nwave)) kernel = np.exp(-xx**2 / (2 * sigma)) kernel /= sum(kernel) for i in range(nspec): for j in range(nwave): Rdata[i, :, j] = kernel #- Convolve the data with the resolution matrix convflux = np.empty_like(flux) for i in range(nspec): convflux[i] = Resolution(Rdata[i]).dot(flux[i]) # create a fake fibermap fibermap = io.empty_fibermap(nspec, nwave) for i in range(0, nspec): fibermap['OBJTYPE'][i] = 'FAKE' io.write_fibermap(self.testfibermap, fibermap) #- write out the frame frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0, fibermap=fibermap, meta=dict(FLAVOR='flat')) write_frame(self.testframe, frame, fibermap=fibermap) # set program arguments argstr = ['--infile', self.testframe, '--outfile', self.testflat] # run it args = ffscript.parse(options=argstr) ffscript.main(args)
def get_frame_data(nspec=10, wavemin=4000, wavemax=4100, nwave=100, meta={}): """ Return basic test data for desispec.frame object: """ wave, model_flux = get_models(nspec, nwave, wavemin=wavemin, wavemax=wavemax) resol_data = set_resolmatrix(nspec, nwave) calib = np.sin((wave - wavemin) * np.pi / np.max(wave)) flux = np.zeros((nspec, nwave)) for i in range(nspec): flux[i] = Resolution(resol_data[i]).dot(model_flux[i] * calib) sigma = 0.01 # flux += np.random.normal(scale=sigma, size=flux.shape) ivar = np.ones(flux.shape) / sigma**2 mask = np.zeros(flux.shape, dtype=int) fibermap = empty_fibermap(nspec, 1500) fibermap['OBJTYPE'] = 'TGT' fibermap['DESI_TARGET'] = desi_mask.QSO fibermap['DESI_TARGET'][0:3] = desi_mask.STD_FAINT # For flux tests fibermap['FIBER_X'] = np.arange(nspec) * 400. / nspec #mm fibermap['FIBER_Y'] = np.arange(nspec) * 400. / nspec #mm fibermap['DELTA_X'] = 0.005 * np.ones(nspec) #mm fibermap['DELTA_Y'] = 0.003 * np.ones(nspec) #mm if "EXPTIME" not in meta.keys(): meta['EXPTIME'] = 1.0 frame = Frame(wave, flux, ivar, mask, resol_data, fibermap=fibermap, meta=meta) return frame
def _agrad(x): z = x[0] v = x[1] r = x[2] lnA = x[3] line_flux = np.exp(lnA) result = 0.0 chisq = 0.0 grad = np.zeros(4) for cam in cams: wave = postage[cam].wave res = Resolution(postage[cam].R) flux = postage[cam].flux ivar = postage[cam].ivar mask = postage[cam].mask _grad, X2 = doublet_chi2_grad(z, wave, res, flux, ivar, mask, continuum=0.0, v=v, r=r, line_flux=line_flux, linea=linea, lineb=lineb) grad += _grad chisq += X2 break return grad, chisq
def test_throughput(self): """ Test that spectra with different throughputs but the same resolution produce a fiberflat mirroring the variations in throughput """ wave, flux, ivar, mask = _get_data() nspec, nwave = flux.shape #- Setup data for a Resolution matrix sigma = 4.0 ndiag = 21 xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag) Rdata = np.zeros((nspec, ndiag, nwave)) kernel = np.exp(-xx**2 / (2 * sigma)) kernel /= sum(kernel) for i in range(nspec): for j in range(nwave): Rdata[i, :, j] = kernel #- Vary the input flux prior to calculating the fiber flat flux[1] *= 1.1 flux[2] *= 1.2 flux[3] *= 0.8 #- Convolve with the (common) resolution matrix convflux = np.empty_like(flux) for i in range(nspec): convflux[i] = Resolution(Rdata[i]).dot(flux[i]) frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0) ff = compute_fiberflat(frame) #- flux[1] is brighter, so should fiberflat[1]. etc. self.assertTrue(np.allclose(ff.fiberflat[0], ff.fiberflat[1] / 1.1)) self.assertTrue(np.allclose(ff.fiberflat[0], ff.fiberflat[2] / 1.2)) self.assertTrue(np.allclose(ff.fiberflat[0], ff.fiberflat[3] / 0.8))
def spectroperf_resample_spectrum_singleproc(spectra, target_index, wave, wavebin, resampling_matrix, ndiag, flux, ivar, rdata): cinv = None for b in spectra._bands: twave = spectra.wave[b] jj = (twave >= wave[0]) & (twave <= wave[-1]) twave = twave[jj] tivar = spectra.ivar[b][target_index][jj] diag_ivar = scipy.sparse.dia_matrix((tivar, [0]), (twave.size, twave.size)) RR = Resolution(spectra.resolution_data[b][target_index][:, jj]).dot( resampling_matrix[b]) tcinv = RR.T.dot(diag_ivar.dot(RR)) tcinvf = RR.T.dot(tivar * spectra.flux[b][target_index][jj]) if cinv is None: cinv = tcinv cinvf = tcinvf else: cinv += tcinv cinvf += tcinvf cinv = cinv.todense() decorrelate_divide_and_conquer(cinv, cinvf, wavebin, flux[target_index], ivar[target_index], rdata[target_index])
def _X2(x): z = x[0] v = x[1] r = x[2] lnA = x[3] line_flux = np.exp(lnA) result = 0.0 for cam in cams: wave = postage[cam].wave res = Resolution(postage[cam].R) flux = postage[cam].flux ivar = postage[cam].ivar mask = postage[cam].mask _, _, X2, _ = doublet_chi2(z, wave, res, flux, ivar, mask, continuum=0.0, sigmav=v, r=r, line_flux=line_flux, linea=linea, lineb=lineb) # X2 = doublet(z, twave, sigmav=v, r=r, linea=linea, lineb=lineb, index=5514) result += X2 break return result
def compute_uniform_sky(frame, nsig_clipping=4., max_iterations=100, model_ivar=False, add_variance=True): """Compute a sky model. Sky[fiber,i] = R[fiber,i,j] Flux[j] Input flux are expected to be flatfielded! We don't check this in this routine. Args: frame : Frame object, which includes attributes - wave : 1D wavelength grid in Angstroms - flux : 2D flux[nspec, nwave] density - ivar : 2D inverse variance of flux - mask : 2D inverse mask flux (0=good) - resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers) nsig_clipping : [optional] sigma clipping value for outlier rejection Optional: max_iterations : int , number of iterations model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims. add_variance : evaluate calibration error and add this to the sky model variance returns SkyModel object with attributes wave, flux, ivar, mask """ log = get_logger() log.info("starting") # Grab sky fibers on this frame skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0] assert np.max(skyfibers) < 500 #- indices, not fiber numbers nwave = frame.nwave nfibers = len(skyfibers) current_ivar = frame.ivar[skyfibers].copy() * (frame.mask[skyfibers] == 0) flux = frame.flux[skyfibers] Rsky = frame.R[skyfibers] input_ivar = None if model_ivar: log.info( "use a model of the inverse variance to remove bias due to correlated ivar and flux" ) input_ivar = current_ivar.copy() median_ivar_vs_wave = np.median(current_ivar, axis=0) median_ivar_vs_fiber = np.median(current_ivar, axis=1) median_median_ivar = np.median(median_ivar_vs_fiber) for f in range(current_ivar.shape[0]): threshold = 0.01 current_ivar[f] = median_ivar_vs_fiber[ f] / median_median_ivar * median_ivar_vs_wave # keep input ivar for very low weights ii = (input_ivar[f] <= (threshold * median_ivar_vs_wave)) #log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1])) current_ivar[f][ii] = input_ivar[f][ii] sqrtw = np.sqrt(current_ivar) sqrtwflux = sqrtw * flux chi2 = np.zeros(flux.shape) nout_tot = 0 for iteration in range(max_iterations): # the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters # A_ij = 1/2 d2(chi2)/di/dj # A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w] # the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters # B_i = 1/2 d(chi2)/di # B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w]) # the model is model[fiber]=R[fiber]*sky # and the parameters are the unconvolved sky flux at the wavelength i # so, d(model)/di[fiber,w] = R[fiber][w,i] # this gives # A_ij = sum_fiber sum_wave_w ivar[fiber,w] R[fiber][w,i] R[fiber][w,j] # A = sum_fiber ( diag(sqrt(ivar))*R[fiber] ) ( diag(sqrt(ivar))* R[fiber] )^t # A = sum_fiber sqrtwR[fiber] sqrtwR[fiber]^t # and # B = sum_fiber sum_wave_w ivar[fiber,w] R[fiber][w] * flux[fiber,w] # B = sum_fiber sum_wave_w sqrt(ivar)[fiber,w]*flux[fiber,w] sqrtwR[fiber,wave] #A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr() A = np.zeros((nwave, nwave)) B = np.zeros((nwave)) # diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber SD = scipy.sparse.lil_matrix((nwave, nwave)) # loop on fiber to handle resolution for fiber in range(nfibers): if fiber % 10 == 0: log.info("iter %d sky fiber %d/%d" % (iteration, fiber, nfibers)) R = Rsky[fiber] # diagonal sparse matrix with content = sqrt(ivar) SD.setdiag(sqrtw[fiber]) sqrtwR = SD * R # each row r of R is multiplied by sqrtw[r] A += (sqrtwR.T * sqrtwR).todense() B += sqrtwR.T * sqrtwflux[fiber] log.info("iter %d solving" % iteration) w = A.diagonal() > 0 A_pos_def = A[w, :] A_pos_def = A_pos_def[:, w] parameters = B * 0 try: parameters[w] = cholesky_solve(A_pos_def, B[w]) except: log.info("cholesky failed, trying svd in iteration {}".format( iteration)) parameters[w] = np.linalg.lstsq(A_pos_def, B[w])[0] log.info("iter %d compute chi2" % iteration) for fiber in range(nfibers): # the parameters are directly the unconvolve sky flux # so we simply have to reconvolve it fiber_convolved_sky_flux = Rsky[fiber].dot(parameters) chi2[fiber] = current_ivar[fiber] * (flux[fiber] - fiber_convolved_sky_flux)**2 log.info("rejecting") nout_iter = 0 if iteration < 1: # only remove worst outlier per wave # apply rejection iteratively, only one entry per wave among fibers # find waves with outlier (fastest way) nout_per_wave = np.sum(chi2 > nsig_clipping**2, axis=0) selection = np.where(nout_per_wave > 0)[0] for i in selection: worst_entry = np.argmax(chi2[:, i]) current_ivar[worst_entry, i] = 0 sqrtw[worst_entry, i] = 0 sqrtwflux[worst_entry, i] = 0 nout_iter += 1 else: # remove all of them at once bad = (chi2 > nsig_clipping**2) current_ivar *= (bad == 0) sqrtw *= (bad == 0) sqrtwflux *= (bad == 0) nout_iter += np.sum(bad) nout_tot += nout_iter sum_chi2 = float(np.sum(chi2)) ndf = int(np.sum(chi2 > 0) - nwave) chi2pdf = 0. if ndf > 0: chi2pdf = sum_chi2 / ndf log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d" % (iteration, sum_chi2, ndf, chi2pdf, nout_iter)) if nout_iter == 0: break log.info("nout tot=%d" % nout_tot) # we know have to compute the sky model for all fibers # and propagate the uncertainties # no need to restore the original ivar to compute the model errors when modeling ivar # the sky inverse variances are very similar log.info("compute the parameter covariance") # we may have to use a different method to compute this # covariance try: parameter_covar = cholesky_invert(A) # the above is too slow # maybe invert per block, sandwich by R except np.linalg.linalg.LinAlgError: log.warning( "cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv" ) parameter_covar = np.linalg.pinv(A) log.info("compute mean resolution") # we make an approximation for the variance to save CPU time # we use the average resolution of all fibers in the frame: mean_res_data = np.mean(frame.resolution_data, axis=0) Rmean = Resolution(mean_res_data) log.info("compute convolved sky and ivar") # The parameters are directly the unconvolved sky # First convolve with average resolution : convolved_sky_covar = Rmean.dot(parameter_covar).dot(Rmean.T.todense()) # and keep only the diagonal convolved_sky_var = np.diagonal(convolved_sky_covar) # inverse convolved_sky_ivar = (convolved_sky_var > 0) / (convolved_sky_var + (convolved_sky_var == 0)) # and simply consider it's the same for all spectra cskyivar = np.tile(convolved_sky_ivar, frame.nspec).reshape(frame.nspec, nwave) # The sky model for each fiber (simple convolution with resolution of each fiber) cskyflux = np.zeros(frame.flux.shape) for i in range(frame.nspec): cskyflux[i] = frame.R[i].dot(parameters) # look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1 if skyfibers.size > 1 and add_variance: modified_cskyivar = _model_variance(frame, cskyflux, cskyivar, skyfibers) else: modified_cskyivar = cskyivar.copy() # need to do better here mask = (cskyivar == 0).astype(np.uint32) return SkyModel( frame.wave.copy(), cskyflux, modified_cskyivar, mask, nrej=nout_tot, stat_ivar=cskyivar) # keep a record of the statistical ivar for QA
def __init__(self, wave, flux, ivar, mask=None, resolution_data=None, fibers=None, spectrograph=None, meta=None, fibermap=None, chi2pix=None, scores=None, scores_comments=None, wsigma=None, ndiag=21, suppress_res_warning=False): """ Lightweight wrapper for multiple spectra on a common wavelength grid x.wave, x.flux, x.ivar, x.mask, x.resolution_data, x.header, sp.R Args: wave: 1D[nwave] wavelength in Angstroms flux: 2D[nspec, nwave] flux ivar: 2D[nspec, nwave] inverse variance of flux Optional: mask: 2D[nspec, nwave] integer bitmask of flux. 0=good. resolution_data: 3D[nspec, ndiag, nwave] diagonals of resolution matrix data fibers: ndarray of which fibers these spectra are spectrograph: integer, which spectrograph [0-9] meta: dict-like object (e.g. FITS header) fibermap: fibermap table chi2pix: 2D[nspec, nwave] chi2 of 2D model to pixel-level data for pixels that contributed to each flux bin scores: dictionnary of 1D arrays of size nspec scores_comments: dictionnary of string (explaining the scores) suppress_res_warning: bool to suppress Warning message when the Resolution image is not read Parameters below allow on-the-fly resolution calculation wsigma: 2D[nspec,nwave] sigma widths for each wavelength bin for all fibers Notes: spectrograph input is used only if fibers is None. In this case, it assumes nspec_per_spectrograph = flux.shape[0] and calculates the fibers array for this spectrograph, i.e. fibers = spectrograph * flux.shape[0] + np.arange(flux.shape[0]) Attributes: All input args become object attributes. nspec : number of spectra, flux.shape[0] nwave : number of wavelengths, flux.shape[1] specmin : minimum fiber number R: array of sparse Resolution matrix objects converted from resolution_data fibermap: fibermap table if provided """ assert wave.ndim == 1 assert flux.ndim == 2 assert wave.shape[0] == flux.shape[1] assert ivar.shape == flux.shape assert (mask is None) or mask.shape == flux.shape assert (mask is None) or mask.dtype in \ (int, np.int64, np.int32, np.uint64, np.uint32), "Bad mask type "+str(mask.dtype) self.wave = wave self.flux = flux self.ivar = ivar self.meta = meta self.fibermap = fibermap self.nspec, self.nwave = self.flux.shape self.chi2pix = chi2pix self.scores = scores self.scores_comments = scores_comments self.ndiag = ndiag fibers_per_spectrograph = 500 #- hardcode; could get from desimodel if mask is None: self.mask = np.zeros(flux.shape, dtype=np.uint32) else: self.mask = util.mask32(mask) if resolution_data is not None: if resolution_data.ndim != 3 or \ resolution_data.shape[0] != self.nspec or \ resolution_data.shape[2] != self.nwave: raise ValueError( "Wrong dimensions for resolution_data[nspec, ndiag, nwave]" ) #- Maybe setup non-None identity matrix resolution matrix instead? self.wsigma = wsigma self.resolution_data = resolution_data if resolution_data is not None: self.wsigma = None #ignore width coefficients if resolution data is given explicitly self.ndiag = None self.R = np.array([Resolution(r) for r in resolution_data]) elif wsigma is not None: from desispec.quicklook.qlresolution import QuickResolution assert ndiag is not None r = [] for sigma in wsigma: r.append(QuickResolution(sigma=sigma, ndiag=self.ndiag)) self.R = np.array(r) else: #SK I believe this should be error, but looking at the #tests frame objects are allowed to not to have resolution data # thus I changed value error to a simple warning message. if not suppress_res_warning: log = get_logger() log.warning("Frame object is constructed without resolution data or respective "\ "sigma widths. Resolution will not be available") # raise ValueError("Need either resolution_data or coefficients to generate it") self.spectrograph = spectrograph # Deal with Fibers (these must be set!) if fibers is not None: fibers = np.asarray(fibers) if len(fibers) != self.nspec: raise ValueError("len(fibers) != nspec ({} != {})".format( len(fibers), self.nspec)) if fibermap is not None and np.any(fibers != fibermap['FIBER']): raise ValueError("fibermap doesn't match fibers") if (spectrograph is not None): minfiber = spectrograph * fibers_per_spectrograph maxfiber = (spectrograph + 1) * fibers_per_spectrograph if np.any(fibers < minfiber) or np.any(maxfiber <= fibers): raise ValueError('fibers inconsistent with spectrograph') self.fibers = fibers else: if fibermap is not None: self.fibers = fibermap['FIBER'] elif spectrograph is not None: self.fibers = spectrograph * fibers_per_spectrograph + np.arange( self.nspec, dtype=int) elif (self.meta is not None) and ('FIBERMIN' in self.meta): self.fibers = self.meta['FIBERMIN'] + np.arange(self.nspec, dtype=int) else: raise ValueError("Must set fibers by one of the methods!") if self.meta is not None: self.meta['FIBERMIN'] = np.min(self.fibers)
def compute_sky(frame, nsig_clipping=4.): """Compute a sky model. Input has to correspond to sky fibers only. Input flux are expected to be flatfielded! We don't check this in this routine. Args: frame : Frame object, which includes attributes - wave : 1D wavelength grid in Angstroms - flux : 2D flux[nspec, nwave] density - ivar : 2D inverse variance of flux - mask : 2D inverse mask flux (0=good) - resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers) nsig_clipping : [optional] sigma clipping value for outlier rejection returns SkyModel object with attributes wave, flux, ivar, mask """ log = get_logger() log.info("starting") # Grab sky fibers on this frame skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0] assert np.max(skyfibers) < 500 #- indices, not fiber numbers nwave = frame.nwave nfibers = len(skyfibers) current_ivar = frame.ivar[skyfibers].copy() flux = frame.flux[skyfibers] Rsky = frame.R[skyfibers] sqrtw = np.sqrt(current_ivar) sqrtwflux = sqrtw * flux chi2 = np.zeros(flux.shape) #debug #nfibers=min(nfibers,2) nout_tot = 0 for iteration in range(20): A = scipy.sparse.lil_matrix((nwave, nwave)).tocsr() B = np.zeros((nwave)) # diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber SD = scipy.sparse.lil_matrix((nwave, nwave)) # loop on fiber to handle resolution for fiber in range(nfibers): if fiber % 10 == 0: log.info("iter %d fiber %d" % (iteration, fiber)) R = Rsky[fiber] # diagonal sparse matrix with content = sqrt(ivar) SD.setdiag(sqrtw[fiber]) sqrtwR = SD * R # each row r of R is multiplied by sqrtw[r] A = A + (sqrtwR.T * sqrtwR).tocsr() B += sqrtwR.T * sqrtwflux[fiber] log.info("iter %d solving" % iteration) skyflux = cholesky_solve(A.todense(), B) log.info("iter %d compute chi2" % iteration) for fiber in range(nfibers): S = Rsky[fiber].dot(skyflux) chi2[fiber] = current_ivar[fiber] * (flux[fiber] - S)**2 log.info("rejecting") nout_iter = 0 if iteration < 1: # only remove worst outlier per wave # apply rejection iteratively, only one entry per wave among fibers # find waves with outlier (fastest way) nout_per_wave = np.sum(chi2 > nsig_clipping**2, axis=0) selection = np.where(nout_per_wave > 0)[0] for i in selection: worst_entry = np.argmax(chi2[:, i]) current_ivar[worst_entry, i] = 0 sqrtw[worst_entry, i] = 0 sqrtwflux[worst_entry, i] = 0 nout_iter += 1 else: # remove all of them at once bad = (chi2 > nsig_clipping**2) current_ivar *= (bad == 0) sqrtw *= (bad == 0) sqrtwflux *= (bad == 0) nout_iter += np.sum(bad) nout_tot += nout_iter sum_chi2 = float(np.sum(chi2)) ndf = int(np.sum(chi2 > 0) - nwave) chi2pdf = 0. if ndf > 0: chi2pdf = sum_chi2 / ndf log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d" % (iteration, sum_chi2, ndf, chi2pdf, nout_iter)) if nout_iter == 0: break log.info("nout tot=%d" % nout_tot) # solve once again to get deconvolved sky variance skyflux, skycovar = cholesky_solve_and_invert(A.todense(), B) #- sky inverse variance, but incomplete and not needed anyway # skyvar=np.diagonal(skycovar) # skyivar=(skyvar>0)/(skyvar+(skyvar==0)) # Use diagonal of skycovar convolved with mean resolution of all fibers # first compute average resolution mean_res_data = np.mean(frame.resolution_data, axis=0) R = Resolution(mean_res_data) # compute convolved sky and ivar cskycovar = R.dot(skycovar).dot(R.T.todense()) cskyvar = np.diagonal(cskycovar) cskyivar = (cskyvar > 0) / (cskyvar + (cskyvar == 0)) # convert cskyivar to 2D; today it is the same for all spectra, # but that may not be the case in the future cskyivar = np.tile(cskyivar, frame.nspec).reshape(frame.nspec, nwave) # Convolved sky cskyflux = np.zeros(frame.flux.shape) for i in range(frame.nspec): cskyflux[i] = frame.R[i].dot(skyflux) # need to do better here mask = (cskyivar == 0).astype(np.uint32) return SkyModel(frame.wave.copy(), cskyflux, cskyivar, mask, nrej=nout_tot)
def __init__(self, wave, flux, ivar, mask=None, resolution_data=None, fibers=None, spectrograph=None, meta=None, fibermap=None, chi2pix=None): """ Lightweight wrapper for multiple spectra on a common wavelength grid x.wave, x.flux, x.ivar, x.mask, x.resolution_data, x.header, sp.R Args: wave: 1D[nwave] wavelength in Angstroms flux: 2D[nspec, nwave] flux ivar: 2D[nspec, nwave] inverse variance of flux Optional: mask: 2D[nspec, nwave] integer bitmask of flux. 0=good. resolution_data: 3D[nspec, ndiag, nwave] diagonals of resolution matrix data fibers: ndarray of which fibers these spectra are spectrograph: integer, which spectrograph [0-9] meta: dict-like object (e.g. FITS header) fibermap: fibermap table chi2pix: 2D[nspec, nwave] chi2 of 2D model to pixel-level data for pixels that contributed to each flux bin Notes: spectrograph input is used only if fibers is None. In this case, it assumes nspec_per_spectrograph = flux.shape[0] and calculates the fibers array for this spectrograph, i.e. fibers = spectrograph * flux.shape[0] + np.arange(flux.shape[0]) Attributes: All input args become object attributes. nspec : number of spectra, flux.shape[0] nwave : number of wavelengths, flux.shape[1] specmin : minimum fiber number R: array of sparse Resolution matrix objects converted from resolution_data fibermap: fibermap table if provided """ assert wave.ndim == 1 assert flux.ndim == 2 assert wave.shape[0] == flux.shape[1] assert ivar.shape == flux.shape assert (mask is None) or mask.shape == flux.shape assert (mask is None) or mask.dtype in \ (int, np.int64, np.int32, np.uint64, np.uint32), "Bad mask type "+str(mask.dtype) self.wave = wave self.flux = flux self.ivar = ivar self.meta = meta self.fibermap = fibermap self.nspec, self.nwave = self.flux.shape self.chi2pix = chi2pix fibers_per_spectrograph = 500 #- hardcode; could get from desimodel if mask is None: self.mask = np.zeros(flux.shape, dtype=np.uint32) else: self.mask = util.mask32(mask) if resolution_data is not None: if resolution_data.ndim != 3 or \ resolution_data.shape[0] != self.nspec or \ resolution_data.shape[2] != self.nwave: raise ValueError( "Wrong dimensions for resolution_data[nspec, ndiag, nwave]" ) #- Maybe setup non-None identity matrix resolution matrix instead? self.resolution_data = resolution_data if resolution_data is not None: self.R = np.array([Resolution(r) for r in resolution_data]) self.spectrograph = spectrograph # Deal with Fibers (these must be set!) if fibers is not None: fibers = np.asarray(fibers) if len(fibers) != self.nspec: raise ValueError("len(fibers) != nspec ({} != {})".format( len(fibers), self.nspec)) if fibermap is not None and np.any(fibers != fibermap['FIBER']): raise ValueError("fibermap doesn't match fibers") if (spectrograph is not None): minfiber = spectrograph * fibers_per_spectrograph maxfiber = (spectrograph + 1) * fibers_per_spectrograph if np.any(fibers < minfiber) or np.any(maxfiber <= fibers): raise ValueError('fibers inconsistent with spectrograph') self.fibers = fibers else: if fibermap is not None: self.fibers = fibermap['FIBER'] elif spectrograph is not None: self.fibers = spectrograph * fibers_per_spectrograph + np.arange( self.nspec, dtype=int) elif (self.meta is not None) and ('FIBERMIN' in self.meta): self.fibers = self.meta['FIBERMIN'] + np.arange(self.nspec, dtype=int) else: raise ValueError("Must set fibers by one of the methods!") if self.meta is not None: self.meta['FIBERMIN'] = np.min(self.fibers)
def sim_source_spectra(allinfo, allzbest, infofile='source-truth.fits', debug=False): """Build the residual (source) spectra. No redshift-fitting. """ from desispec.io import read_spectra, write_spectra from desispec.spectra import Spectra from desispec.interpolation import resample_flux from desispec.resolution import Resolution from redrock.external.desi import DistTargetsDESI from redrock.templates import find_templates, Template assert(np.all(allinfo['TARGETID'] == allzbest['TARGETID'])) nsim = len(allinfo) # Select the subset of objects for which we got the correct lens (BGS) # redshift. these = np.where((allzbest['SPECTYPE'] == 'GALAXY') * (np.abs(allzbest['Z'] - allinfo['LENS_Z']) < 0.003))[0] print('Selecting {}/{} lenses with the correct redshift'.format(len(these), nsim)) if len(these) == 0: raise ValueError('No spectra passed the cuts!') allinfo = allinfo[these] allzbest = allzbest[these] print('Writing {}'.format(infofile)) allinfo.write(infofile, overwrite=True) tempfile = find_templates()[0] rrtemp = Template(tempfile) # loop on each chunk of lens+source spectra nchunk = len(set(allinfo['CHUNK'])) for ichunk in set(allinfo['CHUNK']): I = np.where(allinfo['CHUNK'] == ichunk)[0] info = allinfo[I] zbest = allzbest[I] specfile = 'lenssource-spectra-chunk{:03d}.fits'.format(ichunk) sourcefile = 'source-spectra-chunk{:03d}.fits'.format(ichunk) spectra = read_spectra(specfile).select(targets=info['TARGETID']) for igal, zz in enumerate(zbest): zwave = rrtemp.wave * (1 + zz['Z']) zflux = rrtemp.flux.T.dot(zz['COEFF']).T #/ (1 + zz['Z']) if debug: fig, ax = plt.subplots() for band in spectra.bands: R = Resolution(spectra.resolution_data[band][igal]) # use fastspecfit here modelflux = R.dot(resample_flux(spectra.wave[band], zwave, zflux)) if debug: ax.plot(spectra.wave[band], spectra.flux[band][igal, :]) ax.plot(spectra.wave[band], modelflux) ax.set_ylim(np.median(spectra.flux['r'][igal, :]) + np.std(spectra.flux['r'][igal, :]) * np.array([-1.5, 3])) #ax.set_xlim(4500, 5500) spectra.flux[band][igal, :] -= modelflux # subtract if debug: qafile = 'source-spectra-chunk{:03d}-{}.png'.format(ichunk, igal) fig.savefig(qafile) plt.close() print('Writing {} spectra to {}'.format(len(zbest), sourcefile)) write_spectra(outfile=sourcefile, spec=spectra) return allinfo
def sim_spectra(wave, flux, program, spectra_filename, obsconditions=None, sourcetype=None, targetid=None, redshift=None, expid=0, seed=0, skyerr=0.0, ra=None, dec=None, meta=None, fibermap_columns=None, fullsim=False,use_poisson=True): """ Simulate spectra from an input set of wavelength and flux and writes a FITS file in the Spectra format that can be used as input to the redshift fitter. Args: wave : 1D np.array of wavelength in Angstrom (in vacuum) in observer frame (i.e. redshifted) flux : 1D or 2D np.array. 1D array must have same size as wave, 2D array must have shape[1]=wave.size flux has to be in units of 10^-17 ergs/s/cm2/A spectra_filename : path to output FITS file in the Spectra format program : dark, lrg, qso, gray, grey, elg, bright, mws, bgs ignored if obsconditions is not None Optional: obsconditions : dictionnary of observation conditions with SEEING EXPTIME AIRMASS MOONFRAC MOONALT MOONSEP sourcetype : list of string, allowed values are (sky,elg,lrg,qso,bgs,star), type of sources, used for fiber aperture loss , default is star targetid : list of targetids for each target. default of None has them generated as str(range(nspec)) redshift : list/array with each index being the redshifts for that target expid : this expid number will be saved in the Spectra fibermap seed : random seed skyerr : fractional sky subtraction error ra : numpy array with targets RA (deg) dec : numpy array with targets Dec (deg) meta : dictionnary, saved in primary fits header of the spectra file fibermap_columns : add these columns to the fibermap fullsim : if True, write full simulation data in extra file per camera use_poisson : if False, do not use numpy.random.poisson to simulate the Poisson noise. This is useful to get reproducible random realizations. """ log = get_logger() if len(flux.shape)==1 : flux=flux.reshape((1,flux.size)) nspec=flux.shape[0] log.info("Starting simulation of {} spectra".format(nspec)) if sourcetype is None : sourcetype = np.array(["star" for i in range(nspec)]) log.debug("sourcetype = {}".format(sourcetype)) tileid = 0 telera = 0 teledec = 0 dateobs = time.gmtime() night = desisim.obs.get_night(utc=dateobs) program = program.lower() frame_fibermap = desispec.io.fibermap.empty_fibermap(nspec) frame_fibermap.meta["FLAVOR"]="custom" frame_fibermap.meta["NIGHT"]=night frame_fibermap.meta["EXPID"]=expid # add DESI_TARGET tm = desitarget.targetmask.desi_mask frame_fibermap['DESI_TARGET'][sourcetype=="star"]=tm.STD_FAINT frame_fibermap['DESI_TARGET'][sourcetype=="lrg"]=tm.LRG frame_fibermap['DESI_TARGET'][sourcetype=="elg"]=tm.ELG frame_fibermap['DESI_TARGET'][sourcetype=="qso"]=tm.QSO frame_fibermap['DESI_TARGET'][sourcetype=="sky"]=tm.SKY frame_fibermap['DESI_TARGET'][sourcetype=="bgs"]=tm.BGS_ANY if fibermap_columns is not None : for k in fibermap_columns.keys() : frame_fibermap[k] = fibermap_columns[k] if targetid is None: targetid = np.arange(nspec).astype(int) # add TARGETID frame_fibermap['TARGETID'] = targetid # spectra fibermap has two extra fields : night and expid # This would be cleaner if desispec would provide the spectra equivalent # of desispec.io.empty_fibermap() spectra_fibermap = desispec.io.empty_fibermap(nspec) spectra_fibermap = desispec.io.util.add_columns(spectra_fibermap, ['NIGHT', 'EXPID', 'TILEID'], [np.int32(night), np.int32(expid), np.int32(tileid)], ) for s in range(nspec): for tp in frame_fibermap.dtype.fields: spectra_fibermap[s][tp] = frame_fibermap[s][tp] if ra is not None : spectra_fibermap["TARGET_RA"] = ra spectra_fibermap["FIBER_RA"] = ra if dec is not None : spectra_fibermap["TARGET_DEC"] = dec spectra_fibermap["FIBER_DEC"] = dec if obsconditions is None: if program in ['dark', 'lrg', 'qso']: obsconditions = desisim.simexp.reference_conditions['DARK'] elif program in ['elg', 'gray', 'grey']: obsconditions = desisim.simexp.reference_conditions['GRAY'] elif program in ['mws', 'bgs', 'bright']: obsconditions = desisim.simexp.reference_conditions['BRIGHT'] else: raise ValueError('unknown program {}'.format(program)) elif isinstance(obsconditions, str): try: obsconditions = desisim.simexp.reference_conditions[obsconditions.upper()] except KeyError: raise ValueError('obsconditions {} not in {}'.format( obsconditions.upper(), list(desisim.simexp.reference_conditions.keys()))) try: params = desimodel.io.load_desiparams() wavemin = params['ccd']['b']['wavemin'] wavemax = params['ccd']['z']['wavemax'] except KeyError: wavemin = desimodel.io.load_throughput('b').wavemin wavemax = desimodel.io.load_throughput('z').wavemax if wave[0] > wavemin: log.warning('Minimum input wavelength {}>{}; padding with zeros'.format( wave[0], wavemin)) dwave = wave[1] - wave[0] npad = int((wave[0] - wavemin)/dwave + 1) wavepad = np.arange(npad) * dwave wavepad += wave[0] - dwave - wavepad[-1] fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype) wave = np.concatenate([wavepad, wave]) flux = np.hstack([fluxpad, flux]) assert flux.shape[1] == len(wave) assert np.allclose(dwave, np.diff(wave)) assert wave[0] <= wavemin if wave[-1] < wavemax: log.warning('Maximum input wavelength {}<{}; padding with zeros'.format( wave[-1], wavemax)) dwave = wave[-1] - wave[-2] npad = int( (wavemax - wave[-1])/dwave + 1 ) wavepad = wave[-1] + dwave + np.arange(npad)*dwave fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype) wave = np.concatenate([wave, wavepad]) flux = np.hstack([flux, fluxpad]) assert flux.shape[1] == len(wave) assert np.allclose(dwave, np.diff(wave)) assert wavemax <= wave[-1] ii = (wavemin <= wave) & (wave <= wavemax) flux_unit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm ** 2 ) wave = wave[ii]*u.Angstrom flux = flux[:,ii]*flux_unit sim = desisim.simexp.simulate_spectra(wave, flux, fibermap=frame_fibermap, obsconditions=obsconditions, redshift=redshift, seed=seed, psfconvolve=True) random_state = np.random.RandomState(seed) sim.generate_random_noise(random_state,use_poisson=use_poisson) scale=1e17 specdata = None resolution={} for camera in sim.instrument.cameras: R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1]) skyscale = skyerr * random_state.normal(size=sim.num_fibers) if fullsim : for table in sim.camera_output : band = table.meta['name'].strip()[0] table_filename=spectra_filename.replace(".fits","-fullsim-{}.fits".format(band)) table.write(table_filename,format="fits",overwrite=True) print("wrote",table_filename) for table in sim.camera_output : wave = table['wavelength'].astype(float) flux = (table['observed_flux']+table['random_noise_electrons']*table['flux_calibration']).T.astype(float) if np.any(skyscale): flux += ((table['num_sky_electrons']*skyscale)*table['flux_calibration']).T.astype(float) ivar = table['flux_inverse_variance'].T.astype(float) band = table.meta['name'].strip()[0] flux = flux * scale ivar = ivar / scale**2 mask = np.zeros(flux.shape).astype(int) spec = Spectra([band], {band : wave}, {band : flux}, {band : ivar}, resolution_data={band : resolution[band]}, mask={band : mask}, fibermap=spectra_fibermap, meta=meta, single=True) if specdata is None : specdata = spec else : specdata.update(spec) desispec.io.write_spectra(spectra_filename, specdata) log.info('Wrote '+spectra_filename) # need to clear the simulation buffers that keeps growing otherwise # because of a different number of fibers each time ... desisim.specsim._simulators.clear() desisim.specsim._simdefaults.clear()
def main(args): # Set up the logger if args.verbose: log = get_logger(DEBUG) else: log = get_logger() # Make sure all necessary environment variables are set DESI_SPECTRO_REDUX_DIR = "./quickGen" if 'DESI_SPECTRO_REDUX' not in os.environ: log.info('DESI_SPECTRO_REDUX environment is not set.') else: DESI_SPECTRO_REDUX_DIR = os.environ['DESI_SPECTRO_REDUX'] if os.path.exists(DESI_SPECTRO_REDUX_DIR): if not os.path.isdir(DESI_SPECTRO_REDUX_DIR): raise RuntimeError("Path %s Not a directory" % DESI_SPECTRO_REDUX_DIR) else: try: os.makedirs(DESI_SPECTRO_REDUX_DIR) except: raise SPECPROD_DIR = 'specprod' if 'SPECPROD' not in os.environ: log.info('SPECPROD environment is not set.') else: SPECPROD_DIR = os.environ['SPECPROD'] prod_Dir = specprod_root() if os.path.exists(prod_Dir): if not os.path.isdir(prod_Dir): raise RuntimeError("Path %s Not a directory" % prod_Dir) else: try: os.makedirs(prod_Dir) except: raise # Initialize random number generator to use. np.random.seed(args.seed) random_state = np.random.RandomState(args.seed) # Derive spectrograph number from nstart if needed if args.spectrograph is None: args.spectrograph = args.nstart / 500 # Read fibermapfile to get object type, night and expid if args.fibermap: log.info("Reading fibermap file {}".format(args.fibermap)) fibermap = read_fibermap(args.fibermap) objtype = get_source_types(fibermap) stdindx = np.where(objtype == 'STD') # match STD with STAR mwsindx = np.where(objtype == 'MWS_STAR') # match MWS_STAR with STAR bgsindx = np.where(objtype == 'BGS') # match BGS with LRG objtype[stdindx] = 'STAR' objtype[mwsindx] = 'STAR' objtype[bgsindx] = 'LRG' NIGHT = fibermap.meta['NIGHT'] EXPID = fibermap.meta['EXPID'] else: # Create a blank fake fibermap fibermap = empty_fibermap(args.nspec) targetids = random_state.randint(2**62, size=args.nspec) fibermap['TARGETID'] = targetids night = get_night() expid = 0 log.info("Initializing SpecSim with config {}".format(args.config)) desiparams = load_desiparams() qsim = get_simulator(args.config, num_fibers=1) if args.simspec: # Read the input file log.info('Reading input file {}'.format(args.simspec)) simspec = desisim.io.read_simspec(args.simspec) nspec = simspec.nspec if simspec.flavor == 'arc': log.warning("quickgen doesn't generate flavor=arc outputs") return else: wavelengths = simspec.wave spectra = simspec.flux if nspec < args.nspec: log.info("Only {} spectra in input file".format(nspec)) args.nspec = nspec else: # Initialize the output truth table. spectra = [] wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value npix = len(wavelengths) truth = dict() meta = Table() truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10)) truth['FLUX'] = np.zeros((args.nspec, npix)) truth['WAVE'] = wavelengths jj = list() for thisobj in set(true_objtype): ii = np.where(true_objtype == thisobj)[0] nobj = len(ii) truth['OBJTYPE'][ii] = thisobj log.info('Generating {} template'.format(thisobj)) # Generate the templates if thisobj == 'ELG': elg = desisim.templates.ELG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = elg.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_elg, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'LRG': lrg = desisim.templates.LRG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = lrg.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_lrg, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'QSO': qso = desisim.templates.QSO(wave=wavelengths) flux, tmpwave, meta1 = qso.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_qso) elif thisobj == 'BGS': bgs = desisim.templates.BGS(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = bgs.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_bgs, rmagrange=args.rmagrange_bgs, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'STD': std = desisim.templates.STD(wave=wavelengths) flux, tmpwave, meta1 = std.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'QSO_BAD': # use STAR template no color cuts star = desisim.templates.STAR(wave=wavelengths) flux, tmpwave, meta1 = star.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'MWS_STAR' or thisobj == 'MWS': mwsstar = desisim.templates.MWS_STAR(wave=wavelengths) flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'WD': wd = desisim.templates.WD(wave=wavelengths) flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'SKY': flux = np.zeros((nobj, npix)) meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32))) elif thisobj == 'TEST': flux = np.zeros((args.nspec, npix)) indx = np.where(wave > 5800.0 - 1E-6)[0][0] ref_integrated_flux = 1E-10 ref_cst_flux_density = 1E-17 single_line = (np.arange(args.nspec) % 2 == 0).astype( np.float32) continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32) for spec in range(args.nspec): flux[spec, indx] = single_line[ spec] * ref_integrated_flux / np.gradient(wavelengths)[ indx] # single line flux[spec] += continuum[ spec] * ref_cst_flux_density # flat continuum meta1 = Table( dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32), LINE=wave[indx] * np.ones(args.nspec, dtype=np.float32), LINEFLUX=single_line * ref_integrated_flux, CONSTFLUXDENSITY=continuum * ref_cst_flux_density)) else: log.fatal('Unknown object type {}'.format(thisobj)) sys.exit(1) # Pack it in. truth['FLUX'][ii] = flux meta = vstack([meta, meta1]) jj.append(ii.tolist()) # Sanity check on units; templates currently return ergs, not 1e-17 ergs... # assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6) # Sort the metadata table. jj = sum(jj, []) meta_new = Table() for k in range(args.nspec): index = int(np.where(np.array(jj) == k)[0]) meta_new = vstack([meta_new, meta[index]]) meta = meta_new # Add TARGETID and the true OBJTYPE to the metadata table. meta.add_column( Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE')) meta.add_column(Column(targetids, name='TARGETID')) # Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z meta.rename_column('REDSHIFT', 'TRUEZ') # explicitly set location on focal plane if needed to support airmass # variations when using specsim v0.5 if qsim.source.focal_xy is None: qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm')) # Set simulation parameters from the simspec header or desiparams bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX'] gray_objects = ['gray', 'grey'] if args.simspec is None: object_type = objtype flavor = None elif simspec.flavor == 'science': object_type = None flavor = simspec.header['PROGRAM'] else: object_type = None flavor = simspec.flavor log.warning( 'Maybe using an outdated simspec file with flavor={}'.format( flavor)) # Set airmass if args.airmass is not None: qsim.atmosphere.airmass = args.airmass elif args.simspec and 'AIRMASS' in simspec.header: qsim.atmosphere.airmass = simspec.header['AIRMASS'] else: qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2 # Set exptime if args.exptime is not None: qsim.observation.exposure_time = args.exptime * u.s elif args.simspec and 'EXPTIME' in simspec.header: qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s elif objtype in bright_objects: qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s else: qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s # Set Moon Phase if args.moon_phase is not None: qsim.atmosphere.moon.moon_phase = args.moon_phase elif args.simspec and 'MOONFRAC' in simspec.header: qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC'] elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_phase = 0.7 elif flavor in gray_objects: qsim.atmosphere.moon.moon_phase = 0.1 else: qsim.atmosphere.moon.moon_phase = 0.5 # Set Moon Zenith if args.moon_zenith is not None: qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg elif args.simspec and 'MOONALT' in simspec.header: qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_zenith = 30 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.moon_zenith = 80 * u.deg else: qsim.atmosphere.moon.moon_zenith = 100 * u.deg # Set Moon - Object Angle if args.moon_angle is not None: qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg elif args.simspec and 'MOONSEP' in simspec.header: qsim.atmosphere.moon.separation_angle = simspec.header[ 'MOONSEP'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.separation_angle = 50 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.separation_angle = 60 * u.deg else: qsim.atmosphere.moon.separation_angle = 60 * u.deg # Initialize per-camera output arrays that will be saved waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {} maxbin = 0 nmax = args.nspec for camera in qsim.instrument.cameras: # Lookup this camera's resolution matrix and convert to the sparse # format used in desispec. R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [args.nspec, 1, 1]) waves[camera.name] = (camera.output_wavelength.to( u.Angstrom).value.astype(np.float32)) nwave = len(waves[camera.name]) maxbin = max(maxbin, len(waves[camera.name])) nobj = np.zeros((nmax, 3, maxbin)) # object photons nsky = np.zeros((nmax, 3, maxbin)) # sky photons nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky) cframe_observedflux = np.zeros( (nmax, 3, maxbin)) # calibrated object flux cframe_ivar = np.zeros( (nmax, 3, maxbin)) # inverse variance of calibrated object flux cframe_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to calibrated flux sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky sky_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to sky only frame_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky trueflux[camera.name] = np.empty( (args.nspec, nwave)) # calibrated flux noisyflux[camera.name] = np.empty( (args.nspec, nwave)) # observed flux with noise obsivar[camera.name] = np.empty( (args.nspec, nwave)) # inverse variance of flux if args.simspec: for i in range(10): cn = camera.name + str(i) if cn in simspec.cameras: dw = np.gradient(simspec.cameras[cn].wave) break else: raise RuntimeError( 'Unable to find a {} camera in input simspec'.format( camera)) else: sflux = np.empty((args.nspec, npix)) #- Check if input simspec is for a continuum flat lamp instead of science #- This does not convolve to per-fiber resolution if args.simspec: if simspec.flavor == 'flat': log.info("Simulating flat lamp exposure") for i, camera in enumerate(qsim.instrument.cameras): channel = camera.name #- from simspec, b/r/z not b0/r1/z9 assert camera.output_wavelength.unit == u.Angstrom num_pixels = len(waves[channel]) phot = list() for j in range(10): cn = camera.name + str(j) if cn in simspec.cameras: camwave = simspec.cameras[cn].wave dw = np.gradient(camwave) phot.append(simspec.cameras[cn].phot) if len(phot) == 0: raise RuntimeError( 'Unable to find a {} camera in input simspec'.format( camera)) else: phot = np.vstack(phot) meanspec = resample_flux(waves[channel], camwave, np.average(phot / dw, axis=0)) fiberflat = random_state.normal(loc=1.0, scale=1.0 / np.sqrt(meanspec), size=(nspec, num_pixels)) ivar = np.tile(meanspec, [nspec, 1]) mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32) for kk in range((args.nspec + args.nstart - 1) // 500 + 1): camera = channel + str(kk) outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID, camera) start = max(500 * kk, args.nstart) end = min(500 * (kk + 1), nmax) if (args.spectrograph <= kk): log.info( "Writing files for channel:{}, spectrograph:{}, spectra:{} to {}" .format(channel, kk, start, end)) ff = FiberFlat(waves[channel], fiberflat[start:end, :], ivar[start:end, :], mask[start:end, :], meanspec, header=dict(CAMERA=camera)) write_fiberflat(outfile, ff) filePath = desispec.io.findfile("fiberflat", NIGHT, EXPID, camera) log.info("Wrote file {}".format(filePath)) sys.exit(0) # Repeat the simulation for all spectra fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom) for j in range(args.nspec): thisobjtype = objtype[j] sys.stdout.flush() if flavor == 'arc': qsim.source.update_in('Quickgen source {0}'.format, 'perfect', wavelengths * u.Angstrom, spectra * fluxunits) else: qsim.source.update_in('Quickgen source {0}'.format(j), thisobjtype.lower(), wavelengths * u.Angstrom, spectra[j, :] * fluxunits) qsim.source.update_out() qsim.simulate() qsim.generate_random_noise(random_state) for i, output in enumerate(qsim.camera_output): assert output['observed_flux'].unit == 1e17 * fluxunits # Extract the simulation results needed to create our uncalibrated # frame output file. num_pixels = len(output) nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0] nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0] nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0] # Get results for our flux-calibrated output file. cframe_observedflux[ j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0] cframe_ivar[ j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0] # Fill brick arrays from the results. camera = output.meta['name'] trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0] noisyflux[camera][j][:] = 1e17 * ( output['observed_flux'][:, 0] + output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:, 0] # Use the same noise realization in the cframe and frame, without any # additional noise from sky subtraction for now. frame_rand_noise[ j, i, :num_pixels] = output['random_noise_electrons'][:, 0] cframe_rand_noise[j, i, :num_pixels] = 1e17 * ( output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) # The sky output file represents a model fit to ~40 sky fibers. # We reduce the variance by a factor of 25 to account for this and # give the sky an independent (Gaussian) noise realization. sky_ivar[ j, i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] - output['num_source_electrons'][:, 0]) sky_rand_noise[j, i, :num_pixels] = random_state.normal( scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]), size=num_pixels) armName = {"b": 0, "r": 1, "z": 2} for channel in 'brz': #Before writing, convert from counts/bin to counts/A (as in Pixsim output) #Quicksim Default: #FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang] #COUNTS_OBJ - object counts in 0.5 Ang bin #COUNTS_SKY - sky counts in 0.5 Ang bin num_pixels = len(waves[channel]) dwave = np.gradient(waves[channel]) nobj[:, armName[channel], :num_pixels] /= dwave frame_rand_noise[:, armName[channel], :num_pixels] /= dwave nivar[:, armName[channel], :num_pixels] *= dwave**2 nsky[:, armName[channel], :num_pixels] /= dwave sky_rand_noise[:, armName[channel], :num_pixels] /= dwave sky_ivar[:, armName[channel], :num_pixels] /= dwave**2 # Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra # Looping over spectrograph for ii in range((args.nspec + args.nstart - 1) // 500 + 1): start = max(500 * ii, args.nstart) # first spectrum for a given spectrograph end = min(500 * (ii + 1), nmax) # last spectrum for the spectrograph if (args.spectrograph <= ii): camera = "{}{}".format(channel, ii) log.info( "Writing files for channel:{}, spectrograph:{}, spectra:{} to {}" .format(channel, ii, start, end)) num_pixels = len(waves[channel]) # Write frame file framefileName = desispec.io.findfile("frame", NIGHT, EXPID, camera) frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \ nsky[start:end,armName[channel],:num_pixels] + \ frame_rand_noise[start:end,armName[channel],:num_pixels] frame_ivar = nivar[start:end, armName[channel], :num_pixels] sh1 = frame_flux.shape[ 0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave) # for example if nstart =400, nspec=150: two spectrographs: # 400-499=> 0 spectrograph, 500-549 => 1 if (args.nstart == start): resol = resolution[channel][:sh1, :, :] else: resol = resolution[channel][-sh1:, :, :] # must create desispec.Frame object frame=Frame(waves[channel], frame_flux, frame_ivar,\ resolution_data=resol, spectrograph=ii, \ fibermap=fibermap[start:end], \ meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.write_frame(framefileName, frame) framefilePath = desispec.io.findfile("frame", NIGHT, EXPID, camera) log.info("Wrote file {}".format(framefilePath)) if args.frameonly or simspec.flavor == 'arc': continue # Write cframe file cframeFileName = desispec.io.findfile("cframe", NIGHT, EXPID, camera) cframeFlux = cframe_observedflux[ start:end, armName[channel], :num_pixels] + cframe_rand_noise[ start:end, armName[channel], :num_pixels] cframeIvar = cframe_ivar[start:end, armName[channel], :num_pixels] # must create desispec.Frame object cframe = Frame(waves[channel], cframeFlux, cframeIvar, \ resolution_data=resol, spectrograph=ii, fibermap=fibermap[start:end], meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.frame.write_frame(cframeFileName, cframe) cframefilePath = desispec.io.findfile("cframe", NIGHT, EXPID, camera) log.info("Wrote file {}".format(cframefilePath)) # Write sky file skyfileName = desispec.io.findfile("sky", NIGHT, EXPID, camera) skyflux=nsky[start:end,armName[channel],:num_pixels] + \ sky_rand_noise[start:end,armName[channel],:num_pixels] skyivar = sky_ivar[start:end, armName[channel], :num_pixels] skymask = np.zeros(skyflux.shape, dtype=np.uint32) # must create desispec.Sky object skymodel = SkyModel(waves[channel], skyflux, skyivar, skymask, header=dict(CAMERA=camera)) desispec.io.sky.write_sky(skyfileName, skymodel) skyfilePath = desispec.io.findfile("sky", NIGHT, EXPID, camera) log.info("Wrote file {}".format(skyfilePath)) # Write calib file calibVectorFile = desispec.io.findfile("calib", NIGHT, EXPID, camera) flux = cframe_observedflux[start:end, armName[channel], :num_pixels] phot = nobj[start:end, armName[channel], :num_pixels] calibration = np.zeros_like(phot) jj = (flux > 0) calibration[jj] = phot[jj] / flux[jj] #- TODO: what should calibivar be? #- For now, model it as the noise of combining ~10 spectra calibivar = 10 / cframe_ivar[start:end, armName[channel], :num_pixels] #mask=(1/calibivar>0).astype(int)?? mask = np.zeros(calibration.shape, dtype=np.uint32) # write flux calibration fluxcalib = FluxCalib(waves[channel], calibration, calibivar, mask) write_flux_calibration(calibVectorFile, fluxcalib) calibfilePath = desispec.io.findfile("calib", NIGHT, EXPID, camera) log.info("Wrote file {}".format(calibfilePath))
def compute_polynomial_times_sky(frame, nsig_clipping=4.,max_iterations=30,model_ivar=False,add_variance=True,angular_variation_deg=1,chromatic_variation_deg=1) : """Compute a sky model. Sky[fiber,i] = R[fiber,i,j] Polynomial(x[fiber],y[fiber],wavelength[j]) Flux[j] Input flux are expected to be flatfielded! We don't check this in this routine. Args: frame : Frame object, which includes attributes - wave : 1D wavelength grid in Angstroms - flux : 2D flux[nspec, nwave] density - ivar : 2D inverse variance of flux - mask : 2D inverse mask flux (0=good) - resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers) nsig_clipping : [optional] sigma clipping value for outlier rejection Optional: max_iterations : int , number of iterations model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims. add_variance : evaluate calibration error and add this to the sky model variance returns SkyModel object with attributes wave, flux, ivar, mask """ log=get_logger() log.info("starting") # Grab sky fibers on this frame skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0] assert np.max(skyfibers) < 500 #- indices, not fiber numbers nwave=frame.nwave nfibers=len(skyfibers) current_ivar=frame.ivar[skyfibers].copy()*(frame.mask[skyfibers]==0) flux = frame.flux[skyfibers] Rsky = frame.R[skyfibers] input_ivar=None if model_ivar : log.info("use a model of the inverse variance to remove bias due to correlated ivar and flux") input_ivar=current_ivar.copy() median_ivar_vs_wave = np.median(current_ivar,axis=0) median_ivar_vs_fiber = np.median(current_ivar,axis=1) median_median_ivar = np.median(median_ivar_vs_fiber) for f in range(current_ivar.shape[0]) : threshold=0.01 current_ivar[f] = median_ivar_vs_fiber[f]/median_median_ivar * median_ivar_vs_wave # keep input ivar for very low weights ii=(input_ivar[f]<=(threshold*median_ivar_vs_wave)) #log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1])) current_ivar[f][ii] = input_ivar[f][ii] # need focal plane coordinates x = frame.fibermap["DESIGN_X"] y = frame.fibermap["DESIGN_Y"] # normalize for numerical stability xm = np.mean(x) ym = np.mean(y) xs = np.std(x) ys = np.std(y) if xs==0 : xs = 1 if ys==0 : ys = 1 x = (x-xm)/xs y = (y-ym)/ys w = (frame.wave-frame.wave[0])/(frame.wave[-1]-frame.wave[0])*2.-1 # precompute the monomials for the sky fibers log.debug("compute monomials for deg={} and {}".format(angular_variation_deg,chromatic_variation_deg)) monomials=[] for dx in range(angular_variation_deg+1) : for dy in range(angular_variation_deg+1-dx) : xypol = (x**dx)*(y**dy) for dw in range(chromatic_variation_deg+1) : wpol=w**dw monomials.append(np.outer(xypol,wpol)) ncoef=len(monomials) coef=np.zeros((ncoef)) allfibers_monomials=np.array(monomials) log.debug("shape of allfibers_monomials = {}".format(allfibers_monomials.shape)) skyfibers_monomials = allfibers_monomials[:,skyfibers,:] log.debug("shape of skyfibers_monomials = {}".format(skyfibers_monomials.shape)) sqrtw=np.sqrt(current_ivar) sqrtwflux=sqrtw*flux chi2=np.zeros(flux.shape) Pol = np.ones(flux.shape,dtype=float) coef[0] = 1. nout_tot=0 previous_chi2=-10. for iteration in range(max_iterations) : # the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters # A_ij = 1/2 d2(chi2)/di/dj # A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w] # the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters # B_i = 1/2 d(chi2)/di # B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w]) # the model is model[fiber]=R[fiber]*Pol(x,y,wave)*sky # the parameters are the unconvolved sky flux at the wavelength i # and the polynomial coefficients A=np.zeros((nwave,nwave),dtype=float) B=np.zeros((nwave),dtype=float) D=scipy.sparse.lil_matrix((nwave,nwave)) D2=scipy.sparse.lil_matrix((nwave,nwave)) Pol /= coef[0] # force constant term to 1. # solving for the deconvolved mean sky spectrum # loop on fiber to handle resolution for fiber in range(nfibers) : if fiber%10==0 : log.info("iter %d sky fiber (1st fit) %d/%d"%(iteration,fiber,nfibers)) D.setdiag(sqrtw[fiber]) D2.setdiag(Pol[fiber]) sqrtwRP = D.dot(Rsky[fiber]).dot(D2) # each row r of R is multiplied by sqrtw[r] A += (sqrtwRP.T*sqrtwRP).todense() B += sqrtwRP.T*sqrtwflux[fiber] log.info("iter %d solving"%iteration) w = A.diagonal()>0 A_pos_def = A[w,:] A_pos_def = A_pos_def[:,w] parameters = B*0 try: parameters[w]=cholesky_solve(A_pos_def,B[w]) except: log.info("cholesky failed, trying svd in iteration {}".format(iteration)) parameters[w]=np.linalg.lstsq(A_pos_def,B[w])[0] # parameters = the deconvolved mean sky spectrum # now evaluate the polynomial coefficients Ap=np.zeros((ncoef,ncoef),dtype=float) Bp=np.zeros((ncoef),dtype=float) D2.setdiag(parameters) for fiber in range(nfibers) : if fiber%10==0 : log.info("iter %d sky fiber (2nd fit) %d/%d"%(iteration,fiber,nfibers)) D.setdiag(sqrtw[fiber]) sqrtwRSM = D.dot(Rsky[fiber]).dot(D2).dot(skyfibers_monomials[:,fiber,:].T) Ap += sqrtwRSM.T.dot(sqrtwRSM) Bp += sqrtwRSM.T.dot(sqrtwflux[fiber]) # Add huge prior on zeroth angular order terms to converge faster # (because those terms are degenerate with the mean deconvolved spectrum) weight=1e24 Ap[0,0] += weight Bp[0] += weight # force 0th term to 1 for i in range(1,chromatic_variation_deg+1) : Ap[i,i] += weight # force other wavelength terms to 0 coef=cholesky_solve(Ap,Bp) log.info("pol coef = {}".format(coef)) # recompute the polynomial values Pol = skyfibers_monomials.T.dot(coef).T # chi2 and outlier rejection log.info("iter %d compute chi2"%iteration) for fiber in range(nfibers) : chi2[fiber]=current_ivar[fiber]*(flux[fiber]-Rsky[fiber].dot(Pol[fiber]*parameters))**2 log.info("rejecting") nout_iter=0 if iteration<1 : # only remove worst outlier per wave # apply rejection iteratively, only one entry per wave among fibers # find waves with outlier (fastest way) nout_per_wave=np.sum(chi2>nsig_clipping**2,axis=0) selection=np.where(nout_per_wave>0)[0] for i in selection : worst_entry=np.argmax(chi2[:,i]) current_ivar[worst_entry,i]=0 sqrtw[worst_entry,i]=0 sqrtwflux[worst_entry,i]=0 nout_iter += 1 else : # remove all of them at once bad=(chi2>nsig_clipping**2) current_ivar *= (bad==0) sqrtw *= (bad==0) sqrtwflux *= (bad==0) nout_iter += np.sum(bad) nout_tot += nout_iter sum_chi2=float(np.sum(chi2)) ndf=int(np.sum(chi2>0)-nwave) chi2pdf=0. if ndf>0 : chi2pdf=sum_chi2/ndf log.info("iter #%d chi2=%g ndf=%d chi2pdf=%f delta=%f nout=%d"%(iteration,sum_chi2,ndf,chi2pdf,abs(sum_chi2-previous_chi2),nout_iter)) if nout_iter == 0 and abs(sum_chi2-previous_chi2)<0.2 : break previous_chi2 = sum_chi2+0. log.info("nout tot=%d"%nout_tot) # we know have to compute the sky model for all fibers # and propagate the uncertainties # no need to restore the original ivar to compute the model errors when modeling ivar # the sky inverse variances are very similar # we ignore here the fact that we have fit a angular variation, # so the sky model uncertainties are inaccurate log.info("compute the parameter covariance") try : parameter_covar=cholesky_invert(A) except np.linalg.linalg.LinAlgError : log.warning("cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv") parameter_covar = np.linalg.pinv(A) log.info("compute mean resolution") # we make an approximation for the variance to save CPU time # we use the average resolution of all fibers in the frame: mean_res_data=np.mean(frame.resolution_data,axis=0) Rmean = Resolution(mean_res_data) log.info("compute convolved sky and ivar") # The parameters are directly the unconvolved sky # First convolve with average resolution : convolved_sky_covar=Rmean.dot(parameter_covar).dot(Rmean.T.todense()) # and keep only the diagonal convolved_sky_var=np.diagonal(convolved_sky_covar) # inverse convolved_sky_ivar=(convolved_sky_var>0)/(convolved_sky_var+(convolved_sky_var==0)) # and simply consider it's the same for all spectra cskyivar = np.tile(convolved_sky_ivar, frame.nspec).reshape(frame.nspec, nwave) # The sky model for each fiber (simple convolution with resolution of each fiber) cskyflux = np.zeros(frame.flux.shape) Pol = allfibers_monomials.T.dot(coef).T for fiber in range(frame.nspec): cskyflux[fiber] = frame.R[fiber].dot(Pol[fiber]*parameters) # look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1 if skyfibers.size > 1 and add_variance : modified_cskyivar = _model_variance(frame,cskyflux,cskyivar,skyfibers) else : modified_cskyivar = cskyivar.copy() # need to do better here mask = (cskyivar==0).astype(np.uint32) return SkyModel(frame.wave.copy(), cskyflux, modified_cskyivar, mask, nrej=nout_tot, stat_ivar = cskyivar) # keep a record of the statistical ivar for QA
def spectroperf_resample_spectrum_multiproc(shm_in_wave, shm_in_flux, shm_in_ivar, shm_in_rdata, in_nwave, in_ndiag, in_bands, target_indices, wave, wavebin, resampling_matrix, ndiag, ntarget, shm_flux, shm_ivar, shm_rdata): nwave = wave.size # manipulate shared memory as np arrays # input shared memory in_wave = list() in_flux = list() in_ivar = list() in_rdata = list() nbands = len(shm_in_wave) for b in range(nbands): in_wave.append( np.array(shm_in_wave[b], copy=False).reshape(in_nwave[b])) in_flux.append( np.array(shm_in_flux[b], copy=False).reshape( (ntarget, in_nwave[b]))) in_ivar.append( np.array(shm_in_ivar[b], copy=False).reshape( (ntarget, in_nwave[b]))) in_rdata.append( np.array(shm_in_rdata[b], copy=False).reshape( (ntarget, in_ndiag[b], in_nwave[b]))) # output shared memory flux = np.array(shm_flux, copy=False).reshape(ntarget, nwave) ivar = np.array(shm_ivar, copy=False).reshape(ntarget, nwave) rdata = np.array(shm_rdata, copy=False).reshape(ntarget, ndiag, nwave) for target_index in target_indices: cinv = None for b in range(nbands): twave = in_wave[b] jj = (twave >= wave[0]) & (twave <= wave[-1]) twave = twave[jj] tivar = in_ivar[b][target_index][jj] diag_ivar = scipy.sparse.dia_matrix((tivar, [0]), (twave.size, twave.size)) RR = Resolution(in_rdata[b][target_index][:, jj]).dot( resampling_matrix[in_bands[b]]) tcinv = RR.T.dot(diag_ivar.dot(RR)) tcinvf = RR.T.dot(tivar * in_flux[b][target_index][jj]) if cinv is None: cinv = tcinv cinvf = tcinvf else: cinv += tcinv cinvf += tcinvf cinv = cinv.todense() decorrelate_divide_and_conquer(cinv, cinvf, wavebin, flux[target_index], ivar[target_index], rdata[target_index])
def compute_uniform_sky(frame, nsig_clipping=4.,max_iterations=100,model_ivar=False,add_variance=True) : """Compute a sky model. Sky[fiber,i] = R[fiber,i,j] Flux[j] Input flux are expected to be flatfielded! We don't check this in this routine. Args: frame : Frame object, which includes attributes - wave : 1D wavelength grid in Angstroms - flux : 2D flux[nspec, nwave] density - ivar : 2D inverse variance of flux - mask : 2D inverse mask flux (0=good) - resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers) nsig_clipping : [optional] sigma clipping value for outlier rejection Optional: max_iterations : int , number of iterations model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims. add_variance : evaluate calibration error and add this to the sky model variance returns SkyModel object with attributes wave, flux, ivar, mask """ log=get_logger() log.info("starting") # Grab sky fibers on this frame skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0] assert np.max(skyfibers) < 500 #- indices, not fiber numbers nwave=frame.nwave nfibers=len(skyfibers) current_ivar=frame.ivar[skyfibers].copy()*(frame.mask[skyfibers]==0) flux = frame.flux[skyfibers] Rsky = frame.R[skyfibers] input_ivar=None if model_ivar : log.info("use a model of the inverse variance to remove bias due to correlated ivar and flux") input_ivar=current_ivar.copy() median_ivar_vs_wave = np.median(current_ivar,axis=0) median_ivar_vs_fiber = np.median(current_ivar,axis=1) median_median_ivar = np.median(median_ivar_vs_fiber) for f in range(current_ivar.shape[0]) : threshold=0.01 current_ivar[f] = median_ivar_vs_fiber[f]/median_median_ivar * median_ivar_vs_wave # keep input ivar for very low weights ii=(input_ivar[f]<=(threshold*median_ivar_vs_wave)) #log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1])) current_ivar[f][ii] = input_ivar[f][ii] sqrtw=np.sqrt(current_ivar) sqrtwflux=sqrtw*flux chi2=np.zeros(flux.shape) nout_tot=0 for iteration in range(max_iterations) : # the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters # A_ij = 1/2 d2(chi2)/di/dj # A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w] # the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters # B_i = 1/2 d(chi2)/di # B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w]) # the model is model[fiber]=R[fiber]*sky # and the parameters are the unconvolved sky flux at the wavelength i # so, d(model)/di[fiber,w] = R[fiber][w,i] # this gives # A_ij = sum_fiber sum_wave_w ivar[fiber,w] R[fiber][w,i] R[fiber][w,j] # A = sum_fiber ( diag(sqrt(ivar))*R[fiber] ) ( diag(sqrt(ivar))* R[fiber] )^t # A = sum_fiber sqrtwR[fiber] sqrtwR[fiber]^t # and # B = sum_fiber sum_wave_w ivar[fiber,w] R[fiber][w] * flux[fiber,w] # B = sum_fiber sum_wave_w sqrt(ivar)[fiber,w]*flux[fiber,w] sqrtwR[fiber,wave] #A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr() A=np.zeros((nwave,nwave)) B=np.zeros((nwave)) # diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber SD=scipy.sparse.lil_matrix((nwave,nwave)) # loop on fiber to handle resolution for fiber in range(nfibers) : if fiber%10==0 : log.info("iter %d sky fiber %d/%d"%(iteration,fiber,nfibers)) R = Rsky[fiber] # diagonal sparse matrix with content = sqrt(ivar) SD.setdiag(sqrtw[fiber]) sqrtwR = SD*R # each row r of R is multiplied by sqrtw[r] A += (sqrtwR.T*sqrtwR).todense() B += sqrtwR.T*sqrtwflux[fiber] log.info("iter %d solving"%iteration) w = A.diagonal()>0 A_pos_def = A[w,:] A_pos_def = A_pos_def[:,w] parameters = B*0 try: parameters[w]=cholesky_solve(A_pos_def,B[w]) except: log.info("cholesky failed, trying svd in iteration {}".format(iteration)) parameters[w]=np.linalg.lstsq(A_pos_def,B[w])[0] log.info("iter %d compute chi2"%iteration) for fiber in range(nfibers) : # the parameters are directly the unconvolve sky flux # so we simply have to reconvolve it fiber_convolved_sky_flux = Rsky[fiber].dot(parameters) chi2[fiber]=current_ivar[fiber]*(flux[fiber]-fiber_convolved_sky_flux)**2 log.info("rejecting") nout_iter=0 if iteration<1 : # only remove worst outlier per wave # apply rejection iteratively, only one entry per wave among fibers # find waves with outlier (fastest way) nout_per_wave=np.sum(chi2>nsig_clipping**2,axis=0) selection=np.where(nout_per_wave>0)[0] for i in selection : worst_entry=np.argmax(chi2[:,i]) current_ivar[worst_entry,i]=0 sqrtw[worst_entry,i]=0 sqrtwflux[worst_entry,i]=0 nout_iter += 1 else : # remove all of them at once bad=(chi2>nsig_clipping**2) current_ivar *= (bad==0) sqrtw *= (bad==0) sqrtwflux *= (bad==0) nout_iter += np.sum(bad) nout_tot += nout_iter sum_chi2=float(np.sum(chi2)) ndf=int(np.sum(chi2>0)-nwave) chi2pdf=0. if ndf>0 : chi2pdf=sum_chi2/ndf log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d"%(iteration,sum_chi2,ndf,chi2pdf,nout_iter)) if nout_iter == 0 : break log.info("nout tot=%d"%nout_tot) # we know have to compute the sky model for all fibers # and propagate the uncertainties # no need to restore the original ivar to compute the model errors when modeling ivar # the sky inverse variances are very similar log.info("compute the parameter covariance") # we may have to use a different method to compute this # covariance try : parameter_covar=cholesky_invert(A) # the above is too slow # maybe invert per block, sandwich by R except np.linalg.linalg.LinAlgError : log.warning("cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv") parameter_covar = np.linalg.pinv(A) log.info("compute mean resolution") # we make an approximation for the variance to save CPU time # we use the average resolution of all fibers in the frame: mean_res_data=np.mean(frame.resolution_data,axis=0) Rmean = Resolution(mean_res_data) log.info("compute convolved sky and ivar") # The parameters are directly the unconvolved sky # First convolve with average resolution : convolved_sky_covar=Rmean.dot(parameter_covar).dot(Rmean.T.todense()) # and keep only the diagonal convolved_sky_var=np.diagonal(convolved_sky_covar) # inverse convolved_sky_ivar=(convolved_sky_var>0)/(convolved_sky_var+(convolved_sky_var==0)) # and simply consider it's the same for all spectra cskyivar = np.tile(convolved_sky_ivar, frame.nspec).reshape(frame.nspec, nwave) # The sky model for each fiber (simple convolution with resolution of each fiber) cskyflux = np.zeros(frame.flux.shape) for i in range(frame.nspec): cskyflux[i] = frame.R[i].dot(parameters) # look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1 if skyfibers.size > 1 and add_variance : modified_cskyivar = _model_variance(frame,cskyflux,cskyivar,skyfibers) else : modified_cskyivar = cskyivar.copy() # need to do better here mask = (cskyivar==0).astype(np.uint32) return SkyModel(frame.wave.copy(), cskyflux, modified_cskyivar, mask, nrej=nout_tot, stat_ivar = cskyivar) # keep a record of the statistical ivar for QA
def compute_sky(frame, nsig_clipping=4.) : """Compute a sky model. Input has to correspond to sky fibers only. Input flux are expected to be flatfielded! We don't check this in this routine. Args: frame : Frame object, which includes attributes - wave : 1D wavelength grid in Angstroms - flux : 2D flux[nspec, nwave] density - ivar : 2D inverse variance of flux - mask : 2D inverse mask flux (0=good) - resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers) nsig_clipping : [optional] sigma clipping value for outlier rejection returns SkyModel object with attributes wave, flux, ivar, mask """ log=get_logger() log.info("starting") # Grab sky fibers on this frame skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0] assert np.max(skyfibers) < 500 #- indices, not fiber numbers nwave=frame.nwave nfibers=len(skyfibers) current_ivar=frame.ivar[skyfibers].copy() flux = frame.flux[skyfibers] Rsky = frame.R[skyfibers] sqrtw=np.sqrt(current_ivar) sqrtwflux=sqrtw*flux chi2=np.zeros(flux.shape) #debug #nfibers=min(nfibers,2) nout_tot=0 for iteration in range(20) : A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr() B=np.zeros((nwave)) # diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber SD=scipy.sparse.lil_matrix((nwave,nwave)) # loop on fiber to handle resolution for fiber in range(nfibers) : if fiber%10==0 : log.info("iter %d fiber %d"%(iteration,fiber)) R = Rsky[fiber] # diagonal sparse matrix with content = sqrt(ivar) SD.setdiag(sqrtw[fiber]) sqrtwR = SD*R # each row r of R is multiplied by sqrtw[r] A = A+(sqrtwR.T*sqrtwR).tocsr() B += sqrtwR.T*sqrtwflux[fiber] log.info("iter %d solving"%iteration) skyflux=cholesky_solve(A.todense(),B) log.info("iter %d compute chi2"%iteration) for fiber in range(nfibers) : S = Rsky[fiber].dot(skyflux) chi2[fiber]=current_ivar[fiber]*(flux[fiber]-S)**2 log.info("rejecting") nout_iter=0 if iteration<1 : # only remove worst outlier per wave # apply rejection iteratively, only one entry per wave among fibers # find waves with outlier (fastest way) nout_per_wave=np.sum(chi2>nsig_clipping**2,axis=0) selection=np.where(nout_per_wave>0)[0] for i in selection : worst_entry=np.argmax(chi2[:,i]) current_ivar[worst_entry,i]=0 sqrtw[worst_entry,i]=0 sqrtwflux[worst_entry,i]=0 nout_iter += 1 else : # remove all of them at once bad=(chi2>nsig_clipping**2) current_ivar *= (bad==0) sqrtw *= (bad==0) sqrtwflux *= (bad==0) nout_iter += np.sum(bad) nout_tot += nout_iter sum_chi2=float(np.sum(chi2)) ndf=int(np.sum(chi2>0)-nwave) chi2pdf=0. if ndf>0 : chi2pdf=sum_chi2/ndf log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d"%(iteration,sum_chi2,ndf,chi2pdf,nout_iter)) if nout_iter == 0 : break log.info("nout tot=%d"%nout_tot) # solve once again to get deconvolved sky variance skyflux,skycovar=cholesky_solve_and_invert(A.todense(),B) #- sky inverse variance, but incomplete and not needed anyway # skyvar=np.diagonal(skycovar) # skyivar=(skyvar>0)/(skyvar+(skyvar==0)) # Use diagonal of skycovar convolved with mean resolution of all fibers # first compute average resolution mean_res_data=np.mean(frame.resolution_data,axis=0) R = Resolution(mean_res_data) # compute convolved sky and ivar cskycovar=R.dot(skycovar).dot(R.T.todense()) cskyvar=np.diagonal(cskycovar) cskyivar=(cskyvar>0)/(cskyvar+(cskyvar==0)) # convert cskyivar to 2D; today it is the same for all spectra, # but that may not be the case in the future cskyivar = np.tile(cskyivar, frame.nspec).reshape(frame.nspec, nwave) # Convolved sky cskyflux = np.zeros(frame.flux.shape) for i in range(frame.nspec): cskyflux[i] = frame.R[i].dot(skyflux) # need to do better here mask = (cskyivar==0).astype(np.uint32) return SkyModel(frame.wave.copy(), cskyflux, cskyivar, mask, nrej=nout_tot)
def compute_non_uniform_sky(frame, nsig_clipping=4.,max_iterations=10,model_ivar=False,add_variance=True,angular_variation_deg=1) : """Compute a sky model. Sky[fiber,i] = R[fiber,i,j] ( Flux_0[j] + x[fiber]*Flux_x[j] + y[fiber]*Flux_y[j] + ... ) Input flux are expected to be flatfielded! We don't check this in this routine. Args: frame : Frame object, which includes attributes - wave : 1D wavelength grid in Angstroms - flux : 2D flux[nspec, nwave] density - ivar : 2D inverse variance of flux - mask : 2D inverse mask flux (0=good) - resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers) nsig_clipping : [optional] sigma clipping value for outlier rejection Optional: max_iterations : int , number of iterations model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims. add_variance : evaluate calibration error and add this to the sky model variance angular_variation_deg : degree of 2D polynomial correction as a function of fiber focal plane coordinates (default=1). One set of coefficients per wavelength returns SkyModel object with attributes wave, flux, ivar, mask """ log=get_logger() log.info("starting") # Grab sky fibers on this frame skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0] assert np.max(skyfibers) < 500 #- indices, not fiber numbers nwave=frame.nwave nfibers=len(skyfibers) current_ivar=frame.ivar[skyfibers].copy()*(frame.mask[skyfibers]==0) flux = frame.flux[skyfibers] Rsky = frame.R[skyfibers] # need focal plane coordinates of fibers x = frame.fibermap["DESIGN_X"][skyfibers] y = frame.fibermap["DESIGN_Y"][skyfibers] # normalize for numerical stability xm = np.mean(frame.fibermap["DESIGN_X"]) ym = np.mean(frame.fibermap["DESIGN_Y"]) xs = np.std(frame.fibermap["DESIGN_X"]) ys = np.std(frame.fibermap["DESIGN_Y"]) if xs==0 : xs = 1 if ys==0 : ys = 1 x = (x-xm)/xs y = (y-ym)/ys # precompute the monomials for the sky fibers log.debug("compute monomials for deg={}".format(angular_variation_deg)) monomials=[] for dx in range(angular_variation_deg+1) : for dy in range(angular_variation_deg+1-dx) : monomials.append((x**dx)*(y**dy)) ncoef=len(monomials) monomials=np.array(monomials) input_ivar=None if model_ivar : log.info("use a model of the inverse variance to remove bias due to correlated ivar and flux") input_ivar=current_ivar.copy() median_ivar_vs_wave = np.median(current_ivar,axis=0) median_ivar_vs_fiber = np.median(current_ivar,axis=1) median_median_ivar = np.median(median_ivar_vs_fiber) for f in range(current_ivar.shape[0]) : threshold=0.01 current_ivar[f] = median_ivar_vs_fiber[f]/median_median_ivar * median_ivar_vs_wave # keep input ivar for very low weights ii=(input_ivar[f]<=(threshold*median_ivar_vs_wave)) #log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1])) current_ivar[f][ii] = input_ivar[f][ii] sqrtw=np.sqrt(current_ivar) sqrtwflux=sqrtw*flux chi2=np.zeros(flux.shape) nout_tot=0 for iteration in range(max_iterations) : # the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters # A_ij = 1/2 d2(chi2)/di/dj # A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w] # the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters # B_i = 1/2 d(chi2)/di # B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w]) # with x_fiber,y_fiber the fiber coordinates in the focal plane (or sky) # the unconvolved sky flux at wavelength i is a polynomial of x_fiber,y_fiber # sky(fiber,i) = pol(x_fiber,y_fiber,p) = sum_p a_ip * x_fiber**degx(p) y_fiber**degy(p) # sky(fiber,i) = sum_p monom[fiber,p] * a_ip # the convolved sky flux at wavelength w is # model[fiber,w] = sum_i R[fiber][w,i] sum_p monom[fiber,p] * a_ip # model[fiber,w] = sum_p monom[fiber,p] R[fiber][w,i] a_ip # # so, the matrix A is composed of blocks (p,k) corresponding to polynomial coefficient indices where # A[pk] = sum_fiber monom[fiber,p]*monom[fiber,k] sqrtwR[fiber] sqrtwR[fiber]^t # similarily # B[p] = sum_fiber monom[fiber,p] * sum_wave_w (sqrt(ivar)[fiber,w]*flux[fiber,w]) sqrtwR[fiber,wave] A=np.zeros((nwave*ncoef,nwave*ncoef)) B=np.zeros((nwave*ncoef)) # diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber SD=scipy.sparse.lil_matrix((nwave,nwave)) # loop on fiber to handle resolution for fiber in range(nfibers) : if fiber%10==0 : log.info("iter %d sky fiber %d/%d"%(iteration,fiber,nfibers)) R = Rsky[fiber] # diagonal sparse matrix with content = sqrt(ivar) SD.setdiag(sqrtw[fiber]) sqrtwR = SD*R # each row r of R is multiplied by sqrtw[r] #wRtR=(sqrtwR.T*sqrtwR).tocsr() wRtR=(sqrtwR.T*sqrtwR).todense() wRtF=sqrtwR.T*sqrtwflux[fiber] # loop on polynomial coefficients (double loop for A) # fill only blocks of A and B for p in range(ncoef) : for k in range(ncoef) : A[p*nwave:(p+1)*nwave,k*nwave:(k+1)*nwave] += monomials[p,fiber]*monomials[k,fiber]*wRtR B[p*nwave:(p+1)*nwave] += monomials[p,fiber]*wRtF log.info("iter %d solving"%iteration) w = A.diagonal()>0 A_pos_def = A[w,:] A_pos_def = A_pos_def[:,w] parameters = B*0 try: parameters[w]=cholesky_solve(A_pos_def,B[w]) except: log.info("cholesky failed, trying svd in iteration {}".format(iteration)) parameters[w]=np.linalg.lstsq(A_pos_def,B[w])[0] log.info("iter %d compute chi2"%iteration) for fiber in range(nfibers) : # loop on polynomial indices unconvolved_fiber_sky_flux = np.zeros(nwave) for p in range(ncoef) : unconvolved_fiber_sky_flux += monomials[p,fiber]*parameters[p*nwave:(p+1)*nwave] # then convolve fiber_convolved_sky_flux = Rsky[fiber].dot(unconvolved_fiber_sky_flux) chi2[fiber]=current_ivar[fiber]*(flux[fiber]-fiber_convolved_sky_flux)**2 log.info("rejecting") nout_iter=0 if iteration<1 : # only remove worst outlier per wave # apply rejection iteratively, only one entry per wave among fibers # find waves with outlier (fastest way) nout_per_wave=np.sum(chi2>nsig_clipping**2,axis=0) selection=np.where(nout_per_wave>0)[0] for i in selection : worst_entry=np.argmax(chi2[:,i]) current_ivar[worst_entry,i]=0 sqrtw[worst_entry,i]=0 sqrtwflux[worst_entry,i]=0 nout_iter += 1 else : # remove all of them at once bad=(chi2>nsig_clipping**2) current_ivar *= (bad==0) sqrtw *= (bad==0) sqrtwflux *= (bad==0) nout_iter += np.sum(bad) nout_tot += nout_iter sum_chi2=float(np.sum(chi2)) ndf=int(np.sum(chi2>0)-nwave) chi2pdf=0. if ndf>0 : chi2pdf=sum_chi2/ndf log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d"%(iteration,sum_chi2,ndf,chi2pdf,nout_iter)) if nout_iter == 0 : break log.info("nout tot=%d"%nout_tot) # we know have to compute the sky model for all fibers # and propagate the uncertainties # no need to restore the original ivar to compute the model errors when modeling ivar # the sky inverse variances are very similar # is there a different method to compute this ? log.info("compute covariance") try : parameter_covar=cholesky_invert(A) except np.linalg.linalg.LinAlgError : log.warning("cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv") parameter_covar = np.linalg.pinv(A) log.info("compute mean resolution") # we make an approximation for the variance to save CPU time # we use the average resolution of all fibers in the frame: mean_res_data=np.mean(frame.resolution_data,axis=0) Rmean = Resolution(mean_res_data) log.info("compute convolved sky and ivar") cskyflux = np.zeros(frame.flux.shape) cskyivar = np.zeros(frame.flux.shape) log.info("compute convolved parameter covariance") # The covariance of the parameters is composed of ncoef*ncoef blocks each of size nwave*nwave # A block (p,k) is the covariance of the unconvolved spectra p and k , corresponding to the polynomial indices p and k # We first sandwich each block with the average resolution. convolved_parameter_covar=np.zeros((ncoef,ncoef,nwave)) for p in range(ncoef) : for k in range(ncoef) : convolved_parameter_covar[p,k] = np.diagonal(Rmean.dot(parameter_covar[p*nwave:(p+1)*nwave,k*nwave:(k+1)*nwave]).dot(Rmean.T.todense())) ''' import astropy.io.fits as pyfits pyfits.writeto("convolved_parameter_covar.fits",convolved_parameter_covar,overwrite=True) # other approach log.info("dense Rmean...") Rmean=Rmean.todense() log.info("invert Rinv...") Rinv=np.linalg.inv(Rmean) # check this print("0?",np.max(np.abs(Rinv.dot(Rmean)-np.eye(Rmean.shape[0])))/np.max(np.abs(Rmean))) convolved_parameter_ivar=np.zeros((ncoef,ncoef,nwave)) for p in range(ncoef) : for k in range(ncoef) : convolved_parameter_ivar[p,k] = np.diagonal(Rinv.T.dot(A[p*nwave:(p+1)*nwave,k*nwave:(k+1)*nwave]).dot(Rinv)) # solve for each wave separately convolved_parameter_covar=np.zeros((ncoef,ncoef,nwave)) for i in range(nwave) : print("inverting ivar of wave %d/%d"%(i,nwave)) convolved_parameter_covar[:,:,i] = cholesky_invert(convolved_parameter_ivar[:,:,i]) pyfits.writeto("convolved_parameter_covar_bis.fits",convolved_parameter_covar,overwrite=True) import sys sys.exit(12) ''' # Now we compute the sky model variance for each fiber individually # accounting for its focal plane coordinates # so that a target fiber distant for a sky fiber will naturally have a larger # sky model variance log.info("compute sky and variance per fiber") for i in range(frame.nspec): # compute monomials M = [] xi=(frame.fibermap["DESIGN_X"][i]-xm)/xs yi=(frame.fibermap["DESIGN_Y"][i]-ym)/ys for dx in range(angular_variation_deg+1) : for dy in range(angular_variation_deg+1-dx) : M.append((xi**dx)*(yi**dy)) M = np.array(M) unconvolved_fiber_sky_flux=np.zeros(nwave) convolved_fiber_skyvar=np.zeros(nwave) for p in range(ncoef) : unconvolved_fiber_sky_flux += M[p]*parameters[p*nwave:(p+1)*nwave] for k in range(ncoef) : convolved_fiber_skyvar += M[p]*M[k]*convolved_parameter_covar[p,k] # convolve sky model with this fiber's resolution cskyflux[i] = frame.R[i].dot(unconvolved_fiber_sky_flux) # save inverse of variance cskyivar[i] = (convolved_fiber_skyvar>0)/(convolved_fiber_skyvar+(convolved_fiber_skyvar==0)) # look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1 if skyfibers.size > 1 and add_variance : modified_cskyivar = _model_variance(frame,cskyflux,cskyivar,skyfibers) else : modified_cskyivar = cskyivar.copy() # need to do better here mask = (cskyivar==0).astype(np.uint32) return SkyModel(frame.wave.copy(), cskyflux, modified_cskyivar, mask, nrej=nout_tot, stat_ivar = cskyivar) # keep a record of the statistical ivar for QA
def test_resolution(self, n = 100): dense = np.arange(n*n).reshape(n,n) R1 = Resolution(dense) assert scipy.sparse.isspmatrix_dia(R1),'Resolution is not recognized as a scipy.sparse.dia_matrix.' assert len(R1.offsets) == desispec.resolution.default_ndiag, 'Resolution.offsets has wrong size' R2 = Resolution(R1) assert np.array_equal(R1.toarray(),R2.toarray()),'Constructor broken for dia_matrix input.' R3 = Resolution(R1.data) assert np.array_equal(R1.toarray(),R3.toarray()),'Constructor broken for array data input.' sparse = scipy.sparse.dia_matrix((R1.data[::-1],R1.offsets[::-1]),(n,n)) R4 = Resolution(sparse) assert np.array_equal(R1.toarray(),R4.toarray()),'Constructor broken for permuted offsets input.' R5 = Resolution(R1.to_fits_array()) assert np.array_equal(R1.toarray(),R5.toarray()),'to_fits_array() is broken.' #- test different sizes of input diagonals for ndiag in [3,5,11]: R6 = Resolution(np.ones((ndiag, n))) assert len(R6.offsets) == ndiag, 'Constructor broken for ndiag={}'.format(ndiag) #- An even number if diagonals is not allowed try: ndiag = 10 R7 = Resolution(np.ones((ndiag, n))) raise RuntimeError('Incorrectly created Resolution with even number of diagonals') except ValueError, err: #- it correctly raised an error, so pass pass
def compute_polynomial_times_sky(frame, nsig_clipping=4., max_iterations=30, model_ivar=False, add_variance=True, angular_variation_deg=1, chromatic_variation_deg=1): """Compute a sky model. Sky[fiber,i] = R[fiber,i,j] Polynomial(x[fiber],y[fiber],wavelength[j]) Flux[j] Input flux are expected to be flatfielded! We don't check this in this routine. Args: frame : Frame object, which includes attributes - wave : 1D wavelength grid in Angstroms - flux : 2D flux[nspec, nwave] density - ivar : 2D inverse variance of flux - mask : 2D inverse mask flux (0=good) - resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers) nsig_clipping : [optional] sigma clipping value for outlier rejection Optional: max_iterations : int , number of iterations model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims. add_variance : evaluate calibration error and add this to the sky model variance returns SkyModel object with attributes wave, flux, ivar, mask """ log = get_logger() log.info("starting") # Grab sky fibers on this frame skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0] assert np.max(skyfibers) < 500 #- indices, not fiber numbers nwave = frame.nwave nfibers = len(skyfibers) current_ivar = frame.ivar[skyfibers].copy() * (frame.mask[skyfibers] == 0) flux = frame.flux[skyfibers] Rsky = frame.R[skyfibers] input_ivar = None if model_ivar: log.info( "use a model of the inverse variance to remove bias due to correlated ivar and flux" ) input_ivar = current_ivar.copy() median_ivar_vs_wave = np.median(current_ivar, axis=0) median_ivar_vs_fiber = np.median(current_ivar, axis=1) median_median_ivar = np.median(median_ivar_vs_fiber) for f in range(current_ivar.shape[0]): threshold = 0.01 current_ivar[f] = median_ivar_vs_fiber[ f] / median_median_ivar * median_ivar_vs_wave # keep input ivar for very low weights ii = (input_ivar[f] <= (threshold * median_ivar_vs_wave)) #log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1])) current_ivar[f][ii] = input_ivar[f][ii] # need focal plane coordinates x = frame.fibermap["FIBERASSIGN_X"] y = frame.fibermap["FIBERASSIGN_Y"] # normalize for numerical stability xm = np.mean(x) ym = np.mean(y) xs = np.std(x) ys = np.std(y) if xs == 0: xs = 1 if ys == 0: ys = 1 x = (x - xm) / xs y = (y - ym) / ys w = (frame.wave - frame.wave[0]) / (frame.wave[-1] - frame.wave[0]) * 2. - 1 # precompute the monomials for the sky fibers log.debug("compute monomials for deg={} and {}".format( angular_variation_deg, chromatic_variation_deg)) monomials = [] for dx in range(angular_variation_deg + 1): for dy in range(angular_variation_deg + 1 - dx): xypol = (x**dx) * (y**dy) for dw in range(chromatic_variation_deg + 1): wpol = w**dw monomials.append(np.outer(xypol, wpol)) ncoef = len(monomials) coef = np.zeros((ncoef)) allfibers_monomials = np.array(monomials) log.debug("shape of allfibers_monomials = {}".format( allfibers_monomials.shape)) skyfibers_monomials = allfibers_monomials[:, skyfibers, :] log.debug("shape of skyfibers_monomials = {}".format( skyfibers_monomials.shape)) sqrtw = np.sqrt(current_ivar) sqrtwflux = sqrtw * flux chi2 = np.zeros(flux.shape) Pol = np.ones(flux.shape, dtype=float) coef[0] = 1. nout_tot = 0 previous_chi2 = -10. for iteration in range(max_iterations): # the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters # A_ij = 1/2 d2(chi2)/di/dj # A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w] # the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters # B_i = 1/2 d(chi2)/di # B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w]) # the model is model[fiber]=R[fiber]*Pol(x,y,wave)*sky # the parameters are the unconvolved sky flux at the wavelength i # and the polynomial coefficients A = np.zeros((nwave, nwave), dtype=float) B = np.zeros((nwave), dtype=float) D = scipy.sparse.lil_matrix((nwave, nwave)) D2 = scipy.sparse.lil_matrix((nwave, nwave)) Pol /= coef[0] # force constant term to 1. # solving for the deconvolved mean sky spectrum # loop on fiber to handle resolution for fiber in range(nfibers): if fiber % 10 == 0: log.info("iter %d sky fiber (1st fit) %d/%d" % (iteration, fiber, nfibers)) D.setdiag(sqrtw[fiber]) D2.setdiag(Pol[fiber]) sqrtwRP = D.dot(Rsky[fiber]).dot( D2) # each row r of R is multiplied by sqrtw[r] A += (sqrtwRP.T * sqrtwRP).todense() B += sqrtwRP.T * sqrtwflux[fiber] log.info("iter %d solving" % iteration) w = A.diagonal() > 0 A_pos_def = A[w, :] A_pos_def = A_pos_def[:, w] parameters = B * 0 try: parameters[w] = cholesky_solve(A_pos_def, B[w]) except: log.info("cholesky failed, trying svd in iteration {}".format( iteration)) parameters[w] = np.linalg.lstsq(A_pos_def, B[w])[0] # parameters = the deconvolved mean sky spectrum # now evaluate the polynomial coefficients Ap = np.zeros((ncoef, ncoef), dtype=float) Bp = np.zeros((ncoef), dtype=float) D2.setdiag(parameters) for fiber in range(nfibers): if fiber % 10 == 0: log.info("iter %d sky fiber (2nd fit) %d/%d" % (iteration, fiber, nfibers)) D.setdiag(sqrtw[fiber]) sqrtwRSM = D.dot(Rsky[fiber]).dot(D2).dot( skyfibers_monomials[:, fiber, :].T) Ap += sqrtwRSM.T.dot(sqrtwRSM) Bp += sqrtwRSM.T.dot(sqrtwflux[fiber]) # Add huge prior on zeroth angular order terms to converge faster # (because those terms are degenerate with the mean deconvolved spectrum) weight = 1e24 Ap[0, 0] += weight Bp[0] += weight # force 0th term to 1 for i in range(1, chromatic_variation_deg + 1): Ap[i, i] += weight # force other wavelength terms to 0 coef = cholesky_solve(Ap, Bp) log.info("pol coef = {}".format(coef)) # recompute the polynomial values Pol = skyfibers_monomials.T.dot(coef).T # chi2 and outlier rejection log.info("iter %d compute chi2" % iteration) for fiber in range(nfibers): chi2[fiber] = current_ivar[fiber] * ( flux[fiber] - Rsky[fiber].dot(Pol[fiber] * parameters))**2 log.info("rejecting") nout_iter = 0 if iteration < 1: # only remove worst outlier per wave # apply rejection iteratively, only one entry per wave among fibers # find waves with outlier (fastest way) nout_per_wave = np.sum(chi2 > nsig_clipping**2, axis=0) selection = np.where(nout_per_wave > 0)[0] for i in selection: worst_entry = np.argmax(chi2[:, i]) current_ivar[worst_entry, i] = 0 sqrtw[worst_entry, i] = 0 sqrtwflux[worst_entry, i] = 0 nout_iter += 1 else: # remove all of them at once bad = (chi2 > nsig_clipping**2) current_ivar *= (bad == 0) sqrtw *= (bad == 0) sqrtwflux *= (bad == 0) nout_iter += np.sum(bad) nout_tot += nout_iter sum_chi2 = float(np.sum(chi2)) ndf = int(np.sum(chi2 > 0) - nwave) chi2pdf = 0. if ndf > 0: chi2pdf = sum_chi2 / ndf log.info("iter #%d chi2=%g ndf=%d chi2pdf=%f delta=%f nout=%d" % (iteration, sum_chi2, ndf, chi2pdf, abs(sum_chi2 - previous_chi2), nout_iter)) if nout_iter == 0 and abs(sum_chi2 - previous_chi2) < 0.2: break previous_chi2 = sum_chi2 + 0. log.info("nout tot=%d" % nout_tot) # we know have to compute the sky model for all fibers # and propagate the uncertainties # no need to restore the original ivar to compute the model errors when modeling ivar # the sky inverse variances are very similar # we ignore here the fact that we have fit a angular variation, # so the sky model uncertainties are inaccurate log.info("compute the parameter covariance") try: parameter_covar = cholesky_invert(A) except np.linalg.linalg.LinAlgError: log.warning( "cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv" ) parameter_covar = np.linalg.pinv(A) log.info("compute mean resolution") # we make an approximation for the variance to save CPU time # we use the average resolution of all fibers in the frame: mean_res_data = np.mean(frame.resolution_data, axis=0) Rmean = Resolution(mean_res_data) log.info("compute convolved sky and ivar") # The parameters are directly the unconvolved sky # First convolve with average resolution : convolved_sky_covar = Rmean.dot(parameter_covar).dot(Rmean.T.todense()) # and keep only the diagonal convolved_sky_var = np.diagonal(convolved_sky_covar) # inverse convolved_sky_ivar = (convolved_sky_var > 0) / (convolved_sky_var + (convolved_sky_var == 0)) # and simply consider it's the same for all spectra cskyivar = np.tile(convolved_sky_ivar, frame.nspec).reshape(frame.nspec, nwave) # The sky model for each fiber (simple convolution with resolution of each fiber) cskyflux = np.zeros(frame.flux.shape) Pol = allfibers_monomials.T.dot(coef).T for fiber in range(frame.nspec): cskyflux[fiber] = frame.R[fiber].dot(Pol[fiber] * parameters) # look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1 if skyfibers.size > 1 and add_variance: modified_cskyivar = _model_variance(frame, cskyflux, cskyivar, skyfibers) else: modified_cskyivar = cskyivar.copy() # need to do better here mask = (cskyivar == 0).astype(np.uint32) return SkyModel( frame.wave.copy(), cskyflux, modified_cskyivar, mask, nrej=nout_tot, stat_ivar=cskyivar) # keep a record of the statistical ivar for QA
def myspecupdate(spectra_in, other) : # Does the other Spectra object have any data? if other.num_spectra() == 0: return # Do we have new bands to add? newbands = [] for b in other.bands: if b not in spectra_in.bands: newbands.append(b) else: if not np.allclose(spectra_in.wave[b], other.wave[b]): raise RuntimeError("band {} has an incompatible wavelength grid".format(b)) bands = list(spectra_in.bands) bands.extend(newbands) # Are we adding mask data in this update? add_mask = False if other.mask is None: if spectra_in.mask is not None: raise RuntimeError("existing spectra has a mask, cannot " "update it to a spectra with no mask") else: if spectra_in.mask is None: add_mask = True # Are we adding resolution data in this update? ndiag = {} add_res = False if other.resolution_data is None: if spectra_in.resolution_data is not None: raise RuntimeError("existing spectra has resolution data, cannot " "update it to a spectra with none") else: if spectra_in.resolution_data is not None: for b in spectra_in.bands: ndiag[b] = spectra_in.resolution_data[b].shape[1] for b in other.bands: odiag = other.resolution_data[b].shape[1] if b not in spectra_in.bands: ndiag[b] = odiag else: if odiag != ndiag[b]: raise RuntimeError("Resolution matrices for a" " given band must have the same dimensoins") else: add_res = True for b in other.bands: ndiag[b] = other.resolution_data[b].shape[1] # Are we adding extra data in this update? add_extra = False if other.extra is None: if spectra_in.extra is not None: raise RuntimeError("existing spectra has extra data, cannot " "update it to a spectra with none") else: if spectra_in.extra is None: add_extra = True # Compute which targets / exposures are new nother = len(other.fibermap) exists = np.zeros(nother, dtype=np.int) indx_original = [] # EA modif : check_exists = True if ( (spectra_in.fibermap is None) or ("EXPID" not in spectra_in.fibermap.keys()) or ("EXPID" not in other.fibermap.keys()) or ("FIBER" not in spectra_in.fibermap.keys()) or ("FIBER" not in other.fibermap.keys()) ) : check_exists = False if check_exists : for r in range(nother): expid = other.fibermap[r]["EXPID"] fiber = other.fibermap[r]["FIBER"] for i, row in enumerate(spectra_in.fibermap): if (expid == row["EXPID"]) and (fiber == row["FIBER"]): indx_original.append(i) exists[r] += 1 if len(np.where(exists > 1)[0]) > 0: raise RuntimeError("found duplicate spectra (same EXPID and FIBER) in the fibermap") indx_exists = np.where(exists == 1)[0] indx_new = np.where(exists == 0)[0] # Make new data arrays of the correct size to hold both the old and # new data nupdate = len(indx_exists) nnew = len(indx_new) if spectra_in.fibermap is None: nold = 0 newfmap = other.fibermap.copy() else: nold = len(spectra_in.fibermap) newfmap = encode_table(np.zeros( (nold + nnew, ), dtype=spectra_in.fibermap.dtype)) newwave = {} newflux = {} newivar = {} newmask = None if add_mask or spectra_in.mask is not None: newmask = {} newres = None newR = None if add_res or spectra_in.resolution_data is not None: newres = {} newR = {} newextra = None if add_extra or spectra_in.extra is not None: newextra = {} for b in bands: nwave = None if b in spectra_in.bands: nwave = spectra_in.wave[b].shape[0] newwave[b] = spectra_in.wave[b] else: nwave = other.wave[b].shape[0] newwave[b] = other.wave[b].astype(spectra_in._ftype) newflux[b] = np.zeros( (nold + nnew, nwave), dtype=spectra_in._ftype) newivar[b] = np.zeros( (nold + nnew, nwave), dtype=spectra_in._ftype) if newmask is not None: newmask[b] = np.zeros( (nold + nnew, nwave), dtype=np.uint32) newmask[b][:,:] = specmask["NODATA"] if newres is not None: newres[b] = np.zeros( (nold + nnew, ndiag[b], nwave), dtype=spectra_in._ftype) if newextra is not None: newextra[b] = {} # Copy the old data if nold > 0: # We have some data (i.e. we are not starting with an empty Spectra) newfmap[:nold] = spectra_in.fibermap for b in spectra_in.bands: newflux[b][:nold,:] = spectra_in.flux[b] newivar[b][:nold,:] = spectra_in.ivar[b] if spectra_in.mask is not None: newmask[b][:nold,:] = spectra_in.mask[b] elif add_mask: newmask[b][:nold,:] = 0 if spectra_in.resolution_data is not None: newres[b][:nold,:,:] = spectra_in.resolution_data[b] if spectra_in.extra is not None: for ex in spectra_in.extra[b].items(): newextra[b][ex[0]] = np.zeros( newflux[b].shape, dtype=spectra_in._ftype) newextra[b][ex[0]][:nold,:] = ex[1] # Update existing spectra for i, s in enumerate(indx_exists): row = indx_original[i] for b in other.bands: newflux[b][row,:] = other.flux[b][s,:].astype(spectra_in._ftype) newivar[b][row,:] = other.ivar[b][s,:].astype(spectra_in._ftype) if other.mask is not None: newmask[b][row,:] = other.mask[b][s,:] else: newmask[b][row,:] = 0 if other.resolution_data is not None: newres[b][row,:,:] = other.resolution_data[b][s,:,:].astype(spectra_in._ftype) if other.extra is not None: for ex in other.extra[b].items(): if ex[0] not in newextra[b]: newextra[b][ex[0]] = np.zeros(newflux[b].shape, dtype=spectra_in._ftype) newextra[b][ex[0]][row,:] = ex[1][s,:].astype(spectra_in._ftype) # Append new spectra if nnew > 0: newfmap[nold:] = other.fibermap[indx_new] for b in other.bands: newflux[b][nold:,:] = other.flux[b][indx_new].astype(spectra_in._ftype) newivar[b][nold:,:] = other.ivar[b][indx_new].astype(spectra_in._ftype) if other.mask is not None: newmask[b][nold:,:] = other.mask[b][indx_new] else: newmask[b][nold:,:] = 0 if other.resolution_data is not None: newres[b][nold:,:,:] = other.resolution_data[b][indx_new].astype(spectra_in._ftype) if other.extra is not None: for ex in other.extra[b].items(): if ex[0] not in newextra[b]: newextra[b][ex[0]] = np.zeros(newflux[b].shape, dtype=spectra_in._ftype) newextra[b][ex[0]][nold:,:] = ex[1][indx_new].astype(spectra_in._ftype) # Update all sparse resolution matrices for b in bands: if newres is not None: newR[b] = np.array( [ Resolution(r) for r in newres[b] ] ) # Swap data into place spectra_in._bands = bands spectra_in.wave = newwave spectra_in.fibermap = newfmap spectra_in.flux = newflux spectra_in.ivar = newivar spectra_in.mask = newmask spectra_in.resolution_data = newres spectra_in.R = newR spectra_in.extra = newextra return spectra_in
def compute_non_uniform_sky(frame, nsig_clipping=4., max_iterations=10, model_ivar=False, add_variance=True, angular_variation_deg=1): """Compute a sky model. Sky[fiber,i] = R[fiber,i,j] ( Flux_0[j] + x[fiber]*Flux_x[j] + y[fiber]*Flux_y[j] + ... ) Input flux are expected to be flatfielded! We don't check this in this routine. Args: frame : Frame object, which includes attributes - wave : 1D wavelength grid in Angstroms - flux : 2D flux[nspec, nwave] density - ivar : 2D inverse variance of flux - mask : 2D inverse mask flux (0=good) - resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers) nsig_clipping : [optional] sigma clipping value for outlier rejection Optional: max_iterations : int , number of iterations model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims. add_variance : evaluate calibration error and add this to the sky model variance angular_variation_deg : degree of 2D polynomial correction as a function of fiber focal plane coordinates (default=1). One set of coefficients per wavelength returns SkyModel object with attributes wave, flux, ivar, mask """ log = get_logger() log.info("starting") # Grab sky fibers on this frame skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0] assert np.max(skyfibers) < 500 #- indices, not fiber numbers nwave = frame.nwave nfibers = len(skyfibers) current_ivar = frame.ivar[skyfibers].copy() * (frame.mask[skyfibers] == 0) flux = frame.flux[skyfibers] Rsky = frame.R[skyfibers] # need focal plane coordinates of fibers x = frame.fibermap["FIBERASSIGN_X"][skyfibers] y = frame.fibermap["FIBERASSIGN_Y"][skyfibers] # normalize for numerical stability xm = np.mean(frame.fibermap["FIBERASSIGN_X"]) ym = np.mean(frame.fibermap["FIBERASSIGN_Y"]) xs = np.std(frame.fibermap["FIBERASSIGN_X"]) ys = np.std(frame.fibermap["FIBERASSIGN_Y"]) if xs == 0: xs = 1 if ys == 0: ys = 1 x = (x - xm) / xs y = (y - ym) / ys # precompute the monomials for the sky fibers log.debug("compute monomials for deg={}".format(angular_variation_deg)) monomials = [] for dx in range(angular_variation_deg + 1): for dy in range(angular_variation_deg + 1 - dx): monomials.append((x**dx) * (y**dy)) ncoef = len(monomials) monomials = np.array(monomials) input_ivar = None if model_ivar: log.info( "use a model of the inverse variance to remove bias due to correlated ivar and flux" ) input_ivar = current_ivar.copy() median_ivar_vs_wave = np.median(current_ivar, axis=0) median_ivar_vs_fiber = np.median(current_ivar, axis=1) median_median_ivar = np.median(median_ivar_vs_fiber) for f in range(current_ivar.shape[0]): threshold = 0.01 current_ivar[f] = median_ivar_vs_fiber[ f] / median_median_ivar * median_ivar_vs_wave # keep input ivar for very low weights ii = (input_ivar[f] <= (threshold * median_ivar_vs_wave)) #log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1])) current_ivar[f][ii] = input_ivar[f][ii] sqrtw = np.sqrt(current_ivar) sqrtwflux = sqrtw * flux chi2 = np.zeros(flux.shape) nout_tot = 0 for iteration in range(max_iterations): # the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters # A_ij = 1/2 d2(chi2)/di/dj # A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w] # the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters # B_i = 1/2 d(chi2)/di # B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w]) # with x_fiber,y_fiber the fiber coordinates in the focal plane (or sky) # the unconvolved sky flux at wavelength i is a polynomial of x_fiber,y_fiber # sky(fiber,i) = pol(x_fiber,y_fiber,p) = sum_p a_ip * x_fiber**degx(p) y_fiber**degy(p) # sky(fiber,i) = sum_p monom[fiber,p] * a_ip # the convolved sky flux at wavelength w is # model[fiber,w] = sum_i R[fiber][w,i] sum_p monom[fiber,p] * a_ip # model[fiber,w] = sum_p monom[fiber,p] R[fiber][w,i] a_ip # # so, the matrix A is composed of blocks (p,k) corresponding to polynomial coefficient indices where # A[pk] = sum_fiber monom[fiber,p]*monom[fiber,k] sqrtwR[fiber] sqrtwR[fiber]^t # similarily # B[p] = sum_fiber monom[fiber,p] * sum_wave_w (sqrt(ivar)[fiber,w]*flux[fiber,w]) sqrtwR[fiber,wave] A = np.zeros((nwave * ncoef, nwave * ncoef)) B = np.zeros((nwave * ncoef)) # diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber SD = scipy.sparse.lil_matrix((nwave, nwave)) # loop on fiber to handle resolution for fiber in range(nfibers): if fiber % 10 == 0: log.info("iter %d sky fiber %d/%d" % (iteration, fiber, nfibers)) R = Rsky[fiber] # diagonal sparse matrix with content = sqrt(ivar) SD.setdiag(sqrtw[fiber]) sqrtwR = SD * R # each row r of R is multiplied by sqrtw[r] #wRtR=(sqrtwR.T*sqrtwR).tocsr() wRtR = (sqrtwR.T * sqrtwR).todense() wRtF = sqrtwR.T * sqrtwflux[fiber] # loop on polynomial coefficients (double loop for A) # fill only blocks of A and B for p in range(ncoef): for k in range(ncoef): A[p * nwave:(p + 1) * nwave, k * nwave:(k + 1) * nwave] += monomials[p, fiber] * monomials[k, fiber] * wRtR B[p * nwave:(p + 1) * nwave] += monomials[p, fiber] * wRtF log.info("iter %d solving" % iteration) w = A.diagonal() > 0 A_pos_def = A[w, :] A_pos_def = A_pos_def[:, w] parameters = B * 0 try: parameters[w] = cholesky_solve(A_pos_def, B[w]) except: log.info("cholesky failed, trying svd in iteration {}".format( iteration)) parameters[w] = np.linalg.lstsq(A_pos_def, B[w])[0] log.info("iter %d compute chi2" % iteration) for fiber in range(nfibers): # loop on polynomial indices unconvolved_fiber_sky_flux = np.zeros(nwave) for p in range(ncoef): unconvolved_fiber_sky_flux += monomials[ p, fiber] * parameters[p * nwave:(p + 1) * nwave] # then convolve fiber_convolved_sky_flux = Rsky[fiber].dot( unconvolved_fiber_sky_flux) chi2[fiber] = current_ivar[fiber] * (flux[fiber] - fiber_convolved_sky_flux)**2 log.info("rejecting") nout_iter = 0 if iteration < 1: # only remove worst outlier per wave # apply rejection iteratively, only one entry per wave among fibers # find waves with outlier (fastest way) nout_per_wave = np.sum(chi2 > nsig_clipping**2, axis=0) selection = np.where(nout_per_wave > 0)[0] for i in selection: worst_entry = np.argmax(chi2[:, i]) current_ivar[worst_entry, i] = 0 sqrtw[worst_entry, i] = 0 sqrtwflux[worst_entry, i] = 0 nout_iter += 1 else: # remove all of them at once bad = (chi2 > nsig_clipping**2) current_ivar *= (bad == 0) sqrtw *= (bad == 0) sqrtwflux *= (bad == 0) nout_iter += np.sum(bad) nout_tot += nout_iter sum_chi2 = float(np.sum(chi2)) ndf = int(np.sum(chi2 > 0) - nwave) chi2pdf = 0. if ndf > 0: chi2pdf = sum_chi2 / ndf log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d" % (iteration, sum_chi2, ndf, chi2pdf, nout_iter)) if nout_iter == 0: break log.info("nout tot=%d" % nout_tot) # we know have to compute the sky model for all fibers # and propagate the uncertainties # no need to restore the original ivar to compute the model errors when modeling ivar # the sky inverse variances are very similar # is there a different method to compute this ? log.info("compute covariance") try: parameter_covar = cholesky_invert(A) except np.linalg.linalg.LinAlgError: log.warning( "cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv" ) parameter_covar = np.linalg.pinv(A) log.info("compute mean resolution") # we make an approximation for the variance to save CPU time # we use the average resolution of all fibers in the frame: mean_res_data = np.mean(frame.resolution_data, axis=0) Rmean = Resolution(mean_res_data) log.info("compute convolved sky and ivar") cskyflux = np.zeros(frame.flux.shape) cskyivar = np.zeros(frame.flux.shape) log.info("compute convolved parameter covariance") # The covariance of the parameters is composed of ncoef*ncoef blocks each of size nwave*nwave # A block (p,k) is the covariance of the unconvolved spectra p and k , corresponding to the polynomial indices p and k # We first sandwich each block with the average resolution. convolved_parameter_covar = np.zeros((ncoef, ncoef, nwave)) for p in range(ncoef): for k in range(ncoef): convolved_parameter_covar[p, k] = np.diagonal( Rmean.dot(parameter_covar[p * nwave:(p + 1) * nwave, k * nwave:(k + 1) * nwave]).dot( Rmean.T.todense())) ''' import astropy.io.fits as pyfits pyfits.writeto("convolved_parameter_covar.fits",convolved_parameter_covar,overwrite=True) # other approach log.info("dense Rmean...") Rmean=Rmean.todense() log.info("invert Rinv...") Rinv=np.linalg.inv(Rmean) # check this print("0?",np.max(np.abs(Rinv.dot(Rmean)-np.eye(Rmean.shape[0])))/np.max(np.abs(Rmean))) convolved_parameter_ivar=np.zeros((ncoef,ncoef,nwave)) for p in range(ncoef) : for k in range(ncoef) : convolved_parameter_ivar[p,k] = np.diagonal(Rinv.T.dot(A[p*nwave:(p+1)*nwave,k*nwave:(k+1)*nwave]).dot(Rinv)) # solve for each wave separately convolved_parameter_covar=np.zeros((ncoef,ncoef,nwave)) for i in range(nwave) : print("inverting ivar of wave %d/%d"%(i,nwave)) convolved_parameter_covar[:,:,i] = cholesky_invert(convolved_parameter_ivar[:,:,i]) pyfits.writeto("convolved_parameter_covar_bis.fits",convolved_parameter_covar,overwrite=True) import sys sys.exit(12) ''' # Now we compute the sky model variance for each fiber individually # accounting for its focal plane coordinates # so that a target fiber distant for a sky fiber will naturally have a larger # sky model variance log.info("compute sky and variance per fiber") for i in range(frame.nspec): # compute monomials M = [] xi = (frame.fibermap["FIBERASSIGN_X"][i] - xm) / xs yi = (frame.fibermap["FIBERASSIGN_Y"][i] - ym) / ys for dx in range(angular_variation_deg + 1): for dy in range(angular_variation_deg + 1 - dx): M.append((xi**dx) * (yi**dy)) M = np.array(M) unconvolved_fiber_sky_flux = np.zeros(nwave) convolved_fiber_skyvar = np.zeros(nwave) for p in range(ncoef): unconvolved_fiber_sky_flux += M[p] * parameters[p * nwave:(p + 1) * nwave] for k in range(ncoef): convolved_fiber_skyvar += M[p] * M[ k] * convolved_parameter_covar[p, k] # convolve sky model with this fiber's resolution cskyflux[i] = frame.R[i].dot(unconvolved_fiber_sky_flux) # save inverse of variance cskyivar[i] = (convolved_fiber_skyvar > 0) / ( convolved_fiber_skyvar + (convolved_fiber_skyvar == 0)) # look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1 if skyfibers.size > 1 and add_variance: modified_cskyivar = _model_variance(frame, cskyflux, cskyivar, skyfibers) else: modified_cskyivar = cskyivar.copy() # need to do better here mask = (cskyivar == 0).astype(np.uint32) return SkyModel( frame.wave.copy(), cskyflux, modified_cskyivar, mask, nrej=nout_tot, stat_ivar=cskyivar) # keep a record of the statistical ivar for QA
def main(args=None): ''' Converts simspec -> frame files; see fastframe --help for usage options ''' #- TODO: use desiutil.log if isinstance(args, (list, tuple, type(None))): args = parse(args) print('Reading files') simspec = desisim.io.read_simspec(args.simspec) if simspec.flavor == 'arc': print('arc exposure; no frames to output') return fibermap = simspec.fibermap obs = simspec.obs night = simspec.header['NIGHT'] expid = simspec.header['EXPID'] firstspec = args.firstspec nspec = min(args.nspec, len(fibermap) - firstspec) print('Simulating spectra {}-{}'.format(firstspec, firstspec + nspec)) wave = simspec.wave['brz'] flux = simspec.flux ii = slice(firstspec, firstspec + nspec) if simspec.flavor == 'science': sim = desisim.simexp.simulate_spectra(wave, flux[ii], fibermap=fibermap[ii], obsconditions=obs, dwave_out=1.0) elif simspec.flavor in ['arc', 'flat', 'calib']: x = fibermap['X_TARGET'] y = fibermap['Y_TARGET'] fiber_area = desisim.simexp.fiber_area_arcsec2(fibermap['X_TARGET'], fibermap['Y_TARGET']) surface_brightness = (flux.T / fiber_area).T config = desisim.simexp._specsim_config_for_wave(wave, dwave_out=1.0) # sim = specsim.simulator.Simulator(config, num_fibers=nspec) sim = desisim.specsim.get_simulator(config, num_fibers=nspec) sim.observation.exposure_time = simspec.header['EXPTIME'] * u.s sbunit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm**2 * u.arcsec**2) xy = np.vstack([x, y]).T * u.mm sim.simulate(calibration_surface_brightness=surface_brightness[ii] * sbunit, focal_positions=xy[ii]) else: raise ValueError('Unknown simspec flavor {}'.format(simspec.flavor)) sim.generate_random_noise() for i, results in enumerate(sim.camera_output): results = sim.camera_output[i] wave = results['wavelength'] phot = (results['num_source_electrons'] + \ results['num_sky_electrons'] + \ results['num_dark_electrons'] + \ results['random_noise_electrons']).T ivar = 1.0 / results['variance_electrons'].T R = Resolution( sim.instrument.cameras[i].get_output_resolution_matrix()) Rdata = np.tile(R.data.T, nspec).T.reshape(nspec, R.data.shape[0], R.data.shape[1]) assert np.all(Rdata[0] == R.data) assert phot.shape == (nspec, len(wave)) for spectro in range(10): imin = max(firstspec, spectro * 500) - firstspec imax = min(firstspec + nspec, (spectro + 1) * 500) - firstspec if imax <= imin: continue xphot = phot[imin:imax] xivar = ivar[imin:imax] xfibermap = fibermap[ii][imin:imax] camera = '{}{}'.format(sim.camera_names[i], spectro) meta = simspec.header.copy() meta['CAMERA'] = camera frame = Frame(wave, xphot, xivar, resolution_data=Rdata[0:imax - imin], spectrograph=spectro, fibermap=xfibermap, meta=meta) outfile = desispec.io.findfile('frame', night, expid, camera, outdir=args.outdir) print('writing {}'.format(outfile)) desispec.io.write_frame(outfile, frame)
# be in the next spectrograph file ('b0' has 500 total, 'b1' has 500 total, etc.) for n in range(nadd_start, nadd_stop): # First, get the overall wavelength range of the spectrum # Individual cameras have 0.5 A resolutions # We make the final coadded spectrum uniformly spaced with 1.0 A bins spectrograph = int(n/500) ncameras = cameras[spectrograph::int(np.ceil(nspec/500.))] global_wavelength_grid = np.arange(cframes['b{:d}'.format(spectrograph)].wave[0], cframes['z{:d}'.format(spectrograph)].wave[-1] + 0.5, 1.0, dtype=np.float32) coadd_all_bands = Spectrum(global_wavelength_grid) for band in ncameras: nn = n - 500*int(n / 500) band_specobj = Spectrum(cframes[band].wave, flux=cframes[band].flux[nn], ivar=cframes[band].ivar[nn], resolution=Resolution(cframes[band].resolution_data[nn])) coadd_all_bands += band_specobj print('Spectra %d: Added band %s' % (n, band)) # Finalize the overall spectrum coadd_all_bands.finalize() print('{}/{} coadded'.format(n+1, nspec)) # And write the output outfile = args.outdir + "spectra-%s-expid%03d-%05d.fits" % (args.night, args.expid, n) write_coadd_spectra(outfile, coadd_all_bands, meta[n], simspecfile) # fibermap=fiberap[n] print('written to {}'.format(outfile))
def main(args): # Set up the logger if args.verbose: log = get_logger(DEBUG) else: log = get_logger() # Make sure all necessary environment variables are set DESI_SPECTRO_REDUX_DIR="./quickGen" if 'DESI_SPECTRO_REDUX' not in os.environ: log.info('DESI_SPECTRO_REDUX environment is not set.') else: DESI_SPECTRO_REDUX_DIR=os.environ['DESI_SPECTRO_REDUX'] if os.path.exists(DESI_SPECTRO_REDUX_DIR): if not os.path.isdir(DESI_SPECTRO_REDUX_DIR): raise RuntimeError("Path %s Not a directory"%DESI_SPECTRO_REDUX_DIR) else: try: os.makedirs(DESI_SPECTRO_REDUX_DIR) except: raise SPECPROD_DIR='specprod' if 'SPECPROD' not in os.environ: log.info('SPECPROD environment is not set.') else: SPECPROD_DIR=os.environ['SPECPROD'] prod_Dir=specprod_root() if os.path.exists(prod_Dir): if not os.path.isdir(prod_Dir): raise RuntimeError("Path %s Not a directory"%prod_Dir) else: try: os.makedirs(prod_Dir) except: raise # Initialize random number generator to use. np.random.seed(args.seed) random_state = np.random.RandomState(args.seed) # Derive spectrograph number from nstart if needed if args.spectrograph is None: args.spectrograph = args.nstart / 500 # Read fibermapfile to get object type, night and expid if args.fibermap: log.info("Reading fibermap file {}".format(args.fibermap)) fibermap=read_fibermap(args.fibermap) objtype = get_source_types(fibermap) stdindx=np.where(objtype=='STD') # match STD with STAR mwsindx=np.where(objtype=='MWS_STAR') # match MWS_STAR with STAR bgsindx=np.where(objtype=='BGS') # match BGS with LRG objtype[stdindx]='STAR' objtype[mwsindx]='STAR' objtype[bgsindx]='LRG' NIGHT=fibermap.meta['NIGHT'] EXPID=fibermap.meta['EXPID'] else: # Create a blank fake fibermap fibermap = empty_fibermap(args.nspec) targetids = random_state.randint(2**62, size=args.nspec) fibermap['TARGETID'] = targetids night = get_night() expid = 0 log.info("Initializing SpecSim with config {}".format(args.config)) desiparams = load_desiparams() qsim = get_simulator(args.config, num_fibers=1) if args.simspec: # Read the input file log.info('Reading input file {}'.format(args.simspec)) simspec = desisim.io.read_simspec(args.simspec) nspec = simspec.nspec if simspec.flavor == 'arc': log.warning("quickgen doesn't generate flavor=arc outputs") return else: wavelengths = simspec.wave spectra = simspec.flux if nspec < args.nspec: log.info("Only {} spectra in input file".format(nspec)) args.nspec = nspec else: # Initialize the output truth table. spectra = [] wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value npix = len(wavelengths) truth = dict() meta = Table() truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10)) truth['FLUX'] = np.zeros((args.nspec, npix)) truth['WAVE'] = wavelengths jj = list() for thisobj in set(true_objtype): ii = np.where(true_objtype == thisobj)[0] nobj = len(ii) truth['OBJTYPE'][ii] = thisobj log.info('Generating {} template'.format(thisobj)) # Generate the templates if thisobj == 'ELG': elg = desisim.templates.ELG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = elg.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_elg,sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'LRG': lrg = desisim.templates.LRG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = lrg.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_lrg,sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'QSO': qso = desisim.templates.QSO(wave=wavelengths) flux, tmpwave, meta1 = qso.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_qso) elif thisobj == 'BGS': bgs = desisim.templates.BGS(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = bgs.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_bgs,rmagrange=args.rmagrange_bgs,sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj =='STD': std = desisim.templates.STD(wave=wavelengths) flux, tmpwave, meta1 = std.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'QSO_BAD': # use STAR template no color cuts star = desisim.templates.STAR(wave=wavelengths) flux, tmpwave, meta1 = star.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'MWS_STAR' or thisobj == 'MWS': mwsstar = desisim.templates.MWS_STAR(wave=wavelengths) flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'WD': wd = desisim.templates.WD(wave=wavelengths) flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'SKY': flux = np.zeros((nobj, npix)) meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32))) elif thisobj == 'TEST': flux = np.zeros((args.nspec, npix)) indx = np.where(wave>5800.0-1E-6)[0][0] ref_integrated_flux = 1E-10 ref_cst_flux_density = 1E-17 single_line = (np.arange(args.nspec)%2 == 0).astype(np.float32) continuum = (np.arange(args.nspec)%2 == 1).astype(np.float32) for spec in range(args.nspec) : flux[spec,indx] = single_line[spec]*ref_integrated_flux/np.gradient(wavelengths)[indx] # single line flux[spec] += continuum[spec]*ref_cst_flux_density # flat continuum meta1 = Table(dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32), LINE=wave[indx]*np.ones(args.nspec, dtype=np.float32), LINEFLUX=single_line*ref_integrated_flux, CONSTFLUXDENSITY=continuum*ref_cst_flux_density)) else: log.fatal('Unknown object type {}'.format(thisobj)) sys.exit(1) # Pack it in. truth['FLUX'][ii] = flux meta = vstack([meta, meta1]) jj.append(ii.tolist()) # Sanity check on units; templates currently return ergs, not 1e-17 ergs... # assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6) # Sort the metadata table. jj = sum(jj,[]) meta_new = Table() for k in range(args.nspec): index = int(np.where(np.array(jj) == k)[0]) meta_new = vstack([meta_new, meta[index]]) meta = meta_new # Add TARGETID and the true OBJTYPE to the metadata table. meta.add_column(Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE')) meta.add_column(Column(targetids, name='TARGETID')) # Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z meta.rename_column('REDSHIFT', 'TRUEZ') # explicitly set location on focal plane if needed to support airmass # variations when using specsim v0.5 if qsim.source.focal_xy is None: qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm')) # Set simulation parameters from the simspec header or desiparams bright_objects = ['bgs','mws','bright','BGS','MWS','BRIGHT_MIX'] gray_objects = ['gray','grey'] if args.simspec is None: object_type = objtype flavor = None elif simspec.flavor == 'science': object_type = None flavor = simspec.header['PROGRAM'] else: object_type = None flavor = simspec.flavor log.warning('Maybe using an outdated simspec file with flavor={}'.format(flavor)) # Set airmass if args.airmass is not None: qsim.atmosphere.airmass = args.airmass elif args.simspec and 'AIRMASS' in simspec.header: qsim.atmosphere.airmass = simspec.header['AIRMASS'] else: qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2 # Set exptime if args.exptime is not None: qsim.observation.exposure_time = args.exptime * u.s elif args.simspec and 'EXPTIME' in simspec.header: qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s elif objtype in bright_objects: qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s else: qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s # Set Moon Phase if args.moon_phase is not None: qsim.atmosphere.moon.moon_phase = args.moon_phase elif args.simspec and 'MOONFRAC' in simspec.header: qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC'] elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_phase = 0.7 elif flavor in gray_objects: qsim.atmosphere.moon.moon_phase = 0.1 else: qsim.atmosphere.moon.moon_phase = 0.5 # Set Moon Zenith if args.moon_zenith is not None: qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg elif args.simspec and 'MOONALT' in simspec.header: qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_zenith = 30 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.moon_zenith = 80 * u.deg else: qsim.atmosphere.moon.moon_zenith = 100 * u.deg # Set Moon - Object Angle if args.moon_angle is not None: qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg elif args.simspec and 'MOONSEP' in simspec.header: qsim.atmosphere.moon.separation_angle = simspec.header['MOONSEP'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.separation_angle = 50 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.separation_angle = 60 * u.deg else: qsim.atmosphere.moon.separation_angle = 60 * u.deg # Initialize per-camera output arrays that will be saved waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {} maxbin = 0 nmax= args.nspec for camera in qsim.instrument.cameras: # Lookup this camera's resolution matrix and convert to the sparse # format used in desispec. R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [args.nspec, 1, 1]) waves[camera.name] = (camera.output_wavelength.to(u.Angstrom).value.astype(np.float32)) nwave = len(waves[camera.name]) maxbin = max(maxbin, len(waves[camera.name])) nobj = np.zeros((nmax,3,maxbin)) # object photons nsky = np.zeros((nmax,3,maxbin)) # sky photons nivar = np.zeros((nmax,3,maxbin)) # inverse variance (object+sky) cframe_observedflux = np.zeros((nmax,3,maxbin)) # calibrated object flux cframe_ivar = np.zeros((nmax,3,maxbin)) # inverse variance of calibrated object flux cframe_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to calibrated flux sky_ivar = np.zeros((nmax,3,maxbin)) # inverse variance of sky sky_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to sky only frame_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to nobj+nsky trueflux[camera.name] = np.empty((args.nspec, nwave)) # calibrated flux noisyflux[camera.name] = np.empty((args.nspec, nwave)) # observed flux with noise obsivar[camera.name] = np.empty((args.nspec, nwave)) # inverse variance of flux if args.simspec: for i in range(10): cn = camera.name + str(i) if cn in simspec.cameras: dw = np.gradient(simspec.cameras[cn].wave) break else: raise RuntimeError('Unable to find a {} camera in input simspec'.format(camera)) else: sflux = np.empty((args.nspec, npix)) #- Check if input simspec is for a continuum flat lamp instead of science #- This does not convolve to per-fiber resolution if args.simspec: if simspec.flavor == 'flat': log.info("Simulating flat lamp exposure") for i,camera in enumerate(qsim.instrument.cameras): channel = camera.name #- from simspec, b/r/z not b0/r1/z9 assert camera.output_wavelength.unit == u.Angstrom num_pixels = len(waves[channel]) phot = list() for j in range(10): cn = camera.name + str(j) if cn in simspec.cameras: camwave = simspec.cameras[cn].wave dw = np.gradient(camwave) phot.append(simspec.cameras[cn].phot) if len(phot) == 0: raise RuntimeError('Unable to find a {} camera in input simspec'.format(camera)) else: phot = np.vstack(phot) meanspec = resample_flux( waves[channel], camwave, np.average(phot/dw, axis=0)) fiberflat = random_state.normal(loc=1.0, scale=1.0 / np.sqrt(meanspec), size=(nspec, num_pixels)) ivar = np.tile(meanspec, [nspec, 1]) mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32) for kk in range((args.nspec+args.nstart-1)//500+1): camera = channel+str(kk) outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID, camera) start=max(500*kk,args.nstart) end=min(500*(kk+1),nmax) if (args.spectrograph <= kk): log.info("Writing files for channel:{}, spectrograph:{}, spectra:{} to {}".format(channel,kk,start,end)) ff = FiberFlat( waves[channel], fiberflat[start:end,:], ivar[start:end,:], mask[start:end,:], meanspec, header=dict(CAMERA=camera)) write_fiberflat(outfile, ff) filePath=desispec.io.findfile("fiberflat",NIGHT,EXPID,camera) log.info("Wrote file {}".format(filePath)) sys.exit(0) # Repeat the simulation for all spectra fluxunits = 1e-17 * u.erg / (u.s * u.cm ** 2 * u.Angstrom) for j in range(args.nspec): thisobjtype = objtype[j] sys.stdout.flush() if flavor == 'arc': qsim.source.update_in( 'Quickgen source {0}'.format, 'perfect', wavelengths * u.Angstrom, spectra * fluxunits) else: qsim.source.update_in( 'Quickgen source {0}'.format(j), thisobjtype.lower(), wavelengths * u.Angstrom, spectra[j, :] * fluxunits) qsim.source.update_out() qsim.simulate() qsim.generate_random_noise(random_state) for i, output in enumerate(qsim.camera_output): assert output['observed_flux'].unit == 1e17 * fluxunits # Extract the simulation results needed to create our uncalibrated # frame output file. num_pixels = len(output) nobj[j, i, :num_pixels] = output['num_source_electrons'][:,0] nsky[j, i, :num_pixels] = output['num_sky_electrons'][:,0] nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:,0] # Get results for our flux-calibrated output file. cframe_observedflux[j, i, :num_pixels] = 1e17 * output['observed_flux'][:,0] cframe_ivar[j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:,0] # Fill brick arrays from the results. camera = output.meta['name'] trueflux[camera][j][:] = 1e17 * output['observed_flux'][:,0] noisyflux[camera][j][:] = 1e17 * (output['observed_flux'][:,0] + output['flux_calibration'][:,0] * output['random_noise_electrons'][:,0]) obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,0] # Use the same noise realization in the cframe and frame, without any # additional noise from sky subtraction for now. frame_rand_noise[j, i, :num_pixels] = output['random_noise_electrons'][:,0] cframe_rand_noise[j, i, :num_pixels] = 1e17 * ( output['flux_calibration'][:,0] * output['random_noise_electrons'][:,0]) # The sky output file represents a model fit to ~40 sky fibers. # We reduce the variance by a factor of 25 to account for this and # give the sky an independent (Gaussian) noise realization. sky_ivar[j, i, :num_pixels] = 25.0 / ( output['variance_electrons'][:,0] - output['num_source_electrons'][:,0]) sky_rand_noise[j, i, :num_pixels] = random_state.normal( scale=1.0 / np.sqrt(sky_ivar[j,i,:num_pixels]),size=num_pixels) armName={"b":0,"r":1,"z":2} for channel in 'brz': #Before writing, convert from counts/bin to counts/A (as in Pixsim output) #Quicksim Default: #FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang] #COUNTS_OBJ - object counts in 0.5 Ang bin #COUNTS_SKY - sky counts in 0.5 Ang bin num_pixels = len(waves[channel]) dwave=np.gradient(waves[channel]) nobj[:,armName[channel],:num_pixels]/=dwave frame_rand_noise[:,armName[channel],:num_pixels]/=dwave nivar[:,armName[channel],:num_pixels]*=dwave**2 nsky[:,armName[channel],:num_pixels]/=dwave sky_rand_noise[:,armName[channel],:num_pixels]/=dwave sky_ivar[:,armName[channel],:num_pixels]/=dwave**2 # Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra # Looping over spectrograph for ii in range((args.nspec+args.nstart-1)//500+1): start=max(500*ii,args.nstart) # first spectrum for a given spectrograph end=min(500*(ii+1),nmax) # last spectrum for the spectrograph if (args.spectrograph <= ii): camera = "{}{}".format(channel, ii) log.info("Writing files for channel:{}, spectrograph:{}, spectra:{} to {}".format(channel,ii,start,end)) num_pixels = len(waves[channel]) # Write frame file framefileName=desispec.io.findfile("frame",NIGHT,EXPID,camera) frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \ nsky[start:end,armName[channel],:num_pixels] + \ frame_rand_noise[start:end,armName[channel],:num_pixels] frame_ivar=nivar[start:end,armName[channel],:num_pixels] sh1=frame_flux.shape[0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave) # for example if nstart =400, nspec=150: two spectrographs: # 400-499=> 0 spectrograph, 500-549 => 1 if (args.nstart==start): resol=resolution[channel][:sh1,:,:] else: resol=resolution[channel][-sh1:,:,:] # must create desispec.Frame object frame=Frame(waves[channel], frame_flux, frame_ivar,\ resolution_data=resol, spectrograph=ii, \ fibermap=fibermap[start:end], \ meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.write_frame(framefileName, frame) framefilePath=desispec.io.findfile("frame",NIGHT,EXPID,camera) log.info("Wrote file {}".format(framefilePath)) if args.frameonly or simspec.flavor == 'arc': continue # Write cframe file cframeFileName=desispec.io.findfile("cframe",NIGHT,EXPID,camera) cframeFlux=cframe_observedflux[start:end,armName[channel],:num_pixels]+cframe_rand_noise[start:end,armName[channel],:num_pixels] cframeIvar=cframe_ivar[start:end,armName[channel],:num_pixels] # must create desispec.Frame object cframe = Frame(waves[channel], cframeFlux, cframeIvar, \ resolution_data=resol, spectrograph=ii, fibermap=fibermap[start:end], meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.frame.write_frame(cframeFileName,cframe) cframefilePath=desispec.io.findfile("cframe",NIGHT,EXPID,camera) log.info("Wrote file {}".format(cframefilePath)) # Write sky file skyfileName=desispec.io.findfile("sky",NIGHT,EXPID,camera) skyflux=nsky[start:end,armName[channel],:num_pixels] + \ sky_rand_noise[start:end,armName[channel],:num_pixels] skyivar=sky_ivar[start:end,armName[channel],:num_pixels] skymask=np.zeros(skyflux.shape, dtype=np.uint32) # must create desispec.Sky object skymodel = SkyModel(waves[channel], skyflux, skyivar, skymask, header=dict(CAMERA=camera)) desispec.io.sky.write_sky(skyfileName, skymodel) skyfilePath=desispec.io.findfile("sky",NIGHT,EXPID,camera) log.info("Wrote file {}".format(skyfilePath)) # Write calib file calibVectorFile=desispec.io.findfile("calib",NIGHT,EXPID,camera) flux = cframe_observedflux[start:end,armName[channel],:num_pixels] phot = nobj[start:end,armName[channel],:num_pixels] calibration = np.zeros_like(phot) jj = (flux>0) calibration[jj] = phot[jj] / flux[jj] #- TODO: what should calibivar be? #- For now, model it as the noise of combining ~10 spectra calibivar=10/cframe_ivar[start:end,armName[channel],:num_pixels] #mask=(1/calibivar>0).astype(int)?? mask=np.zeros(calibration.shape, dtype=np.uint32) # write flux calibration fluxcalib = FluxCalib(waves[channel], calibration, calibivar, mask) write_flux_calibration(calibVectorFile, fluxcalib) calibfilePath=desispec.io.findfile("calib",NIGHT,EXPID,camera) log.info("Wrote file {}".format(calibfilePath))