def test_edges(self):
'''Test for large edge effects in resampling'''
x = np.arange(0.0, 100)
y = np.sin(x/20)
xx = np.linspace(1, 99, 23)
yy = resample_flux(xx, x, y)
diff = np.abs(yy - np.interp(xx, x, y))
self.assertLess(np.max(np.abs(diff)), 1e-2)
def test_resample(self):
n = 100
x = np.arange(n)
y = np.ones(n)
# we need in this test to make sure we have the same boundaries of the edges bins
# to obtain the same flux density on the edges
# because the resampling routine considers the flux is 0 outside of the input bins
nout = n//2
stepout = n/float(nout)
xout = np.arange(nout)*stepout+stepout/2-0.5
yout = resample_flux(xout, x, y)
self.assertTrue(np.all(yout == 1.0))
def test_flux_conservation(self):
n = 100
x = np.arange(n)
#y = 1+np.sin(x/20.0)
y = 0.4*x+10 # only exact for linear relation
y[n//2+1] += 10
# xout must have edges including bin half width equal
# or larger than input to get the same integrated flux
xout = np.arange(0,n+1,2)
yout = resample_flux(xout, x, y)
fluxin = np.sum(y*np.gradient(x))
fluxout = np.sum(yout*np.gradient(xout))
self.assertAlmostEqual(fluxin, fluxout)
def test_weighted_resample(self):
n = 100
x = np.arange(n)
y = 1+np.sin(x/20.0)
y[n//2+1] += 10
ivar = np.ones(n)
for rebin in (2, 3, 5):
xout = np.arange(0,n+1,rebin)
yout, ivout = resample_flux(xout, x, y, ivar)
self.assertEqual(len(xout), len(yout))
self.assertEqual(len(xout), len(ivout))
# we have to compare the variance of ouput bins that
# are fully contained in input
self.assertAlmostEqual(ivout[ivout.size//2], ivar[ivar.size//2]*rebin)
# check sum of weights is conserved
ivar_in = np.sum(ivar)
ivar_out = np.sum(ivout)
self.assertAlmostEqual(ivar_in,ivar_out)
def test_non_uniform_grid(self):
n = 100
x = np.arange(n)+1.
y = np.ones(n)
# we need in this test to make sure we have the same boundaries of the edges bins
# to obtain the same flux density on the edges
# because the resampling routine considers the flux is 0 outside of the input bins
# we consider here a logarithmic output grid
nout = n//2
lstepout = (log(x[-1])-log(x[0]))/float(nout)
xout = np.exp(np.arange(nout)*lstepout)-0.5
xout[0] = x[0]-0.5+(xout[1]-xout[0])/2 # same edge of first bin
offset = x[-1]+0.5-(xout[-1]-xout[-2])/2 - xout[-1]
xout[-2:] += offset # same edge of last bin
yout = resample_flux(xout, x, y)
zero = np.max(np.abs(yout-1))
self.assertAlmostEqual(zero,0.)
def _resample_flux(args):
return resample_flux(*args)
def desi_qso_templates(z_wind=0.2, zmnx=(0.4,4.), outfil=None, N_perz=500,
boss_pca_fil=None, wvmnx=(3500., 10000.),
rebin_wave=None, rstate=None,
sdss_pca_fil=None, no_write=False, redshift=None,
seed=None, old_read=False, ipad=40, cosmo=None):
""" Generate QSO templates for DESI
Rebins to input wavelength array (or log10 in wvmnx)
Parameters
----------
z_wind : float, optional
Window for sampling PCAs
zmnx : tuple, optional
Min/max for generation
N_perz : int, optional
Number of draws per redshift window
old_read : bool, optional
Read the files the old way
seed : int, optional
Seed for the random number state
rebin_wave : ndarray, optional
Input wavelengths for rebinning
wvmnx : tuple, optional
Wavelength limits for rebinning (not used with rebin_wave)
redshift : ndarray, optional
Redshifts desired for the templates
ipad : int, optional
Padding for enabling enough models
cosmo: astropy.cosmology.core, optional
Cosmology inistantiation from astropy.cosmology.code
Returns
-------
wave : ndarray
Wavelengths that the spectra were rebinned to
flux : ndarray (2D; flux vs. model)
z : ndarray
Redshifts
"""
# Cosmology
if cosmo is None:
from astropy import cosmology
cosmo = cosmology.core.FlatLambdaCDM(70., 0.3)
if old_read:
# PCA values
if boss_pca_fil is None:
boss_pca_fil = 'BOSS_DR10Lya_PCA_values_nocut.fits.gz'
hdu = fits.open(boss_pca_fil)
boss_pca_coeff = hdu[1].data
if sdss_pca_fil is None:
sdss_pca_fil = 'SDSS_DR7Lya_PCA_values_nocut.fits.gz'
hdu2 = fits.open(sdss_pca_fil)
sdss_pca_coeff = hdu2[1].data
# Open the BOSS catalog file
boss_cat_fil = os.environ.get('BOSSPATH')+'/DR10/BOSSLyaDR10_cat_v2.1.fits.gz'
bcat_hdu = fits.open(boss_cat_fil)
t_boss = bcat_hdu[1].data
boss_zQSO = t_boss['z_pipe']
# Open the SDSS catalog file
sdss_cat_fil = os.environ.get('SDSSPATH')+'/DR7_QSO/dr7_qso.fits.gz'
scat_hdu = fits.open(sdss_cat_fil)
t_sdss = scat_hdu[1].data
sdss_zQSO = t_sdss['z']
if len(sdss_pca_coeff) != len(sdss_zQSO):
print('Need to finish running the SDSS models!')
sdss_zQSO = sdss_zQSO[0:len(sdss_pca_coeff)]
# Eigenvectors
eigen, eigen_wave = fbq.read_qso_eigen()
else:
infile = lvmsim.io.find_basis_template('qso')
with fits.open(infile) as hdus:
hdu_names = [hdus[ii].name for ii in range(len(hdus))]
boss_pca_coeff = hdus[hdu_names.index('BOSS_PCA')].data
sdss_pca_coeff = hdus[hdu_names.index('SDSS_PCA')].data
boss_zQSO = hdus[hdu_names.index('BOSS_Z')].data
sdss_zQSO = hdus[hdu_names.index('SDSS_Z')].data
eigen = hdus[hdu_names.index('SDSS_EIGEN')].data
eigen_wave = hdus[hdu_names.index('SDSS_EIGEN_WAVE')].data
# Fiddle with the eigen-vectors
npix = len(eigen_wave)
chkpix = np.where((eigen_wave > 900.) & (eigen_wave < 5000.) )[0]
lambda_912 = 911.76
pix912 = np.argmin( np.abs(eigen_wave-lambda_912) )
# Loop on redshift. If the
if redshift is None:
z0 = np.arange(zmnx[0],zmnx[1],z_wind)
z1 = z0 + z_wind
else:
if np.isscalar(redshift):
z0 = np.array([redshift])
else:
z0 = redshift.copy()
z1 = z0.copy() #+ z_wind
pca_list = ['PCA0', 'PCA1', 'PCA2', 'PCA3']
PCA_mean = np.zeros(4)
PCA_sig = np.zeros(4)
PCA_rand = np.zeros((4,N_perz*ipad))
final_spec = np.zeros((npix, N_perz * len(z0)))
final_wave = np.zeros((npix, N_perz * len(z0)))
final_z = np.zeros(N_perz * len(z0))
# Random state
if rstate is None:
rstate = np.random.RandomState(seed)
for ii in range(len(z0)):
# BOSS or SDSS?
if z0[ii] > 2.15:
zQSO = boss_zQSO
pca_coeff = boss_pca_coeff
else:
zQSO = sdss_zQSO
pca_coeff = sdss_pca_coeff
# Random z values and wavelengths
zrand = rstate.uniform( z0[ii], z1[ii], N_perz*ipad)
wave = np.outer(eigen_wave, 1+zrand)
# MFP (Worseck+14)
mfp = 37. * ( (1+zrand)/5. )**(-5.4) # Physical Mpc
# Grab PCA mean + sigma
if redshift is None:
idx = np.where( (zQSO >= z0[ii]) & (zQSO < z1[ii]) )[0]
else:
# Hack by @moustakas: add a little jitter to get the set of QSOs
# that are *nearest* in redshift to the desired output redshift.
idx = np.where( (zQSO >= z0[ii]-0.01) & (zQSO < z1[ii]+0.01) )[0]
if len(idx) == 0:
idx = np.array([(np.abs(zQSO-zrand[0])).argmin()])
#pdb.set_trace()
log.debug('Making z=({:g},{:g}) with {:d} input quasars'.format(z0[ii],z1[ii],len(idx)))
# Get PCA stats and random values
for jj,ipca in enumerate(pca_list):
if jj == 0: # Use bounds for PCA0 [avoids negative values]
xmnx = perc(pca_coeff[ipca][idx], per=95)
PCA_rand[jj, :] = rstate.uniform(xmnx[0], xmnx[1], N_perz*ipad)
else:
PCA_mean[jj] = np.mean(pca_coeff[ipca][idx])
PCA_sig[jj] = np.std(pca_coeff[ipca][idx])
# Draws
PCA_rand[jj, :] = rstate.uniform( PCA_mean[jj] - 2*PCA_sig[jj],
PCA_mean[jj] + 2*PCA_sig[jj], N_perz*ipad)
# Generate the templates (ipad*N_perz)
spec = np.dot(eigen.T, PCA_rand)
# Take first good N_perz
# Truncate, MFP, Fill
ngd = 0
nbad = 0
for kk in range(ipad*N_perz):
# Any zero values?
mn = np.min(spec[chkpix, kk])
if mn < 0.:
nbad += 1
continue
# MFP
if z0[ii] > 2.39:
z912 = wave[0:pix912,kk]/lambda_912 - 1.
phys_dist = np.fabs( cosmo.lookback_distance(z912) -
cosmo.lookback_distance(zrand[kk]) ) # Mpc
spec[0:pix912, kk] = spec[0:pix912,kk] * np.exp(-phys_dist.value/mfp[kk])
# Write
final_spec[:, ii*N_perz+ngd] = spec[:,kk]
final_wave[:, ii*N_perz+ngd] = wave[:,kk]
final_z[ii*N_perz+ngd] = zrand[kk]
ngd += 1
if ngd == N_perz:
break
if ngd != N_perz:
print('Did not make enough!')
#pdb.set_trace()
log.warning('Did not make enough qso templates. ngd = {}, N_perz = {}'.format(ngd,N_perz))
# Rebin
if rebin_wave is None:
light = 2.99792458e5 # [km/s]
velpixsize = 10. # [km/s]
pixsize = velpixsize/light/np.log(10) # [pixel size in log-10 A]
minwave = np.log10(wvmnx[0]) # minimum wavelength [log10-A]
maxwave = np.log10(wvmnx[1]) # maximum wavelength [log10-A]
r_npix = np.round((maxwave-minwave)/pixsize+1)
log_wave = minwave+np.arange(r_npix)*pixsize # constant log-10 spacing
else:
log_wave = np.log10(rebin_wave)
r_npix = len(log_wave)
totN = N_perz * len(z0)
rebin_spec = np.zeros((r_npix, totN))
for ii in range(totN):
# Interpolate (in log space)
rebin_spec[:, ii] = resample_flux(log_wave, np.log10(final_wave[:, ii]), final_spec[:, ii])
#f1d = interp1d(np.log10(final_wave[:,ii]), final_spec[:,ii])
#rebin_spec[:,ii] = f1d(log_wave)
if outfil is None:
return 10.**log_wave, rebin_spec, final_z
# Transpose for consistency
out_spec = np.array(rebin_spec.T, dtype='float32')
# Write
hdu = fits.PrimaryHDU(out_spec)
hdu.header.set('PROJECT', 'DESI QSO TEMPLATES')
hdu.header.set('VERSION', '1.1')
hdu.header.set('OBJTYPE', 'QSO')
hdu.header.set('DISPAXIS', 1, 'dispersion axis')
hdu.header.set('CRPIX1', 1, 'reference pixel number')
hdu.header.set('CRVAL1', minwave, 'reference log10(Ang)')
hdu.header.set('CDELT1', pixsize, 'delta log10(Ang)')
hdu.header.set('LOGLAM', 1, 'log10 spaced wavelengths?')
hdu.header.set('AIRORVAC', 'vac', ' wavelengths in vacuum (vac) or air')
hdu.header.set('VELSCALE', velpixsize, ' pixel size in km/s')
hdu.header.set('WAVEUNIT', 'Angstrom', ' wavelength units')
hdu.header.set('BUNIT', '1e-17 erg/s/cm2/A', ' flux unit')
idval = list(range(totN))
col0 = fits.Column(name=str('TEMPLATEID'),format=str('J'), array=idval)
col1 = fits.Column(name=str('Z'),format=str('E'),array=final_z)
cols = fits.ColDefs([col0, col1])
tbhdu = fits.BinTableHDU.from_columns(cols)
tbhdu.header.set('EXTNAME','METADATA')
hdulist = fits.HDUList([hdu, tbhdu])
hdulist.writeto(outfil, clobber=True)
return final_wave, final_spec, final_z
def main(args):
# Set up the logger
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
# Make sure all necessary environment variables are set
setup_envs()
# Initialize random number generator to use.
np.random.seed(args.seed)
random_state = np.random.RandomState(args.seed)
# Derive spectrograph number from nstart if needed
if args.spectrograph is None:
args.spectrograph = args.nstart / args.n_fibers
# Read fibermapfile to get object type, night and expid
fibermap, objtype, night, expid = get_fibermap(args.fibermap,
log=log,
nspec=args.nspec)
# Initialize the spectral simulator
log.info("Initializing SpecSim with config {}".format(args.config))
lvmparams = load_lvmparams(config=args.config, telescope=args.telescope)
qsim = get_simulator(args.config, num_fibers=1, params=lvmparams)
if args.simspec:
# Read the input file
log.info('Reading input file {}'.format(args.simspec))
simspec = lvmsim.io.read_simspec(args.simspec)
nspec = simspec.nspec
if simspec.flavor == 'arc':
# - TODO: do we need quickgen to support arcs? For full pipeline
# - arcs are used to measure PSF but aren't extracted except for
# - debugging.
# - TODO: if we do need arcs, this needs to be redone.
# - conversion from phot to flux doesn't include throughput,
# - and arc lines are rebinned to nearest 0.2 A.
# Create full wavelength and flux arrays for arc exposure
wave_b = np.array(simspec.wave['b'])
wave_r = np.array(simspec.wave['r'])
wave_z = np.array(simspec.wave['z'])
phot_b = np.array(simspec.phot['b'][0])
phot_r = np.array(simspec.phot['r'][0])
phot_z = np.array(simspec.phot['z'][0])
sim_wave = np.concatenate((wave_b, wave_r, wave_z))
sim_phot = np.concatenate((phot_b, phot_r, phot_z))
wavelengths = np.arange(3533., 9913.1, 0.2)
phot = np.zeros(len(wavelengths))
for i in range(len(sim_wave)):
wavelength = sim_wave[i]
flux_index = np.argmin(abs(wavelength - wavelengths))
phot[flux_index] = sim_phot[i]
# Convert photons to flux: following specter conversion method
dw = np.gradient(wavelengths)
exptime = 5. # typical BOSS exposure time in s
fibarea = const.pi * (1.07e-2 /
2)**2 # cross-sectional fiber area in cm^2
hc = 1.e17 * const.h * const.c # convert to erg A
spectra = (hc * exptime * fibarea * dw * phot) / wavelengths
else:
wavelengths = simspec.wave['brz']
spectra = simspec.flux
if nspec < args.nspec:
log.info("Only {} spectra in input file".format(nspec))
args.nspec = nspec
else:
# Initialize the output truth table.
spectra = []
wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value
npix = len(wavelengths)
truth = dict()
meta = Table()
truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10))
truth['FLUX'] = np.zeros((args.nspec, npix))
truth['WAVE'] = wavelengths
jj = list()
for thisobj in set(true_objtype):
ii = np.where(true_objtype == thisobj)[0]
nobj = len(ii)
truth['OBJTYPE'][ii] = thisobj
log.info('Generating {} template'.format(thisobj))
# Generate the templates
if thisobj == 'ELG':
elg = lvmsim.templates.ELG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = elg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_elg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'LRG':
lrg = lvmsim.templates.LRG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = lrg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_lrg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'QSO':
qso = lvmsim.templates.QSO(wave=wavelengths)
flux, tmpwave, meta1 = qso.make_templates(
nmodel=nobj, seed=args.seed, zrange=args.zrange_qso)
elif thisobj == 'BGS':
bgs = lvmsim.templates.BGS(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = bgs.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_bgs,
rmagrange=args.rmagrange_bgs,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'STD':
fstd = lvmsim.templates.FSTD(wave=wavelengths)
flux, tmpwave, meta1 = fstd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'QSO_BAD': # use STAR template no color cuts
star = lvmsim.templates.STAR(wave=wavelengths)
flux, tmpwave, meta1 = star.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'MWS_STAR' or thisobj == 'MWS':
mwsstar = lvmsim.templates.MWS_STAR(wave=wavelengths)
flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'WD':
wd = lvmsim.templates.WD(wave=wavelengths)
flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'SKY':
flux = np.zeros((nobj, npix))
meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32)))
elif thisobj == 'TEST':
flux = np.zeros((args.nspec, npix))
indx = np.where(wave > 5800.0 - 1E-6)[0][0]
ref_integrated_flux = 1E-10
ref_cst_flux_density = 1E-17
single_line = (np.arange(args.nspec) % 2 == 0).astype(
np.float32)
continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32)
for spec in range(args.nspec):
flux[spec, indx] = single_line[
spec] * ref_integrated_flux / np.gradient(wavelengths)[
indx] # single line
flux[spec] += continuum[
spec] * ref_cst_flux_density # flat continuum
meta1 = Table(
dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32),
LINE=wave[indx] *
np.ones(args.nspec, dtype=np.float32),
LINEFLUX=single_line * ref_integrated_flux,
CONSTFLUXDENSITY=continuum * ref_cst_flux_density))
else:
log.fatal('Unknown object type {}'.format(thisobj))
sys.exit(1)
# Pack it in.
truth['FLUX'][ii] = flux
meta = vstack([meta, meta1])
jj.append(ii.tolist())
# Sanity check on units; templates currently return ergs, not 1e-17 ergs...
# assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6)
# Sort the metadata table.
jj = sum(jj, [])
meta_new = Table()
for k in range(args.nspec):
index = int(np.where(np.array(jj) == k)[0])
meta_new = vstack([meta_new, meta[index]])
meta = meta_new
# Add TARGETID and the true OBJTYPE to the metadata table.
meta.add_column(
Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE'))
meta.add_column(Column(targetids, name='TARGETID'))
# Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z
meta.rename_column('REDSHIFT', 'TRUEZ')
# ---------- end simspec
# explicitly set location on focal plane if needed to support airmass
# variations when using specsim v0.5
if qsim.source.focal_xy is None:
qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm'))
# Set simulation parameters from the simspec header or lvmparams
bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX']
gray_objects = ['gray', 'grey']
if args.simspec is None:
object_type = objtype
flavor = None
elif simspec.flavor == 'science':
object_type = None
flavor = simspec.header['PROGRAM']
else:
object_type = None
flavor = simspec.flavor
log.warning(
'Maybe using an outdated simspec file with flavor={}'.format(
flavor))
# Set airmass
if args.airmass is not None:
qsim.atmosphere.airmass = args.airmass
elif args.simspec and 'AIRMASS' in simspec.header:
qsim.atmosphere.airmass = simspec.header['AIRMASS']
else:
qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2
# Set site location
if args.location is not None:
qsim.observation.observatory = args.location
else:
qsim.observation.observatory = 'APO'
# Set exptime
if args.exptime is not None:
qsim.observation.exposure_time = args.exptime * u.s
elif args.simspec and 'EXPTIME' in simspec.header:
qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s
elif objtype in bright_objects:
qsim.observation.exposure_time = lvmparams['exptime_bright'] * u.s
else:
qsim.observation.exposure_time = lvmparams['exptime_dark'] * u.s
# Set Moon Phase
if args.moon_phase is not None:
qsim.atmosphere.moon.moon_phase = args.moon_phase
elif args.simspec and 'MOONFRAC' in simspec.header:
qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC']
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_phase = 0.7
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_phase = 0.1
else:
qsim.atmosphere.moon.moon_phase = 0.5
# Set Moon Zenith
if args.moon_zenith is not None:
qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg
elif args.simspec and 'MOONALT' in simspec.header:
qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_zenith = 30 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_zenith = 80 * u.deg
else:
qsim.atmosphere.moon.moon_zenith = 100 * u.deg
# Set Moon - Object Angle
if args.moon_angle is not None:
qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg
elif args.simspec and 'MOONSEP' in simspec.header:
qsim.atmosphere.moon.separation_angle = simspec.header[
'MOONSEP'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.separation_angle = 50 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
else:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
# Initialize per-camera output arrays that will be saved
waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {}
maxbin = 0
nmax = args.nspec
for camera in qsim.instrument.cameras:
# Lookup this camera's resolution matrix and convert to the sparse format used in lvmspec.
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(),
[args.nspec, 1, 1])
waves[camera.name] = (camera.output_wavelength.to(
u.Angstrom).value.astype(np.float32))
nwave = len(waves[camera.name])
maxbin = max(maxbin, len(waves[camera.name]))
nobj = np.zeros((nmax, 3, maxbin)) # object photons
nsky = np.zeros((nmax, 3, maxbin)) # sky photons
nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky)
cframe_observedflux = np.zeros(
(nmax, 3, maxbin)) # calibrated object flux
cframe_ivar = np.zeros(
(nmax, 3, maxbin)) # inverse variance of calibrated object flux
cframe_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to calibrated flux
sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky
sky_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to sky only
frame_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky
trueflux[camera.name] = np.empty(
(args.nspec, nwave)) # calibrated flux
noisyflux[camera.name] = np.empty(
(args.nspec, nwave)) # observed flux with noise
obsivar[camera.name] = np.empty(
(args.nspec, nwave)) # inverse variance of flux
if args.simspec:
dw = np.gradient(simspec.wave[camera.name])
else:
sflux = np.empty((args.nspec, npix))
# - Check if input simspec is for a continuum flat lamp instead of science
# - This does not convolve to per-fiber resolution
if args.simspec:
if simspec.flavor == 'flat':
log.info("Simulating flat lamp exposure")
for i, camera in enumerate(qsim.instrument.cameras):
channel = camera.name
assert camera.output_wavelength.unit == u.Angstrom
num_pixels = len(waves[channel])
dw = np.gradient(simspec.wave[channel])
meanspec = resample_flux(
waves[channel], simspec.wave[channel],
np.average(simspec.phot[channel] / dw, axis=0))
fiberflat = random_state.normal(loc=1.0,
scale=1.0 / np.sqrt(meanspec),
size=(nspec, num_pixels))
ivar = np.tile(meanspec, [nspec, 1])
mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32)
for kk in range((args.nspec + args.nstart - 1) //
args.n_fibers + 1):
camera = channel + str(kk)
outfile = lvmspec.io.findfile('fiberflat', night, expid,
camera)
start = max(args.n_fibers * kk, args.nstart)
end = min(args.n_fibers * (kk + 1), nmax)
if (args.spectrograph <= kk):
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, kk, start, end))
ff = FiberFlat(waves[channel],
fiberflat[start:end, :],
ivar[start:end, :],
mask[start:end, :],
meanspec,
header=dict(CAMERA=camera))
write_fiberflat(outfile, ff)
filePath = lvmspec.io.findfile("fiberflat", night, expid,
camera)
log.info("Wrote file {}".format(filePath))
sys.exit(0)
# Repeat the simulation for all spectra
scale = 1e-17
fluxunits = scale * u.erg / (u.s * u.cm**2 * u.Angstrom)
for j in range(args.nspec):
thisobjtype = objtype[j]
sys.stdout.flush()
if flavor == 'arc':
qsim.source.update_in('Quickgen source {0}'.format, 'perfect',
wavelengths * u.Angstrom,
spectra * fluxunits)
else:
qsim.source.update_in('Quickgen source {0}'.format(j),
thisobjtype.lower(),
wavelengths * u.Angstrom,
spectra[j, :] * fluxunits)
qsim.source.update_out()
qsim.simulate()
qsim.generate_random_noise(random_state)
for i, output in enumerate(qsim.camera_output):
assert output['observed_flux'].unit == 1e17 * fluxunits
# Extract the simulation results needed to create our uncalibrated
# frame output file.
num_pixels = len(output)
nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0]
nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0]
nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0]
# Get results for our flux-calibrated output file.
cframe_observedflux[
j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0]
cframe_ivar[
j,
i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0]
# Fill brick arrays from the results.
camera = output.meta['name']
trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0]
noisyflux[camera][j][:] = 1e17 * (
output['observed_flux'][:, 0] +
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,
0]
# Use the same noise realization in the cframe and frame, without any
# additional noise from sky subtraction for now.
frame_rand_noise[
j, i, :num_pixels] = output['random_noise_electrons'][:, 0]
cframe_rand_noise[j, i, :num_pixels] = 1e17 * (
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
# The sky output file represents a model fit to ~40 sky fibers.
# We reduce the variance by a factor of 25 to account for this and
# give the sky an independent (Gaussian) noise realization.
sky_ivar[
j,
i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] -
output['num_source_electrons'][:, 0])
sky_rand_noise[j, i, :num_pixels] = random_state.normal(
scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]),
size=num_pixels)
armName = {"b": 0, "r": 1, "z": 2}
for channel in 'brz':
# Before writing, convert from counts/bin to counts/A (as in Pixsim output)
# Quicksim Default:
# FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang]
# COUNTS_OBJ - object counts in 0.5 Ang bin
# COUNTS_SKY - sky counts in 0.5 Ang bin
num_pixels = len(waves[channel])
dwave = np.gradient(waves[channel])
nobj[:, armName[channel], :num_pixels] /= dwave
frame_rand_noise[:, armName[channel], :num_pixels] /= dwave
nivar[:, armName[channel], :num_pixels] *= dwave**2
nsky[:, armName[channel], :num_pixels] /= dwave
sky_rand_noise[:, armName[channel], :num_pixels] /= dwave
sky_ivar[:, armName[channel], :num_pixels] /= dwave**2
# Now write the outputs in DESI standard file system. None of the output file can have more than args.n_fibers spectra
# Looping over spectrograph
for ii in range((args.nspec + args.nstart - 1) // args.n_fibers + 1):
start = max(args.n_fibers * ii,
args.nstart) # first spectrum for a given spectrograph
end = min(args.n_fibers * (ii + 1),
nmax) # last spectrum for the spectrograph
if (args.spectrograph <= ii):
camera = "{}{}".format(channel, ii)
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, ii, start, end))
num_pixels = len(waves[channel])
# Write frame file
framefileName = lvmspec.io.findfile("frame", night, expid,
camera)
frame_flux = nobj[start:end, armName[channel], :num_pixels] + \
nsky[start:end, armName[channel], :num_pixels] + \
frame_rand_noise[start:end, armName[channel], :num_pixels]
frame_ivar = nivar[start:end, armName[channel], :num_pixels]
# required for slicing the resolution metric, resolusion matrix has (nspec, ndiag, wave)
# for example if nstart =400, nspec=150: two spectrographs:
# 400-499=> 0 spectrograph, 500-549 => 1
sh1 = frame_flux.shape[0]
if (args.nstart == start):
resol = resolution[channel][:sh1, :, :]
else:
resol = resolution[channel][-sh1:, :, :]
# must create lvmspec.Frame object
frame = Frame(waves[channel],
frame_flux,
frame_ivar,
resolution_data=resol,
spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor))
lvmspec.io.write_frame(framefileName, frame)
framefilePath = lvmspec.io.findfile("frame", night, expid,
camera)
log.info("Wrote file {}".format(framefilePath))
if args.frameonly or simspec.flavor == 'arc':
continue
# Write cframe file
cframeFileName = lvmspec.io.findfile("cframe", night, expid,
camera)
cframeFlux = cframe_observedflux[start:end, armName[channel], :num_pixels] + \
cframe_rand_noise[start:end, armName[channel], :num_pixels]
cframeIvar = cframe_ivar[start:end,
armName[channel], :num_pixels]
# must create lvmspec.Frame object
cframe = Frame(waves[channel],
cframeFlux,
cframeIvar,
resolution_data=resol,
spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor))
lvmspec.io.frame.write_frame(cframeFileName, cframe)
cframefilePath = lvmspec.io.findfile("cframe", night, expid,
camera)
log.info("Wrote file {}".format(cframefilePath))
# Write sky file
skyfileName = lvmspec.io.findfile("sky", night, expid, camera)
skyflux = nsky[start:end, armName[channel], :num_pixels] + \
sky_rand_noise[start:end, armName[channel], :num_pixels]
skyivar = sky_ivar[start:end, armName[channel], :num_pixels]
skymask = np.zeros(skyflux.shape, dtype=np.uint32)
# must create lvmspec.Sky object
skymodel = SkyModel(waves[channel],
skyflux,
skyivar,
skymask,
header=dict(CAMERA=camera))
lvmspec.io.sky.write_sky(skyfileName, skymodel)
skyfilePath = lvmspec.io.findfile("sky", night, expid, camera)
log.info("Wrote file {}".format(skyfilePath))
# Write calib file
calibVectorFile = lvmspec.io.findfile("calib", night, expid,
camera)
flux = cframe_observedflux[start:end,
armName[channel], :num_pixels]
phot = nobj[start:end, armName[channel], :num_pixels]
calibration = np.zeros_like(phot)
jj = (flux > 0)
calibration[jj] = phot[jj] / flux[jj]
# - TODO: what should calibivar be?
# - For now, model it as the noise of combining ~10 spectra
calibivar = 10 / cframe_ivar[start:end,
armName[channel], :num_pixels]
# mask=(1/calibivar>0).astype(int)??
mask = np.zeros(calibration.shape, dtype=np.uint32)
# write flux calibration
fluxcalib = FluxCalib(waves[channel], calibration, calibivar,
mask)
write_flux_calibration(calibVectorFile, fluxcalib)
calibfilePath = lvmspec.io.findfile("calib", night, expid,
camera)
log.info("Wrote file {}".format(calibfilePath))