def setUp(self):
self.nspec = 5
self.nwave = 10
self.wave = np.arange(self.nwave)
self.fiberflat = np.random.uniform(size=(self.nspec, self.nwave))
self.ivar = np.ones(self.fiberflat.shape)
self.mask = np.zeros(self.fiberflat.shape, dtype=np.uint32)
self.meanspec = np.random.uniform(size=self.nwave)
self.ff = FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask,
self.meanspec)
def run_pa(self, frame, outputfile):
from lvmspec.fiberflat import FiberFlat
import lvmspec.io.fiberflat as ffIO
from lvmspec.linalg import cholesky_solve
nwave = frame.nwave
nfibers = frame.nspec
wave = frame.wave #- this will become part of output too
flux = frame.flux
sumFlux = np.zeros((nwave))
realFlux = np.zeros(flux.shape)
ivar = frame.ivar * (frame.mask == 0)
#deconv
for fib in range(nfibers):
Rf = frame.R[fib].todense()
B = flux[fib]
realFlux[fib] = cholesky_solve(Rf, B)
sumFlux += realFlux[fib]
#iflux=nfibers/sumFlux
flat = np.zeros(flux.shape)
flat_ivar = np.zeros(ivar.shape)
avg = sumFlux / nfibers
for fib in range(nfibers):
Rf = frame.R[fib]
# apply and reconvolute
M = Rf.dot(avg)
M0 = (M == 0)
flat[fib] = (~M0) * flux[fib] / (M + M0) + M0
flat_ivar[fib] = ivar[fib] * M**2
fibflat = FiberFlat(frame.wave.copy(), flat, flat_ivar,
frame.mask.copy(), avg)
#fiberflat=compute_fiberflat(input_frame)
ffIO.write_fiberflat(outputfile, fibflat, header=frame.meta)
log.info("Wrote fiberflat file {}".format(outputfile))
def _write_fiberflat(self):
"""Write a fake fiberflat"""
fiberflat = np.ones((self.nspec, self.nwave))
ivar = np.ones((self.nspec, self.nwave))
mask = np.zeros((self.nspec, self.nwave), dtype=int)
meanspec = np.ones(self.nwave)
ff = FiberFlat(self.wave, fiberflat, ivar, mask, meanspec)
io.write_fiberflat(self.fiberflatfile, ff)
def get_fiberflat_from_frame(frame):
from lvmspec.fiberflat import FiberFlat
flux = frame.flux
fiberflat = np.ones_like(flux)
ffivar = 2 * np.ones_like(flux)
fiberflat[0] *= 0.8
fiberflat[1] *= 1.2
ff = FiberFlat(frame.wave, fiberflat, ffivar)
return ff
def test_apply_fiberflat_ivar(self):
'''test error propagation in apply_fiberflat'''
wave = np.arange(5000, 5010)
nwave = len(wave)
nspec = 3
flux = np.random.uniform(0.9, 1.0, size=(nspec, nwave))
ivar = np.ones_like(flux)
origframe = Frame(wave, flux, ivar, spectrograph=0)
fiberflat = np.ones_like(flux)
ffmask = np.zeros_like(flux)
fiberflat[0] *= 0.5
fiberflat[1] *= 1.5
#- ff with essentially no error
ffivar = 1e20 * np.ones_like(flux)
ff = FiberFlat(wave, fiberflat, ffivar)
frame = copy.deepcopy(origframe)
apply_fiberflat(frame, ff)
self.assertTrue(np.allclose(frame.ivar, fiberflat**2))
#- ff with large error
ffivar = np.ones_like(flux)
ff = FiberFlat(wave, fiberflat, ffivar)
frame = copy.deepcopy(origframe)
apply_fiberflat(frame, ff)
#- c = a/b
#- (sigma_c/c)^2 = (sigma_a/a)^2 + (sigma_b/b)^2
var = frame.flux**2 * (1.0/(origframe.ivar * origframe.flux**2) + \
1.0/(ff.ivar * ff.fiberflat**2))
self.assertTrue(np.allclose(frame.ivar, 1 / var))
#- ff.ivar=0 should result in frame.ivar=0, even if ff.fiberflat=0 too
ffivar = np.ones_like(flux)
ffivar[0, 0:5] = 0.0
fiberflat[0, 0:5] = 0.0
ff = FiberFlat(wave, fiberflat, ffivar)
frame = copy.deepcopy(origframe)
apply_fiberflat(frame, ff)
self.assertTrue(np.all(frame.ivar[0, 0:5] == 0.0))
def test_dimensions(self):
#- check dimensionality mismatches
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.wave, self.ivar, self.mask,
self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.wave, self.ivar, self.mask,
self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask,
self.fiberflat)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat[0:2], self.ivar, self.mask,
self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.fiberflat, self.fiberflat, self.ivar, self.mask,
self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.wave, self.mask,
self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask[0:2, :],
self.meanspec)
fibers = np.arange(self.nspec)
FiberFlat(self.wave,
self.fiberflat,
self.ivar,
self.mask,
self.meanspec,
fibers=fibers)
with self.assertRaises(ValueError):
FiberFlat(self.wave,
self.fiberflat,
self.ivar,
self.mask,
self.meanspec,
fibers=fibers[1:])
def test_apply_fiberflat(self):
'''test apply_fiberflat interface and changes to flux and mask'''
wave = np.arange(5000, 5050)
nwave = len(wave)
nspec = 3
flux = np.random.uniform(size=(nspec, nwave))
ivar = np.ones_like(flux)
frame = Frame(wave, flux, ivar, spectrograph=0)
fiberflat = np.ones_like(flux)
ffivar = 2 * np.ones_like(flux)
ffmask = np.zeros_like(flux)
fiberflat[0] *= 0.8
fiberflat[1] *= 1.2
fiberflat[2, 0:10] = 0 #- bad fiberflat
ffivar[2, 10:20] = 0 #- bad fiberflat
ffmask[2, 20:30] = 1 #- bad fiberflat
ff = FiberFlat(wave, fiberflat, ffivar)
origframe = copy.deepcopy(frame)
apply_fiberflat(frame, ff)
#- was fiberflat applied?
self.assertTrue(np.all(frame.flux[0] == origframe.flux[0] / 0.8))
self.assertTrue(np.all(frame.flux[1] == origframe.flux[1] / 1.2))
self.assertTrue(np.all(frame.flux[2] == origframe.flux[2]))
#- did mask get set?
ii = (ff.fiberflat == 0)
self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0))
ii = (ff.ivar == 0)
self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0))
ii = (ff.mask != 0)
self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0))
#- Should fail if frame and ff don't have a common wavelength grid
frame.wave = frame.wave + 0.1
with self.assertRaises(ValueError):
apply_fiberflat(frame, ff)
def main(args):
# Set up the logger
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
# Make sure all necessary environment variables are set
setup_envs()
# Initialize random number generator to use.
np.random.seed(args.seed)
random_state = np.random.RandomState(args.seed)
# Derive spectrograph number from nstart if needed
if args.spectrograph is None:
args.spectrograph = args.nstart / args.n_fibers
# Read fibermapfile to get object type, night and expid
fibermap, objtype, night, expid = get_fibermap(args.fibermap,
log=log,
nspec=args.nspec)
# Initialize the spectral simulator
log.info("Initializing SpecSim with config {}".format(args.config))
lvmparams = load_lvmparams(config=args.config, telescope=args.telescope)
qsim = get_simulator(args.config, num_fibers=1, params=lvmparams)
if args.simspec:
# Read the input file
log.info('Reading input file {}'.format(args.simspec))
simspec = lvmsim.io.read_simspec(args.simspec)
nspec = simspec.nspec
if simspec.flavor == 'arc':
# - TODO: do we need quickgen to support arcs? For full pipeline
# - arcs are used to measure PSF but aren't extracted except for
# - debugging.
# - TODO: if we do need arcs, this needs to be redone.
# - conversion from phot to flux doesn't include throughput,
# - and arc lines are rebinned to nearest 0.2 A.
# Create full wavelength and flux arrays for arc exposure
wave_b = np.array(simspec.wave['b'])
wave_r = np.array(simspec.wave['r'])
wave_z = np.array(simspec.wave['z'])
phot_b = np.array(simspec.phot['b'][0])
phot_r = np.array(simspec.phot['r'][0])
phot_z = np.array(simspec.phot['z'][0])
sim_wave = np.concatenate((wave_b, wave_r, wave_z))
sim_phot = np.concatenate((phot_b, phot_r, phot_z))
wavelengths = np.arange(3533., 9913.1, 0.2)
phot = np.zeros(len(wavelengths))
for i in range(len(sim_wave)):
wavelength = sim_wave[i]
flux_index = np.argmin(abs(wavelength - wavelengths))
phot[flux_index] = sim_phot[i]
# Convert photons to flux: following specter conversion method
dw = np.gradient(wavelengths)
exptime = 5. # typical BOSS exposure time in s
fibarea = const.pi * (1.07e-2 /
2)**2 # cross-sectional fiber area in cm^2
hc = 1.e17 * const.h * const.c # convert to erg A
spectra = (hc * exptime * fibarea * dw * phot) / wavelengths
else:
wavelengths = simspec.wave['brz']
spectra = simspec.flux
if nspec < args.nspec:
log.info("Only {} spectra in input file".format(nspec))
args.nspec = nspec
else:
# Initialize the output truth table.
spectra = []
wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value
npix = len(wavelengths)
truth = dict()
meta = Table()
truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10))
truth['FLUX'] = np.zeros((args.nspec, npix))
truth['WAVE'] = wavelengths
jj = list()
for thisobj in set(true_objtype):
ii = np.where(true_objtype == thisobj)[0]
nobj = len(ii)
truth['OBJTYPE'][ii] = thisobj
log.info('Generating {} template'.format(thisobj))
# Generate the templates
if thisobj == 'ELG':
elg = lvmsim.templates.ELG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = elg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_elg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'LRG':
lrg = lvmsim.templates.LRG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = lrg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_lrg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'QSO':
qso = lvmsim.templates.QSO(wave=wavelengths)
flux, tmpwave, meta1 = qso.make_templates(
nmodel=nobj, seed=args.seed, zrange=args.zrange_qso)
elif thisobj == 'BGS':
bgs = lvmsim.templates.BGS(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = bgs.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_bgs,
rmagrange=args.rmagrange_bgs,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'STD':
fstd = lvmsim.templates.FSTD(wave=wavelengths)
flux, tmpwave, meta1 = fstd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'QSO_BAD': # use STAR template no color cuts
star = lvmsim.templates.STAR(wave=wavelengths)
flux, tmpwave, meta1 = star.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'MWS_STAR' or thisobj == 'MWS':
mwsstar = lvmsim.templates.MWS_STAR(wave=wavelengths)
flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'WD':
wd = lvmsim.templates.WD(wave=wavelengths)
flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'SKY':
flux = np.zeros((nobj, npix))
meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32)))
elif thisobj == 'TEST':
flux = np.zeros((args.nspec, npix))
indx = np.where(wave > 5800.0 - 1E-6)[0][0]
ref_integrated_flux = 1E-10
ref_cst_flux_density = 1E-17
single_line = (np.arange(args.nspec) % 2 == 0).astype(
np.float32)
continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32)
for spec in range(args.nspec):
flux[spec, indx] = single_line[
spec] * ref_integrated_flux / np.gradient(wavelengths)[
indx] # single line
flux[spec] += continuum[
spec] * ref_cst_flux_density # flat continuum
meta1 = Table(
dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32),
LINE=wave[indx] *
np.ones(args.nspec, dtype=np.float32),
LINEFLUX=single_line * ref_integrated_flux,
CONSTFLUXDENSITY=continuum * ref_cst_flux_density))
else:
log.fatal('Unknown object type {}'.format(thisobj))
sys.exit(1)
# Pack it in.
truth['FLUX'][ii] = flux
meta = vstack([meta, meta1])
jj.append(ii.tolist())
# Sanity check on units; templates currently return ergs, not 1e-17 ergs...
# assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6)
# Sort the metadata table.
jj = sum(jj, [])
meta_new = Table()
for k in range(args.nspec):
index = int(np.where(np.array(jj) == k)[0])
meta_new = vstack([meta_new, meta[index]])
meta = meta_new
# Add TARGETID and the true OBJTYPE to the metadata table.
meta.add_column(
Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE'))
meta.add_column(Column(targetids, name='TARGETID'))
# Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z
meta.rename_column('REDSHIFT', 'TRUEZ')
# ---------- end simspec
# explicitly set location on focal plane if needed to support airmass
# variations when using specsim v0.5
if qsim.source.focal_xy is None:
qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm'))
# Set simulation parameters from the simspec header or lvmparams
bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX']
gray_objects = ['gray', 'grey']
if args.simspec is None:
object_type = objtype
flavor = None
elif simspec.flavor == 'science':
object_type = None
flavor = simspec.header['PROGRAM']
else:
object_type = None
flavor = simspec.flavor
log.warning(
'Maybe using an outdated simspec file with flavor={}'.format(
flavor))
# Set airmass
if args.airmass is not None:
qsim.atmosphere.airmass = args.airmass
elif args.simspec and 'AIRMASS' in simspec.header:
qsim.atmosphere.airmass = simspec.header['AIRMASS']
else:
qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2
# Set site location
if args.location is not None:
qsim.observation.observatory = args.location
else:
qsim.observation.observatory = 'APO'
# Set exptime
if args.exptime is not None:
qsim.observation.exposure_time = args.exptime * u.s
elif args.simspec and 'EXPTIME' in simspec.header:
qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s
elif objtype in bright_objects:
qsim.observation.exposure_time = lvmparams['exptime_bright'] * u.s
else:
qsim.observation.exposure_time = lvmparams['exptime_dark'] * u.s
# Set Moon Phase
if args.moon_phase is not None:
qsim.atmosphere.moon.moon_phase = args.moon_phase
elif args.simspec and 'MOONFRAC' in simspec.header:
qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC']
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_phase = 0.7
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_phase = 0.1
else:
qsim.atmosphere.moon.moon_phase = 0.5
# Set Moon Zenith
if args.moon_zenith is not None:
qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg
elif args.simspec and 'MOONALT' in simspec.header:
qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_zenith = 30 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_zenith = 80 * u.deg
else:
qsim.atmosphere.moon.moon_zenith = 100 * u.deg
# Set Moon - Object Angle
if args.moon_angle is not None:
qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg
elif args.simspec and 'MOONSEP' in simspec.header:
qsim.atmosphere.moon.separation_angle = simspec.header[
'MOONSEP'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.separation_angle = 50 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
else:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
# Initialize per-camera output arrays that will be saved
waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {}
maxbin = 0
nmax = args.nspec
for camera in qsim.instrument.cameras:
# Lookup this camera's resolution matrix and convert to the sparse format used in lvmspec.
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(),
[args.nspec, 1, 1])
waves[camera.name] = (camera.output_wavelength.to(
u.Angstrom).value.astype(np.float32))
nwave = len(waves[camera.name])
maxbin = max(maxbin, len(waves[camera.name]))
nobj = np.zeros((nmax, 3, maxbin)) # object photons
nsky = np.zeros((nmax, 3, maxbin)) # sky photons
nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky)
cframe_observedflux = np.zeros(
(nmax, 3, maxbin)) # calibrated object flux
cframe_ivar = np.zeros(
(nmax, 3, maxbin)) # inverse variance of calibrated object flux
cframe_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to calibrated flux
sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky
sky_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to sky only
frame_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky
trueflux[camera.name] = np.empty(
(args.nspec, nwave)) # calibrated flux
noisyflux[camera.name] = np.empty(
(args.nspec, nwave)) # observed flux with noise
obsivar[camera.name] = np.empty(
(args.nspec, nwave)) # inverse variance of flux
if args.simspec:
dw = np.gradient(simspec.wave[camera.name])
else:
sflux = np.empty((args.nspec, npix))
# - Check if input simspec is for a continuum flat lamp instead of science
# - This does not convolve to per-fiber resolution
if args.simspec:
if simspec.flavor == 'flat':
log.info("Simulating flat lamp exposure")
for i, camera in enumerate(qsim.instrument.cameras):
channel = camera.name
assert camera.output_wavelength.unit == u.Angstrom
num_pixels = len(waves[channel])
dw = np.gradient(simspec.wave[channel])
meanspec = resample_flux(
waves[channel], simspec.wave[channel],
np.average(simspec.phot[channel] / dw, axis=0))
fiberflat = random_state.normal(loc=1.0,
scale=1.0 / np.sqrt(meanspec),
size=(nspec, num_pixels))
ivar = np.tile(meanspec, [nspec, 1])
mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32)
for kk in range((args.nspec + args.nstart - 1) //
args.n_fibers + 1):
camera = channel + str(kk)
outfile = lvmspec.io.findfile('fiberflat', night, expid,
camera)
start = max(args.n_fibers * kk, args.nstart)
end = min(args.n_fibers * (kk + 1), nmax)
if (args.spectrograph <= kk):
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, kk, start, end))
ff = FiberFlat(waves[channel],
fiberflat[start:end, :],
ivar[start:end, :],
mask[start:end, :],
meanspec,
header=dict(CAMERA=camera))
write_fiberflat(outfile, ff)
filePath = lvmspec.io.findfile("fiberflat", night, expid,
camera)
log.info("Wrote file {}".format(filePath))
sys.exit(0)
# Repeat the simulation for all spectra
scale = 1e-17
fluxunits = scale * u.erg / (u.s * u.cm**2 * u.Angstrom)
for j in range(args.nspec):
thisobjtype = objtype[j]
sys.stdout.flush()
if flavor == 'arc':
qsim.source.update_in('Quickgen source {0}'.format, 'perfect',
wavelengths * u.Angstrom,
spectra * fluxunits)
else:
qsim.source.update_in('Quickgen source {0}'.format(j),
thisobjtype.lower(),
wavelengths * u.Angstrom,
spectra[j, :] * fluxunits)
qsim.source.update_out()
qsim.simulate()
qsim.generate_random_noise(random_state)
for i, output in enumerate(qsim.camera_output):
assert output['observed_flux'].unit == 1e17 * fluxunits
# Extract the simulation results needed to create our uncalibrated
# frame output file.
num_pixels = len(output)
nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0]
nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0]
nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0]
# Get results for our flux-calibrated output file.
cframe_observedflux[
j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0]
cframe_ivar[
j,
i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0]
# Fill brick arrays from the results.
camera = output.meta['name']
trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0]
noisyflux[camera][j][:] = 1e17 * (
output['observed_flux'][:, 0] +
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,
0]
# Use the same noise realization in the cframe and frame, without any
# additional noise from sky subtraction for now.
frame_rand_noise[
j, i, :num_pixels] = output['random_noise_electrons'][:, 0]
cframe_rand_noise[j, i, :num_pixels] = 1e17 * (
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
# The sky output file represents a model fit to ~40 sky fibers.
# We reduce the variance by a factor of 25 to account for this and
# give the sky an independent (Gaussian) noise realization.
sky_ivar[
j,
i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] -
output['num_source_electrons'][:, 0])
sky_rand_noise[j, i, :num_pixels] = random_state.normal(
scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]),
size=num_pixels)
armName = {"b": 0, "r": 1, "z": 2}
for channel in 'brz':
# Before writing, convert from counts/bin to counts/A (as in Pixsim output)
# Quicksim Default:
# FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang]
# COUNTS_OBJ - object counts in 0.5 Ang bin
# COUNTS_SKY - sky counts in 0.5 Ang bin
num_pixels = len(waves[channel])
dwave = np.gradient(waves[channel])
nobj[:, armName[channel], :num_pixels] /= dwave
frame_rand_noise[:, armName[channel], :num_pixels] /= dwave
nivar[:, armName[channel], :num_pixels] *= dwave**2
nsky[:, armName[channel], :num_pixels] /= dwave
sky_rand_noise[:, armName[channel], :num_pixels] /= dwave
sky_ivar[:, armName[channel], :num_pixels] /= dwave**2
# Now write the outputs in DESI standard file system. None of the output file can have more than args.n_fibers spectra
# Looping over spectrograph
for ii in range((args.nspec + args.nstart - 1) // args.n_fibers + 1):
start = max(args.n_fibers * ii,
args.nstart) # first spectrum for a given spectrograph
end = min(args.n_fibers * (ii + 1),
nmax) # last spectrum for the spectrograph
if (args.spectrograph <= ii):
camera = "{}{}".format(channel, ii)
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, ii, start, end))
num_pixels = len(waves[channel])
# Write frame file
framefileName = lvmspec.io.findfile("frame", night, expid,
camera)
frame_flux = nobj[start:end, armName[channel], :num_pixels] + \
nsky[start:end, armName[channel], :num_pixels] + \
frame_rand_noise[start:end, armName[channel], :num_pixels]
frame_ivar = nivar[start:end, armName[channel], :num_pixels]
# required for slicing the resolution metric, resolusion matrix has (nspec, ndiag, wave)
# for example if nstart =400, nspec=150: two spectrographs:
# 400-499=> 0 spectrograph, 500-549 => 1
sh1 = frame_flux.shape[0]
if (args.nstart == start):
resol = resolution[channel][:sh1, :, :]
else:
resol = resolution[channel][-sh1:, :, :]
# must create lvmspec.Frame object
frame = Frame(waves[channel],
frame_flux,
frame_ivar,
resolution_data=resol,
spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor))
lvmspec.io.write_frame(framefileName, frame)
framefilePath = lvmspec.io.findfile("frame", night, expid,
camera)
log.info("Wrote file {}".format(framefilePath))
if args.frameonly or simspec.flavor == 'arc':
continue
# Write cframe file
cframeFileName = lvmspec.io.findfile("cframe", night, expid,
camera)
cframeFlux = cframe_observedflux[start:end, armName[channel], :num_pixels] + \
cframe_rand_noise[start:end, armName[channel], :num_pixels]
cframeIvar = cframe_ivar[start:end,
armName[channel], :num_pixels]
# must create lvmspec.Frame object
cframe = Frame(waves[channel],
cframeFlux,
cframeIvar,
resolution_data=resol,
spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor))
lvmspec.io.frame.write_frame(cframeFileName, cframe)
cframefilePath = lvmspec.io.findfile("cframe", night, expid,
camera)
log.info("Wrote file {}".format(cframefilePath))
# Write sky file
skyfileName = lvmspec.io.findfile("sky", night, expid, camera)
skyflux = nsky[start:end, armName[channel], :num_pixels] + \
sky_rand_noise[start:end, armName[channel], :num_pixels]
skyivar = sky_ivar[start:end, armName[channel], :num_pixels]
skymask = np.zeros(skyflux.shape, dtype=np.uint32)
# must create lvmspec.Sky object
skymodel = SkyModel(waves[channel],
skyflux,
skyivar,
skymask,
header=dict(CAMERA=camera))
lvmspec.io.sky.write_sky(skyfileName, skymodel)
skyfilePath = lvmspec.io.findfile("sky", night, expid, camera)
log.info("Wrote file {}".format(skyfilePath))
# Write calib file
calibVectorFile = lvmspec.io.findfile("calib", night, expid,
camera)
flux = cframe_observedflux[start:end,
armName[channel], :num_pixels]
phot = nobj[start:end, armName[channel], :num_pixels]
calibration = np.zeros_like(phot)
jj = (flux > 0)
calibration[jj] = phot[jj] / flux[jj]
# - TODO: what should calibivar be?
# - For now, model it as the noise of combining ~10 spectra
calibivar = 10 / cframe_ivar[start:end,
armName[channel], :num_pixels]
# mask=(1/calibivar>0).astype(int)??
mask = np.zeros(calibration.shape, dtype=np.uint32)
# write flux calibration
fluxcalib = FluxCalib(waves[channel], calibration, calibivar,
mask)
write_flux_calibration(calibVectorFile, fluxcalib)
calibfilePath = lvmspec.io.findfile("calib", night, expid,
camera)
log.info("Wrote file {}".format(calibfilePath))
class TestFiberFlatObject(unittest.TestCase):
def setUp(self):
self.nspec = 5
self.nwave = 10
self.wave = np.arange(self.nwave)
self.fiberflat = np.random.uniform(size=(self.nspec, self.nwave))
self.ivar = np.ones(self.fiberflat.shape)
self.mask = np.zeros(self.fiberflat.shape, dtype=np.uint32)
self.meanspec = np.random.uniform(size=self.nwave)
self.ff = FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask,
self.meanspec)
def test_init(self):
for key in ('wave', 'fiberflat', 'ivar', 'mask', 'meanspec'):
x = self.ff.__getattribute__(key)
y = self.__getattribute__(key)
self.assertTrue(np.all(x == y), key)
self.assertEqual(self.nspec, self.ff.nspec)
self.assertEqual(self.nwave, self.ff.nwave)
def test_dimensions(self):
#- check dimensionality mismatches
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.wave, self.ivar, self.mask,
self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.wave, self.ivar, self.mask,
self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask,
self.fiberflat)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat[0:2], self.ivar, self.mask,
self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.fiberflat, self.fiberflat, self.ivar, self.mask,
self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.wave, self.mask,
self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask[0:2, :],
self.meanspec)
fibers = np.arange(self.nspec)
FiberFlat(self.wave,
self.fiberflat,
self.ivar,
self.mask,
self.meanspec,
fibers=fibers)
with self.assertRaises(ValueError):
FiberFlat(self.wave,
self.fiberflat,
self.ivar,
self.mask,
self.meanspec,
fibers=fibers[1:])
def test_slice(self):
x = self.ff[1]
x = self.ff[1:2]
x = self.ff[[1, 2, 3]]
x = self.ff[self.ff.fibers < 3]