def setUp(self):
self.nspec = 5
self.nwave = 10
self.wave = np.arange(self.nwave)
self.fiberflat = np.random.uniform(size=(self.nspec, self.nwave))
self.ivar = np.ones(self.fiberflat.shape)
self.mask = np.zeros(self.fiberflat.shape, dtype=np.uint32)
self.meanspec = np.random.uniform(size=self.nwave)
self.ff = FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.meanspec)
def test_dimensions(self):
#- check dimensionality mismatches
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.wave, self.ivar, self.mask, self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.wave, self.ivar, self.mask, self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.fiberflat)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat[0:2], self.ivar, self.mask, self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.fiberflat, self.fiberflat, self.ivar, self.mask, self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.wave, self.mask, self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask[0:2, :], self.meanspec)
fibers = np.arange(self.nspec)
FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.meanspec, fibers=fibers)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.meanspec, fibers=fibers[1:])
def test_dimensions(self):
#- check dimensionality mismatches
self.assertRaises(
ValueError, lambda x: FiberFlat(*x),
(self.wave, self.wave, self.ivar, self.mask, self.meanspec))
self.assertRaises(
ValueError, lambda x: FiberFlat(*x),
(self.wave, self.fiberflat, self.ivar, self.mask, self.fiberflat))
self.assertRaises(ValueError, lambda x: FiberFlat(*x),
(self.wave, self.fiberflat[0:2], self.ivar,
self.mask, self.meanspec))
def test_fiberflat_rw(self):
nspec, nwave, ndiag = 10, 20, 3
flat = np.random.uniform(size=(nspec, nwave))
ivar = np.random.uniform(size=(nspec, nwave))
mask = np.zeros(shape=(nspec, nwave), dtype=int)
meanspec = np.random.uniform(size=(nwave, ))
wave = np.arange(nwave)
ff = FiberFlat(wave, flat, ivar, mask, meanspec)
desispec.io.write_fiberflat(self.testfile, ff)
xff = desispec.io.read_fiberflat(self.testfile)
self.assertTrue(
np.all(ff.fiberflat.astype('f4').astype('f8') == xff.fiberflat))
self.assertTrue(np.all(ff.ivar.astype('f4').astype('f8') == xff.ivar))
self.assertTrue(np.all(ff.mask == xff.mask))
self.assertTrue(
np.all(ff.meanspec.astype('f4').astype('f8') == xff.meanspec))
self.assertTrue(np.all(ff.wave.astype('f4').astype('f8') == xff.wave))
self.assertTrue(xff.fiberflat.dtype.isnative)
self.assertTrue(xff.ivar.dtype.isnative)
self.assertTrue(xff.mask.dtype.isnative)
self.assertTrue(xff.meanspec.dtype.isnative)
self.assertTrue(xff.wave.dtype.isnative)
class TestFiberFlatObject(unittest.TestCase):
def setUp(self):
self.nspec = 5
self.nwave = 10
self.wave = np.arange(self.nwave)
self.fiberflat = np.random.uniform(size=(self.nspec, self.nwave))
self.ivar = np.ones(self.fiberflat.shape)
self.mask = np.zeros(self.fiberflat.shape)
self.meanspec = np.random.uniform(size=self.nwave)
self.ff = FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.meanspec)
def test_init(self):
for key in ('wave', 'fiberflat', 'ivar', 'mask', 'meanspec'):
x = self.ff.__getattribute__(key)
y = self.__getattribute__(key)
self.assertTrue(np.all(x == y), key)
self.assertEqual(self.nspec, self.ff.nspec)
self.assertEqual(self.nwave, self.ff.nwave)
def test_dimensions(self):
#- check dimensionality mismatches
self.assertRaises(ValueError, lambda x: FiberFlat(*x), (self.wave, self.wave, self.ivar, self.mask, self.meanspec))
self.assertRaises(ValueError, lambda x: FiberFlat(*x), (self.wave, self.fiberflat, self.ivar, self.mask, self.fiberflat))
self.assertRaises(ValueError, lambda x: FiberFlat(*x), (self.wave, self.fiberflat[0:2], self.ivar, self.mask, self.meanspec))
def test_slice(self):
x = self.ff[1]
x = self.ff[1:2]
x = self.ff[[1,2,3]]
x = self.ff[self.ff.fibers<3]
def run_pa(self, frame, outputfile):
from desispec.fiberflat import FiberFlat
import desispec.io.fiberflat as ffIO
from desispec.linalg import cholesky_solve
nwave = frame.nwave
nfibers = frame.nspec
wave = frame.wave #- this will become part of output too
flux = frame.flux
sumFlux = np.zeros((nwave))
realFlux = np.zeros(flux.shape)
ivar = frame.ivar * (frame.mask == 0)
#deconv
for fib in range(nfibers):
Rf = frame.R[fib].todense()
B = flux[fib]
realFlux[fib] = cholesky_solve(Rf, B)
sumFlux += realFlux[fib]
#iflux=nfibers/sumFlux
flat = np.zeros(flux.shape)
flat_ivar = np.zeros(ivar.shape)
avg = sumFlux / nfibers
for fib in range(nfibers):
Rf = frame.R[fib]
# apply and reconvolute
M = Rf.dot(avg)
M0 = (M == 0)
flat[fib] = (~M0) * flux[fib] / (M + M0) + M0
flat_ivar[fib] = ivar[fib] * M**2
fibflat = FiberFlat(frame.wave.copy(), flat, flat_ivar,
frame.mask.copy(), avg)
#fiberflat=compute_fiberflat(input_frame)
ffIO.write_fiberflat(outputfile, fibflat, header=frame.meta)
log.info("Wrote fiberflat file {}".format(outputfile))
def read_fiberflat(filename):
"""Read fiberflat from filename
Args:
filename (str): Name of fiberflat file, or (night, expid, camera) tuple
Returns:
FiberFlat object with attributes
fiberflat, ivar, mask, meanspec, wave, header
Notes:
fiberflat, ivar, mask are 2D [nspec, nwave]
meanspec and wave are 1D [nwave]
"""
#- check if outfile is (night, expid, camera) tuple instead
if isinstance(filename, (tuple, list)) and len(filename) == 3:
night, expid, camera = filename
filename = findfile('fiberflat', night, expid, camera)
header = fits.getheader(filename, 0)
fiberflat = native_endian(fits.getdata(filename, 0)).astype('f8')
ivar = native_endian(fits.getdata(filename, "IVAR").astype('f8'))
mask = native_endian(fits.getdata(filename, "MASK", uint=True))
meanspec = native_endian(fits.getdata(filename, "MEANSPEC").astype('f8'))
wave = native_endian(fits.getdata(filename, "WAVELENGTH").astype('f8'))
return FiberFlat(wave, fiberflat, ivar, mask, meanspec, header=header)
def _write_fiberflat(self):
"""Write a fake fiberflat"""
fiberflat = np.ones((self.nspec, self.nwave))
ivar = np.ones((self.nspec, self.nwave))
mask = np.zeros((self.nspec, self.nwave), dtype=int)
meanspec = np.ones(self.nwave)
ff = FiberFlat(self.wave, fiberflat, ivar, mask, meanspec)
io.write_fiberflat(self.fiberflatfile, ff)
def get_fiberflat_from_frame(frame):
from desispec.fiberflat import FiberFlat
flux = frame.flux
fiberflat = np.ones_like(flux)
ffivar = 2 * np.ones_like(flux)
fiberflat[0] *= 0.8
fiberflat[1] *= 1.2
ff = FiberFlat(frame.wave, fiberflat, ffivar)
return ff
def setUp(self):
self.nspec = 5
self.nwave = 10
self.wave = np.arange(self.nwave)
self.fiberflat = np.random.uniform(size=(self.nspec, self.nwave))
self.ivar = np.ones(self.fiberflat.shape)
self.mask = np.zeros(self.fiberflat.shape)
self.meanspec = np.random.uniform(size=self.nwave)
self.ff = FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.meanspec)
def test_apply_fiberflat_ivar(self):
'''test error propagation in apply_fiberflat'''
wave = np.arange(5000, 5010)
nwave = len(wave)
nspec = 3
flux = np.random.uniform(0.9, 1.0, size=(nspec, nwave))
ivar = np.ones_like(flux)
origframe = Frame(wave, flux, ivar, spectrograph=0)
fiberflat = np.ones_like(flux)
ffmask = np.zeros_like(flux)
fiberflat[0] *= 0.5
fiberflat[1] *= 1.5
#- ff with essentially no error
ffivar = 1e20 * np.ones_like(flux)
ff = FiberFlat(wave, fiberflat, ffivar)
frame = copy.deepcopy(origframe)
apply_fiberflat(frame, ff)
self.assertTrue(np.allclose(frame.ivar, fiberflat**2))
#- ff with large error
ffivar = np.ones_like(flux)
ff = FiberFlat(wave, fiberflat, ffivar)
frame = copy.deepcopy(origframe)
apply_fiberflat(frame, ff)
#- c = a/b
#- (sigma_c/c)^2 = (sigma_a/a)^2 + (sigma_b/b)^2
var = frame.flux**2 * (1.0/(origframe.ivar * origframe.flux**2) + \
1.0/(ff.ivar * ff.fiberflat**2))
self.assertTrue(np.allclose(frame.ivar, 1 / var))
#- ff.ivar=0 should result in frame.ivar=0, even if ff.fiberflat=0 too
ffivar = np.ones_like(flux)
ffivar[0, 0:5] = 0.0
fiberflat[0, 0:5] = 0.0
ff = FiberFlat(wave, fiberflat, ffivar)
frame = copy.deepcopy(origframe)
apply_fiberflat(frame, ff)
self.assertTrue(np.all(frame.ivar[0, 0:5] == 0.0))
class TestFiberFlatObject(unittest.TestCase):
def setUp(self):
self.nspec = 5
self.nwave = 10
self.wave = np.arange(self.nwave)
self.fiberflat = np.random.uniform(size=(self.nspec, self.nwave))
self.ivar = np.ones(self.fiberflat.shape)
self.mask = np.zeros(self.fiberflat.shape, dtype=np.uint32)
self.meanspec = np.random.uniform(size=self.nwave)
self.ff = FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.meanspec)
def test_init(self):
for key in ('wave', 'fiberflat', 'ivar', 'mask', 'meanspec'):
x = self.ff.__getattribute__(key)
y = self.__getattribute__(key)
self.assertTrue(np.all(x == y), key)
self.assertEqual(self.nspec, self.ff.nspec)
self.assertEqual(self.nwave, self.ff.nwave)
def test_dimensions(self):
#- check dimensionality mismatches
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.wave, self.ivar, self.mask, self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.wave, self.ivar, self.mask, self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.fiberflat)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat[0:2], self.ivar, self.mask, self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.fiberflat, self.fiberflat, self.ivar, self.mask, self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.wave, self.mask, self.meanspec)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask[0:2, :], self.meanspec)
fibers = np.arange(self.nspec)
FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.meanspec, fibers=fibers)
with self.assertRaises(ValueError):
FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.meanspec, fibers=fibers[1:])
def test_slice(self):
x = self.ff[1]
x = self.ff[1:2]
x = self.ff[[1,2,3]]
x = self.ff[self.ff.fibers<3]
def _write_fiberflat(self, camera=None):
"""Write a fake fiberflat"""
fiberflat = np.ones((self.nspec, self.nwave))
ivar = np.ones((self.nspec, self.nwave))
mask = np.zeros((self.nspec, self.nwave), dtype=int)
meanspec = np.ones(self.nwave)
ff = FiberFlat(self.wave, fiberflat, ivar, mask, meanspec)
if camera is not None:
hdr = fits.Header()
hdr['CAMERA'] = camera
else:
hdr = None
io.write_fiberflat(self.fiberflatfile, ff, hdr)
def test_apply_fiberflat(self):
'''test apply_fiberflat interface and changes to flux and mask'''
wave = np.arange(5000, 5050)
nwave = len(wave)
nspec = 3
flux = np.random.uniform(size=(nspec, nwave))
ivar = np.ones_like(flux)
frame = Frame(wave, flux, ivar, spectrograph=0)
fiberflat = np.ones_like(flux)
ffivar = 2 * np.ones_like(flux)
ffmask = np.zeros_like(flux)
fiberflat[0] *= 0.8
fiberflat[1] *= 1.2
fiberflat[2, 0:10] = 0 #- bad fiberflat
ffivar[2, 10:20] = 0 #- bad fiberflat
ffmask[2, 20:30] = 1 #- bad fiberflat
ff = FiberFlat(wave, fiberflat, ffivar)
origframe = copy.deepcopy(frame)
apply_fiberflat(frame, ff)
#- was fiberflat applied?
self.assertTrue(np.all(frame.flux[0] == origframe.flux[0] / 0.8))
self.assertTrue(np.all(frame.flux[1] == origframe.flux[1] / 1.2))
self.assertTrue(np.all(frame.flux[2] == origframe.flux[2]))
#- did mask get set?
ii = (ff.fiberflat == 0)
self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0))
ii = (ff.ivar == 0)
self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0))
ii = (ff.mask != 0)
self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0))
#- Should fail if frame and ff don't have a common wavelength grid
frame.wave = frame.wave + 0.1
with self.assertRaises(ValueError):
apply_fiberflat(frame, ff)
def qproc_compute_fiberflat(qframe,
niter_meanspec=4,
nsig_clipping=3.,
spline_res_clipping=20.,
spline_res_flat=5.):
"""
Fast estimation of fiberflat
"""
log = get_logger()
t0 = time.time()
log.info("Starting...")
twave = np.mean(qframe.wave, axis=0)
tflux = np.zeros(qframe.flux.shape)
tivar = np.zeros(qframe.flux.shape)
if qframe.mask is not None:
qframe.ivar *= (qframe.mask == 0)
for i in range(qframe.flux.shape[0]):
jj = (qframe.ivar[i] > 0)
tflux[i] = np.interp(twave, qframe.wave[i, jj], qframe.flux[i, jj])
tivar[i] = np.interp(twave,
qframe.wave[i, jj],
qframe.ivar[i, jj],
left=0,
right=0)
# iterative loop to a absorb constant term in fiber (should have more parameters)
a = np.ones(tflux.shape[0])
for iter in range(niter_meanspec):
mflux = np.median(a[:, np.newaxis] * tflux, axis=0)
for i in range(qframe.flux.shape[0]):
a[i] = np.median(tflux[i, mflux > 0] / mflux[mflux > 0])
# trivial fiberflat
fflat = tflux / (mflux + (mflux == 0))
fivar = tivar * mflux**2
mask = np.zeros((fflat.shape), dtype='uint32')
chi2 = 0
# spline fit to reject outliers and smooth the flat
for fiber in range(fflat.shape[0]):
# iterative spline fit
max_rej_it = 5 # not more than 5 pixels at a time
max_bad = 1000
nbad_tot = 0
for loop in range(20):
good = (fivar[fiber] > 0)
splineflat = spline_fit(twave,
twave[good],
fflat[fiber, good],
required_resolution=spline_res_clipping,
input_ivar=fivar[fiber, good],
max_resolution=3 * spline_res_clipping)
fchi2 = fivar[fiber] * (fflat[fiber] - splineflat)**2
bad = np.where(fchi2 > nsig_clipping**2)[0]
if bad.size > 0:
if bad.size > max_rej_it: # not more than 5 pixels at a time
ii = np.argsort(fchi2[bad])
bad = bad[ii[-max_rej_it:]]
fivar[fiber, bad] = 0
nbad_tot += len(bad)
#log.warning("iteration {} rejecting {} pixels (tot={}) from fiber {}".format(loop,len(bad),nbad_tot,fiber))
if nbad_tot >= max_bad:
fivar[fiber, :] = 0
log.warning(
"1st pass: rejecting fiber {} due to too many (new) bad pixels"
.format(fiber))
else:
break
chi2 += np.sum(fchi2)
min_ivar = 0.1 * np.median(fivar[fiber])
med_flat = np.median(fflat[fiber])
good = (fivar[fiber] > 0)
splineflat = spline_fit(twave,
twave[good],
fflat[fiber, good],
required_resolution=spline_res_flat,
input_ivar=fivar[fiber, good],
max_resolution=3 * spline_res_flat)
fflat[fiber] = splineflat # replace by spline
ii = np.where(fivar[fiber] > min_ivar)[0]
if ii.size < 2:
fflat[fiber] = 1
fivar[fiber] = 0
# set flat in unkown edges to median value of fiber (and ivar to 0)
b = ii[0]
e = ii[-1] + 1
fflat[fiber, :b] = med_flat # default
fivar[fiber, :b] = 0
mask[fiber, :b] = 1 # need to change this
fflat[fiber, e:] = med_flat # default
fivar[fiber, e:] = 0
mask[fiber, e:] = 1 # need to change this
# internal interpolation
bad = (fivar[fiber][b:e] <= min_ivar)
good = (fivar[fiber][b:e] > min_ivar)
fflat[fiber][b:e][bad] = np.interp(twave[b:e][bad], twave[b:e][good],
fflat[fiber][b:e][good])
ndata = np.sum(fivar > 0)
if ndata > 0:
chi2pdf = chi2 / ndata
else:
chi2pdf = 0
t1 = time.time()
log.info(" done in {:3.1f} sec".format(t1 - t0))
# return a fiberflat object ...
return FiberFlat(twave, fflat, fivar, mask, mflux, chi2pdf=chi2pdf)
def main(args):
# Set up the logger
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
# Make sure all necessary environment variables are set
DESI_SPECTRO_REDUX_DIR = "./quickGen"
if 'DESI_SPECTRO_REDUX' not in os.environ:
log.info('DESI_SPECTRO_REDUX environment is not set.')
else:
DESI_SPECTRO_REDUX_DIR = os.environ['DESI_SPECTRO_REDUX']
if os.path.exists(DESI_SPECTRO_REDUX_DIR):
if not os.path.isdir(DESI_SPECTRO_REDUX_DIR):
raise RuntimeError("Path %s Not a directory" %
DESI_SPECTRO_REDUX_DIR)
else:
try:
os.makedirs(DESI_SPECTRO_REDUX_DIR)
except:
raise
SPECPROD_DIR = 'specprod'
if 'SPECPROD' not in os.environ:
log.info('SPECPROD environment is not set.')
else:
SPECPROD_DIR = os.environ['SPECPROD']
prod_Dir = specprod_root()
if os.path.exists(prod_Dir):
if not os.path.isdir(prod_Dir):
raise RuntimeError("Path %s Not a directory" % prod_Dir)
else:
try:
os.makedirs(prod_Dir)
except:
raise
# Initialize random number generator to use.
np.random.seed(args.seed)
random_state = np.random.RandomState(args.seed)
# Derive spectrograph number from nstart if needed
if args.spectrograph is None:
args.spectrograph = args.nstart / 500
# Read fibermapfile to get object type, night and expid
if args.fibermap:
log.info("Reading fibermap file {}".format(args.fibermap))
fibermap = read_fibermap(args.fibermap)
objtype = get_source_types(fibermap)
stdindx = np.where(objtype == 'STD') # match STD with STAR
mwsindx = np.where(objtype == 'MWS_STAR') # match MWS_STAR with STAR
bgsindx = np.where(objtype == 'BGS') # match BGS with LRG
objtype[stdindx] = 'STAR'
objtype[mwsindx] = 'STAR'
objtype[bgsindx] = 'LRG'
NIGHT = fibermap.meta['NIGHT']
EXPID = fibermap.meta['EXPID']
else:
# Create a blank fake fibermap
fibermap = empty_fibermap(args.nspec)
targetids = random_state.randint(2**62, size=args.nspec)
fibermap['TARGETID'] = targetids
night = get_night()
expid = 0
log.info("Initializing SpecSim with config {}".format(args.config))
desiparams = load_desiparams()
qsim = get_simulator(args.config, num_fibers=1)
if args.simspec:
# Read the input file
log.info('Reading input file {}'.format(args.simspec))
simspec = desisim.io.read_simspec(args.simspec)
nspec = simspec.nspec
if simspec.flavor == 'arc':
log.warning("quickgen doesn't generate flavor=arc outputs")
return
else:
wavelengths = simspec.wave
spectra = simspec.flux
if nspec < args.nspec:
log.info("Only {} spectra in input file".format(nspec))
args.nspec = nspec
else:
# Initialize the output truth table.
spectra = []
wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value
npix = len(wavelengths)
truth = dict()
meta = Table()
truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10))
truth['FLUX'] = np.zeros((args.nspec, npix))
truth['WAVE'] = wavelengths
jj = list()
for thisobj in set(true_objtype):
ii = np.where(true_objtype == thisobj)[0]
nobj = len(ii)
truth['OBJTYPE'][ii] = thisobj
log.info('Generating {} template'.format(thisobj))
# Generate the templates
if thisobj == 'ELG':
elg = desisim.templates.ELG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = elg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_elg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'LRG':
lrg = desisim.templates.LRG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = lrg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_lrg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'QSO':
qso = desisim.templates.QSO(wave=wavelengths)
flux, tmpwave, meta1 = qso.make_templates(
nmodel=nobj, seed=args.seed, zrange=args.zrange_qso)
elif thisobj == 'BGS':
bgs = desisim.templates.BGS(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = bgs.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_bgs,
rmagrange=args.rmagrange_bgs,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'STD':
std = desisim.templates.STD(wave=wavelengths)
flux, tmpwave, meta1 = std.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'QSO_BAD': # use STAR template no color cuts
star = desisim.templates.STAR(wave=wavelengths)
flux, tmpwave, meta1 = star.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'MWS_STAR' or thisobj == 'MWS':
mwsstar = desisim.templates.MWS_STAR(wave=wavelengths)
flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'WD':
wd = desisim.templates.WD(wave=wavelengths)
flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'SKY':
flux = np.zeros((nobj, npix))
meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32)))
elif thisobj == 'TEST':
flux = np.zeros((args.nspec, npix))
indx = np.where(wave > 5800.0 - 1E-6)[0][0]
ref_integrated_flux = 1E-10
ref_cst_flux_density = 1E-17
single_line = (np.arange(args.nspec) % 2 == 0).astype(
np.float32)
continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32)
for spec in range(args.nspec):
flux[spec, indx] = single_line[
spec] * ref_integrated_flux / np.gradient(wavelengths)[
indx] # single line
flux[spec] += continuum[
spec] * ref_cst_flux_density # flat continuum
meta1 = Table(
dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32),
LINE=wave[indx] *
np.ones(args.nspec, dtype=np.float32),
LINEFLUX=single_line * ref_integrated_flux,
CONSTFLUXDENSITY=continuum * ref_cst_flux_density))
else:
log.fatal('Unknown object type {}'.format(thisobj))
sys.exit(1)
# Pack it in.
truth['FLUX'][ii] = flux
meta = vstack([meta, meta1])
jj.append(ii.tolist())
# Sanity check on units; templates currently return ergs, not 1e-17 ergs...
# assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6)
# Sort the metadata table.
jj = sum(jj, [])
meta_new = Table()
for k in range(args.nspec):
index = int(np.where(np.array(jj) == k)[0])
meta_new = vstack([meta_new, meta[index]])
meta = meta_new
# Add TARGETID and the true OBJTYPE to the metadata table.
meta.add_column(
Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE'))
meta.add_column(Column(targetids, name='TARGETID'))
# Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z
meta.rename_column('REDSHIFT', 'TRUEZ')
# explicitly set location on focal plane if needed to support airmass
# variations when using specsim v0.5
if qsim.source.focal_xy is None:
qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm'))
# Set simulation parameters from the simspec header or desiparams
bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX']
gray_objects = ['gray', 'grey']
if args.simspec is None:
object_type = objtype
flavor = None
elif simspec.flavor == 'science':
object_type = None
flavor = simspec.header['PROGRAM']
else:
object_type = None
flavor = simspec.flavor
log.warning(
'Maybe using an outdated simspec file with flavor={}'.format(
flavor))
# Set airmass
if args.airmass is not None:
qsim.atmosphere.airmass = args.airmass
elif args.simspec and 'AIRMASS' in simspec.header:
qsim.atmosphere.airmass = simspec.header['AIRMASS']
else:
qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2
# Set exptime
if args.exptime is not None:
qsim.observation.exposure_time = args.exptime * u.s
elif args.simspec and 'EXPTIME' in simspec.header:
qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s
elif objtype in bright_objects:
qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s
else:
qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s
# Set Moon Phase
if args.moon_phase is not None:
qsim.atmosphere.moon.moon_phase = args.moon_phase
elif args.simspec and 'MOONFRAC' in simspec.header:
qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC']
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_phase = 0.7
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_phase = 0.1
else:
qsim.atmosphere.moon.moon_phase = 0.5
# Set Moon Zenith
if args.moon_zenith is not None:
qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg
elif args.simspec and 'MOONALT' in simspec.header:
qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_zenith = 30 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_zenith = 80 * u.deg
else:
qsim.atmosphere.moon.moon_zenith = 100 * u.deg
# Set Moon - Object Angle
if args.moon_angle is not None:
qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg
elif args.simspec and 'MOONSEP' in simspec.header:
qsim.atmosphere.moon.separation_angle = simspec.header[
'MOONSEP'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.separation_angle = 50 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
else:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
# Initialize per-camera output arrays that will be saved
waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {}
maxbin = 0
nmax = args.nspec
for camera in qsim.instrument.cameras:
# Lookup this camera's resolution matrix and convert to the sparse
# format used in desispec.
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(),
[args.nspec, 1, 1])
waves[camera.name] = (camera.output_wavelength.to(
u.Angstrom).value.astype(np.float32))
nwave = len(waves[camera.name])
maxbin = max(maxbin, len(waves[camera.name]))
nobj = np.zeros((nmax, 3, maxbin)) # object photons
nsky = np.zeros((nmax, 3, maxbin)) # sky photons
nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky)
cframe_observedflux = np.zeros(
(nmax, 3, maxbin)) # calibrated object flux
cframe_ivar = np.zeros(
(nmax, 3, maxbin)) # inverse variance of calibrated object flux
cframe_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to calibrated flux
sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky
sky_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to sky only
frame_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky
trueflux[camera.name] = np.empty(
(args.nspec, nwave)) # calibrated flux
noisyflux[camera.name] = np.empty(
(args.nspec, nwave)) # observed flux with noise
obsivar[camera.name] = np.empty(
(args.nspec, nwave)) # inverse variance of flux
if args.simspec:
for i in range(10):
cn = camera.name + str(i)
if cn in simspec.cameras:
dw = np.gradient(simspec.cameras[cn].wave)
break
else:
raise RuntimeError(
'Unable to find a {} camera in input simspec'.format(
camera))
else:
sflux = np.empty((args.nspec, npix))
#- Check if input simspec is for a continuum flat lamp instead of science
#- This does not convolve to per-fiber resolution
if args.simspec:
if simspec.flavor == 'flat':
log.info("Simulating flat lamp exposure")
for i, camera in enumerate(qsim.instrument.cameras):
channel = camera.name #- from simspec, b/r/z not b0/r1/z9
assert camera.output_wavelength.unit == u.Angstrom
num_pixels = len(waves[channel])
phot = list()
for j in range(10):
cn = camera.name + str(j)
if cn in simspec.cameras:
camwave = simspec.cameras[cn].wave
dw = np.gradient(camwave)
phot.append(simspec.cameras[cn].phot)
if len(phot) == 0:
raise RuntimeError(
'Unable to find a {} camera in input simspec'.format(
camera))
else:
phot = np.vstack(phot)
meanspec = resample_flux(waves[channel], camwave,
np.average(phot / dw, axis=0))
fiberflat = random_state.normal(loc=1.0,
scale=1.0 / np.sqrt(meanspec),
size=(nspec, num_pixels))
ivar = np.tile(meanspec, [nspec, 1])
mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32)
for kk in range((args.nspec + args.nstart - 1) // 500 + 1):
camera = channel + str(kk)
outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID,
camera)
start = max(500 * kk, args.nstart)
end = min(500 * (kk + 1), nmax)
if (args.spectrograph <= kk):
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, kk, start, end))
ff = FiberFlat(waves[channel],
fiberflat[start:end, :],
ivar[start:end, :],
mask[start:end, :],
meanspec,
header=dict(CAMERA=camera))
write_fiberflat(outfile, ff)
filePath = desispec.io.findfile("fiberflat", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(filePath))
sys.exit(0)
# Repeat the simulation for all spectra
fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom)
for j in range(args.nspec):
thisobjtype = objtype[j]
sys.stdout.flush()
if flavor == 'arc':
qsim.source.update_in('Quickgen source {0}'.format, 'perfect',
wavelengths * u.Angstrom,
spectra * fluxunits)
else:
qsim.source.update_in('Quickgen source {0}'.format(j),
thisobjtype.lower(),
wavelengths * u.Angstrom,
spectra[j, :] * fluxunits)
qsim.source.update_out()
qsim.simulate()
qsim.generate_random_noise(random_state)
for i, output in enumerate(qsim.camera_output):
assert output['observed_flux'].unit == 1e17 * fluxunits
# Extract the simulation results needed to create our uncalibrated
# frame output file.
num_pixels = len(output)
nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0]
nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0]
nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0]
# Get results for our flux-calibrated output file.
cframe_observedflux[
j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0]
cframe_ivar[
j,
i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0]
# Fill brick arrays from the results.
camera = output.meta['name']
trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0]
noisyflux[camera][j][:] = 1e17 * (
output['observed_flux'][:, 0] +
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,
0]
# Use the same noise realization in the cframe and frame, without any
# additional noise from sky subtraction for now.
frame_rand_noise[
j, i, :num_pixels] = output['random_noise_electrons'][:, 0]
cframe_rand_noise[j, i, :num_pixels] = 1e17 * (
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
# The sky output file represents a model fit to ~40 sky fibers.
# We reduce the variance by a factor of 25 to account for this and
# give the sky an independent (Gaussian) noise realization.
sky_ivar[
j,
i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] -
output['num_source_electrons'][:, 0])
sky_rand_noise[j, i, :num_pixels] = random_state.normal(
scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]),
size=num_pixels)
armName = {"b": 0, "r": 1, "z": 2}
for channel in 'brz':
#Before writing, convert from counts/bin to counts/A (as in Pixsim output)
#Quicksim Default:
#FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang]
#COUNTS_OBJ - object counts in 0.5 Ang bin
#COUNTS_SKY - sky counts in 0.5 Ang bin
num_pixels = len(waves[channel])
dwave = np.gradient(waves[channel])
nobj[:, armName[channel], :num_pixels] /= dwave
frame_rand_noise[:, armName[channel], :num_pixels] /= dwave
nivar[:, armName[channel], :num_pixels] *= dwave**2
nsky[:, armName[channel], :num_pixels] /= dwave
sky_rand_noise[:, armName[channel], :num_pixels] /= dwave
sky_ivar[:, armName[channel], :num_pixels] /= dwave**2
# Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra
# Looping over spectrograph
for ii in range((args.nspec + args.nstart - 1) // 500 + 1):
start = max(500 * ii,
args.nstart) # first spectrum for a given spectrograph
end = min(500 * (ii + 1),
nmax) # last spectrum for the spectrograph
if (args.spectrograph <= ii):
camera = "{}{}".format(channel, ii)
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, ii, start, end))
num_pixels = len(waves[channel])
# Write frame file
framefileName = desispec.io.findfile("frame", NIGHT, EXPID,
camera)
frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \
nsky[start:end,armName[channel],:num_pixels] + \
frame_rand_noise[start:end,armName[channel],:num_pixels]
frame_ivar = nivar[start:end, armName[channel], :num_pixels]
sh1 = frame_flux.shape[
0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave)
# for example if nstart =400, nspec=150: two spectrographs:
# 400-499=> 0 spectrograph, 500-549 => 1
if (args.nstart == start):
resol = resolution[channel][:sh1, :, :]
else:
resol = resolution[channel][-sh1:, :, :]
# must create desispec.Frame object
frame=Frame(waves[channel], frame_flux, frame_ivar,\
resolution_data=resol, spectrograph=ii, \
fibermap=fibermap[start:end], \
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.write_frame(framefileName, frame)
framefilePath = desispec.io.findfile("frame", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(framefilePath))
if args.frameonly or simspec.flavor == 'arc':
continue
# Write cframe file
cframeFileName = desispec.io.findfile("cframe", NIGHT, EXPID,
camera)
cframeFlux = cframe_observedflux[
start:end,
armName[channel], :num_pixels] + cframe_rand_noise[
start:end, armName[channel], :num_pixels]
cframeIvar = cframe_ivar[start:end,
armName[channel], :num_pixels]
# must create desispec.Frame object
cframe = Frame(waves[channel], cframeFlux, cframeIvar, \
resolution_data=resol, spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.frame.write_frame(cframeFileName, cframe)
cframefilePath = desispec.io.findfile("cframe", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(cframefilePath))
# Write sky file
skyfileName = desispec.io.findfile("sky", NIGHT, EXPID, camera)
skyflux=nsky[start:end,armName[channel],:num_pixels] + \
sky_rand_noise[start:end,armName[channel],:num_pixels]
skyivar = sky_ivar[start:end, armName[channel], :num_pixels]
skymask = np.zeros(skyflux.shape, dtype=np.uint32)
# must create desispec.Sky object
skymodel = SkyModel(waves[channel],
skyflux,
skyivar,
skymask,
header=dict(CAMERA=camera))
desispec.io.sky.write_sky(skyfileName, skymodel)
skyfilePath = desispec.io.findfile("sky", NIGHT, EXPID, camera)
log.info("Wrote file {}".format(skyfilePath))
# Write calib file
calibVectorFile = desispec.io.findfile("calib", NIGHT, EXPID,
camera)
flux = cframe_observedflux[start:end,
armName[channel], :num_pixels]
phot = nobj[start:end, armName[channel], :num_pixels]
calibration = np.zeros_like(phot)
jj = (flux > 0)
calibration[jj] = phot[jj] / flux[jj]
#- TODO: what should calibivar be?
#- For now, model it as the noise of combining ~10 spectra
calibivar = 10 / cframe_ivar[start:end,
armName[channel], :num_pixels]
#mask=(1/calibivar>0).astype(int)??
mask = np.zeros(calibration.shape, dtype=np.uint32)
# write flux calibration
fluxcalib = FluxCalib(waves[channel], calibration, calibivar,
mask)
write_flux_calibration(calibVectorFile, fluxcalib)
calibfilePath = desispec.io.findfile("calib", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(calibfilePath))
def qproc_compute_fiberflat(qframe,niter_meanspec=4,nsig_clipping=3.,spline_res_clipping=20.,spline_res_flat=5.) :
"""
Fast estimation of fiberflat
"""
log = get_logger()
t0=time.time()
log.info("Starting...")
twave=np.mean(qframe.wave,axis=0)
tflux=np.zeros(qframe.flux.shape)
tivar=np.zeros(qframe.flux.shape)
if qframe.mask is not None :
qframe.ivar *= (qframe.mask==0)
for i in range(qframe.flux.shape[0]) :
jj=(qframe.ivar[i]>0)
tflux[i]=np.interp(twave,qframe.wave[i,jj],qframe.flux[i,jj])
tivar[i]=np.interp(twave,qframe.wave[i,jj],qframe.ivar[i,jj],left=0,right=0)
# iterative loop to a absorb constant term in fiber
if 1 : # simple scaling per fiber
a=np.ones(tflux.shape[0])
for iter in range(niter_meanspec) :
mflux=np.median(a[:,np.newaxis]*tflux,axis=0)
for i in range(qframe.flux.shape[0]) :
a[i] = np.median(tflux[i,mflux>0]/mflux[mflux>0])
else : # polynomial fit does not improve much and s more fragile
x=np.linspace(-1,1,tflux.shape[1])
pol=np.ones(tflux.shape)
for iteration in range(niter_meanspec) :
if iteration>0 :
for i in range(tflux.shape[0]) :
jj=(mflux>0)&(tivar[i]>0)
c = np.polyfit(x[jj],tflux[i,jj]/mflux[jj],1,w=mflux[jj]**2*tivar[i,jj])
pol[i] = np.poly1d(c)(x)
mflux=np.median(pol*tflux,axis=0)
# trivial fiberflat
fflat=tflux/(mflux+(mflux==0))
fivar=tivar*mflux**2
mask=np.zeros((fflat.shape), dtype='uint32')
chi2=0
# special case with test slit
mask_lines = ( qframe.flux.shape[0]<50 )
if mask_lines :
log.warning("Will interpolate over absorption lines in input continuum spectrum from illumination bench")
# spline fit to reject outliers and smooth the flat
for fiber in range(fflat.shape[0]) :
# iterative spline fit
max_rej_it=5# not more than 5 pixels at a time
max_bad=1000
nbad_tot=0
for loop in range(20) :
good=(fivar[fiber]>0)
splineflat = spline_fit(twave,twave[good],fflat[fiber,good],required_resolution=spline_res_clipping,input_ivar=fivar[fiber,good],max_resolution=3*spline_res_clipping)
fchi2 = fivar[fiber]*(fflat[fiber]-splineflat)**2
bad=np.where(fchi2>nsig_clipping**2)[0]
if bad.size>0 :
if bad.size>max_rej_it : # not more than 5 pixels at a time
ii=np.argsort(fchi2[bad])
bad=bad[ii[-max_rej_it:]]
fivar[fiber,bad] = 0
nbad_tot += len(bad)
#log.warning("iteration {} rejecting {} pixels (tot={}) from fiber {}".format(loop,len(bad),nbad_tot,fiber))
if nbad_tot>=max_bad:
fivar[fiber,:]=0
log.warning("1st pass: rejecting fiber {} due to too many (new) bad pixels".format(fiber))
else :
break
chi2 += np.sum(fchi2)
min_ivar = 0.1*np.median(fivar[fiber])
med_flat = np.median(fflat[fiber])
good=(fivar[fiber]>0)
splineflat = spline_fit(twave,twave[good],fflat[fiber,good],required_resolution=spline_res_flat,input_ivar=fivar[fiber,good],max_resolution=3*spline_res_flat)
fflat[fiber] = splineflat # replace by spline
ii=np.where(fivar[fiber]>min_ivar)[0]
if ii.size<2 :
fflat[fiber] = 1
fivar[fiber] = 0
# set flat in unknown edges to median value of fiber (and ivar to 0)
b=ii[0]
e=ii[-1]+1
fflat[fiber,:b]=med_flat # default
fivar[fiber,:b]=0
mask[fiber,:b]=1 # need to change this
fflat[fiber,e:]=med_flat # default
fivar[fiber,e:]=0
mask[fiber,e:]=1 # need to change this
# internal interpolation
bad=(fivar[fiber][b:e]<=min_ivar)
good=(fivar[fiber][b:e]>min_ivar)
fflat[fiber][b:e][bad]=np.interp(twave[b:e][bad],twave[b:e][good],fflat[fiber][b:e][good])
# special case with test slit
if mask_lines :
if qframe.meta["camera"].upper()[0] == "B" :
jj=((twave>3900)&(twave<3960))|((twave>4350)&(twave<4440))|(twave>5800)
elif qframe.meta["camera"].upper()[0] == "R" :
jj=(twave<5750)
else :
jj=(twave<7550)|(twave>9800)
if np.sum(jj)>0 :
njj=np.logical_not(jj)
fflat[fiber,jj] = np.interp(twave[jj],twave[njj],fflat[fiber,njj])
ndata=np.sum(fivar>0)
if ndata>0 :
chi2pdf = chi2/ndata
else :
chi2pdf = 0
t1=time.time()
log.info(" done in {:3.1f} sec".format(t1-t0))
# return a fiberflat object ...
return FiberFlat(twave, fflat, fivar, mask, mflux,chi2pdf=chi2pdf)
