def difference(s0, s1):
diff = dict()
ivar = dict()
mask = dict()
wave = dict()
common = list(set(s1.bands).intersection(s0.bands))
for dindex in common:
# diff[dindex] = ma.array(data=s1.flux[dindex],mask=s1.mask[dindex])
# diff[dindex] = diff[dindex] - ma.array(data=s0.flux[dindex],mask=s0.mask[dindex])
diff[dindex] = s0.flux[dindex]-s1.flux[dindex]
# ivar0 = ma.array(data=s0.ivar[dindex],mask=s0.mask[dindex])
# ivar1 = ma.array(data=s1.ivar[dindex],mask=s1.mask[dindex])
# ivar[dindex] = 1/(1/ivar0 + 1/ivar1)
ivar[dindex] = 1/(1/s0.ivar[dindex] + 1/s1.ivar[dindex])
mask[dindex] = 1-(1-s0.mask[dindex])*(1-s1.mask[dindex]).astype('int')
wave[dindex] = s1.wave[dindex]
ans = Spectra(bands=common, wave=wave, flux=diff,ivar=ivar,mask=mask,fibermap = s0.fibermap, resolution_data=s0.resolution_data)
return ans
def _random_spectra(self, ns=3, nw=10):
wave = np.linspace(5000, 5100, nw)
flux = np.random.uniform(0, 1, size=(ns,nw))
ivar = np.random.uniform(0, 1, size=(ns,nw))
#mask = np.zeros((ns,nw),dtype=int)
mask = None
rdat = np.ones((ns,3,nw))
rdat[:,0] *= 0.25
rdat[:,1] *= 0.5
rdat[:,2] *= 0.25
fmap = empty_fibermap(ns)
fmap["TARGETID"][:]=12 # same target
return Spectra(bands=["x"],wave={"x":wave},flux={"x":flux},ivar={"x":ivar}, mask=None, resolution_data={"x":rdat} , fibermap=fmap)
def difference(s0, s1):
diff = dict()
ivar = dict()
mask = dict()
for dindex in s1.bands:
diff[dindex] = ma.array(data=s1.flux[dindex], mask=s1.mask[dindex])
diff[dindex] = diff[dindex] - ma.array(data=s0.flux[dindex],
mask=s0.mask[dindex])
ivar0 = ma.array(data=s0.ivar[dindex], mask=s0.mask[dindex])
ivar1 = ma.array(data=s1.ivar[dindex], mask=s1.mask[dindex])
ivar[dindex] = 1 / (1 / ivar0 + 1 / ivar1)
mask[dindex] = diff[dindex].mask.astype('int')
return Spectra(bands=s1.bands.copy(),
wave=dict(s1.wave),
flux=diff,
ivar=ivar,
mask=mask,
fibermap=s0.fibermap)
def myspecselect(thespec,
nights=None,
bands=None,
targets=None,
fibers=None,
expids=None,
indices=None,
invert=False,
remove_scores=False,
clean_fiberstatus=False,
output_indices=False):
"""
Select a subset of the data.
This filters the data based on a logical AND of the different
criteria, optionally inverting that selection.
Args:
nights (list): optional list of nights to select.
bands (list): optional list of bands to select.
targets (list): optional list of target IDs to select.
fibers (list): list/array of fiber indices to select.
ADDED=> expids (list): list/array of individual exposures to select.
ADDED =>indices (list) : list of raw (arbitrary) indices in the Spectra object to select.
invert (bool): after combining all criteria, invert selection.
remove_scores (bool): probably tmp trick, TODO
output_indices (bool): if True, also returns indices of kept spectra
Returns (Spectra):
a new Spectra object containing the selected data.
"""
keep_fiberstatus = None
if clean_fiberstatus == False:
keep_fiberstatus = [True for x in range(thespec.num_spectra())]
else:
keep_fiberstatus = [(x == 0) for x in thespec.fibermap["FIBERSTATUS"]]
keep_bands = None
if bands is None:
keep_bands = thespec.bands
else:
keep_bands = [x for x in thespec.bands if x in bands]
if len(keep_bands) == 0:
print("myspecselect: no valid bands were selected.")
return None
keep_nights = None
if nights is None:
keep_nights = [True for x in range(thespec.num_spectra())]
else:
keep_nights = [(x in nights) for x in thespec.fibermap["NIGHT"]]
if sum(keep_nights) == 0:
print("myspecselect: no valid nights were selected.")
return None
keep_targets = None
if targets is None:
keep_targets = [True for x in range(thespec.num_spectra())]
else:
keep_targets = [(x in targets) for x in thespec.fibermap["TARGETID"]]
if sum(keep_targets) == 0:
print("myspecselect: no valid targets were selected.")
return None
keep_fibers = None
if fibers is None:
keep_fibers = [True for x in range(thespec.num_spectra())]
else:
keep_fibers = [(x in fibers) for x in thespec.fibermap["FIBER"]]
if sum(keep_fibers) == 0:
print("myspecselect: no valid fibers were selected.")
return None
keep_expids = None
if expids is None:
keep_expids = [True for x in range(thespec.num_spectra())]
else:
keep_expids = [(x in expids) for x in thespec.fibermap["EXPID"]]
if sum(keep_expids) == 0:
print("myspecselect: no valid expids were selected.")
return None
keep_indices = None
if indices is None:
keep_indices = [True for x in range(thespec.num_spectra())]
else:
keep_indices = [(x in indices) for x in range(thespec.num_spectra())]
if sum(keep_indices) == 0:
print("myspecselect: no valid indices were selected.")
return None
keep_rows = [(x and y and z and t and u and v) for x, y, z, t, u, v in zip(
keep_nights, keep_targets, keep_fibers, keep_expids, keep_indices,
keep_fiberstatus)]
if invert:
keep_rows = [not x for x in keep_rows]
keep = [i for i, x in enumerate(keep_rows) if x]
if len(keep) == 0:
print("myspecselect: selection has no spectra.")
return None
keep_wave = {}
keep_flux = {}
keep_ivar = {}
keep_mask = None
keep_res = None
keep_extra = None
if thespec.mask is not None:
keep_mask = {}
if thespec.resolution_data is not None:
keep_res = {}
if thespec.extra is not None:
keep_extra = {}
for b in keep_bands:
keep_wave[b] = thespec.wave[b]
keep_flux[b] = thespec.flux[b][keep, :]
keep_ivar[b] = thespec.ivar[b][keep, :]
if thespec.mask is not None:
keep_mask[b] = thespec.mask[b][keep, :]
if thespec.resolution_data is not None:
keep_res[b] = thespec.resolution_data[b][keep, :, :]
if thespec.extra is not None:
keep_extra[b] = {}
for ex in thespec.extra[b].items():
keep_extra[b][ex[0]] = ex[1][keep, :]
keep_scores = None
if not remove_scores:
if thespec.scores is not None: keep_scores = thespec.scores[keep]
ret = Spectra(keep_bands,
keep_wave,
keep_flux,
keep_ivar,
mask=keep_mask,
resolution_data=keep_res,
fibermap=thespec.fibermap[keep],
meta=thespec.meta,
extra=keep_extra,
single=thespec._single,
scores=keep_scores)
if output_indices:
return (ret, keep)
return ret
def fast_resample_spectra(spectra, wave):
"""
Fast resampling of spectra file.
The output resolution = Id. The neighboring
flux bins are correlated.
Args:
spectra: desispec.spectra.Spectra object
wave: 1D numy array with new wavelenght grid
Returns:
desispec.spectra.Spectra object, resolution data=Id
"""
log = get_logger()
log.debug("Resampling to wave grid: {}".format(wave))
nwave = wave.size
b = spectra._bands[0]
ntarget = spectra.flux[b].shape[0]
nres = spectra.resolution_data[b].shape[1]
ivar = np.zeros((ntarget, nwave), dtype=spectra.flux[b].dtype)
flux = np.zeros((ntarget, nwave), dtype=spectra.ivar[b].dtype)
if spectra.mask is not None:
mask = np.zeros((ntarget, nwave), dtype=spectra.mask[b].dtype)
else:
mask = None
rdata = np.ones((ntarget, 1, nwave),
dtype=spectra.resolution_data[b].dtype
) # pointless for this resampling
bands = ""
for b in spectra._bands:
if spectra.mask is not None:
tivar = spectra.ivar[b] * (spectra.mask[b] == 0)
else:
tivar = spectra.ivar[b]
for i in range(ntarget):
ivar[i] += resample_flux(wave, spectra.wave[b], tivar[i])
flux[i] += resample_flux(wave, spectra.wave[b],
tivar[i] * spectra.flux[b][i])
bands += b
for i in range(ntarget):
ok = (ivar[i] > 0)
flux[i, ok] /= ivar[i, ok]
if spectra.mask is not None:
dmask = {
bands: mask,
}
else:
dmask = None
res = Spectra(bands=[
bands,
],
wave={
bands: wave,
},
flux={
bands: flux,
},
ivar={
bands: ivar,
},
mask=dmask,
resolution_data={
bands: rdata,
},
fibermap=spectra.fibermap,
meta=spectra.meta,
extra=spectra.extra,
scores=spectra.scores)
return res
def spectroperf_resample_spectra(spectra, wave, nproc=1):
"""
Resampling of spectra file using the spectrophotometic approach
Args:
spectra: desispec.spectra.Spectra object
wave: 1D numy array with new wavelenght grid
Returns:
desispec.spectra.Spectra object
"""
log = get_logger()
log.debug("resampling to wave grid of size {}: {}".format(wave.size, wave))
b = spectra._bands[0]
ntarget = spectra.flux[b].shape[0]
nwave = wave.size
if spectra.mask is not None:
mask = np.zeros((ntarget, nwave), dtype=spectra.mask[b].dtype)
else:
mask = None
# number of diagonals is the max of the number of diagonals in the
# input spectra cameras
ndiag = 0
for b in spectra._bands:
ndiag = max(ndiag, spectra.resolution_data[b].shape[1])
dw = np.gradient(wave)
wavebin = np.min(dw[dw > 0.]) # min wavelength bin size
log.debug("min wavelength bin= {:2.1f} A; ndiag= {:d}".format(
wavebin, ndiag))
log.debug("compute resampling matrices")
resampling_matrix = dict()
for b in spectra._bands:
twave = spectra.wave[b]
jj = np.where((twave >= wave[0]) & (twave <= wave[-1]))[0]
twave = spectra.wave[b][jj]
resampling_matrix[b] = get_resampling_matrix(wave, twave)
if nproc == 1:
# allocate array
flux = np.zeros((ntarget, nwave), dtype=float)
ivar = np.zeros((ntarget, nwave), dtype=float)
rdata = np.zeros((ntarget, ndiag, nwave), dtype=float)
# simply loop on targets
for target_index in range(ntarget):
log.debug("resampling {}/{}".format(target_index + 1, ntarget))
t0 = time.time()
spectroperf_resample_spectrum_singleproc(spectra, target_index,
wave, wavebin,
resampling_matrix, ndiag,
flux, ivar, rdata)
t1 = time.time()
log.debug("done one spectrum in {} sec".format(t1 - t0))
else:
log.debug("allocate shared memory")
# input
shm_in_wave = list()
shm_in_flux = list()
shm_in_ivar = list()
shm_in_rdata = list()
in_nwave = list()
in_ndiag = list()
for b in spectra._bands:
shm_in_wave.append(
multiprocessing.Array('d', spectra.wave[b], lock=False))
shm_in_flux.append(
multiprocessing.Array('d', spectra.flux[b].ravel(),
lock=False))
shm_in_ivar.append(
multiprocessing.Array('d', spectra.ivar[b].ravel(),
lock=False))
shm_in_rdata.append(
multiprocessing.Array('d',
spectra.resolution_data[b].ravel(),
lock=False))
in_nwave.append(spectra.wave[b].size)
in_ndiag.append(spectra.resolution_data[b].shape[1])
# output
shm_flux = multiprocessing.Array('d', ntarget * nwave, lock=False)
shm_ivar = multiprocessing.Array('d', ntarget * nwave, lock=False)
shm_rdata = multiprocessing.Array('d',
ntarget * ndiag * nwave,
lock=False)
# manipulate shared memory as np arrays
flux = np.array(shm_flux, copy=False).reshape(ntarget, nwave)
ivar = np.array(shm_ivar, copy=False).reshape(ntarget, nwave)
rdata = np.array(shm_rdata, copy=False).reshape(ntarget, ndiag, nwave)
# split targets per process
target_indices = np.array_split(np.arange(ntarget), nproc)
# loop on processes
procs = list()
for proc_index in range(nproc):
log.debug("starting process #{}".format(proc_index + 1))
proc = multiprocessing.Process(
target=spectroperf_resample_spectrum_multiproc,
args=(shm_in_wave, shm_in_flux, shm_in_ivar, shm_in_rdata,
in_nwave, in_ndiag, spectra._bands,
target_indices[proc_index], wave, wavebin,
resampling_matrix, ndiag, ntarget, shm_flux, shm_ivar,
shm_rdata))
proc.start()
procs.append(proc)
# wait for the processes to finish
log.info("waiting for the {} processes to finish ...".format(nproc))
for proc in procs:
proc.join()
log.info("all done!")
bands = ""
for b in spectra._bands:
bands += b
if spectra.mask is not None:
dmask = {
bands: mask,
}
else:
dmask = None
res = Spectra(bands=[
bands,
],
wave={
bands: wave,
},
flux={
bands: flux,
},
ivar={
bands: ivar,
},
mask=dmask,
resolution_data={
bands: rdata,
},
fibermap=spectra.fibermap,
meta=spectra.meta,
extra=spectra.extra,
scores=spectra.scores)
return res
def coadd_cameras(spectra, cosmics_nsig=0.):
#check_alignement_of_camera_wavelength(spectra)
log = get_logger()
# ordering
mwave = [np.mean(spectra.wave[b]) for b in spectra._bands]
sbands = np.array(
spectra._bands)[np.argsort(mwave)] # bands sorted by inc. wavelength
log.debug("wavelength sorted cameras= {}".format(sbands))
# create wavelength array
wave = None
tolerance = 0.0001 #A , tolerance
for b in sbands:
if wave is None:
wave = spectra.wave[b]
else:
wave = np.append(
wave, spectra.wave[b][spectra.wave[b] > wave[-1] + tolerance])
nwave = wave.size
# check alignment
number_of_overlapping_cameras = np.zeros(nwave)
for b in spectra._bands:
windices = np.argmin((np.tile(wave, (spectra.wave[b].size, 1)) -
np.tile(spectra.wave[b], (wave.size, 1)).T)**2,
axis=1)
dist = np.sqrt(np.max(spectra.wave[b] - wave[windices]))
log.debug("camera {} max dist= {}A".format(b, dist))
if dist > tolerance:
log.error(
"Cannot directly coadd the camera spectra because wavelength are not aligned, use --lin-step or --log10-step to resample to a common grid"
)
sys.exit(12)
number_of_overlapping_cameras[windices] += 1
# targets
targets = np.unique(spectra.fibermap["TARGETID"])
ntarget = targets.size
log.debug("number of targets= {}".format(ntarget))
# ndiag = max of all cameras
ndiag = 0
for b in sbands:
ndiag = max(ndiag, spectra.resolution_data[b].shape[1])
log.debug("ndiag= {}".format(ndiag))
b = sbands[0]
flux = np.zeros((ntarget, nwave), dtype=spectra.flux[b].dtype)
ivar = np.zeros((ntarget, nwave), dtype=spectra.ivar[b].dtype)
if spectra.mask is not None:
ivar_unmasked = np.zeros((ntarget, nwave), dtype=spectra.ivar[b].dtype)
mask = np.zeros((ntarget, nwave), dtype=spectra.mask[b].dtype)
else:
ivar_unmasked = ivar
mask = None
rdata = np.zeros((ntarget, ndiag, nwave),
dtype=spectra.resolution_data[b].dtype)
for b in spectra._bands:
log.debug("coadding band '{}'".format(b))
# indices
windices = np.argmin((np.tile(wave, (spectra.wave[b].size, 1)) -
np.tile(spectra.wave[b], (wave.size, 1)).T)**2,
axis=1)
band_ndiag = spectra.resolution_data[b].shape[1]
for i, tid in enumerate(targets):
jj = np.where(spectra.fibermap["TARGETID"] == tid)[0]
if cosmics_nsig is not None and cosmics_nsig > 0:
# interpolate over bad measurements
# to be able to compute gradient next
# to a bad pixel and identify oulier
# many cosmics residuals are on edge
# of cosmic ray trace, and so can be
# next to a masked flux bin
grad = []
gradvar = []
for j in jj:
if spectra.mask is not None:
ttivar = spectra.ivar[b][j] * (spectra.mask[b][j] == 0)
else:
ttivar = spectra.ivar[b][j]
good = (ttivar > 0)
bad = (ttivar <= 0)
ttflux = spectra.flux[b][j]
ttflux[bad] = np.interp(spectra.wave[b][bad],
spectra.wave[b][good],
ttflux[good])
ttivar = spectra.ivar[b][j]
ttivar[bad] = np.interp(spectra.wave[b][bad],
spectra.wave[b][good],
ttivar[good])
ttvar = 1. / ttivar
ttflux[1:] = ttflux[1:] - ttflux[:-1]
ttvar[1:] = ttvar[1:] + ttvar[:-1]
ttflux[0] = 0
grad.append(ttflux)
gradvar.append(ttvar)
ivar_unmasked[i, windices] += np.sum(spectra.ivar[b][jj], axis=0)
if spectra.mask is not None:
ivarjj = spectra.ivar[b][jj] * (spectra.mask[b][jj] == 0)
else:
ivarjj = spectra.ivar[b][jj]
if cosmics_nsig is not None and cosmics_nsig > 0 and len(grad) > 0:
grad = np.array(grad)
gradivar = 1 / np.array(gradvar)
nspec = grad.shape[0]
meangrad = np.sum(gradivar * grad, axis=0) / np.sum(gradivar)
deltagrad = grad - meangrad
chi2 = np.sum(gradivar * deltagrad**2, axis=0) / (nspec - 1)
for l in np.where(chi2 > cosmics_nsig**2)[0]:
k = np.argmax(gradivar[:, l] * deltagrad[:, l]**2)
log.debug("masking spec {} wave={}".format(
k, spectra.wave[b][l]))
ivarjj[k][l] = 0.
ivar[i, windices] += np.sum(ivarjj, axis=0)
flux[i, windices] += np.sum(ivarjj * spectra.flux[b][jj], axis=0)
for r in range(band_ndiag):
rdata[i, r + (ndiag - band_ndiag) // 2, windices] += np.sum(
(spectra.ivar[b][jj] * spectra.resolution_data[b][jj, r]),
axis=0)
if spectra.mask is not None:
# this deserves some attention ...
tmpmask = np.bitwise_and.reduce(spectra.mask[b][jj], axis=0)
# directly copy mask where no overlap
jj = (number_of_overlapping_cameras[windices] == 1)
mask[i, windices[jj]] = tmpmask[jj]
# 'and' in overlapping regions
jj = (number_of_overlapping_cameras[windices] > 1)
mask[i, windices[jj]] = mask[i, windices[jj]] & tmpmask[jj]
for i, tid in enumerate(targets):
ok = (ivar[i] > 0)
if np.sum(ok) > 0:
flux[i][ok] /= ivar[i][ok]
ok = (ivar_unmasked[i] > 0)
if np.sum(ok) > 0:
rdata[i][:, ok] /= ivar_unmasked[i][ok]
fibermap = coadd_fibermap(spectra.fibermap)
bands = ""
for b in sbands:
bands += b
if spectra.mask is not None:
dmask = {
bands: mask,
}
else:
dmask = None
res = Spectra(bands=[
bands,
],
wave={
bands: wave,
},
flux={
bands: flux,
},
ivar={
bands: ivar,
},
mask=dmask,
resolution_data={
bands: rdata,
},
fibermap=fibermap,
meta=spectra.meta,
extra=spectra.extra,
scores=None)
return res
def _coadd_targets(spectra, targetids=None):
'''
Coadds individual exposures of the same targets; returns new Spectra object
Parameters
----------
spectra : :class:`desispec.spectra.Spectra`
targetids : (optional) array-like subset of target IDs to keep
Returns
-------
coadded_spectra : :class:`desispec.spectra.Spectra` where individual
spectra of each target have been combined into a single spectrum
per camera.
Note: coadds per camera but not across cameras.
'''
if targetids is None:
targetids = spectra.target_ids()
#- Create output arrays to fill
ntargets = spectra.num_targets()
wave = dict()
flux = dict()
ivar = dict()
rdat = dict()
for channel in spectra.bands:
wave[channel] = spectra.wave[channel].copy()
nwave = len(wave[channel])
flux[channel] = np.zeros((ntargets, nwave))
ivar[channel] = np.zeros((ntargets, nwave))
ndiag = spectra.resolution_data[channel].shape[1]
rdat[channel] = np.zeros((ntargets, ndiag, nwave))
#- Loop over targets, coadding all spectra for each target
fibermap = Table(dtype=spectra.fibermap.dtype)
for i, targetid in enumerate(targetids):
ii = np.where(spectra.fibermap['TARGETID'] == targetid)[0]
fibermap.add_row(spectra.fibermap[ii[0]])
for channel in spectra.bands:
if len(ii) > 1:
outwave, outflux, outivar, outrdat = _coadd(
spectra.wave[channel], spectra.flux[channel][ii],
spectra.ivar[channel][ii],
spectra.resolution_data[channel][ii])
else:
outwave, outflux, outivar, outrdat = (
spectra.wave[channel], spectra.flux[channel][ii[0]],
spectra.ivar[channel][ii[0]],
spectra.resolution_data[channel][ii[0]])
flux[channel][i] = outflux
ivar[channel][i] = outivar
rdat[channel][i] = outrdat
return Spectra(spectra.bands,
wave,
flux,
ivar,
mask=None,
resolution_data=rdat,
fibermap=fibermap,
meta=spectra.meta)
def sim_spectra(wave,
flux,
program,
spectra_filename,
obsconditions=None,
sourcetype=None,
targetid=None,
redshift=None,
expid=0,
seed=0,
skyerr=0.0,
ra=None,
dec=None):
"""
Simulate spectra from an input set of wavelength and flux and writes a FITS file in the Spectra format that can
be used as input to the redshift fitter.
Args:
wave : 1D np.array of wavelength in Angstrom (in vacuum) in observer frame (i.e. redshifted)
flux : 1D or 2D np.array. 1D array must have same size as wave, 2D array must have shape[1]=wave.size
flux has to be in units of 10^-17 ergs/s/cm2/A
spectra_filename : path to output FITS file in the Spectra format
program : dark, lrg, qso, gray, grey, elg, bright, mws, bgs
ignored if obsconditions is not None
Optional:
obsconditions : dictionnary of observation conditions with SEEING EXPTIME AIRMASS MOONFRAC MOONALT MOONSEP
sourcetype : list of string, allowed values are (sky,elg,lrg,qso,bgs,star), type of sources, used for fiber aperture loss , default is star
targetid : list of targetids for each target. default of None has them generated as str(range(nspec))
redshift : list/array with each index being the redshifts for that target
expid : this expid number will be saved in the Spectra fibermap
seed : random seed
skyerr : fractional sky subtraction error
ra : numpy array with targets RA (deg)
dec : numpy array with targets Dec (deg)
"""
log = get_logger()
if len(flux.shape) == 1:
flux = flux.reshape((1, flux.size))
nspec = flux.shape[0]
log.info("Starting simulation of {} spectra".format(nspec))
if sourcetype is None:
sourcetype = np.array(["star" for i in range(nspec)])
log.debug("sourcetype = {}".format(sourcetype))
tileid = 0
telera = 0
teledec = 0
dateobs = time.gmtime()
night = desisim.obs.get_night(utc=dateobs)
program = program.lower()
frame_fibermap = desispec.io.fibermap.empty_fibermap(nspec)
frame_fibermap.meta["FLAVOR"] = "custom"
frame_fibermap.meta["NIGHT"] = night
frame_fibermap.meta["EXPID"] = expid
# add DESI_TARGET
tm = desitarget.targetmask.desi_mask
frame_fibermap['DESI_TARGET'][sourcetype == "star"] = tm.STD_FSTAR
frame_fibermap['DESI_TARGET'][sourcetype == "lrg"] = tm.LRG
frame_fibermap['DESI_TARGET'][sourcetype == "elg"] = tm.ELG
frame_fibermap['DESI_TARGET'][sourcetype == "qso"] = tm.QSO
frame_fibermap['DESI_TARGET'][sourcetype == "sky"] = tm.SKY
frame_fibermap['DESI_TARGET'][sourcetype == "bgs"] = tm.BGS_ANY
if targetid is None:
targetid = np.arange(nspec).astype(int)
# add TARGETID
frame_fibermap['TARGETID'] = targetid
# spectra fibermap has two extra fields : night and expid
# This would be cleaner if desispec would provide the spectra equivalent
# of desispec.io.empty_fibermap()
spectra_fibermap = desispec.io.empty_fibermap(nspec)
spectra_fibermap = desispec.io.util.add_columns(
spectra_fibermap,
['NIGHT', 'EXPID', 'TILEID'],
[np.int32(night), np.int32(expid),
np.int32(tileid)],
)
for s in range(nspec):
for tp in frame_fibermap.dtype.fields:
spectra_fibermap[s][tp] = frame_fibermap[s][tp]
if ra is not None:
spectra_fibermap["RA_TARGET"] = ra
spectra_fibermap["RA_OBS"] = ra
if dec is not None:
spectra_fibermap["DEC_TARGET"] = dec
spectra_fibermap["DEC_OBS"] = dec
if obsconditions is None:
if program in ['dark', 'lrg', 'qso']:
obsconditions = desisim.simexp.reference_conditions['DARK']
elif program in ['elg', 'gray', 'grey']:
obsconditions = desisim.simexp.reference_conditions['GRAY']
elif program in ['mws', 'bgs', 'bright']:
obsconditions = desisim.simexp.reference_conditions['BRIGHT']
else:
raise ValueError('unknown program {}'.format(program))
elif isinstance(obsconditions, str):
try:
obsconditions = desisim.simexp.reference_conditions[
obsconditions.upper()]
except KeyError:
raise ValueError('obsconditions {} not in {}'.format(
obsconditions.upper(),
list(desisim.simexp.reference_conditions.keys())))
try:
params = desimodel.io.load_desiparams()
wavemin = params['ccd']['b']['wavemin']
wavemax = params['ccd']['z']['wavemax']
except KeyError:
wavemin = desimodel.io.load_throughput('b').wavemin
wavemax = desimodel.io.load_throughput('z').wavemax
if wave[0] > wavemin:
log.warning(
'Minimum input wavelength {}>{}; padding with zeros'.format(
wave[0], wavemin))
dwave = wave[1] - wave[0]
npad = int((wave[0] - wavemin) / dwave + 1)
wavepad = np.arange(npad) * dwave
wavepad += wave[0] - dwave - wavepad[-1]
fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype)
wave = np.concatenate([wavepad, wave])
flux = np.hstack([fluxpad, flux])
assert flux.shape[1] == len(wave)
assert np.allclose(dwave, np.diff(wave))
assert wave[0] <= wavemin
if wave[-1] < wavemax:
log.warning(
'Maximum input wavelength {}<{}; padding with zeros'.format(
wave[-1], wavemax))
dwave = wave[-1] - wave[-2]
npad = int((wavemax - wave[-1]) / dwave + 1)
wavepad = wave[-1] + dwave + np.arange(npad) * dwave
fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype)
wave = np.concatenate([wave, wavepad])
flux = np.hstack([flux, fluxpad])
assert flux.shape[1] == len(wave)
assert np.allclose(dwave, np.diff(wave))
assert wavemax <= wave[-1]
ii = (wavemin <= wave) & (wave <= wavemax)
flux_unit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm**2)
wave = wave[ii] * u.Angstrom
flux = flux[:, ii] * flux_unit
sim = desisim.simexp.simulate_spectra(wave,
flux,
fibermap=frame_fibermap,
obsconditions=obsconditions,
redshift=redshift,
seed=seed,
psfconvolve=True)
random_state = np.random.RandomState(seed)
sim.generate_random_noise(random_state)
scale = 1e17
specdata = None
resolution = {}
for camera in sim.instrument.cameras:
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1])
skyscale = skyerr * random_state.normal(size=sim.num_fibers)
for table in sim.camera_output:
wave = table['wavelength'].astype(float)
flux = (table['observed_flux'] + table['random_noise_electrons'] *
table['flux_calibration']).T.astype(float)
if np.any(skyscale):
flux += ((table['num_sky_electrons'] * skyscale) *
table['flux_calibration']).T.astype(float)
ivar = table['flux_inverse_variance'].T.astype(float)
band = table.meta['name'].strip()[0]
flux = flux * scale
ivar = ivar / scale**2
mask = np.zeros(flux.shape).astype(int)
spec = Spectra([band], {band: wave}, {band: flux}, {band: ivar},
resolution_data={band: resolution[band]},
mask={band: mask},
fibermap=spectra_fibermap,
meta=None,
single=True)
if specdata is None:
specdata = spec
else:
specdata.update(spec)
desispec.io.write_spectra(spectra_filename, specdata)
log.info('Wrote ' + spectra_filename)
# need to clear the simulation buffers that keeps growing otherwise
# because of a different number of fibers each time ...
desisim.specsim._simulators.clear()
desisim.specsim._simdefaults.clear()
def sim_spectra(wave,
flux,
program,
spectra_filename,
obsconditions=None,
expid=0,
seed=0,
survey='desi'):
'''
Simulate spectra from input observer wavelength and (redshifted) flux and writes a .FITS file in the spectra
format that can be used as input to the redshift fitter.
Args:
Wave : 1D np.array of wavelength in Angstrom (in vacuum) in observer frame (i.e. redshifted)
Flux : 1D or 2D np.array. 1D array must have same size as wave,
2D array must have shape[1] = wave.size for multiple input.
Note: Flux has to be in units of 1e-17 [ergs/s/cm2/A].
spectra_filename: Path to output FITS file in the Spectra format
Optional:
obsconditions: Dictionary of observation conditions: {SEEING, EXPTIME, AIRMASS, MOONFRAC, MOONALT, MOONSEP}
expid: This expid number will be saved in the spectra fibermap
seed: Random seed
'''
log = get_logger()
if len(flux.shape) == 1:
flux = flux.reshape((1, flux.size))
nspec = flux.shape[0]
log.info("Starting simulation of {} spectra".format(nspec))
tileid = 0
telera = 0
teledec = 0
night = desisim.obs.get_night(utc=time.gmtime())
frame_fibermap = desispec.io.fibermap.empty_fibermap(nspec)
frame_fibermap.meta["FLAVOR"] = "custom"
frame_fibermap.meta["NIGHT"] = night
frame_fibermap.meta["EXPID"] = expid
## Add DESI_TARGET and TARGETID
tm = desitarget.desi_mask ## Contains 'templates' for STD_FSTAR.
for spec in range(nspec):
frame_fibermap['DESI_TARGET'][spec] = tm.STD_FSTAR
frame_fibermap['TARGETID'][spec] = spec
## Spectra fibermap has two extra fields: night and expid.
spectra_fibermap = np.zeros(shape=(nspec, ), dtype=spectra_dtype())
for s in range(nspec):
for tp in frame_fibermap.dtype.fields:
spectra_fibermap[s][tp] = frame_fibermap[s][tp]
spectra_fibermap[:]['EXPID'] = expid ## Needed by spectra.
spectra_fibermap[:]['NIGHT'] = night ## Needed by spectra.
program = program.lower()
if obsconditions is None:
if program in ['dark', 'lrg', 'qso']:
"""
E.g.
reference_conditions['DARK']['SEEING'] = 1.1
reference_conditions['DARK']['EXPTIME'] = 1000
reference_conditions['DARK']['AIRMASS'] = 1.0
reference_conditions['DARK']['MOONFRAC'] = 0.0
reference_conditions['DARK']['MOONALT'] = -60
reference_conditions['DARK']['MOONSEP'] = 180
"""
obsconditions = desisim.simexp.reference_conditions['DARK']
elif program in ['elg', 'gray', 'grey']:
obsconditions = desisim.simexp.reference_conditions['GRAY']
elif program in ['mws', 'bgs', 'bright']:
obsconditions = desisim.simexp.reference_conditions['BRIGHT']
else:
raise ValueError('Unknown program {}'.format(program))
elif isinstance(obsconditions, str):
try:
obsconditions = desisim.simexp.reference_conditions[
obsconditions.upper()]
except KeyError:
raise ValueError(
'Input observation conditions {} are not in {}'.format(
obsconditions.upper(),
list(desisim.simexp.reference_conditions.keys())))
if survey == 'pfs':
wavemin = 3796. ## [A]
wavemax = 12605. ## [A]
elif survey == 'desi':
wavemin = desimodel.io.load_throughput('b').wavemin
wavemax = desimodel.io.load_throughput('z').wavemax
elif survey == 'beast':
wavemin = 3796. ## [A]
wavemax = 12605. ## [A]
else:
raise ValueError("\n\nSurvey %s is not available." % survey)
log.info("Setting wave limits to survey: {} ... {} to {} [A]".format(
survey, wavemin, wavemax))
ii = (wavemin <= wave) & (wave <= wavemax)
flux_unit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm**2)
wave = wave[
ii] * u.Angstrom ## Dimensionful quantities; only between wavelength limits.
flux = flux[:, ii] * flux_unit
sim = desisim.simexp.simulate_spectra(wave,
flux,
fibermap=frame_fibermap,
obsconditions=obsconditions,
survey=survey)
## Add random noise.
random_state = np.random.RandomState(seed)
sim.generate_random_noise(random_state)
specdata = None
scale = 1.e17
resolution = {}
## Methods for sim object.
for camera in sim.instrument.cameras:
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1])
for table in sim.camera_output:
wave = table['wavelength'].astype(float)
flux = (table['observed_flux'] + table['random_noise_electrons'] *
table['flux_calibration']).T.astype(float)
ivar = table['flux_inverse_variance'].T.astype(float)
band = table.meta['name'].strip()[0]
flux = flux * scale
ivar = ivar / scale**2
mask = np.zeros(flux.shape).astype(int)
## Create spectra object for redrock.
spec = Spectra([band], {band: wave}, {band: flux}, {band: ivar},
resolution_data={band: resolution[band]},
mask={band: mask},
fibermap=spectra_fibermap,
meta=None,
single=True)
if specdata is None:
specdata = spec
else:
specdata.update(spec)
log.info("Writing to: %s" % spectra_filename)
desispec.io.write_spectra(spectra_filename, specdata)
log.info('Successfully created %s.' % spectra_filename)
classification['TARGETID'] = allfmap[idx]['TARGETID']
classification['CNNPRED'] = pred[idx]
classification['CNNLABEL'] = label_names_arr[
labels[idx]]
# Merge the classification and redrock fit to the fibermap.
fmap = join(allfmap[idx],
allzbest[idx],
keys='TARGETID')
fmap = join(fmap, classification, keys='TARGETID')
# Pack data into Spectra and write to FITS.
cand_spectra = Spectra(
bands=['brz'],
wave={'brz': allwave},
flux={'brz': allflux[idx]},
ivar={'brz': allivar[idx]},
resolution_data={'brz': allres[idx]},
fibermap=fmap)
outfits = '{}/transient_candidate_spectra_{}_{}.fits'.format(
out_path, obsdate, tile_number)
write_spectra(outfits, cand_spectra)
print('Output file saved in {}'.format(outfits))
# Make a plot of up to 16 transients
selection = sorted(
np.random.choice(idx.flatten(),
size=np.minimum(len(idx), 16),
replace=False))
def rrtemp_to_spectra(infile, nrshifts=None, tempfluxes=None, tempwaves=None, zbest=None):
'''
returns a list of files of spectra class objects setup to run through redrock
infile: must be a quickspectra infile with keys IVAR, MASK, RESOLUTION
fileloc: file location
tempfluxes: must be a list of redshift template fluxes
tempwaves: must be a list of redshift template waves
zbest: if tempfluxes and tempwaves are not provided the zbest data must be so the templates can be found
'''
fileloc = os.path.dirname(infile)
spectra = desispec.io.read_spectra(infile)
if tempfluxes == None and tempwaves == None:
tempfile = redrock.templates.find_templates()[0]
rrtemp = redrock.templates.Template(tempfile, wave=spectra.wave)
tempfluxes = []
tempwaves = []
for ii in range(len(zbest)):
ncoeff = rrtemp.flux.shape[0]
coeff = zbest['COEFF'][ii][:ncoeff]
tempfluxes.append(rrtemp.flux.T.dot(coeff))
tempwaves.append(rrtemp.wave * (1+zbest[ii]['Z']))
if nrshifts == None:
nrshifts = len(tempfluxes)
spectra_fibermap = Table.read(infile, 'FIBERMAP')
ivarb = fits.getdata(infile, 'B_IVAR')
ivarr = fits.getdata(infile, 'R_IVAR')
ivarz = fits.getdata(infile, 'Z_IVAR')
maskb = fits.getdata(infile, 'B_MASK')
maskr = fits.getdata(infile, 'R_MASK')
maskz = fits.getdata(infile, 'Z_MASK')
resb = fits.getdata(infile, 'B_RESOLUTION')
resr = fits.getdata(infile, 'R_RESOLUTION')
resz = fits.getdata(infile, 'Z_RESOLUTION')
spectra_fibermaps = []
specfiles = []
reszbests = []
for row in spectra_fibermap:
spectra_fibermaps.append(Table(row))
specdata = []
for ii in range(nrshifts):
ivar = {'b': np.array([ivarb[ii]]), 'r': np.array([ivarr[ii]]), 'z': np.array([ivarz[ii]])}
mask = {'b': np.array([maskb[ii]]), 'r': np.array([maskr[ii]]), 'z': np.array([maskz[ii]])}
res = {'b': np.array([resb[ii]]), 'r': np.array([resr[ii]]), 'z': np.array([resz[ii]])}
netflux = {'b': None, 'r': None, 'z': None}
bands = []
for band in spectra.bands:
R = Resolution(spectra.resolution_data[band][0])
txflux = R.dot(resample_flux(spectra.wave[band], tempwaves[ii], tempfluxes[ii]))
netflux[band] = np.array([spectra.flux[band][ii] - txflux])
spec = Spectra(spectra.bands, spectra.wave, netflux, ivar, resolution_data=res, mask=mask,
fibermap=spectra_fibermaps[ii], meta=None, single=True)
residualoutfile = os.path.join(fileloc, 'residualdata-spectra-class-{}-{}'.format(infile[len(fileloc)+1:-5], ii))
specfile = desispec.io.write_spectra(outfile=residualoutfile, spec=spec)
specfiles.append(specfile)
return specfiles
# Apply the DESITrIP preprocessing to selected spectra.
rewave, reflux, reivar = rebin_flux(allwave, allflux, allivar, allzbest['Z'],
minwave=2500., maxwave=9500., nbins=150,
log=True, clip=True)
rsflux = rescale_flux(reflux)
# Run the classification.
if args.tfmodel is not None:
pred = classifier.predict(rsflux)
# Create output: selected target spectra.
selected_spectra = Spectra(bands=['brz'],
wave={'brz' : allwave},
flux={'brz' : allflux},
ivar={'brz' : allivar},
mask={'brz' : allmask},
resolution_data={'brz' : allres},
fibermap=allfmap)
write_spectra('selected-{}-{}.fits'.format(args.tile, args.date), selected_spectra)
# Append preprocess spectra to output.
hx = fits.HDUList()
hdu_rewave = fits.PrimaryHDU(rewave)
hdu_rewave.header['EXTNAME'] = 'REWAVE'
hdu_rewave.header['BUNIT'] = 'Angstrom'
hdu_rewave.header['AIRORVAC'] = ('vac', 'Vacuum wavelengths')
hx.append(hdu_rewave)
def sim_spectra(wave,
flux,
program,
spectra_filename,
obsconditions=None,
sourcetype=None,
expid=0,
seed=0):
"""
Simulate spectra from an input set of wavelength and flux and writes a FITS file in the Spectra format that can
be used as input to the redshift fitter.
Args:
wave : 1D np.array of wavelength in Angstrom (in vacuum) in observer frame (i.e. redshifted)
flux : 1D or 2D np.array. 1D array must have same size as wave, 2D array must have shape[1]=wave.size
flux has to be in units of 10^-17 ergs/s/cm2/A
spectra_filename : path to output FITS file in the Spectra format
Optional:
obsconditions : dictionnary of observation conditions with SEEING EXPTIME AIRMASS MOONFRAC MOONALT MOONSEP
sourcetype : list of string, allowed values are (sky,elg,lrg,qso,bgs,star), type of sources, used for fiber aperture loss , default is star
expid : this expid number will be saved in the Spectra fibermap
seed : random seed
"""
log = get_logger()
if len(flux.shape) == 1:
flux = flux.reshape((1, flux.size))
nspec = flux.shape[0]
log.info("Starting simulation of {} spectra".format(nspec))
if sourcetype is None:
sourcetype = np.array(["star" for i in range(nspec)])
log.debug("sourcetype = {}".format(sourcetype))
tileid = 0
telera = 0
teledec = 0
dateobs = time.gmtime()
night = desisim.obs.get_night(utc=dateobs)
program = program.lower()
frame_fibermap = desispec.io.fibermap.empty_fibermap(nspec)
frame_fibermap.meta["FLAVOR"] = "custom"
frame_fibermap.meta["NIGHT"] = night
frame_fibermap.meta["EXPID"] = expid
# add DESI_TARGET
tm = desitarget.desi_mask
frame_fibermap['DESI_TARGET'][sourcetype == "star"] = tm.STD_FSTAR
frame_fibermap['DESI_TARGET'][sourcetype == "lrg"] = tm.LRG
frame_fibermap['DESI_TARGET'][sourcetype == "elg"] = tm.ELG
frame_fibermap['DESI_TARGET'][sourcetype == "qso"] = tm.QSO
frame_fibermap['DESI_TARGET'][sourcetype == "sky"] = tm.SKY
frame_fibermap['DESI_TARGET'][sourcetype == "bgs"] = tm.BGS_ANY
# add dummy TARGETID
frame_fibermap['TARGETID'] = np.arange(nspec).astype(int)
# spectra fibermap has two extra fields : night and expid
spectra_fibermap = np.zeros(shape=(nspec, ), dtype=spectra_dtype())
for s in range(nspec):
for tp in frame_fibermap.dtype.fields:
spectra_fibermap[s][tp] = frame_fibermap[s][tp]
spectra_fibermap[:]['EXPID'] = expid # needed by spectra
spectra_fibermap[:]['NIGHT'] = night # needed by spectra
if obsconditions is None:
if program in ['dark', 'lrg', 'qso']:
obsconditions = desisim.simexp.reference_conditions['DARK']
elif program in ['elg', 'gray', 'grey']:
obsconditions = desisim.simexp.reference_conditions['GRAY']
elif program in ['mws', 'bgs', 'bright']:
obsconditions = desisim.simexp.reference_conditions['BRIGHT']
else:
raise ValueError('unknown program {}'.format(program))
elif isinstance(obsconditions, str):
try:
obsconditions = desisim.simexp.reference_conditions[
obsconditions.upper()]
except KeyError:
raise ValueError('obsconditions {} not in {}'.format(
obsconditions.upper(),
list(desisim.simexp.reference_conditions.keys())))
wavemin = desimodel.io.load_throughput('b').wavemin
wavemax = desimodel.io.load_throughput('z').wavemax
ii = (wavemin <= wave) & (wave <= wavemax)
flux_unit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm**2)
wave = wave[ii] * u.Angstrom
flux = flux[:, ii] * flux_unit
random_state = np.random.RandomState(seed)
sim = desisim.simexp.simulate_spectra(wave,
flux,
fibermap=frame_fibermap,
obsconditions=obsconditions)
sim.generate_random_noise(random_state)
scale = 1e17
specdata = None
resolution = {}
for camera in sim.instrument.cameras:
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1])
for table in sim.camera_output:
wave = table['wavelength'].astype(float)
flux = (table['observed_flux'] + table['random_noise_electrons'] *
table['flux_calibration']).T.astype(float)
ivar = table['flux_inverse_variance'].T.astype(float)
band = table.meta['name'].strip()[0]
flux = flux * scale
ivar = ivar / scale**2
mask = np.zeros(flux.shape).astype(int)
spec = Spectra([band], {band: wave}, {band: flux}, {band: ivar},
resolution_data={band: resolution[band]},
mask={band: mask},
fibermap=spectra_fibermap,
meta=None,
single=True)
if specdata is None:
specdata = spec
else:
specdata.update(spec)
desispec.io.write_spectra(spectra_filename, specdata)
log.info('Wrote ' + spectra_filename)