def test_sn_weight():
""" Test sn_weight method """
# Very low S/N first
dspec = dummy_spectra(s2n=0.3, seed=1234)
cat_wave = coadd.new_wave_grid(dspec.data['wave'], wave_method='concatenate')
rspec = dspec.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
smask = rspec.data['sig'].filled(0.) > 0.
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
np.testing.assert_allclose(rms_sn[0], 0.318, atol=0.1) # Noise is random
# Low S/N first
dspec = dummy_spectra(s2n=3., seed=1234)
cat_wave = coadd.new_wave_grid(dspec.data['wave'], wave_method='concatenate')
rspec = dspec.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
smask = rspec.data['sig'].filled(0.) > 0.
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
np.testing.assert_allclose(rms_sn[0], 2.934, atol=0.1) # Noise is random
# High S/N now
dspec2 = dummy_spectra(s2n=10., seed=1234)
cat_wave = coadd.new_wave_grid(dspec2.data['wave'], wave_method='concatenate')
rspec2 = dspec2.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
smask = rspec2.data['sig'].filled(0.) > 0.
fluxes, sigs, wave = coadd.unpack_spec(rspec2)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
np.testing.assert_allclose(rms_sn[0], 9.904, atol=0.1) # Noise is random
def test_scale():
""" Test scale algorithms """
# Hand
dspec = dummy_spectra(s2n=10.)
cat_wave = coadd.new_wave_grid(dspec.data['wave'],
wave_method='concatenate')
rspec = dspec.rebin(cat_wave * units.AA,
all=True,
do_sig=True,
masking='none')
smask = rspec.data['sig'].filled(0.) > 0.
sv_high = rspec.copy()
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
_, _ = coadd.scale_spectra(rspec,
smask,
rms_sn,
hand_scale=[3., 5., 10.],
scale_method='hand')
np.testing.assert_allclose(np.median(rspec.flux.value[rspec.sig > 0.]),
3.,
atol=0.01) # Noise is random
# Median
rspec = sv_high.copy()
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
_, mthd = coadd.scale_spectra(rspec, smask, rms_sn, scale_method='median')
assert mthd == 'median_flux'
np.testing.assert_allclose(np.median(rspec.flux.value[rspec.sig > 0.]),
1.,
atol=0.01) # Noise is random
# Auto-none
dspec = dummy_spectra(s2n=0.1)
rspec = dspec.rebin(cat_wave * units.AA,
all=True,
do_sig=True,
masking='none')
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
an_scls, an_mthd = coadd.scale_spectra(rspec, smask, rms_sn)
assert an_mthd == 'none_SN'
# Auto-median
dspec = dummy_spectra(s2n=1.5)
rspec = dspec.rebin(cat_wave * units.AA,
all=True,
do_sig=True,
masking='none')
rspec.data['flux'][1, :] *= 10.
rspec.data['sig'][1, :] *= 10.
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
am_scls, am_mthd = coadd.scale_spectra(rspec,
smask,
rms_sn,
scale_method='median')
assert am_mthd == 'median_flux'
np.testing.assert_allclose(am_scls[1], 0.1, atol=0.01)
def get_brightest_obj(specobjs_list, echelle=True):
"""
Utility routine to find the brightest object in each exposure given a specobjs_list. This currently only works
for echelle.
Parameters:
specobjs_list: list
List of SpecObjs objects.
Optional Parameters:
echelle: bool, default=True
Returns:
(objid, snr_bar), tuple
objid: ndarray, int, shape (len(list),)
Array of object ids representing the brightest object in each exposure
snr_bar: ndarray, float, shape (len(list),)
Average S/N over all the orders for this object
"""
nexp = len(specobjs_list)
objid = np.zeros(nexp, dtype=int)
snr_bar = np.zeros(nexp)
if echelle:
norders = specobjs_list[0].ech_orderindx.max() + 1
for iexp, sobjs in enumerate(specobjs_list):
uni_objid = np.unique(sobjs.ech_objid)
nobjs = len(uni_objid)
order_snr = np.zeros((norders, nobjs))
for iord in range(norders):
for iobj in range(nobjs):
ind = (sobjs.ech_orderindx == iord) & (sobjs.ech_objid
== uni_objid[iobj])
flux = sobjs[ind][0].optimal['COUNTS']
sig = sobjs[ind][0].optimal['COUNTS_SIG']
wave = sobjs[ind][0].optimal['WAVE']
mask = sobjs[ind][0].optimal['MASK']
rms_sn, weights = coadd.sn_weights(flux,
sig,
mask,
wave,
const_weights=True)
order_snr[iord, iobj] = rms_sn
# Compute the average SNR and find the brightest object
snr_bar_vec = np.mean(order_snr, axis=0)
objid[iexp] = uni_objid[snr_bar_vec.argmax()]
snr_bar[iexp] = snr_bar_vec[snr_bar_vec.argmax()]
else:
msgs.error(
'Automated objid selection is not yet implemented for multislit')
# -- for multislit this routine will:
# Determine which object on the slitmask is brightest and attempt to match them up somehow. The problem is
# how to associate the objects together on the slits if they have moved.
return objid, snr_bar
def test_1dcoadd():
""" Test 1dcoadd method"""
# Setup
dspec = dummy_spectra(s2n=10.)
cat_wave = coadd.new_wave_grid(dspec.data['wave'], wave_method='concatenate')
rspec = dspec.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
smask = rspec.data['sig'].filled(0.) > 0.
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
# Coadd
spec1d = coadd.one_d_coadd(rspec, smask, weights)
assert spec1d.npix == 1740
def optimal_weights(specobjs_list, slitid, objid):
"""
Determine optimal weights for a 2d coadds. This script grabs the information from SpecObjs list for the
object with specified slitid and objid and passes to coadd.sn_weights to determine the optimal weights for
each exposure. This routine will also pass back the trace and the wavelengths (optimally extracted) for each
exposure.
Args:
specobjs_list: list
list of SpecObjs objects contaning the objects that were extracted from each frame that will contribute
to the coadd.
slitid: int
The slitid that has the brightest object whose S/N will be used to determine the weight for each frame.
objid: int
The objid index of the brightest object whose S/N will be used to determine the weight for each frame.
Returns:
(rms_sn, weights, trace_stack, wave_stack)
rms_sn : ndarray, shape = (len(specobjs_list),)
Root mean square S/N value for each input spectra
weights : ndarray, shape (len(specobjs_list),)
Weights to be applied to the spectra. These are signal-to-noise squared weights.
trace_stack: ndarray, shape = (len(specobs_list), nspec)
Traces for each exposure
wave_stack: ndarray, shape = (len(specobs_list), nspec)
Wavelengths (optimally extracted) for each exposure.
"""
nexp = len(specobjs_list)
nspec = specobjs_list[0][0].trace_spat.shape[0]
# Grab the traces, flux, wavelength and noise for this slit and objid.
trace_stack = np.zeros((nexp, nspec), dtype=float)
flux_stack = np.zeros((nexp, nspec), dtype=float)
sig_stack = np.zeros((nexp, nspec), dtype=float)
wave_stack = np.zeros((nexp, nspec), dtype=float)
mask_stack = np.zeros((nexp, nspec), dtype=bool)
for iexp, sobjs in enumerate(specobjs_list):
ithis = (sobjs.slitid == slitid) & (sobjs.objid == objid[iexp])
trace_stack[iexp, :] = sobjs[ithis].trace_spat
flux_stack[iexp, :] = sobjs[ithis][0].optimal['COUNTS']
sig_stack[iexp, :] = sobjs[ithis][0].optimal['COUNTS_SIG']
wave_stack[iexp, :] = sobjs[ithis][0].optimal['WAVE']
mask_stack[iexp, :] = sobjs[ithis][0].optimal['MASK']
# TODO For now just use the zero as the reference for the wavelengths? Perhaps we should be rebinning the data though?
rms_sn, weights = coadd.sn_weights(flux_stack, sig_stack, mask_stack,
wave_stack)
return rms_sn, weights, trace_stack, wave_stack
def optimal_weights(self, slitorderid, objid, const_weights=False):
"""
Determine optimal weights for 2d coadds. This script grabs the information from SpecObjs list for the
object with specified slitid and objid and passes to coadd.sn_weights to determine the optimal weights for
each exposure.
Parameters
----------
slitorderid : :obj:`int`
The slit or order id that has the brightest object whose
S/N will be used to determine the weight for each frame.
objid : `numpy.ndarray`_
Array of object indices with shape = (nexp,) of the
brightest object whose S/N will be used to determine the
weight for each frame.
const_weights : :obj:`bool`
Use constant weights for coadding the exposures.
Default=False
Returns
-------
rms_sn : ndarray, shape = (len(specobjs_list),)
Root mean square S/N value for each input spectra
weights : ndarray, shape (len(specobjs_list),)
Weights to be applied to the spectra. These are
signal-to-noise squared weights.
"""
nexp = len(self.stack_dict['specobjs_list'])
nspec = self.stack_dict['specobjs_list'][0][0].TRACE_SPAT.shape[0]
# Grab the traces, flux, wavelength and noise for this slit and objid.
flux_stack = np.zeros((nspec, nexp), dtype=float)
ivar_stack = np.zeros((nspec, nexp), dtype=float)
wave_stack = np.zeros((nspec, nexp), dtype=float)
mask_stack = np.zeros((nspec, nexp), dtype=bool)
for iexp, sobjs in enumerate(self.stack_dict['specobjs_list']):
ithis = sobjs.slitorder_objid_indices(slitorderid, objid[iexp])
flux_stack[:, iexp] = sobjs[ithis].OPT_COUNTS
ivar_stack[:, iexp] = sobjs[ithis].OPT_COUNTS_IVAR
wave_stack[:, iexp] = sobjs[ithis].OPT_WAVE
mask_stack[:, iexp] = sobjs[ithis].OPT_MASK
# TODO For now just use the zero as the reference for the wavelengths? Perhaps we should be rebinning the data though?
rms_sn, weights = coadd.sn_weights(wave_stack, flux_stack, ivar_stack, mask_stack, self.sn_smooth_npix,
const_weights=const_weights)
return rms_sn, weights.T
def get_brightest_obj(self, specobjs_list, nslits):
"""
Utility routine to find the brightest object in each exposure given a specobjs_list for Echelle reductions.
Args:
specobjs_list: list
List of SpecObjs objects.
echelle: bool, default=True, optional
Returns:
tuple: Returns the following:
- objid: ndarray, int, shape (len(specobjs_list),):
Array of object ids representing the brightest object
in each exposure
- snr_bar: ndarray, float, shape (len(list),): Average
S/N over all the orders for this object
"""
nexp = len(specobjs_list)
objid = np.zeros(nexp, dtype=int)
snr_bar = np.zeros(nexp)
# norders = specobjs_list[0].ech_orderindx.max() + 1
for iexp, sobjs in enumerate(specobjs_list):
uni_objid = np.unique(sobjs.ECH_OBJID)
nobjs = len(uni_objid)
order_snr = np.zeros((nslits, nobjs))
for iord in range(nslits):
for iobj in range(nobjs):
ind = (sobjs.ECH_ORDERINDX == iord) & (sobjs.ECH_OBJID == uni_objid[iobj])
flux = sobjs[ind][0].OPT_COUNTS
ivar = sobjs[ind][0].OPT_COUNTS_IVAR
wave = sobjs[ind][0].OPT_WAVE
mask = sobjs[ind][0].OPT_MASK
rms_sn, weights = coadd.sn_weights(wave, flux, ivar, mask, self.sn_smooth_npix, const_weights=True)
order_snr[iord, iobj] = rms_sn
# Compute the average SNR and find the brightest object
snr_bar_vec = np.mean(order_snr, axis=0)
objid[iexp] = uni_objid[snr_bar_vec.argmax()]
snr_bar[iexp] = snr_bar_vec[snr_bar_vec.argmax()]
self.snr_report(snr_bar)
return objid, None, snr_bar
def compute_weights(all_ra,
all_dec,
all_wave,
all_sci,
all_ivar,
all_idx,
whitelight_img,
dspat,
dwv,
sn_smooth_npix=None,
relative_weights=False):
""" Calculate wavelength dependent optimal weights. The weighting
is currently based on a relative (S/N)^2 at each wavelength
Args:
all_ra (`numpy.ndarray`_):
1D flattened array containing the RA values of each pixel from all spec2d files
all_dec (`numpy.ndarray`_):
1D flattened array containing the DEC values of each pixel from all spec2d files
all_wave (`numpy.ndarray`_):
1D flattened array containing the wavelength values of each pixel from all spec2d files
all_sci (`numpy.ndarray`_):
1D flattened array containing the counts of each pixel from all spec2d files
all_ivar (`numpy.ndarray`_):
1D flattened array containing the inverse variance of each pixel from all spec2d files
all_idx (`numpy.ndarray`_):
1D flattened array containing an integer identifier indicating which spec2d file
each pixel originates from. For example, a 0 would indicate that a pixel originates
from the first spec2d frame listed in the input file. a 1 would indicate that this
pixel originates from the second spec2d file, and so forth.
whitelight_img (`numpy.ndarray`_):
A 2D array containing a whitelight image, that was created with the input all_* arrays.
dspat (float):
The size of each spaxel on the sky (in degrees)
dwv (float):
The size of each wavelength pixel (in Angstroms)
sn_smooth_npix (float, optional):
Number of pixels used for determining smoothly varying S/N ratio weights.
This is currently not required, since a relative weighting scheme with a
polynomial fit is used to calculate the S/N weights.
relative_weights (bool, optional):
Calculate weights by fitting to the ratio of spectra?
Returns:
`numpy.ndarray`_ : a 1D array the same size as all_sci, containing relative wavelength
dependent weights of each input pixel.
"""
msgs.info("Calculating the optimal weights of each pixel")
# Determine number of files
numfiles = np.unique(all_idx).size
# Find the location of the object with the highest S/N in the combined white light image
idx_max = np.unravel_index(np.argmax(whitelight_img), whitelight_img.shape)
msgs.info(
"Highest S/N object located at spaxel (x, y) = {0:d}, {1:d}".format(
idx_max[0], idx_max[1]))
# Generate a master 2D WCS to register all frames
coord_min = [np.min(all_ra), np.min(all_dec), np.min(all_wave)]
coord_dlt = [dspat, dspat, dwv]
whitelightWCS = generate_masterWCS(coord_min, coord_dlt)
# Make the bin edges to be at +/- 1 pixels around the maximum (i.e. summing 9 pixels total)
numwav = int((np.max(all_wave) - np.min(all_wave)) / dwv)
xbins = np.array([idx_max[0] - 1, idx_max[0] + 2]) - 0.5
ybins = np.array([idx_max[1] - 1, idx_max[1] + 2]) - 0.5
spec_bins = np.arange(1 + numwav) - 0.5
bins = (xbins, ybins, spec_bins)
# Extract the spectrum of the highest S/N object
flux_stack = np.zeros((numwav, numfiles))
ivar_stack = np.zeros((numwav, numfiles))
for ff in range(numfiles):
msgs.info(
"Extracting spectrum of highest S/N detection from frame {0:d}/{1:d}"
.format(ff + 1, numfiles))
ww = (all_idx == ff)
# Extract the spectrum
pix_coord = whitelightWCS.wcs_world2pix(
np.vstack((all_ra[ww], all_dec[ww], all_wave[ww] * 1.0E-10)).T, 0)
spec, edges = np.histogramdd(pix_coord, bins=bins, weights=all_sci[ww])
var, edges = np.histogramdd(pix_coord,
bins=bins,
weights=1 / all_ivar[ww])
norm, edges = np.histogramdd(pix_coord, bins=bins)
normspec = (norm > 0) / (norm + (norm == 0))
var_spec = var[0, 0, :]
ivar_spec = (var_spec > 0) / (var_spec + (var_spec == 0))
# Calculate the S/N in a given spectral bin
flux_stack[:, ff] = spec[0, 0, :] * np.sqrt(
normspec
) # Note: sqrt(nrmspec), is because we want the S/N in a _single_ pixel (i.e. not spectral bin)
ivar_stack[:, ff] = ivar_spec
mask_stack = (flux_stack != 0.0) & (ivar_stack != 0.0)
# Obtain a wavelength of each pixel
wcs_res = whitelightWCS.wcs_pix2world(
np.vstack((np.zeros(numwav), np.zeros(numwav), np.arange(numwav))).T,
0)
wave_spec = wcs_res[:, 2] * 1.0E10
# Compute the smoothing scale to use
if sn_smooth_npix is None:
sn_smooth_npix = int(np.round(0.1 * wave_spec.size))
rms_sn, weights = coadd.sn_weights(wave_spec,
flux_stack,
ivar_stack,
mask_stack,
sn_smooth_npix,
relative_weights=relative_weights)
# Because we pass back a weights array, we need to interpolate to assign each detector pixel a weight
all_wghts = np.ones(all_idx.size)
for ff in range(numfiles):
ww = (all_idx == ff)
all_wghts[ww] = interp1d(wave_spec,
weights[:, ff],
kind='cubic',
bounds_error=False,
fill_value="extrapolate")(all_wave[ww])
msgs.info("Optimal weighting complete")
return all_wghts
def get_brightest_obj(self, specobjs_list, spat_ids):
"""
Utility routine to find the brightest object in each exposure given a specobjs_list for MultiSlit reductions.
Args:
specobjs_list: list
List of SpecObjs objects.
spat_ids (`numpy.ndarray`_):
Returns:
tuple: Returns the following:
- objid: ndarray, int, shape (len(specobjs_list),):
Array of object ids representing the brightest object
in each exposure
- slit_idx (int): 0-based index
- spat_id (int): SPAT_ID for slit that highest S/N ratio object is on
(only for pypeline=MultiSlit)
- snr_bar: ndarray, float, shape (len(list),): Average
S/N over all the orders for this object
"""
nexp = len(specobjs_list)
nspec = specobjs_list[0][0].TRACE_SPAT.shape[0]
nslits = spat_ids.size
slit_snr_max = np.full((nslits, nexp), -np.inf)
objid_max = np.zeros((nslits, nexp), dtype=int)
# Loop over each exposure, slit, find the brighest object on that slit for every exposure
for iexp, sobjs in enumerate(specobjs_list):
msgs.info("Working on exposure {}".format(iexp))
for islit, spat_id in enumerate(spat_ids):
ithis = sobjs.SLITID == spat_id
nobj_slit = np.sum(ithis)
if np.any(ithis):
objid_this = sobjs[ithis].OBJID
flux = np.zeros((nspec, nobj_slit))
ivar = np.zeros((nspec, nobj_slit))
wave = np.zeros((nspec, nobj_slit))
mask = np.zeros((nspec, nobj_slit), dtype=bool)
for iobj, spec in enumerate(sobjs[ithis]):
flux[:, iobj] = spec.OPT_COUNTS
ivar[:, iobj] = spec.OPT_COUNTS_IVAR
wave[:, iobj] = spec.OPT_WAVE
mask[:, iobj] = spec.OPT_MASK
rms_sn, weights = coadd.sn_weights(wave, flux, ivar, mask, None, const_weights=True)
imax = np.argmax(rms_sn)
slit_snr_max[islit, iexp] = rms_sn[imax]
objid_max[islit, iexp] = objid_this[imax]
# Find the highest snr object among all the slits
slit_snr = np.mean(slit_snr_max, axis=1)
slitid = slit_snr.argmax()
snr_bar_mean = slit_snr[slitid]
snr_bar = slit_snr_max[slitid, :]
objid = objid_max[slitid, :]
if (snr_bar_mean == -np.inf):
msgs.error('You do not appear to have a unique reference object that was traced as the highest S/N '
'ratio on the same slit of every exposure')
self.snr_report(snr_bar, slitid=slitid)
return objid, slitid, spat_ids[slitid], snr_bar