def test_sn_weight():
""" Test sn_weight method """
# Very low S/N first
dspec = dummy_spectra(s2n=0.3, seed=1234)
cat_wave = coadd.new_wave_grid(dspec.data['wave'], wave_method='concatenate')
rspec = dspec.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
smask = rspec.data['sig'].filled(0.) > 0.
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
np.testing.assert_allclose(rms_sn[0], 0.318, atol=0.1) # Noise is random
# Low S/N first
dspec = dummy_spectra(s2n=3., seed=1234)
cat_wave = coadd.new_wave_grid(dspec.data['wave'], wave_method='concatenate')
rspec = dspec.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
smask = rspec.data['sig'].filled(0.) > 0.
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
np.testing.assert_allclose(rms_sn[0], 2.934, atol=0.1) # Noise is random
# High S/N now
dspec2 = dummy_spectra(s2n=10., seed=1234)
cat_wave = coadd.new_wave_grid(dspec2.data['wave'], wave_method='concatenate')
rspec2 = dspec2.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
smask = rspec2.data['sig'].filled(0.) > 0.
fluxes, sigs, wave = coadd.unpack_spec(rspec2)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
np.testing.assert_allclose(rms_sn[0], 9.904, atol=0.1) # Noise is random
def test_scale():
""" Test scale algorithms """
# Hand
dspec = dummy_spectra(s2n=10.)
cat_wave = coadd.new_wave_grid(dspec.data['wave'],
wave_method='concatenate')
rspec = dspec.rebin(cat_wave * units.AA,
all=True,
do_sig=True,
masking='none')
smask = rspec.data['sig'].filled(0.) > 0.
sv_high = rspec.copy()
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
_, _ = coadd.scale_spectra(rspec,
smask,
rms_sn,
hand_scale=[3., 5., 10.],
scale_method='hand')
np.testing.assert_allclose(np.median(rspec.flux.value[rspec.sig > 0.]),
3.,
atol=0.01) # Noise is random
# Median
rspec = sv_high.copy()
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
_, mthd = coadd.scale_spectra(rspec, smask, rms_sn, scale_method='median')
assert mthd == 'median_flux'
np.testing.assert_allclose(np.median(rspec.flux.value[rspec.sig > 0.]),
1.,
atol=0.01) # Noise is random
# Auto-none
dspec = dummy_spectra(s2n=0.1)
rspec = dspec.rebin(cat_wave * units.AA,
all=True,
do_sig=True,
masking='none')
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
an_scls, an_mthd = coadd.scale_spectra(rspec, smask, rms_sn)
assert an_mthd == 'none_SN'
# Auto-median
dspec = dummy_spectra(s2n=1.5)
rspec = dspec.rebin(cat_wave * units.AA,
all=True,
do_sig=True,
masking='none')
rspec.data['flux'][1, :] *= 10.
rspec.data['sig'][1, :] *= 10.
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
am_scls, am_mthd = coadd.scale_spectra(rspec,
smask,
rms_sn,
scale_method='median')
assert am_mthd == 'median_flux'
np.testing.assert_allclose(am_scls[1], 0.1, atol=0.01)
def order_median_scale(spectra, nsig=3.0, niter=5, overlapfrac=0.03, num_min_pixels=50, SN_MIN_MEDSCALE=0.5, debug=False):
'''
Scale different orders using the median of overlap regions. It starts from the reddest order, i.e. scale H to K,
and then scale J to H+K, etc.
Parameters:
spectra: XSpectrum1D spectra
nsig: float
sigma used for sigma_clipping median
niter: int
number of iterations for sigma_clipping median
overlapfrac: float
minmum overlap fraction (number of overlapped pixels devided by number of pixels of the whole spectrum) between orders.
num_min_pixels: int
minum required good pixels. The code only scale orders when the overlapped
pixels > max(num_min_pixels,overlapfrac*len(wave))
SN_MIN_MEDSCALE: float
Maximum RMS S/N allowed to automatically apply median scaling
Show QA plot if debug=True
Return:
No return, but the spectra is already scaled after executing this function.
'''
norder = spectra.nspec
fluxes, sigs, wave = coadd.unpack_spec(spectra, all_wave=False)
fluxes_raw = fluxes.copy()
# scaling spectrum order by order. We use the reddest order as the reference since slit loss in redder is smaller
for i in range(norder - 1):
iord = norder - i - 1
sn_iord_iref = fluxes[iord] * (1. / sigs[iord])
sn_iord_scale = fluxes[iord - 1] * (1. / sigs[iord - 1])
allok = (sigs[iord - 1, :] > 0) & (sigs[iord, :] > 0) & (sn_iord_iref > SN_MIN_MEDSCALE) & (
sn_iord_scale > SN_MIN_MEDSCALE)
if sum(allok) > np.maximum(num_min_pixels, len(wave) * overlapfrac):
# Ratio
med_flux = spectra.data['flux'][iord, allok] / spectra.data['flux'][iord - 1, allok]
# Clip
mn_scale, med_scale, std_scale = stats.sigma_clipped_stats(med_flux, sigma=nsig, iters=niter)
med_scale = np.minimum(med_scale, 5.0)
spectra.data['flux'][iord - 1, :] *= med_scale
spectra.data['sig'][iord - 1, :] *= med_scale
msgs.info('Scaled %s order by a factor of %s'%(iord,str(med_scale)))
if debug:
plt.plot(wave, spectra.data['flux'][iord], 'r-', label='reference spectrum')
plt.plot(wave, fluxes_raw[iord - 1], 'k-', label='raw spectrum')
plt.plot(spectra.data['wave'][iord - 1, :], spectra.data['flux'][iord - 1, :], 'b-',
label='scaled spectrum')
mny, medy, stdy = stats.sigma_clipped_stats(fluxes[iord, allok], sigma=nsig, iters=niter)
plt.ylim([0.1 * medy, 4.0 * medy])
plt.xlim([np.min(wave[sigs[iord - 1, :] > 0]), np.max(wave[sigs[iord, :] > 0])])
plt.legend()
plt.xlabel('wavelength')
plt.ylabel('Flux')
plt.show()
else:
msgs.warn('Not enough overlap region for sticking different orders.')
def test_1dcoadd():
""" Test 1dcoadd method"""
# Setup
dspec = dummy_spectra(s2n=10.)
cat_wave = coadd.new_wave_grid(dspec.data['wave'], wave_method='concatenate')
rspec = dspec.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
smask = rspec.data['sig'].filled(0.) > 0.
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
# Coadd
spec1d = coadd.one_d_coadd(rspec, smask, weights)
assert spec1d.npix == 1740
def ech_coadd(files,
objids=None,
extract='OPT',
flux=True,
giantcoadd=False,
orderscale='median',
mergeorder=True,
wave_grid_method='velocity',
niter=5,
wave_grid_min=None,
wave_grid_max=None,
v_pix=None,
scale_method='auto',
do_offset=False,
sigrej_final=3.,
do_var_corr=False,
SN_MIN_MEDSCALE=0.5,
overlapfrac=0.01,
num_min_pixels=10,
phot_scale_dicts=None,
qafile=None,
outfile=None,
do_cr=True,
debug=False,
**kwargs):
"""
routines for coadding spectra observed with echelle spectrograph.
parameters:
files (list): file names
objids (str): objid
extract (str): 'OPT' or 'BOX'
flux (bool): fluxed or not
giantcoadd (bool): coadding order by order or do it at once?
wave_grid_method (str): default velocity
niter (int): number of iteration for rejections
wave_grid_min (float): min wavelength, None means it will find the min value from your spectra
wave_grid_max (float): max wavelength, None means it will find the max value from your spectra
v_pix (float): delta velocity, see coadd.py
scale_method (str): see coadd.py
do_offset (str): see coadd.py, not implemented yet.
sigrej_final (float): see coadd.py
do_var_corr (bool): see coadd.py, default False. It seems True will results in a large error
SN_MIN_MEDSCALE (float): minimum SNR for scaling different orders
overlapfrac (float): minimum overlap fraction for scaling different orders.
qafile (str): name of qafile
outfile (str): name of coadded spectrum
do_cr (bool): remove cosmic rays?
debug (bool): show debug plots?
kwargs: see coadd.py
returns:
spec1d: coadded XSpectrum1D
"""
nfile = len(files)
if nfile <= 1:
msgs.info('Only one spectrum exits coadding...')
return
fname = files[0]
ext_final = fits.getheader(fname, -1)
norder = ext_final['ECHORDER'] + 1
msgs.info('spectrum {:s} has {:d} orders'.format(fname, norder))
if norder <= 1:
msgs.error(
'The number of orders have to be greater than one for echelle. Longslit data?'
)
if giantcoadd:
msgs.info('Coadding all orders and exposures at once')
spectra = load.ech_load_spec(files,
objid=objids,
order=None,
extract=extract,
flux=flux)
wave_grid = np.zeros((2, spectra.nspec))
for i in range(spectra.nspec):
wave_grid[0, i] = spectra[i].wvmin.value
wave_grid[1, i] = spectra[i].wvmax.value
ech_kwargs = {
'echelle': True,
'wave_grid_min': np.min(wave_grid),
'wave_grid_max': np.max(wave_grid),
'v_pix': v_pix
}
kwargs.update(ech_kwargs)
# Coadding
spec1d = coadd.coadd_spectra(spectra,
wave_grid_method=wave_grid_method,
niter=niter,
scale_method=scale_method,
do_offset=do_offset,
sigrej_final=sigrej_final,
do_var_corr=do_var_corr,
qafile=qafile,
outfile=outfile,
do_cr=do_cr,
debug=debug,
**kwargs)
else:
msgs.info('Coadding individual orders first and then merge order')
spectra_list = []
# Keywords for Table
rsp_kwargs = {}
rsp_kwargs['wave_tag'] = '{:s}_WAVE'.format(extract)
rsp_kwargs['flux_tag'] = '{:s}_FLAM'.format(extract)
rsp_kwargs['sig_tag'] = '{:s}_FLAM_SIG'.format(extract)
#wave_grid = np.zeros((2,norder))
for iord in range(norder):
spectra = load.ech_load_spec(files,
objid=objids,
order=iord,
extract=extract,
flux=flux)
ech_kwargs = {
'echelle': False,
'wave_grid_min': spectra.wvmin.value,
'wave_grid_max': spectra.wvmax.value,
'v_pix': v_pix
}
#wave_grid[0,iord] = spectra.wvmin.value
#wave_grid[1,iord] = spectra.wvmax.value
kwargs.update(ech_kwargs)
# Coadding the individual orders
if qafile is not None:
qafile_iord = qafile + '_%s' % str(iord)
else:
qafile_iord = None
spec1d_iord = coadd.coadd_spectra(
spectra,
wave_grid_method=wave_grid_method,
niter=niter,
scale_method=scale_method,
do_offset=do_offset,
sigrej_final=sigrej_final,
do_var_corr=do_var_corr,
qafile=qafile_iord,
outfile=None,
do_cr=do_cr,
debug=debug,
**kwargs)
spectrum = spec_from_array(spec1d_iord.wavelength,
spec1d_iord.flux, spec1d_iord.sig,
**rsp_kwargs)
spectra_list.append(spectrum)
spectra_coadd = collate(spectra_list)
# Rebin the spectra
# ToDo: we should read in JFH's wavelength grid here.
# Join into one XSpectrum1D object
# Final wavelength array
kwargs['wave_grid_min'] = np.min(
spectra_coadd.data['wave'][spectra_coadd.data['wave'] > 0])
kwargs['wave_grid_max'] = np.max(
spectra_coadd.data['wave'][spectra_coadd.data['wave'] > 0])
wave_final = coadd.new_wave_grid(spectra_coadd.data['wave'],
wave_method=wave_grid_method,
**kwargs)
# The rebin function in linetools can not work on collated spectra (i.e. filled 0).
# Thus I have to rebin the spectra first and then collate again.
spectra_list_new = []
for i in range(spectra_coadd.nspec):
speci = spectra_list[i].rebin(wave_final * units.AA,
all=True,
do_sig=True,
grow_bad_sig=True,
masking='none')
spectra_list_new.append(speci)
spectra_coadd_rebin = collate(spectra_list_new)
## Note
if orderscale == 'photometry':
# Only tested on NIRES.
if phot_scale_dicts is not None:
spectra_coadd_rebin = order_phot_scale(spectra_coadd_rebin,
phot_scale_dicts,
debug=debug)
else:
msgs.warn(
'No photometric information is provided. Will use median scale.'
)
orderscale = 'median'
elif orderscale == 'median':
#rmask = spectra_coadd_rebin.data['sig'].filled(0.) > 0.
#sn2, weights = coadd.sn_weights(fluxes, sigs, rmask, wave)
## scaling different orders
order_median_scale(spectra_coadd_rebin,
nsig=sigrej_final,
niter=niter,
overlapfrac=overlapfrac,
num_min_pixels=num_min_pixels,
SN_MIN_MEDSCALE=SN_MIN_MEDSCALE,
debug=debug)
else:
msgs.warn('No any scaling is performed between different orders.')
if mergeorder:
fluxes, sigs, wave = coadd.unpack_spec(spectra_coadd_rebin,
all_wave=False)
## Megering orders
msgs.info('Merging different orders')
## ToDo: Joe claimed not to use pixel depedent weighting.
weights = 1.0 / sigs**2
weights[~np.isfinite(weights)] = 0.0
weight_combine = np.sum(weights, axis=0)
weight_norm = weights / weight_combine
weight_norm[np.isnan(weight_norm)] = 1.0
flux_final = np.sum(fluxes * weight_norm, axis=0)
sig_final = np.sqrt(np.sum((weight_norm * sigs)**2, axis=0))
spec1d_final = spec_from_array(wave_final * units.AA, flux_final,
sig_final, **rsp_kwargs)
if outfile is not None:
msgs.info(
'Saving the final calibrated spectrum as {:s}'.format(
outfile))
coadd.write_to_disk(spec1d_final, outfile)
if (qafile is not None) or (debug):
# plot and save qa
plt.figure(figsize=(12, 6))
ax1 = plt.axes([0.07, 0.13, 0.9, 0.4])
ax2 = plt.axes([0.07, 0.55, 0.9, 0.4])
plt.setp(ax2.get_xticklabels(), visible=False)
medf = np.median(spec1d_final.flux)
ylim = (np.sort([0. - 0.3 * medf, 5 * medf]))
cmap = plt.get_cmap('RdYlBu_r')
for idx in range(spectra_coadd_rebin.nspec):
spectra_coadd_rebin.select = idx
color = cmap(float(idx) / spectra_coadd_rebin.nspec)
ind_good = spectra_coadd_rebin.sig > 0
ax1.plot(spectra_coadd_rebin.wavelength[ind_good],
spectra_coadd_rebin.flux[ind_good],
color=color)
if (np.max(spec1d_final.wavelength) > (9000.0 * units.AA)):
skytrans_file = resource_filename(
'pypeit',
'/data/skisim/atm_transmission_secz1.5_1.6mm.dat')
skycat = np.genfromtxt(skytrans_file, dtype='float')
scale = 0.85 * ylim[1]
ax2.plot(skycat[:, 0] * 1e4,
skycat[:, 1] * scale,
'm-',
alpha=0.5)
ax2.plot(spec1d_final.wavelength,
spec1d_final.sig,
ls='steps-',
color='0.7')
ax2.plot(spec1d_final.wavelength,
spec1d_final.flux,
ls='steps-',
color='b')
ax1.set_xlim([
np.min(spec1d_final.wavelength.value),
np.max(spec1d_final.wavelength.value)
])
ax2.set_xlim([
np.min(spec1d_final.wavelength.value),
np.max(spec1d_final.wavelength.value)
])
ax1.set_ylim(ylim)
ax2.set_ylim(ylim)
ax1.set_xlabel('Wavelength (Angstrom)')
ax1.set_ylabel('Flux')
ax2.set_ylabel('Flux')
plt.tight_layout(pad=0.2, h_pad=0., w_pad=0.2)
if len(qafile.split('.')) == 1:
msgs.info(
"No fomat given for the qafile, save to PDF format.")
qafile = qafile + '.pdf'
if qafile:
plt.savefig(qafile)
msgs.info("Wrote coadd QA: {:s}".format(qafile))
if debug:
plt.show()
plt.close()
### Do NOT remove this part althoug it is deprecated.
# we may need back to using this pieces of code after fixing the coadd.coadd_spectra problem on first order.
#kwargs['echelle'] = True
#kwargs['wave_grid_min'] = np.min(wave_grid)
#kwargs['wave_grid_max'] = np.max(wave_grid)
#spec1d_final = coadd.coadd_spectra(spectra_coadd_rebin, wave_grid_method=wave_grid_method, niter=niter,
# scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final,
# do_var_corr=do_var_corr, qafile=qafile, outfile=outfile,
# do_cr=do_cr, debug=debug, **kwargs)
return spec1d_final
else:
msgs.warn('Skipped merging orders')
if outfile is not None:
for iord in range(len(spectra_list)):
outfile_iord = outfile.replace(
'.fits', '_ORDER{:04d}.fits'.format(iord))
msgs.info(
'Saving the final calibrated spectrum of order {:d} as {:s}'
.format(iord, outfile))
spectra_list[iord].write_to_fits(outfile_iord)
return spectra_list
