def test_sn_weight():
""" Test sn_weight method """
# Very low S/N first
dspec = dummy_spectra(s2n=0.3, seed=1234)
cat_wave = coadd.new_wave_grid(dspec.data['wave'], wave_method='concatenate')
rspec = dspec.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
smask = rspec.data['sig'].filled(0.) > 0.
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
np.testing.assert_allclose(rms_sn[0], 0.318, atol=0.1) # Noise is random
# Low S/N first
dspec = dummy_spectra(s2n=3., seed=1234)
cat_wave = coadd.new_wave_grid(dspec.data['wave'], wave_method='concatenate')
rspec = dspec.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
smask = rspec.data['sig'].filled(0.) > 0.
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
np.testing.assert_allclose(rms_sn[0], 2.934, atol=0.1) # Noise is random
# High S/N now
dspec2 = dummy_spectra(s2n=10., seed=1234)
cat_wave = coadd.new_wave_grid(dspec2.data['wave'], wave_method='concatenate')
rspec2 = dspec2.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
smask = rspec2.data['sig'].filled(0.) > 0.
fluxes, sigs, wave = coadd.unpack_spec(rspec2)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
np.testing.assert_allclose(rms_sn[0], 9.904, atol=0.1) # Noise is random
def test_new_wave_grid():
# Dummy spectrum
dspec = dummy_spectra()
# iref [default]
iref_wave = coadd.new_wave_grid(dspec.data['wave'])
np.testing.assert_allclose(iref_wave[0], 5000.)
np.testing.assert_allclose(iref_wave[-1], 6000.)
# Concatenate
cat_wave = coadd.new_wave_grid(dspec.data['wave'],
wave_method='concatenate')
np.testing.assert_allclose(cat_wave[0], 4000.5)
np.testing.assert_allclose(cat_wave[-1], 6300.8)
# Velocity
vel_wave = coadd.new_wave_grid(dspec.data['wave'], wave_method='velocity')
np.testing.assert_allclose(vel_wave[0], 4000.5)
np.testing.assert_allclose(vel_wave[-1], 6300.25691664)
vel_wave = coadd.new_wave_grid(dspec.data['wave'],
wave_method='velocity',
v_pix=100.)
np.testing.assert_allclose(vel_wave[0], 4000.5)
np.testing.assert_allclose(vel_wave[-1], 6300.6820934900243)
# Pixel
pix_wave = coadd.new_wave_grid(dspec.data['wave'],
wave_method='pixel',
A_pix=2.5)
np.testing.assert_allclose(pix_wave[0], 4000.5)
np.testing.assert_allclose(pix_wave[-1], 6303.0)
def coadd_cos_from_x1dfiles(filenames, wv_array=None, A_pix=0.01 * u.AA):
spec_list = []
#TODO: mask out x1d spectral regions with bad values.
for filename in filenames:
sp = readspec(filename)
import pdb
pdb.set_trace()
# mask =
spec_list += [sp]
# spec_list contains all individual spectra
specs = collate(spec_list) # now all in a single XSpectrum1D object
#rebin
if wv_array is None:
# bring them to a unique native wavelength grid using PYPIT
A_pix = A_pix.to("AA").value
cat_wave = arco.new_wave_grid(specs.data['wave'],
wave_method='pixel',
A_pix=A_pix)
else:
cat_wave = wv_array.to('AA').value
specs = specs.rebin(cat_wave * u.AA,
all=True,
do_sig=True,
masking='none',
grow_bad_sig=True)
# estimate weights for coaddition (PYPYT)
sn2, weights = arco.sn_weight(specs)
# coaddition
spec1d = arco.one_d_coadd(specs, weights)
return spec1d
def test_scale():
""" Test scale algorithms """
# Hand
dspec = dummy_spectra(s2n=10.)
cat_wave = coadd.new_wave_grid(dspec.data['wave'],
wave_method='concatenate')
rspec = dspec.rebin(cat_wave * units.AA,
all=True,
do_sig=True,
masking='none')
smask = rspec.data['sig'].filled(0.) > 0.
sv_high = rspec.copy()
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
_, _ = coadd.scale_spectra(rspec,
smask,
rms_sn,
hand_scale=[3., 5., 10.],
scale_method='hand')
np.testing.assert_allclose(np.median(rspec.flux.value[rspec.sig > 0.]),
3.,
atol=0.01) # Noise is random
# Median
rspec = sv_high.copy()
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
_, mthd = coadd.scale_spectra(rspec, smask, rms_sn, scale_method='median')
assert mthd == 'median_flux'
np.testing.assert_allclose(np.median(rspec.flux.value[rspec.sig > 0.]),
1.,
atol=0.01) # Noise is random
# Auto-none
dspec = dummy_spectra(s2n=0.1)
rspec = dspec.rebin(cat_wave * units.AA,
all=True,
do_sig=True,
masking='none')
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
an_scls, an_mthd = coadd.scale_spectra(rspec, smask, rms_sn)
assert an_mthd == 'none_SN'
# Auto-median
dspec = dummy_spectra(s2n=1.5)
rspec = dspec.rebin(cat_wave * units.AA,
all=True,
do_sig=True,
masking='none')
rspec.data['flux'][1, :] *= 10.
rspec.data['sig'][1, :] *= 10.
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
am_scls, am_mthd = coadd.scale_spectra(rspec,
smask,
rms_sn,
scale_method='median')
assert am_mthd == 'median_flux'
np.testing.assert_allclose(am_scls[1], 0.1, atol=0.01)
def test_cleancr():
""" Test clean CR method"""
# Setup
dspec = dummy_spectra(s2n=10.)
dspec.data['flux'][0, 700] *= 1000. # One bad pixel
dspec.data['sig'][0, 700] *= 500.
cat_wave = coadd.new_wave_grid(dspec.data['wave'], wave_method='concatenate')
rspec = dspec.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
#
smask = rspec.data['sig'].filled(0.) <= 0.
coadd.clean_cr(rspec, smask)
def test_1dcoadd():
""" Test 1dcoadd method"""
# Setup
dspec = dummy_spectra(s2n=10.)
cat_wave = coadd.new_wave_grid(dspec.data['wave'], wave_method='concatenate')
rspec = dspec.rebin(cat_wave*units.AA, all=True, do_sig=True, masking='none')
smask = rspec.data['sig'].filled(0.) > 0.
fluxes, sigs, wave = coadd.unpack_spec(rspec)
rms_sn, weights = coadd.sn_weights(fluxes, sigs, smask, wave)
# Coadd
spec1d = coadd.one_d_coadd(rspec, smask, weights)
assert spec1d.npix == 1740
def coadd_stis_from_x1dfiles(filenames, wv_array=None, rebin=None, debug=True):
"""
Parameters
----------
filenames : list
List of filenames with x1d STIS data
Must be of the same object and same
configuration
wv_array : Quantity array
Wavelength array to perform the co-add
rebin : int, optional
If given, it rebins the current sampling by
rebin number of pixels
Returns
-------
spec1d : XSpectrum1D
Co-added version of all the spectra
"""
spec_list = []
for filename in filenames:
aux = load_single_x1d_stis(filename, debug=debug)
for sp in aux:
spec_list += [sp]
# spec_list contains all echelle orders from different files and multi-extensions
specs = collate(spec_list) # now all in a single XSpectrum1D object
if wv_array is None:
# bring them to a unique native wavelength grid using PYPIT
cat_wave = arco.new_wave_grid(specs.data['wave'],
wave_method='velocity')
else:
cat_wave = wv_array.to('AA').value
if rebin is not None:
rebin = int(rebin)
cat_wave = cat_wave[::rebin]
specs = specs.rebin(cat_wave * u.AA,
all=True,
do_sig=True,
masking='none',
grow_bad_sig=True)
# estimate weights for coaddition (PYPYT)
sn2, weights = arco.sn_weight(specs, smask=None)
# coaddition
spec1d = arco.one_d_coadd(specs, None, weights)
# spec1d = arco.coadd_spectra(specs, wave_grid_method='velocity', scale_method='auto')
return spec1d
def ech_coadd(files,
objids=None,
extract='OPT',
flux=True,
giantcoadd=False,
orderscale='median',
mergeorder=True,
wave_grid_method='velocity',
niter=5,
wave_grid_min=None,
wave_grid_max=None,
v_pix=None,
scale_method='auto',
do_offset=False,
sigrej_final=3.,
do_var_corr=False,
SN_MIN_MEDSCALE=0.5,
overlapfrac=0.01,
num_min_pixels=10,
phot_scale_dicts=None,
qafile=None,
outfile=None,
do_cr=True,
debug=False,
**kwargs):
"""
routines for coadding spectra observed with echelle spectrograph.
parameters:
files (list): file names
objids (str): objid
extract (str): 'OPT' or 'BOX'
flux (bool): fluxed or not
giantcoadd (bool): coadding order by order or do it at once?
wave_grid_method (str): default velocity
niter (int): number of iteration for rejections
wave_grid_min (float): min wavelength, None means it will find the min value from your spectra
wave_grid_max (float): max wavelength, None means it will find the max value from your spectra
v_pix (float): delta velocity, see coadd.py
scale_method (str): see coadd.py
do_offset (str): see coadd.py, not implemented yet.
sigrej_final (float): see coadd.py
do_var_corr (bool): see coadd.py, default False. It seems True will results in a large error
SN_MIN_MEDSCALE (float): minimum SNR for scaling different orders
overlapfrac (float): minimum overlap fraction for scaling different orders.
qafile (str): name of qafile
outfile (str): name of coadded spectrum
do_cr (bool): remove cosmic rays?
debug (bool): show debug plots?
kwargs: see coadd.py
returns:
spec1d: coadded XSpectrum1D
"""
nfile = len(files)
if nfile <= 1:
msgs.info('Only one spectrum exits coadding...')
return
fname = files[0]
ext_final = fits.getheader(fname, -1)
norder = ext_final['ECHORDER'] + 1
msgs.info('spectrum {:s} has {:d} orders'.format(fname, norder))
if norder <= 1:
msgs.error(
'The number of orders have to be greater than one for echelle. Longslit data?'
)
if giantcoadd:
msgs.info('Coadding all orders and exposures at once')
spectra = load.ech_load_spec(files,
objid=objids,
order=None,
extract=extract,
flux=flux)
wave_grid = np.zeros((2, spectra.nspec))
for i in range(spectra.nspec):
wave_grid[0, i] = spectra[i].wvmin.value
wave_grid[1, i] = spectra[i].wvmax.value
ech_kwargs = {
'echelle': True,
'wave_grid_min': np.min(wave_grid),
'wave_grid_max': np.max(wave_grid),
'v_pix': v_pix
}
kwargs.update(ech_kwargs)
# Coadding
spec1d = coadd.coadd_spectra(spectra,
wave_grid_method=wave_grid_method,
niter=niter,
scale_method=scale_method,
do_offset=do_offset,
sigrej_final=sigrej_final,
do_var_corr=do_var_corr,
qafile=qafile,
outfile=outfile,
do_cr=do_cr,
debug=debug,
**kwargs)
else:
msgs.info('Coadding individual orders first and then merge order')
spectra_list = []
# Keywords for Table
rsp_kwargs = {}
rsp_kwargs['wave_tag'] = '{:s}_WAVE'.format(extract)
rsp_kwargs['flux_tag'] = '{:s}_FLAM'.format(extract)
rsp_kwargs['sig_tag'] = '{:s}_FLAM_SIG'.format(extract)
#wave_grid = np.zeros((2,norder))
for iord in range(norder):
spectra = load.ech_load_spec(files,
objid=objids,
order=iord,
extract=extract,
flux=flux)
ech_kwargs = {
'echelle': False,
'wave_grid_min': spectra.wvmin.value,
'wave_grid_max': spectra.wvmax.value,
'v_pix': v_pix
}
#wave_grid[0,iord] = spectra.wvmin.value
#wave_grid[1,iord] = spectra.wvmax.value
kwargs.update(ech_kwargs)
# Coadding the individual orders
if qafile is not None:
qafile_iord = qafile + '_%s' % str(iord)
else:
qafile_iord = None
spec1d_iord = coadd.coadd_spectra(
spectra,
wave_grid_method=wave_grid_method,
niter=niter,
scale_method=scale_method,
do_offset=do_offset,
sigrej_final=sigrej_final,
do_var_corr=do_var_corr,
qafile=qafile_iord,
outfile=None,
do_cr=do_cr,
debug=debug,
**kwargs)
spectrum = spec_from_array(spec1d_iord.wavelength,
spec1d_iord.flux, spec1d_iord.sig,
**rsp_kwargs)
spectra_list.append(spectrum)
spectra_coadd = collate(spectra_list)
# Rebin the spectra
# ToDo: we should read in JFH's wavelength grid here.
# Join into one XSpectrum1D object
# Final wavelength array
kwargs['wave_grid_min'] = np.min(
spectra_coadd.data['wave'][spectra_coadd.data['wave'] > 0])
kwargs['wave_grid_max'] = np.max(
spectra_coadd.data['wave'][spectra_coadd.data['wave'] > 0])
wave_final = coadd.new_wave_grid(spectra_coadd.data['wave'],
wave_method=wave_grid_method,
**kwargs)
# The rebin function in linetools can not work on collated spectra (i.e. filled 0).
# Thus I have to rebin the spectra first and then collate again.
spectra_list_new = []
for i in range(spectra_coadd.nspec):
speci = spectra_list[i].rebin(wave_final * units.AA,
all=True,
do_sig=True,
grow_bad_sig=True,
masking='none')
spectra_list_new.append(speci)
spectra_coadd_rebin = collate(spectra_list_new)
## Note
if orderscale == 'photometry':
# Only tested on NIRES.
if phot_scale_dicts is not None:
spectra_coadd_rebin = order_phot_scale(spectra_coadd_rebin,
phot_scale_dicts,
debug=debug)
else:
msgs.warn(
'No photometric information is provided. Will use median scale.'
)
orderscale = 'median'
elif orderscale == 'median':
#rmask = spectra_coadd_rebin.data['sig'].filled(0.) > 0.
#sn2, weights = coadd.sn_weights(fluxes, sigs, rmask, wave)
## scaling different orders
order_median_scale(spectra_coadd_rebin,
nsig=sigrej_final,
niter=niter,
overlapfrac=overlapfrac,
num_min_pixels=num_min_pixels,
SN_MIN_MEDSCALE=SN_MIN_MEDSCALE,
debug=debug)
else:
msgs.warn('No any scaling is performed between different orders.')
if mergeorder:
fluxes, sigs, wave = coadd.unpack_spec(spectra_coadd_rebin,
all_wave=False)
## Megering orders
msgs.info('Merging different orders')
## ToDo: Joe claimed not to use pixel depedent weighting.
weights = 1.0 / sigs**2
weights[~np.isfinite(weights)] = 0.0
weight_combine = np.sum(weights, axis=0)
weight_norm = weights / weight_combine
weight_norm[np.isnan(weight_norm)] = 1.0
flux_final = np.sum(fluxes * weight_norm, axis=0)
sig_final = np.sqrt(np.sum((weight_norm * sigs)**2, axis=0))
spec1d_final = spec_from_array(wave_final * units.AA, flux_final,
sig_final, **rsp_kwargs)
if outfile is not None:
msgs.info(
'Saving the final calibrated spectrum as {:s}'.format(
outfile))
coadd.write_to_disk(spec1d_final, outfile)
if (qafile is not None) or (debug):
# plot and save qa
plt.figure(figsize=(12, 6))
ax1 = plt.axes([0.07, 0.13, 0.9, 0.4])
ax2 = plt.axes([0.07, 0.55, 0.9, 0.4])
plt.setp(ax2.get_xticklabels(), visible=False)
medf = np.median(spec1d_final.flux)
ylim = (np.sort([0. - 0.3 * medf, 5 * medf]))
cmap = plt.get_cmap('RdYlBu_r')
for idx in range(spectra_coadd_rebin.nspec):
spectra_coadd_rebin.select = idx
color = cmap(float(idx) / spectra_coadd_rebin.nspec)
ind_good = spectra_coadd_rebin.sig > 0
ax1.plot(spectra_coadd_rebin.wavelength[ind_good],
spectra_coadd_rebin.flux[ind_good],
color=color)
if (np.max(spec1d_final.wavelength) > (9000.0 * units.AA)):
skytrans_file = resource_filename(
'pypeit',
'/data/skisim/atm_transmission_secz1.5_1.6mm.dat')
skycat = np.genfromtxt(skytrans_file, dtype='float')
scale = 0.85 * ylim[1]
ax2.plot(skycat[:, 0] * 1e4,
skycat[:, 1] * scale,
'm-',
alpha=0.5)
ax2.plot(spec1d_final.wavelength,
spec1d_final.sig,
ls='steps-',
color='0.7')
ax2.plot(spec1d_final.wavelength,
spec1d_final.flux,
ls='steps-',
color='b')
ax1.set_xlim([
np.min(spec1d_final.wavelength.value),
np.max(spec1d_final.wavelength.value)
])
ax2.set_xlim([
np.min(spec1d_final.wavelength.value),
np.max(spec1d_final.wavelength.value)
])
ax1.set_ylim(ylim)
ax2.set_ylim(ylim)
ax1.set_xlabel('Wavelength (Angstrom)')
ax1.set_ylabel('Flux')
ax2.set_ylabel('Flux')
plt.tight_layout(pad=0.2, h_pad=0., w_pad=0.2)
if len(qafile.split('.')) == 1:
msgs.info(
"No fomat given for the qafile, save to PDF format.")
qafile = qafile + '.pdf'
if qafile:
plt.savefig(qafile)
msgs.info("Wrote coadd QA: {:s}".format(qafile))
if debug:
plt.show()
plt.close()
### Do NOT remove this part althoug it is deprecated.
# we may need back to using this pieces of code after fixing the coadd.coadd_spectra problem on first order.
#kwargs['echelle'] = True
#kwargs['wave_grid_min'] = np.min(wave_grid)
#kwargs['wave_grid_max'] = np.max(wave_grid)
#spec1d_final = coadd.coadd_spectra(spectra_coadd_rebin, wave_grid_method=wave_grid_method, niter=niter,
# scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final,
# do_var_corr=do_var_corr, qafile=qafile, outfile=outfile,
# do_cr=do_cr, debug=debug, **kwargs)
return spec1d_final
else:
msgs.warn('Skipped merging orders')
if outfile is not None:
for iord in range(len(spectra_list)):
outfile_iord = outfile.replace(
'.fits', '_ORDER{:04d}.fits'.format(iord))
msgs.info(
'Saving the final calibrated spectrum of order {:d} as {:s}'
.format(iord, outfile))
spectra_list[iord].write_to_fits(outfile_iord)
return spectra_list