def test_qa():
""" Test QA """
if os.getenv('RUN_ALL_PYPIT_TESTS') is None:
assert True
return
# Setup
#wvmnx = [[5000., 6000.],
# [5000.5, 6000.5],
# [5000.8, 6000.8],
# ]
#npix = [1000, 1000, 1000]
dspec = dummy_spectra(s2n=10.)#, wvmnx=wvmnx, npix=npix)
dspec.data['flux'][0, 700] *= 1000. # One bad pixel
dspec.data['sig'][0, 700] *= 500.
coadd.coadd_spectra(dspec, wave_method='concatenate', qafile='tst.pdf')
def test_coadd():
""" Test full coadd method"""
# Setup
dspec = dummy_spectra(s2n=10.)
dspec.data['flux'][0, 700] *= 1000. # One bad pixel
dspec.data['sig'][0, 700] *= 500.
spec1d = coadd.coadd_spectra(kast_blue, None, dspec, wave_grid_method='concatenate')
assert np.isclose(np.median(spec1d.flux.value), 1., atol=0.003)
def test_coadd_with_fluxing():
""" Test full coadd method with flux scaling"""
scale_dict = dict(filter='DECAM-R', mag=19.0, mag_type='AB')
# Setup
dspec = dummy_spectra(s2n=10.)
dspec.data['flux'][0, 700] *= 1000. # One bad pixel
dspec.data['sig'][0, 700] *= 500.
spec1d = coadd.coadd_spectra(kast_blue, None, dspec, wave_grid_method='concatenate', flux_scale=scale_dict)
# Test
assert np.median(spec1d.flux.value) > 6.61
def ech_coadd(files,
objids=None,
extract='OPT',
flux=True,
giantcoadd=False,
orderscale='median',
mergeorder=True,
wave_grid_method='velocity',
niter=5,
wave_grid_min=None,
wave_grid_max=None,
v_pix=None,
scale_method='auto',
do_offset=False,
sigrej_final=3.,
do_var_corr=False,
SN_MIN_MEDSCALE=0.5,
overlapfrac=0.01,
num_min_pixels=10,
phot_scale_dicts=None,
qafile=None,
outfile=None,
do_cr=True,
debug=False,
**kwargs):
"""
routines for coadding spectra observed with echelle spectrograph.
parameters:
files (list): file names
objids (str): objid
extract (str): 'OPT' or 'BOX'
flux (bool): fluxed or not
giantcoadd (bool): coadding order by order or do it at once?
wave_grid_method (str): default velocity
niter (int): number of iteration for rejections
wave_grid_min (float): min wavelength, None means it will find the min value from your spectra
wave_grid_max (float): max wavelength, None means it will find the max value from your spectra
v_pix (float): delta velocity, see coadd.py
scale_method (str): see coadd.py
do_offset (str): see coadd.py, not implemented yet.
sigrej_final (float): see coadd.py
do_var_corr (bool): see coadd.py, default False. It seems True will results in a large error
SN_MIN_MEDSCALE (float): minimum SNR for scaling different orders
overlapfrac (float): minimum overlap fraction for scaling different orders.
qafile (str): name of qafile
outfile (str): name of coadded spectrum
do_cr (bool): remove cosmic rays?
debug (bool): show debug plots?
kwargs: see coadd.py
returns:
spec1d: coadded XSpectrum1D
"""
nfile = len(files)
if nfile <= 1:
msgs.info('Only one spectrum exits coadding...')
return
fname = files[0]
ext_final = fits.getheader(fname, -1)
norder = ext_final['ECHORDER'] + 1
msgs.info('spectrum {:s} has {:d} orders'.format(fname, norder))
if norder <= 1:
msgs.error(
'The number of orders have to be greater than one for echelle. Longslit data?'
)
if giantcoadd:
msgs.info('Coadding all orders and exposures at once')
spectra = load.ech_load_spec(files,
objid=objids,
order=None,
extract=extract,
flux=flux)
wave_grid = np.zeros((2, spectra.nspec))
for i in range(spectra.nspec):
wave_grid[0, i] = spectra[i].wvmin.value
wave_grid[1, i] = spectra[i].wvmax.value
ech_kwargs = {
'echelle': True,
'wave_grid_min': np.min(wave_grid),
'wave_grid_max': np.max(wave_grid),
'v_pix': v_pix
}
kwargs.update(ech_kwargs)
# Coadding
spec1d = coadd.coadd_spectra(spectra,
wave_grid_method=wave_grid_method,
niter=niter,
scale_method=scale_method,
do_offset=do_offset,
sigrej_final=sigrej_final,
do_var_corr=do_var_corr,
qafile=qafile,
outfile=outfile,
do_cr=do_cr,
debug=debug,
**kwargs)
else:
msgs.info('Coadding individual orders first and then merge order')
spectra_list = []
# Keywords for Table
rsp_kwargs = {}
rsp_kwargs['wave_tag'] = '{:s}_WAVE'.format(extract)
rsp_kwargs['flux_tag'] = '{:s}_FLAM'.format(extract)
rsp_kwargs['sig_tag'] = '{:s}_FLAM_SIG'.format(extract)
#wave_grid = np.zeros((2,norder))
for iord in range(norder):
spectra = load.ech_load_spec(files,
objid=objids,
order=iord,
extract=extract,
flux=flux)
ech_kwargs = {
'echelle': False,
'wave_grid_min': spectra.wvmin.value,
'wave_grid_max': spectra.wvmax.value,
'v_pix': v_pix
}
#wave_grid[0,iord] = spectra.wvmin.value
#wave_grid[1,iord] = spectra.wvmax.value
kwargs.update(ech_kwargs)
# Coadding the individual orders
if qafile is not None:
qafile_iord = qafile + '_%s' % str(iord)
else:
qafile_iord = None
spec1d_iord = coadd.coadd_spectra(
spectra,
wave_grid_method=wave_grid_method,
niter=niter,
scale_method=scale_method,
do_offset=do_offset,
sigrej_final=sigrej_final,
do_var_corr=do_var_corr,
qafile=qafile_iord,
outfile=None,
do_cr=do_cr,
debug=debug,
**kwargs)
spectrum = spec_from_array(spec1d_iord.wavelength,
spec1d_iord.flux, spec1d_iord.sig,
**rsp_kwargs)
spectra_list.append(spectrum)
spectra_coadd = collate(spectra_list)
# Rebin the spectra
# ToDo: we should read in JFH's wavelength grid here.
# Join into one XSpectrum1D object
# Final wavelength array
kwargs['wave_grid_min'] = np.min(
spectra_coadd.data['wave'][spectra_coadd.data['wave'] > 0])
kwargs['wave_grid_max'] = np.max(
spectra_coadd.data['wave'][spectra_coadd.data['wave'] > 0])
wave_final = coadd.new_wave_grid(spectra_coadd.data['wave'],
wave_method=wave_grid_method,
**kwargs)
# The rebin function in linetools can not work on collated spectra (i.e. filled 0).
# Thus I have to rebin the spectra first and then collate again.
spectra_list_new = []
for i in range(spectra_coadd.nspec):
speci = spectra_list[i].rebin(wave_final * units.AA,
all=True,
do_sig=True,
grow_bad_sig=True,
masking='none')
spectra_list_new.append(speci)
spectra_coadd_rebin = collate(spectra_list_new)
## Note
if orderscale == 'photometry':
# Only tested on NIRES.
if phot_scale_dicts is not None:
spectra_coadd_rebin = order_phot_scale(spectra_coadd_rebin,
phot_scale_dicts,
debug=debug)
else:
msgs.warn(
'No photometric information is provided. Will use median scale.'
)
orderscale = 'median'
elif orderscale == 'median':
#rmask = spectra_coadd_rebin.data['sig'].filled(0.) > 0.
#sn2, weights = coadd.sn_weights(fluxes, sigs, rmask, wave)
## scaling different orders
order_median_scale(spectra_coadd_rebin,
nsig=sigrej_final,
niter=niter,
overlapfrac=overlapfrac,
num_min_pixels=num_min_pixels,
SN_MIN_MEDSCALE=SN_MIN_MEDSCALE,
debug=debug)
else:
msgs.warn('No any scaling is performed between different orders.')
if mergeorder:
fluxes, sigs, wave = coadd.unpack_spec(spectra_coadd_rebin,
all_wave=False)
## Megering orders
msgs.info('Merging different orders')
## ToDo: Joe claimed not to use pixel depedent weighting.
weights = 1.0 / sigs**2
weights[~np.isfinite(weights)] = 0.0
weight_combine = np.sum(weights, axis=0)
weight_norm = weights / weight_combine
weight_norm[np.isnan(weight_norm)] = 1.0
flux_final = np.sum(fluxes * weight_norm, axis=0)
sig_final = np.sqrt(np.sum((weight_norm * sigs)**2, axis=0))
spec1d_final = spec_from_array(wave_final * units.AA, flux_final,
sig_final, **rsp_kwargs)
if outfile is not None:
msgs.info(
'Saving the final calibrated spectrum as {:s}'.format(
outfile))
coadd.write_to_disk(spec1d_final, outfile)
if (qafile is not None) or (debug):
# plot and save qa
plt.figure(figsize=(12, 6))
ax1 = plt.axes([0.07, 0.13, 0.9, 0.4])
ax2 = plt.axes([0.07, 0.55, 0.9, 0.4])
plt.setp(ax2.get_xticklabels(), visible=False)
medf = np.median(spec1d_final.flux)
ylim = (np.sort([0. - 0.3 * medf, 5 * medf]))
cmap = plt.get_cmap('RdYlBu_r')
for idx in range(spectra_coadd_rebin.nspec):
spectra_coadd_rebin.select = idx
color = cmap(float(idx) / spectra_coadd_rebin.nspec)
ind_good = spectra_coadd_rebin.sig > 0
ax1.plot(spectra_coadd_rebin.wavelength[ind_good],
spectra_coadd_rebin.flux[ind_good],
color=color)
if (np.max(spec1d_final.wavelength) > (9000.0 * units.AA)):
skytrans_file = resource_filename(
'pypeit',
'/data/skisim/atm_transmission_secz1.5_1.6mm.dat')
skycat = np.genfromtxt(skytrans_file, dtype='float')
scale = 0.85 * ylim[1]
ax2.plot(skycat[:, 0] * 1e4,
skycat[:, 1] * scale,
'm-',
alpha=0.5)
ax2.plot(spec1d_final.wavelength,
spec1d_final.sig,
ls='steps-',
color='0.7')
ax2.plot(spec1d_final.wavelength,
spec1d_final.flux,
ls='steps-',
color='b')
ax1.set_xlim([
np.min(spec1d_final.wavelength.value),
np.max(spec1d_final.wavelength.value)
])
ax2.set_xlim([
np.min(spec1d_final.wavelength.value),
np.max(spec1d_final.wavelength.value)
])
ax1.set_ylim(ylim)
ax2.set_ylim(ylim)
ax1.set_xlabel('Wavelength (Angstrom)')
ax1.set_ylabel('Flux')
ax2.set_ylabel('Flux')
plt.tight_layout(pad=0.2, h_pad=0., w_pad=0.2)
if len(qafile.split('.')) == 1:
msgs.info(
"No fomat given for the qafile, save to PDF format.")
qafile = qafile + '.pdf'
if qafile:
plt.savefig(qafile)
msgs.info("Wrote coadd QA: {:s}".format(qafile))
if debug:
plt.show()
plt.close()
### Do NOT remove this part althoug it is deprecated.
# we may need back to using this pieces of code after fixing the coadd.coadd_spectra problem on first order.
#kwargs['echelle'] = True
#kwargs['wave_grid_min'] = np.min(wave_grid)
#kwargs['wave_grid_max'] = np.max(wave_grid)
#spec1d_final = coadd.coadd_spectra(spectra_coadd_rebin, wave_grid_method=wave_grid_method, niter=niter,
# scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final,
# do_var_corr=do_var_corr, qafile=qafile, outfile=outfile,
# do_cr=do_cr, debug=debug, **kwargs)
return spec1d_final
else:
msgs.warn('Skipped merging orders')
if outfile is not None:
for iord in range(len(spectra_list)):
outfile_iord = outfile.replace(
'.fits', '_ORDER{:04d}.fits'.format(iord))
msgs.info(
'Saving the final calibrated spectrum of order {:d} as {:s}'
.format(iord, outfile))
spectra_list[iord].write_to_fits(outfile_iord)
return spectra_list
def main(args, unit_test=False, path=''):
""" Runs the XSpecGui on an input file
path : str, optional
Mainly for running the unit test
"""
import glob
import yaml
from numpy import isnan
import pdb as debugger
from astropy.io import fits
from pypeit import msgs
from pypeit.core import coadd
from pypeit import specobjs
from pypeit.spectrographs import util
# Load the input file
with open(args.infile, 'r') as infile:
coadd_dict = yaml.load(infile)
# Spectrograph
spectrograph = util.load_spectrograph(coadd_dict.pop('spectrograph'))
# Grab object names in the spectra
filelist = coadd_dict.pop('filenames')
# Allow for wildcards
files = []
for ifl in filelist:
if '*' in ifl:
files += glob.glob(path + ifl)
else:
files += [path + ifl]
# Load spectra
if len(files) == 0:
msgs.error("No files match your input list")
else:
msgs.info("Coadding {:d} data frames".format(len(files)))
# figure out whether it is Echelle or Longslit
header0 = fits.getheader(files[0], 0)
pypeline = header0['PYPELINE']
# also need norder for Echelle data
if pypeline == 'Echelle':
ext_final = fits.getheader(files[0], -1)
norder = ext_final['ECHORDER'] + 1
fdict = {}
for ifile in files:
# Open file
hdulist = fits.open(ifile)
# Grab objects
objects = [hdu.name for hdu in hdulist][1:]
fdict[ifile] = objects
# Global parameters?
if 'global' in coadd_dict.keys():
gparam = coadd_dict.pop('global')
else:
gparam = {}
if args.debug:
gparam['debug'] = True
sv_gparam = gparam.copy()
# Extraction
if 'extract' in coadd_dict.keys():
ex_value = coadd_dict.pop('extract')
else:
ex_value = 'OPT'
msgs.info("Using {:s} extraction".format(ex_value))
# Fluxed data?
if 'flux' in coadd_dict.keys():
flux_value = coadd_dict.pop('flux')
else:
flux_value = True
# Loop on sources
for key in coadd_dict.keys():
# Re-init gparam
gparam = sv_gparam.copy()
iobj = coadd_dict[key]['object']
# Check iobj input
if isinstance(iobj, list):
if len(iobj) != len(files):
raise IOError(
"Input list of object names must have same length as files"
)
#
outfile = coadd_dict[key]['outfile']
# Scale
if 'scale' in coadd_dict[key]:
scale_dict = coadd_dict[key]['scale']
else:
scale_dict = None
# Generate local keywords
try:
local_kwargs = coadd_dict[key]['local']
except KeyError:
local_kwargs = {}
else:
for lkey in local_kwargs:
gparam[lkey] = local_kwargs[lkey]
if unit_test:
return gparam, ex_value, flux_value, iobj, outfile, files, local_kwargs
# Loop on spec1d files
gdfiles = []
extensions = []
gdobj = []
for fkey in fdict:
# Input as str or list
if not isinstance(iobj, list) == 1: # Simple single object
use_obj = iobj
else:
ind = files.index(fkey)
use_obj = iobj[ind]
if pypeline == 'Echelle':
gdfiles.append(fkey)
gdobj += [use_obj]
else:
# Find object indices
# FW: mtch_obj_to_objects will return None when no matching and raise TypeError: cannot unpack non-iterable NoneType object
try:
mtch_obj, idx = specobjs.mtch_obj_to_objects(
use_obj, fdict[fkey], **local_kwargs)
except TypeError:
mtch_obj = None
if mtch_obj is None:
msgs.info("No object {:s} in file {:s}".format(iobj, fkey))
elif len(mtch_obj) == 1:
#Check if optimal extraction is present in all objects.
# If not, warn the user and set ex_value to 'box'.
hdulist = fits.open(fkey)
try: #In case the optimal extraction array is a NaN array
if flux_value is True: # If we have a fluxed spectrum, look for flam
obj_opt = hdulist[mtch_obj[0]].data['OPT_FLAM']
else: # If not, look for counts
obj_opt = hdulist[mtch_obj[0]].data['OPT_COUNTS']
if any(isnan(obj_opt)):
msgs.warn(
"Object {:s} in file {:s} has a NaN array for optimal extraction. Boxcar will be used instead."
.format(mtch_obj[0], fkey))
ex_value = 'box'
except KeyError: #In case the array is absent altogether.
msgs.warn(
"Object {:s} in file {:s} doesn't have an optimal extraction. Boxcar will be used instead."
.format(mtch_obj[0], fkey))
try:
if flux_value is True: # If we have a fluxed spectrum, look for flam
hdulist[mtch_obj[0]].data['BOX_FLAM']
else: # If not, look for counts
hdulist[mtch_obj[0]].data['BOX_COUNTS']
except KeyError:
#In case the boxcar extract is also absent
msgs.error(
"Object {:s} in file {:s} doesn't have a boxcar extraction either. Co-addition cannot be performed"
.format(mtch_obj[0], fkey))
ex_value = 'box'
gdfiles.append(fkey)
gdobj += mtch_obj
extensions.append(idx[0] + 1)
else:
raise ValueError(
"Multiple matches to object {:s} in file {:s}".format(
iobj, fkey))
# Load spectra
if len(gdfiles) == 0:
msgs.error("No files match your input criteria")
# QA file name
exten = outfile.split('.')[-1] # Allow for hdf or fits or whatever
qafile = outfile.replace(exten, 'pdf')
if pypeline == 'Echelle':
# Check whether the scale_dict is in the right shape.
if 'orderscale' in gparam.keys():
orderscale_value = gparam['orderscale']
else:
orderscale_value = 'median'
if (scale_dict is not None) and (orderscale_value == 'photometry'):
if len(scale_dict) != norder:
raise IOError(
"You need to specifiy the photometric information for every order."
)
spec1d = coadd.ech_coadd(gdfiles,
objids=gdobj,
extract=ex_value,
flux=flux_value,
phot_scale_dicts=scale_dict,
outfile=outfile,
qafile=qafile,
**gparam)
else:
spectra = coadd.load_spec(gdfiles,
iextensions=extensions,
extract=ex_value,
flux=flux_value)
# Coadd!
coadd.coadd_spectra(spectrograph,
gdfiles,
spectra,
qafile=qafile,
outfile=outfile,
flux_scale=scale_dict,
**gparam)
def ech_coadd(files,objids=None,extract='OPT',flux=True,giantcoadd=False,
wave_grid_method='velocity', niter=5,wave_grid_min=None, wave_grid_max=None,v_pix=None,
scale_method='auto', do_offset=False, sigrej_final=3.,do_var_corr=False,
qafile=None, outfile=None,do_cr=True, debug=False,**kwargs):
nfile = len(files)
if nfile <=1:
msgs.info('Only one spectrum exits coadding...')
return
fname = files[0]
ext_final = fits.getheader(fname, -1)
norder = ext_final['ORDER'] + 1
msgs.info('spectrum {:s} has {:d} orders'.format(fname, norder))
if norder <= 1:
msgs.error('The number of orders have to be greater than one for echelle. Longslit data?')
if giantcoadd:
msgs.info('Coadding all orders and exposures at once')
spectra = ech_load_spec(files, objid=objids,order=None, extract=extract, flux=flux)
wave_grid = np.zeros((2,spectra.nspec))
for i in range(spectra.nspec):
wave_grid[0, i] = spectra[i].wvmin.value
wave_grid[1, i] = spectra[i].wvmax.value
ech_kwargs = {'echelle': True, 'wave_grid_min': np.min(wave_grid), 'wave_grid_max': np.max(wave_grid),
'v_pix': v_pix}
kwargs.update(ech_kwargs)
# Coadding
spec1d = coadd.coadd_spectra(spectra, wave_grid_method=wave_grid_method, niter=niter,
scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final,
do_var_corr=do_var_corr, qafile=qafile, outfile=outfile,
do_cr=do_cr, debug=debug,**kwargs)
else:
msgs.info('Coadding individual orders first and then merge order')
spectra_list = []
# Keywords for Table
rsp_kwargs = {}
rsp_kwargs['wave_tag'] = '{:s}_WAVE'.format(extract)
rsp_kwargs['flux_tag'] = '{:s}_FLAM'.format(extract)
rsp_kwargs['sig_tag'] = '{:s}_FLAM_SIG'.format(extract)
wave_grid = np.zeros((2,norder))
for iord in range(norder):
spectra = ech_load_spec(files, objid=objids, order=iord, extract=extract, flux=flux)
ech_kwargs = {'echelle': False, 'wave_grid_min': spectra.wvmin.value, 'wave_grid_max': spectra.wvmax.value, 'v_pix': v_pix}
wave_grid[0,iord] = spectra.wvmin.value
wave_grid[1,iord] = spectra.wvmax.value
kwargs.update(ech_kwargs)
# Coadding the individual orders
if qafile is not None:
qafile_iord = qafile+'_%s'%str(iord)
else:
qafile_iord = None
spec1d_iord = coadd.coadd_spectra(spectra, wave_grid_method=wave_grid_method, niter=niter,
scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final,
do_var_corr=do_var_corr, qafile=qafile_iord, outfile=outfile,
do_cr=do_cr, debug=debug, **kwargs)
spectrum = spec_from_array(spec1d_iord.wavelength, spec1d_iord.flux, spec1d_iord.sig,**rsp_kwargs)
spectra_list.append(spectrum)
# Join into one XSpectrum1D object
spectra_coadd = collate(spectra_list)
kwargs['echelle'] = True
kwargs['wave_grid_min'] = np.min(wave_grid)
kwargs['wave_grid_max'] = np.max(wave_grid)
# ToDo: Currently I'm not using the first order due to some problem. Need to add it back after fix the problem.
spec1d = coadd.coadd_spectra(spectra_coadd[1:], wave_grid_method=wave_grid_method, niter=niter,
scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final,
do_var_corr=do_var_corr, qafile=qafile, outfile=outfile,
do_cr=do_cr, debug=debug, **kwargs)
return spec1d