tell_correct=False,
debug=debug,
show=show)
fnames_flux = [f.replace('.fits', '_flux.fits') for f in fnames]
## Let's coadd all the fluxed spectra
# you should get a coadded spectrum named as 'spec1d_stack_{:}.fits'.format(qsoname)
# a straight merge of individual order stacked spectra named as 'spec1d_merge_{:}.fits'.format(qsoname)
# a individual order stacked spectra (multi-extension) named as 'spec1d_order_{:}.fits'.format(qsoname)
# TODO: change the outfile to work with datapath. It's a hard coding on these names in coadd1d
wave_stack, flux_stack, ivar_stack, mask_stack = coadd1d.ech_combspec(
fnames_flux,
objids,
sensfile=sensfile,
ex_value='OPT',
outfile=qsoname,
debug=debug,
show_order_scale=False,
show_exp=True,
show=show)
# run telluric.qso_telluric to get the final results
spec1dfluxfile = '{:}.fits'.format(qsoname)
telloutfile = '{:}_tellmodel.fits'.format(qsoname)
outfile = '{:}_tellcorr.fits'.format(qsoname)
# TODO: add other modes here
TelQSO = telluric.qso_telluric(spec1dfluxfile,
telgridfile,
pca_file,
z_qso,
def flux_coadd_telluric(spec1dfiles,
sensfile,
tell_function,
telloutfile,
outfile,
objids=None,
z_qso=None,
ex_value='OPT',
polyorder=3,
fit_region_min=[9200.0],
fit_region_max=[9700.0],
scale_method=None,
hand_scale=None,
const_weights=False,
wave_grid_min=None,
wave_grid_max=None,
mask_lyman_a=True,
do_flux=False,
do_stack=False,
do_tell=False,
use_exist_sens=True,
show=False,
debug=False):
nfiles = len(spec1dfiles)
if objids is None:
objids = ['OBJ0001'] * nfiles
elif len(objids) != nfiles:
msgs.error(
'The length of objids should be exactly the same as the number of spec1d files.'
)
#fnames = np.sort(fnames)
### Apply the sensfunc to all spectra (only sensfunc but not tellluric)
if do_flux:
apply_sensfunc(spec1dfiles,
sensfile,
extinct_correct=False,
tell_correct=False,
debug=debug,
show=show)
# fnames_flux = [f.replace('.fits', '_flux.fits') for f in fnames]
else:
msgs.warn(
'You skiped the fluxing step, make sure you have applied sensfunction to your 1D spectra.'
)
# The name of the final stacked 1d spectrum
if len(outroot.split('.')) == 1:
outroot = outroot + '.fits'
elif outroot.split('.')[-1] != 'fits':
outroot = outroot + '.fits'
stackfile = os.path.join(sci_path, outroot)
if do_stack:
## Let's coadd all the fluxed spectra
# you should get a coadded spectrum named as '{:}_stack.fits'.format(qsoname)
# a straight merge of individual order stacked spectra named as '{:}_merge.fits'.format(qsoname)
# a individual order stacked spectra (multi-extension) named as '{:}_order.fits'.format(qsoname)
if pypeline == 'Echelle':
wave_stack, flux_stack, ivar_stack, mask_stack = coadd1d.ech_combspec(
fnames,
objids,
show=show,
sensfile=sensfile,
ex_value=ex_value,
outfile=stackfile,
scale_method=scale_method,
hand_scale=hand_scale,
wave_grid_min=wave_grid_min,
wave_grid_max=wave_grid_max,
const_weights=const_weights,
debug=debug)
else:
wave_stack, flux_stack, ivar_stack, mask_stack = coadd1d.multi_combspec(
fnames,
objids,
show=show,
ex_value=ex_value,
outfile=stackfile,
wave_grid_min=wave_grid_min,
wave_grid_max=wave_grid_max,
scale_method=scale_method,
hand_scale=hand_scale,
const_weights=const_weights,
debug=debug,
debug_scale=debug)
elif os.path.exists(stackfile):
msgs.info('Loading stacked 1d spectrum {:}'.format(stackfile))
else:
msgs.warn(
'No stacked 1d spectrum was found. Please set do_stack=True!')
### Telluric correction
if do_tell:
apply_tell_from_file(z_qso,
stackfile,
tell_method=tell_method,
instrument=instrument,
telgridfile=telgridfile,
polyorder=polyorder,
fit_region_min=fit_region_min,
fit_region_max=fit_region_max,
mask_lyman_a=mask_lyman_a,
show=show,
debug=debug)
def flux_tell(sci_path,
stdfile,
spec1dfiles=None,
std_path=None,
fileroot=None,
outroot=None,
z_qso=None,
tell_method='qso',
instrument=None,
star_type=None,
star_mag=None,
star_ra=None,
star_dec=None,
mask_abs_lines=True,
sens_polyorder=8,
objids=None,
ex_value='OPT',
polyorder=3,
fit_region_min=[9200.0],
fit_region_max=[9700.0],
scale_method=None,
hand_scale=None,
const_weights=False,
wave_grid_min=None,
wave_grid_max=None,
mask_lyman_a=True,
do_sens=False,
do_flux=False,
do_stack=False,
do_tell=False,
use_exist_sens=True,
show=False,
debug=False):
if std_path is None:
std_path = sci_path
if fileroot is None:
fileroot = 'spec1d_flux_tell.fits'
if outroot is None:
outroot = fileroot
std1dfile = os.path.join(std_path, stdfile)
header = fits.getheader(std1dfile, 0)
pypeline = header['PYPELINE']
if spec1dfiles is None:
scifiles_all = os.listdir(sci_path)
spec1dfiles = []
for i in range(len(scifiles_all)):
if ('spec1d' in scifiles_all[i]) and (fileroot in scifiles_all[i]):
spec1dfiles.append(scifiles_all[i])
nfiles = len(spec1dfiles)
fnames = []
for ifile in range(nfiles):
fnames.append(os.path.join(sci_path, spec1dfiles[ifile]))
if objids is None:
objids = ['OBJ0001'] * nfiles
elif len(objids) != nfiles:
msgs.error(
'The length of objids should be exactly the same with the number of spec1d files.'
)
#fnames = np.sort(fnames)
### get sensfunc
if (star_ra is None) and (star_dec is None) and (star_mag is None) and (
star_type is None):
star_ra, star_dec = header['RA'], header['DEC']
if '.sens.fits' not in stdfile:
sensfile = std1dfile.replace('.fits', '_sens.fits')
telgridfile = None # value it to None
if do_sens:
if os.path.exists(sensfile) and (use_exist_sens):
msgs.warn(
'{:} is already exists. Skip doing sensfunc.'.format(sensfile))
else:
sensfile, telgridfile = get_sens_from_file(
std1dfile=std1dfile,
instrument=instrument,
star_type=star_type,
star_mag=star_mag,
star_ra=star_ra,
star_dec=star_dec,
sens_polyorder=sens_polyorder,
mask_abs_lines=mask_abs_lines,
show=show,
debug=debug)
if telgridfile is None:
msgs.info('Loading sensfile {:}'.format(sensfile))
if (instrument == 'GNIRS') or (instrument == 'NIRES'):
telgridfile = os.path.join(
basedir,
'Dropbox/PypeIt_Redux/TelFit_MaunaKea_3100_26100_R20000.fits')
elif (instrument == 'XSHOOTER_VIS') or (instrument == 'GMOS-S'):
telgridfile = os.path.join(
basedir,
'Dropbox/PypeIt_Redux/XSHOOTER/TelFit_Paranal_VIS_4900_11100_R25000.fits'
)
elif instrument == 'XSHOOTER_NIR':
telgridfile = os.path.join(
basedir,
'Dropbox/PypeIt_Redux/XSHOOTER/TelFit_Paranal_NIR_9800_25000_R25000.fits'
)
else:
telgridfile = os.path.join(
basedir,
'Dropbox/PypeIt_Redux/TelFit_MaunaKea_3100_26100_R20000.fits')
msgs.warn('No telluric grid is found. Using MaunaKea!')
### Apply the sensfunc to all spectra (only sensfunc but not tellluric)
if do_flux:
apply_sensfunc(fnames,
sensfile,
extinct_correct=False,
tell_correct=False,
debug=debug,
show=show)
# fnames_flux = [f.replace('.fits', '_flux.fits') for f in fnames]
else:
msgs.warn(
'You skiped the fluxing step, make sure you have applied sensfunction to your 1D spectra.'
)
# The name of the final stacked 1d spectrum
if len(outroot.split('.')) == 1:
outroot = outroot + '.fits'
elif outroot.split('.')[-1] != 'fits':
outroot = outroot + '.fits'
stackfile = os.path.join(sci_path, outroot)
if do_stack:
## Let's coadd all the fluxed spectra
# you should get a coadded spectrum named as '{:}_stack.fits'.format(qsoname)
# a straight merge of individual order stacked spectra named as '{:}_merge.fits'.format(qsoname)
# a individual order stacked spectra (multi-extension) named as '{:}_order.fits'.format(qsoname)
if pypeline == 'Echelle':
wave_stack, flux_stack, ivar_stack, mask_stack = coadd1d.ech_combspec(
fnames,
objids,
show=show,
sensfile=sensfile,
ex_value=ex_value,
outfile=stackfile,
scale_method=scale_method,
hand_scale=hand_scale,
wave_grid_min=wave_grid_min,
wave_grid_max=wave_grid_max,
const_weights=const_weights,
debug=debug)
else:
wave_stack, flux_stack, ivar_stack, mask_stack = coadd1d.multi_combspec(
fnames,
objids,
show=show,
ex_value=ex_value,
outfile=stackfile,
wave_grid_min=wave_grid_min,
wave_grid_max=wave_grid_max,
scale_method=scale_method,
hand_scale=hand_scale,
const_weights=const_weights,
debug=debug,
debug_scale=debug)
elif os.path.exists(stackfile):
msgs.info('Loading stacked 1d spectrum {:}'.format(stackfile))
else:
msgs.warn(
'No stacked 1d spectrum was found. Please set do_stack=True!')
### Telluric correction
if do_tell:
apply_tell_from_file(z_qso,
stackfile,
tell_method=tell_method,
instrument=instrument,
telgridfile=telgridfile,
polyorder=polyorder,
fit_region_min=fit_region_min,
fit_region_max=fit_region_max,
mask_lyman_a=mask_lyman_a,
show=show,
debug=debug)
if not os.path.exists(spec1dfiles_flux[0]):
apply_sensfunc(spec1dfiles,
sensfile,
extinct_correct=False,
tell_correct=False,
debug=False,
show=False)
# Now let's coadd the spectrum
spec1dfluxfile = os.path.join(mpia_path,
'spec1d_coadd_{:}.fits'.format(qsoname))
wave_stack, flux_stack, ivar_stack, mask_stack = coadd1d.ech_combspec(
spec1dfiles_flux,
objids,
show=show,
show_exp=True,
sensfile=sensfile,
ex_value='OPT',
outfile=spec1dfluxfile,
debug=debug)
sys.exit(-1)
# This is a pickle file containing the PCA model for the QSOs
pca_file = os.path.join(mpia_path, 'qso_pca_1200_3100.pckl')
# run telluric.qso_telluric to get the final results
telloutfile = os.path.join(mpia_path, '{:}_tellmodel.fits'.format(qsoname))
outfile = os.path.join(mpia_path,
'spec1d_coadd_{:}_tellcorr.fits'.format(qsoname))
# TODO: add other modes here
sensfile = os.path.join(os.getenv('HOME'), 'Dropbox/PypeIt_Redux/XSHOOTER/LTT3218_sens_tell.fits')
if do_sens:
if std1dfile is None:
msgs.error('You need either give a std1dfile to derive sensfunc')
else:
# run telluric.sensfunc_telluric to get the sensfile
TelSens = telluric.sensfunc_telluric(std1dfile, telgridfile, sensfile, mask_abs_lines=True, debug=debug)
## Apply the sensfunc to all spectra (only sensfunc but not tellluric)
# TODO: change show=False to show=show
#apply_sensfunc(fnames, sensfile, extinct_correct=False, tell_correct=False, debug=debug, show=False)
fnames_flux = [f.replace('.fits', '_flux.fits') for f in fnames]
## Let's coadd all the fluxed spectra
# you should get a coadded spectrum named as 'spec1d_stack_{:}.fits'.format(qsoname)
# a straight merge of individual order stacked spectra named as 'spec1d_merge_{:}.fits'.format(qsoname)
# a individual order stacked spectra (multi-extension) named as 'spec1d_order_{:}.fits'.format(qsoname)
# TODO: change the outfile to work with datapath. It's a hard coding on these names in coadd1d
wave_stack, flux_stack, ivar_stack, mask_stack = coadd1d.ech_combspec(fnames_flux[0:11], objids, sensfile=sensfile,
ex_value='OPT', outfile=qsoname, show=show,
show_exp=show_exp, debug=debug)
sys.exit(-1)
# run telluric.qso_telluric to get the final results
spec1dfluxfile = 'spec1d_stack_{:}.fits'.format(qsoname)
telloutfile = 'spec1d_stack_{:}_tellmodel.fits'.format(qsoname)
outfile = 'spec1d_stack_{:}_tellcorr.fits'.format(qsoname)
# TODO: add other modes here
TelQSO = telluric.qso_telluric(spec1dfluxfile, telgridfile, pca_file, z_qso, telloutfile, outfile,
create_bal_mask=None, debug=True, show=show)
def main(args, unit_test=False, path=''):
""" Runs the XSpecGui on an input file
path : str, optional
Mainly for running the unit test
"""
import glob
import yaml
import IPython
import numpy as np
from numpy import isnan
from astropy.io import fits
from pypeit import msgs
from pypeit.core import coadd1d
from pypeit import specobjs
from pypeit.spectrographs import util
# Load the input file
with open(args.infile, 'r') as infile:
coadd_dict = yaml.load(infile)
# Spectrograph
spectrograph = util.load_spectrograph(coadd_dict.pop('spectrograph'))
# Grab object names in the spectra
filelist = coadd_dict.pop('filenames')
# Allow for wildcards
files = []
for ifl in filelist:
if '*' in ifl:
files += glob.glob(path+ifl)
else:
files += [path+ifl]
# Load spectra
if len(files) == 0:
msgs.error("No files match your input list")
else:
msgs.info("Coadding {:d} data frames".format(len(files)))
# figure out whether it is Echelle or Longslit
header0 = fits.getheader(files[0],0)
pypeline = header0['PYPELINE']
# also need norder for Echelle data
if pypeline == 'Echelle':
#ext_final = fits.getheader(files[0], -1)
#norder = ext_final['ECHORDER'] + 1
ext_first = fits.getheader(files[0], 1)
ext_final = fits.getheader(files[0], -1)
norder = abs(ext_final['ECHORDER'] - ext_first['ECHORDER']) + 1
order_vec = np.arange(np.fmax(ext_first['ECHORDER'],ext_final['ECHORDER']),
np.fmin(ext_first['ECHORDER'],ext_final['ECHORDER'])-1,-1)
fdict = {}
for ifile in files:
# Open file
hdulist = fits.open(ifile)
# Grab objects
objects = [hdu.name for hdu in hdulist][1:]
fdict[ifile] = objects
# Global parameters?
if 'global' in coadd_dict.keys():
gparam = coadd_dict.pop('global')
else:
gparam = {}
if args.debug:
gparam['debug'] = True
if args.show:
gparam['show'] = True
sv_gparam = gparam.copy()
# Extraction
if 'extract' in coadd_dict.keys():
ex_value = coadd_dict.pop('extract')
else:
ex_value = 'OPT'
msgs.info("Using {:s} extraction".format(ex_value))
# Fluxed data?
if 'flux' in coadd_dict.keys():
flux_value = coadd_dict.pop('flux')
else:
flux_value = True
# sensfunc weight
if 'sensfile' in coadd_dict.keys():
sensfile = coadd_dict.pop('sensfile')
else:
sensfile = None
# Loop on sources
for key in coadd_dict.keys():
# Re-init gparam
gparam = sv_gparam.copy()
iobj = coadd_dict[key]['object']
# Check iobj input
if isinstance(iobj, list):
if len(iobj) != len(files):
raise IOError("Input list of object names must have same length as files")
#
outfile = coadd_dict[key]['outfile']
# Scale
if 'scale' in coadd_dict[key]:
scale_dict = coadd_dict[key]['scale']
else:
scale_dict = None
# Generate local keywords
try:
local_kwargs = coadd_dict[key]['local']
except KeyError:
local_kwargs = {}
else:
for lkey in local_kwargs:
gparam[lkey] = local_kwargs[lkey]
if unit_test:
return gparam, ex_value, flux_value, iobj, outfile, files, local_kwargs
# Loop on spec1d files
gdfiles = []
extensions = []
gdobj = []
for fkey in fdict:
# Input as str or list
if not isinstance(iobj, list) == 1: # Simple single object
use_obj = iobj
else:
ind = files.index(fkey)
use_obj = iobj[ind]
if pypeline == 'Echelle':
gdfiles.append(fkey)
gdobj += [use_obj]
else:
# Find object indices
# FW: mtch_obj_to_objects will return None when no matching and raise TypeError: cannot unpack non-iterable NoneType object
try:
mtch_obj, idx = specobjs.mtch_obj_to_objects(use_obj, fdict[fkey], **local_kwargs)
except TypeError:
mtch_obj = None
if mtch_obj is None:
msgs.info("No object {:s} in file {:s}".format(iobj, fkey))
elif len(mtch_obj) == 1:
#Check if optimal extraction is present in all objects.
# If not, warn the user and set ex_value to 'box'.
hdulist = fits.open(fkey)
try: #In case the optimal extraction array is a NaN array
if flux_value is True: # If we have a fluxed spectrum, look for flam
obj_opt = hdulist[mtch_obj[0]].data['OPT_FLAM']
else: # If not, look for counts
obj_opt = hdulist[mtch_obj[0]].data['OPT_COUNTS']
if any(isnan(obj_opt)):
msgs.warn("Object {:s} in file {:s} has a NaN array for optimal extraction. Boxcar will be used instead.".format(mtch_obj[0],fkey))
ex_value = 'box'
except KeyError: #In case the array is absent altogether.
msgs.warn("Object {:s} in file {:s} doesn't have an optimal extraction. Boxcar will be used instead.".format(mtch_obj[0],fkey))
try:
if flux_value is True: # If we have a fluxed spectrum, look for flam
hdulist[mtch_obj[0]].data['BOX_FLAM']
else: # If not, look for counts
hdulist[mtch_obj[0]].data['BOX_COUNTS']
except KeyError:
#In case the boxcar extract is also absent
msgs.error("Object {:s} in file {:s} doesn't have a boxcar extraction either. Co-addition cannot be performed".format(mtch_obj[0],fkey))
ex_value = 'box'
gdfiles.append(fkey)
gdobj += mtch_obj
extensions.append(idx[0]+1)
else:
raise ValueError("Multiple matches to object {:s} in file {:s}".format(iobj, fkey))
# Load spectra
if len(gdfiles) == 0:
msgs.error("No files match your input criteria")
# QA file name
exten = outfile.split('.')[-1] # Allow for hdf or fits or whatever
qafile = outfile.replace(exten, 'pdf')
## The following part will be replaced with the new coadd code
if pypeline == 'Echelle':
# Check whether the scale_dict is in the right shape.
if 'orderscale' in gparam.keys():
orderscale_value = gparam['orderscale']
else:
orderscale_value = 'median'
if (scale_dict is not None) and (orderscale_value=='photometry'):
if len(scale_dict) != norder:
raise IOError("You need to specifiy the photometric information for every order.")
wave_stack, flux_stack, ivar_stack, mask_stack = coadd1d.ech_combspec(
gdfiles, gdobj, sensfile=sensfile, ex_value=ex_value, flux_value=flux_value, phot_scale_dicts=scale_dict,
outfile=outfile, qafile=qafile, **gparam)
else:
wave_stack, flux_stack, ivar_stack, mask_stack = coadd1d.multi_combspec(
gdfiles, gdobj, ex_value=ex_value, flux_value=flux_value, phot_scale_dicts=scale_dict,
outfile=outfile, qafile=qafile, **gparam)
tell_correct=False,
debug=debug,
show=False)
fnames_flux = [f.replace('.fits', '_flux.fits') for f in fnames]
## Let's coadd all the fluxed spectra
# you should get a coadded spectrum named as 'spec1d_stack_{:}.fits'.format(qsoname)
# a straight merge of individual order stacked spectra named as 'spec1d_merge_{:}.fits'.format(qsoname)
# a individual order stacked spectra (multi-extension) named as 'spec1d_order_{:}.fits'.format(qsoname)
# TODO: change the outfile to work with datapath. It's a hard coding on these names in coadd1d
wave_stack, flux_stack, ivar_stack, mask_stack = coadd1d.ech_combspec(
fnames_flux[0:7],
objids[0:7],
sensfile=sensfile,
ex_value='OPT',
outfile=qsoname + '_newcoadd',
show_order_scale=False,
debug=False,
show=True,
show_exp=True)
#
# run telluric.qso_telluric to get the final results
spec1dfluxfile = 'spec1d_stack_{:}.fits'.format(qsoname)
telloutfile = 'spec1d_stack_{:}_tellmodel.fits'.format(qsoname)
outfile = 'spec1d_stack_{:}_tellcorr.fits'.format(qsoname)
# TODO: add other modes here
TelQSO = telluric.qso_telluric(spec1dfluxfile,
telgridfile,
pca_file,
z_qso,
'spec1d_flux_s190519_0038-J1007+2115_NIRES_2019May19T055923.665.fits',
datapath +
'spec1d_flux_s190519_0041-J1007+2115_NIRES_2019May19T062048.865.fits',
datapath +
'spec1d_flux_s190519_0042-J1007+2115_NIRES_2019May19T062750.635.fits',
datapath +
'spec1d_flux_s190519_0045-J1007+2115_NIRES_2019May19T064943.885.fits',
datapath +
'spec1d_flux_s190519_0046-J1007+2115_NIRES_2019May19T065646.165.fits',
datapath +
'spec1d_flux_s190519_0049-J1007+2115_NIRES_2019May19T071920.215.fits',
datapath +
'spec1d_flux_s190519_0050-J1007+2115_NIRES_2019May19T072621.985.fits',
datapath +
'spec1d_flux_s190519_0053-J1007+2115_NIRES_2019May19T074819.315.fits',
datapath +
'spec1d_flux_s190519_0054-J1007+2115_NIRES_2019May19T075521.595.fits',
datapath +
'spec1d_flux_s190519_0057-J1007+2115_NIRES_2019May19T081918.265.fits',
datapath +
'spec1d_flux_s190519_0058-J1007+2115_NIRES_2019May19T082620.545.fits'
]
objids = [
'OBJ0001', 'OBJ0001', 'OBJ0001', 'OBJ0001', 'OBJ0001', 'OBJ0001',
'OBJ0001', 'OBJ0001', 'OBJ0001', 'OBJ0001', 'OBJ0001', 'OBJ0001'
]
wave_stack, flux_stack, ivar_stack, mask_stack = coadd1d.ech_combspec(
fnames, objids, debug=False)