def get_specobjs(self):
"""Get the updated version of SpecObjs
Returns:
SpecObjs: SpecObjs Class
"""
if self._use_updates:
msgs.work("Have not updated SpecObjs yet")
return self.specobjs
else:
return None
def flexure_obj_oldbuggyversion(specobjs,
maskslits,
method,
sky_spectrum,
sky_file=None,
mxshft=None):
"""Correct wavelengths for flexure, object by object
Parameters
----------
method : str
Options are: 'boxcar' (recommended) or 'slitpix'.
Returns
-------
flex_list: list
list of dicts containing flexure results. Aligned with
specobjs. Filled with a basically empty dict if the slit is
skipped or there is no object
"""
msgs.work("Consider doing 2 passes in flexure as in LowRedux")
# Load Archive
# skyspec_fil, arx_sky = flexure_archive(spectrograph=spectrograph, skyspec_fil=skyspec_fil)
# Loop on objects
flex_list = []
gdslits = np.where(~maskslits)[0]
for sl in range(len(specobjs)):
# Reset
flex_dict = dict(polyfit=[],
shift=[],
subpix=[],
corr=[],
corr_cen=[],
spec_file=sky_file,
smooth=[],
arx_spec=[],
sky_spec=[])
if sl not in gdslits:
flex_list.append(flex_dict.copy())
continue
msgs.info(
"Working on flexure in slit (if an object was detected): {:d}".
format(sl))
for specobj in specobjs[sl]: # for convenience
if specobj is None:
continue
# Using boxcar
if method in ['boxcar', 'slitcen']:
sky_wave = specobj.BOX_WAVE #.to('AA').value
sky_flux = specobj.BOX_COUNTS_SKY
else:
msgs.error(
"Not ready for this flexure method: {}".format(method))
# Generate 1D spectrum for object
obj_sky = xspectrum1d.XSpectrum1D.from_tuple((sky_wave, sky_flux))
# Calculate the shift
fdict = flex_shift(obj_sky, sky_spectrum, mxshft=mxshft)
# Simple interpolation to apply
npix = len(sky_wave)
x = np.linspace(0., 1., npix)
# Apply
for attr in ['boxcar', 'optimal']:
if not hasattr(specobj, attr):
continue
if 'WAVE' in getattr(specobj, attr).keys():
msgs.info(
"Applying flexure correction to {0:s} extraction for object:"
.format(attr) + msgs.newline() +
"{0:s}".format(str(specobj)))
f = interpolate.interp1d(x,
sky_wave,
bounds_error=False,
fill_value="extrapolate")
getattr(specobj, attr)['WAVE'] = f(x + fdict['shift'] /
(npix - 1)) * units.AA
# Shift sky spec too
cut_sky = fdict['sky_spec']
x = np.linspace(0., 1., cut_sky.npix)
f = interpolate.interp1d(x,
cut_sky.wavelength.value,
bounds_error=False,
fill_value="extrapolate")
twave = f(x + fdict['shift'] / (cut_sky.npix - 1)) * units.AA
new_sky = xspectrum1d.XSpectrum1D.from_tuple((twave, cut_sky.flux))
# Update dict
for key in [
'polyfit', 'shift', 'subpix', 'corr', 'corr_cen', 'smooth',
'arx_spec'
]:
flex_dict[key].append(fdict[key])
flex_dict['sky_spec'].append(new_sky)
flex_list.append(flex_dict.copy())
return flex_list
def flex_shift(obj_skyspec, arx_skyspec, mxshft=20):
""" Calculate shift between object sky spectrum and archive sky spectrum
Parameters
----------
obj_skyspec
arx_skyspec
Returns
-------
flex_dict: dict
Contains flexure info
"""
# TODO None of these routines should have dependencies on XSpectrum1d!
# Determine the brightest emission lines
msgs.warn("If we use Paranal, cut down on wavelength early on")
arx_amp, arx_amp_cont, arx_cent, arx_wid, _, arx_w, arx_yprep, nsig = arc.detect_lines(
arx_skyspec.flux.value)
obj_amp, obj_amp_cont, obj_cent, obj_wid, _, obj_w, obj_yprep, nsig_obj = arc.detect_lines(
obj_skyspec.flux.value)
# Keep only 5 brightest amplitude lines (xxx_keep is array of
# indices within arx_w of the 5 brightest)
arx_keep = np.argsort(arx_amp[arx_w])[-5:]
obj_keep = np.argsort(obj_amp[obj_w])[-5:]
# Calculate wavelength (Angstrom per pixel)
arx_disp = np.append(
arx_skyspec.wavelength.value[1] - arx_skyspec.wavelength.value[0],
arx_skyspec.wavelength.value[1:] - arx_skyspec.wavelength.value[:-1])
#arx_disp = (np.amax(arx_sky.wavelength.value)-np.amin(arx_sky.wavelength.value))/arx_sky.wavelength.size
obj_disp = np.append(
obj_skyspec.wavelength.value[1] - obj_skyspec.wavelength.value[0],
obj_skyspec.wavelength.value[1:] - obj_skyspec.wavelength.value[:-1])
#obj_disp = (np.amax(obj_sky.wavelength.value)-np.amin(obj_sky.wavelength.value))/obj_sky.wavelength.size
# Calculate resolution (lambda/delta lambda_FWHM)..maybe don't need
# this? can just use sigmas
arx_idx = (arx_cent + 0.5).astype(
np.int)[arx_w][arx_keep] # The +0.5 is for rounding
arx_res = arx_skyspec.wavelength.value[arx_idx]/\
(arx_disp[arx_idx]*(2*np.sqrt(2*np.log(2)))*arx_wid[arx_w][arx_keep])
obj_idx = (obj_cent + 0.5).astype(
np.int)[obj_w][obj_keep] # The +0.5 is for rounding
obj_res = obj_skyspec.wavelength.value[obj_idx]/ \
(obj_disp[obj_idx]*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
#obj_res = (obj_sky.wavelength.value[0]+(obj_disp*obj_cent[obj_w][obj_keep]))/(
# obj_disp*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
if not np.all(np.isfinite(obj_res)):
msgs.warn(
'Failed to measure the resolution of the object spectrum, likely due to error '
'in the wavelength image.')
return None
msgs.info("Resolution of Archive={0} and Observation={1}".format(
np.median(arx_res), np.median(obj_res)))
# Determine sigma of gaussian for smoothing
arx_sig2 = np.power(arx_disp[arx_idx] * arx_wid[arx_w][arx_keep], 2)
obj_sig2 = np.power(obj_disp[obj_idx] * obj_wid[obj_w][obj_keep], 2)
arx_med_sig2 = np.median(arx_sig2)
obj_med_sig2 = np.median(obj_sig2)
if obj_med_sig2 >= arx_med_sig2:
smooth_sig = np.sqrt(obj_med_sig2 - arx_med_sig2) # Ang
smooth_sig_pix = smooth_sig / np.median(arx_disp[arx_idx])
arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix * 2 *
np.sqrt(2 * np.log(2)))
else:
msgs.warn("Prefer archival sky spectrum to have higher resolution")
smooth_sig_pix = 0.
msgs.warn("New Sky has higher resolution than Archive. Not smoothing")
#smooth_sig = np.sqrt(arx_med_sig**2-obj_med_sig**2)
#Determine region of wavelength overlap
min_wave = max(np.amin(arx_skyspec.wavelength.value),
np.amin(obj_skyspec.wavelength.value))
max_wave = min(np.amax(arx_skyspec.wavelength.value),
np.amax(obj_skyspec.wavelength.value))
#Smooth higher resolution spectrum by smooth_sig (flux is conserved!)
# if np.median(obj_res) >= np.median(arx_res):
# msgs.warn("New Sky has higher resolution than Archive. Not smoothing")
#obj_sky_newflux = ndimage.gaussian_filter(obj_sky.flux, smooth_sig)
# else:
#tmp = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
# arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix*2*np.sqrt(2*np.log(2)))
#arx_sky.flux = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
# Define wavelengths of overlapping spectra
keep_idx = np.where((obj_skyspec.wavelength.value >= min_wave)
& (obj_skyspec.wavelength.value <= max_wave))[0]
#keep_wave = [i for i in obj_sky.wavelength.value if i>=min_wave if i<=max_wave]
#Rebin both spectra onto overlapped wavelength range
if len(keep_idx) <= 50:
msgs.warn("Not enough overlap between sky spectra")
return None
else: #rebin onto object ALWAYS
keep_wave = obj_skyspec.wavelength[keep_idx]
arx_skyspec = arx_skyspec.rebin(keep_wave)
obj_skyspec = obj_skyspec.rebin(keep_wave)
# Trim edges (rebinning is junk there)
arx_skyspec.data['flux'][0, :2] = 0.
arx_skyspec.data['flux'][0, -2:] = 0.
obj_skyspec.data['flux'][0, :2] = 0.
obj_skyspec.data['flux'][0, -2:] = 0.
# Normalize spectra to unit average sky count
norm = np.sum(obj_skyspec.flux.value) / obj_skyspec.npix
obj_skyspec.flux = obj_skyspec.flux / norm
norm2 = np.sum(arx_skyspec.flux.value) / arx_skyspec.npix
arx_skyspec.flux = arx_skyspec.flux / norm2
if (norm < 0.):
msgs.warn("Bad normalization of object in flexure algorithm")
msgs.warn("Will try the median")
norm = np.median(obj_skyspec.flux.value)
if (norm < 0.):
msgs.warn("Improper sky spectrum for flexure. Is it too faint??")
return None
if (norm2 < 0.):
msgs.warn(
'Bad normalization of archive in flexure. You are probably using wavelengths '
'well beyond the archive.')
return None
# Deal with bad pixels
msgs.work("Need to mask bad pixels")
# Deal with underlying continuum
msgs.work("Consider taking median first [5 pixel]")
everyn = obj_skyspec.npix // 20
bspline_par = dict(everyn=everyn)
mask, ct = utils.robust_polyfit(obj_skyspec.wavelength.value,
obj_skyspec.flux.value,
3,
function='bspline',
sigma=3.,
bspline_par=bspline_par)
obj_sky_cont = utils.func_val(ct, obj_skyspec.wavelength.value, 'bspline')
obj_sky_flux = obj_skyspec.flux.value - obj_sky_cont
mask, ct_arx = utils.robust_polyfit(arx_skyspec.wavelength.value,
arx_skyspec.flux.value,
3,
function='bspline',
sigma=3.,
bspline_par=bspline_par)
arx_sky_cont = utils.func_val(ct_arx, arx_skyspec.wavelength.value,
'bspline')
arx_sky_flux = arx_skyspec.flux.value - arx_sky_cont
# Consider sharpness filtering (e.g. LowRedux)
msgs.work("Consider taking median first [5 pixel]")
#Cross correlation of spectra
#corr = np.correlate(arx_skyspec.flux, obj_skyspec.flux, "same")
corr = np.correlate(arx_sky_flux, obj_sky_flux, "same")
#Create array around the max of the correlation function for fitting for subpixel max
# Restrict to pixels within maxshift of zero lag
lag0 = corr.size // 2
#mxshft = settings.argflag['reduce']['flexure']['maxshift']
max_corr = np.argmax(corr[lag0 - mxshft:lag0 + mxshft]) + lag0 - mxshft
subpix_grid = np.linspace(max_corr - 3., max_corr + 3., 7)
#Fit a 2-degree polynomial to peak of correlation function. JFH added this if/else to not crash for bad slits
if np.any(np.isfinite(corr[subpix_grid.astype(np.int)])):
fit = utils.func_fit(subpix_grid, corr[subpix_grid.astype(np.int)],
'polynomial', 2)
success = True
max_fit = -0.5 * fit[1] / fit[2]
else:
fit = utils.func_fit(subpix_grid, 0.0 * subpix_grid, 'polynomial', 2)
success = False
max_fit = 0.0
msgs.warn('Flexure compensation failed for one of your objects')
#Calculate and apply shift in wavelength
shift = float(max_fit) - lag0
msgs.info("Flexure correction of {:g} pixels".format(shift))
#model = (fit[2]*(subpix_grid**2.))+(fit[1]*subpix_grid)+fit[0]
flex_dict = dict(polyfit=fit,
shift=shift,
subpix=subpix_grid,
corr=corr[subpix_grid.astype(np.int)],
sky_spec=obj_skyspec,
arx_spec=arx_skyspec,
corr_cen=corr.size / 2,
smooth=smooth_sig_pix,
success=success)
# Return
return flex_dict
def flexure_obj(specobjs, maskslits, method, sky_file, mxshft=None):
"""Correct wavelengths for flexure, object by object
Args:
specobjs (pypeit.specobjs.Specobjs):
maskslits (ndarray):
True = masked slit
method (str)
'boxcar' -- Recommneded
'slitpix' --
sky_file (str):
Sky file
mxshft (int, optional):
Passed to flex_shift()
Returns:
list: list of dicts containing flexure results
Aligned with specobjs
Filled with a basically empty dict if the slit is skipped or there is no object
"""
sv_fdict = None
msgs.work("Consider doing 2 passes in flexure as in LowRedux")
# Load Archive
sky_spectrum = load_sky_spectrum(sky_file)
nslits = len(maskslits)
gdslits = np.where(~maskslits)[0]
# Loop on objects
flex_list = []
# Slit/objects to come back to
return_later_sobjs = []
# Loop over slits, and then over objects here
for slit in range(nslits):
msgs.info(
"Working on flexure in slit (if an object was detected): {:d}".
format(slit))
# TODO -- This only will work for MultiSlit
indx = specobjs.slitorder_indices(slit)
this_specobjs = specobjs[indx]
# Reset
flex_dict = dict(polyfit=[],
shift=[],
subpix=[],
corr=[],
corr_cen=[],
spec_file=sky_file,
smooth=[],
arx_spec=[],
sky_spec=[])
# If no objects on this slit append an empty dictionary
if slit not in gdslits:
flex_list.append(flex_dict.copy())
continue
for ss, specobj in enumerate(this_specobjs):
if specobj is None:
continue
if len(specobj._data.keys()
) == 1: # Nothing extracted; only the trace exists
continue
msgs.info(
"Working on flexure for object # {:d}".format(specobj.OBJID) +
"in slit # {:d}".format(specobj.SLITID))
# Using boxcar
if method in ['boxcar', 'slitcen']:
sky_wave = specobj.BOX_WAVE #.to('AA').value
sky_flux = specobj.BOX_COUNTS_SKY
else:
msgs.error(
"Not ready for this flexure method: {}".format(method))
# Generate 1D spectrum for object
obj_sky = xspectrum1d.XSpectrum1D.from_tuple((sky_wave, sky_flux))
# Calculate the shift
fdict = flex_shift(obj_sky, sky_spectrum, mxshft=mxshft)
punt = False
if fdict is None:
msgs.warn(
"Flexure shift calculation failed for this spectrum.")
if sv_fdict is not None:
msgs.warn(
"Will used saved estimate from a previous slit/object")
fdict = copy.deepcopy(sv_fdict)
else:
# One does not exist yet
# Save it for later
return_later_sobjs.append([slit, ss])
punt = True
else:
sv_fdict = copy.deepcopy(fdict)
# Punt?
if punt:
break
# Interpolate
new_sky = specobj.flexure_interp(sky_wave, fdict)
# Update dict
for key in [
'polyfit', 'shift', 'subpix', 'corr', 'corr_cen', 'smooth',
'arx_spec'
]:
flex_dict[key].append(fdict[key])
flex_dict['sky_spec'].append(new_sky)
flex_list.append(flex_dict.copy())
# Do we need to go back?
for items in return_later_sobjs:
if sv_fdict is None:
msgs.info("No flexure corrections could be made")
break
# Setup
slit, ss = items
flex_dict = flex_list[slit]
specobj = specobjs[ss]
sky_wave = specobj.BOX_WAVE #.to('AA').value
# Copy me
fdict = copy.deepcopy(sv_fdict)
# Interpolate
new_sky = specobj.flexure_interp(sky_wave, fdict)
# Update dict
for key in [
'polyfit', 'shift', 'subpix', 'corr', 'corr_cen', 'smooth',
'arx_spec'
]:
flex_dict[key].append(fdict[key])
flex_dict['sky_spec'].append(new_sky)
return flex_list
def standard_sensfunc(wave, flux, ivar, mask_bad, flux_std, mask_balm=None, mask_tell=None,
maxiter=35, upper=2.0, lower=2.0, polyorder=5, balm_mask_wid=50., nresln=20., telluric=True,
resolution=2700., polycorrect=True, debug=False, polyfunc=False, show_QA=False):
"""
Generate a sensitivity function based on observed flux and standard spectrum.
Parameters
----------
wave : ndarray
wavelength as observed
flux : ndarray
counts/s as observed
ivar : ndarray
inverse variance
flux_std : Quantity array
standard star true flux (erg/s/cm^2/A)
msk_bad : ndarray
mask for bad pixels. True is good.
msk_star: ndarray
mask for hydrogen recombination lines. True is good.
msk_tell:ndarray
mask for telluric regions. True is good.
maxiter : integer
maximum number of iterations for polynomial fit
upper : integer
number of sigma for rejection in polynomial
lower : integer
number of sigma for rejection in polynomial
polyorder : integer
order of polynomial fit
balm_mask_wid: float
in units of angstrom
Mask parameter for Balmer absorption. A region equal to
balm_mask_wid is masked.
resolution: integer/float.
spectra resolution
This paramters should be removed in the future. The resolution should be estimated from spectra directly.
debug : bool
if True shows some dubugging plots
Returns
-------
sensfunc
"""
# Create copy of the arrays to avoid modification
wave_obs = wave.copy()
flux_obs = flux.copy()
ivar_obs = ivar.copy()
# preparing arrays
if np.any(np.invert(np.isfinite(ivar_obs))):
msgs.warn("NaN are present in the inverse variance")
# check masks
if mask_tell is None:
mask_tell = np.ones_like(wave_obs,dtype=bool)
if mask_balm is None:
mask_balm = np.ones_like(wave_obs, dtype=bool)
# Removing outliers
# Calculate log of flux_obs setting a floor at TINY
logflux_obs = 2.5 * np.log10(np.maximum(flux_obs, TINY))
# Set a fix value for the variance of logflux
logivar_obs = np.ones_like(logflux_obs) * (10.0 ** 2)
# Calculate log of flux_std model setting a floor at TINY
logflux_std = 2.5 * np.log10(np.maximum(flux_std, TINY))
# Calculate ratio setting a floor at MAGFUNC_MIN and a ceiling at
# MAGFUNC_MAX
magfunc = logflux_std - logflux_obs
magfunc = np.maximum(np.minimum(magfunc, MAGFUNC_MAX), MAGFUNC_MIN)
msk_magfunc = (magfunc < 0.99 * MAGFUNC_MAX) & (magfunc > 0.99 * MAGFUNC_MIN)
# Define two new masks, True is good and False is masked pixel
# mask for all bad pixels on sensfunc
masktot = mask_bad & msk_magfunc & np.isfinite(ivar_obs) & np.isfinite(logflux_obs) & np.isfinite(logflux_std)
logivar_obs[np.invert(masktot)] = 0.0
# mask used for polynomial fit
msk_fit_sens = masktot & mask_tell & mask_balm
# Polynomial fitting to derive a smooth sensfunc (i.e. without telluric)
pypeitFit = fitting.robust_fit(wave_obs[msk_fit_sens], magfunc[msk_fit_sens], polyorder,
function='polynomial', maxiter=maxiter,
lower=lower, upper=upper,
groupbadpix=False,
grow=0, sticky=True, use_mad=True)
magfunc_poly = pypeitFit.eval(wave_obs)
# Polynomial corrections on Hydrogen Recombination lines
if ((sum(msk_fit_sens) > 0.5 * len(msk_fit_sens)) & polycorrect):
## Only correct Hydrogen Recombination lines with polyfit in the telluric free region
balmer_clean = np.zeros_like(wave_obs, dtype=bool)
# Commented out the bluest recombination lines since they are weak for spectroscopic standard stars.
#836.4, 3969.6, 3890.1, 4102.8, 4102.8, 4341.6, 4862.7, \
lines_hydrogen = np.array([5407.0, 6564.6, 8224.8, 8239.2, 8203.6, 8440.3, 8469.6, 8504.8, 8547.7, 8600.8, \
8667.4, 8752.9, 8865.2, 9017.4, 9229.0, 10049.4, 10938.1, 12818.1, 21655.0])
for line_hydrogen in lines_hydrogen:
ihydrogen = np.abs(wave_obs - line_hydrogen) <= balm_mask_wid
balmer_clean[ihydrogen] = True
msk_clean = ((balmer_clean) | (magfunc == MAGFUNC_MAX) | (magfunc == MAGFUNC_MIN)) & \
(magfunc_poly > MAGFUNC_MIN) & (magfunc_poly < MAGFUNC_MAX)
magfunc[msk_clean] = magfunc_poly[msk_clean]
msk_badpix = np.isfinite(ivar_obs) & (ivar_obs > 0)
magfunc[np.invert(msk_badpix)] = magfunc_poly[np.invert(msk_badpix)]
else:
## if half more than half of your spectrum is masked (or polycorrect=False) then do not correct it with polyfit
msgs.warn('No polynomial corrections performed on Hydrogen Recombination line regions')
if not telluric:
# Apply mask to ivar
#logivar_obs[~msk_fit_sens] = 0.
# ToDo
# Compute an effective resolution for the standard. This could be improved
# to setup an array of breakpoints based on the resolution. At the
# moment we are using only one number
msgs.work("Should pull resolution from arc line analysis")
msgs.work("At the moment the resolution is taken as the PixelScale")
msgs.work("This needs to be changed!")
std_pix = np.median(np.abs(wave_obs - np.roll(wave_obs, 1)))
std_res = np.median(wave_obs/resolution) # median resolution in units of Angstrom.
#std_res = std_pix
#resln = std_res
if (nresln * std_res) < std_pix:
msgs.warn("Bspline breakpoints spacing shoud be larger than 1pixel")
msgs.warn("Changing input nresln to fix this")
nresln = std_res / std_pix
# Fit magfunc with bspline
kwargs_bspline = {'bkspace': std_res * nresln}
kwargs_reject = {'maxrej': 5}
msgs.info("Initialize bspline for flux calibration")
# init_bspline = pydl.bspline(wave_obs, bkspace=kwargs_bspline['bkspace'])
init_bspline = bspline.bspline(wave_obs, bkspace=kwargs_bspline['bkspace'])
fullbkpt = init_bspline.breakpoints
# TESTING turning off masking for now
# remove masked regions from breakpoints
msk_obs = np.ones_like(wave_obs).astype(bool)
msk_obs[np.invert(masktot)] = False
msk_bkpt = interpolate.interp1d(wave_obs, msk_obs, kind='nearest',
fill_value='extrapolate')(fullbkpt)
init_breakpoints = fullbkpt[msk_bkpt > 0.999]
# init_breakpoints = fullbkpt
msgs.info("Bspline fit on magfunc. ")
bset1, bmask = fitting.iterfit(wave_obs, magfunc, invvar=logivar_obs, inmask=msk_fit_sens, upper=upper, lower=lower,
fullbkpt=init_breakpoints, maxiter=maxiter, kwargs_bspline=kwargs_bspline,
kwargs_reject=kwargs_reject)
logfit1, _ = bset1.value(wave_obs)
logfit_bkpt, _ = bset1.value(init_breakpoints)
if debug:
# Check for calibration
plt.figure(1)
plt.plot(wave_obs, magfunc, drawstyle='steps-mid', color='black', label='magfunc')
plt.plot(wave_obs, logfit1, color='cornflowerblue', label='logfit1')
plt.plot(wave_obs[np.invert(msk_fit_sens)], magfunc[np.invert(msk_fit_sens)], '+', color='red', markersize=5.0,
label='masked magfunc')
plt.plot(wave_obs[np.invert(msk_fit_sens)], logfit1[np.invert(msk_fit_sens)], '+', color='red', markersize=5.0,
label='masked logfit1')
plt.plot(init_breakpoints, logfit_bkpt, '.', color='green', markersize=4.0, label='breakpoints')
plt.plot(init_breakpoints, np.interp(init_breakpoints, wave_obs, magfunc), '.', color='green',
markersize=4.0,
label='breakpoints')
plt.plot(wave_obs, 1.0 / np.sqrt(logivar_obs), color='orange', label='sigma')
plt.legend()
plt.xlabel('Wavelength [ang]')
plt.ylim(0.0, 1.2 * MAGFUNC_MAX)
plt.title('1st Bspline fit')
plt.show()
# Create sensitivity function
magfunc = np.maximum(np.minimum(logfit1, MAGFUNC_MAX), MAGFUNC_MIN)
if ((sum(msk_fit_sens) > 0.5 * len(msk_fit_sens)) & polycorrect):
msk_clean = ((magfunc == MAGFUNC_MAX) | (magfunc == MAGFUNC_MIN)) & \
(magfunc_poly > MAGFUNC_MIN) & (magfunc_poly<MAGFUNC_MAX)
magfunc[msk_clean] = magfunc_poly[msk_clean]
msk_badpix = np.isfinite(ivar_obs) & (ivar_obs>0)
magfunc[np.invert(msk_badpix)] = magfunc_poly[np.invert(msk_badpix)]
else:
## if half more than half of your spectrum is masked (or polycorrect=False) then do not correct it with polyfit
msgs.warn('No polynomial corrections performed on Hydrogen Recombination line regions')
# Calculate sensfunc
if polyfunc:
sensfunc = 10.0 ** (0.4 * magfunc_poly)
magfunc = magfunc_poly
else:
sensfunc = 10.0 ** (0.4 * magfunc)
if debug:
plt.figure()
magfunc_raw = logflux_std - logflux_obs
plt.plot(wave_obs[masktot],magfunc_raw[masktot] , 'k-',lw=3,label='Raw Magfunc')
plt.plot(wave_obs[masktot],magfunc_poly[masktot] , 'c-',lw=3,label='Polynomial Fit')
plt.plot(wave_obs[np.invert(mask_tell)], magfunc_raw[np.invert(mask_tell)], 's',
color='0.7',label='Telluric Region')
plt.plot(wave_obs[np.invert(mask_balm)], magfunc_raw[np.invert(mask_balm)], 'r+',label='Recombination Line region')
plt.plot(wave_obs[masktot], magfunc[masktot],'b-',label='Final Magfunc')
plt.legend(fancybox=True, shadow=True)
plt.xlim([0.995*np.min(wave_obs[masktot]),1.005*np.max(wave_obs[masktot])])
plt.ylim([0.,1.2*np.max(magfunc[masktot])])
plt.show()
plt.close()
return sensfunc, masktot
def lacosmic(det,
sciframe,
saturation,
nonlinear,
varframe=None,
maxiter=1,
grow=1.5,
remove_compact_obj=True,
sigclip=5.0,
sigfrac=0.3,
objlim=5.0):
"""
Identify cosmic rays using the L.A.Cosmic algorithm
U{http://www.astro.yale.edu/dokkum/lacosmic/}
(article : U{http://arxiv.org/abs/astro-ph/0108003})
This routine is mostly courtesy of Malte Tewes
Args:
det:
sciframe:
saturation:
nonlinear:
varframe:
maxiter:
grow:
remove_compact_obj:
sigclip:
sigfrac:
objlim:
Returns:
ndarray: mask of cosmic rays (0=no CR, 1=CR)
"""
dnum = parse.get_dnum(det)
msgs.info("Detecting cosmic rays with the L.A.Cosmic algorithm")
# msgs.work("Include these parameters in the settings files to be adjusted by the user")
# Set the settings
scicopy = sciframe.copy()
crmask = np.cast['bool'](np.zeros(sciframe.shape))
sigcliplow = sigclip * sigfrac
# Determine if there are saturated pixels
satpix = np.zeros_like(sciframe)
# satlev = settings_det['saturation']*settings_det['nonlinear']
satlev = saturation * nonlinear
wsat = np.where(sciframe >= satlev)
if wsat[0].size == 0: satpix = None
else:
satpix[wsat] = 1.0
satpix = np.cast['bool'](satpix)
# Define the kernels
laplkernel = np.array([[0.0, -1.0, 0.0], [-1.0, 4.0, -1.0],
[0.0, -1.0, 0.0]]) # Laplacian kernal
growkernel = np.ones((3, 3))
for i in range(1, maxiter + 1):
msgs.info("Convolving image with Laplacian kernel")
# Subsample, convolve, clip negative values, and rebin to original size
subsam = utils.subsample(scicopy)
conved = signal.convolve2d(subsam,
laplkernel,
mode="same",
boundary="symm")
cliped = conved.clip(min=0.0)
lplus = utils.rebin_evlist(cliped, np.array(cliped.shape) / 2.0)
msgs.info("Creating noise model")
# Build a custom noise map, and compare this to the laplacian
m5 = ndimage.filters.median_filter(scicopy, size=5, mode='mirror')
if varframe is None:
noise = np.sqrt(np.abs(m5))
else:
noise = np.sqrt(varframe)
msgs.info("Calculating Laplacian signal to noise ratio")
# Laplacian S/N
s = lplus / (2.0 * noise
) # Note that the 2.0 is from the 2x2 subsampling
# Remove the large structures
sp = s - ndimage.filters.median_filter(s, size=5, mode='mirror')
msgs.info("Selecting candidate cosmic rays")
# Candidate cosmic rays (this will include HII regions)
candidates = sp > sigclip
nbcandidates = np.sum(candidates)
msgs.info("{0:5d} candidate pixels".format(nbcandidates))
# At this stage we use the saturated stars to mask the candidates, if available :
if satpix is not None:
msgs.info("Masking saturated pixels")
candidates = np.logical_and(np.logical_not(satpix), candidates)
nbcandidates = np.sum(candidates)
msgs.info(
"{0:5d} candidate pixels not part of saturated stars".format(
nbcandidates))
msgs.info("Building fine structure image")
# We build the fine structure image :
m3 = ndimage.filters.median_filter(scicopy, size=3, mode='mirror')
m37 = ndimage.filters.median_filter(m3, size=7, mode='mirror')
f = m3 - m37
f /= noise
f = f.clip(min=0.01)
msgs.info("Removing suspected compact bright objects")
# Now we have our better selection of cosmics :
if remove_compact_obj:
cosmics = np.logical_and(candidates, sp / f > objlim)
else:
cosmics = candidates
nbcosmics = np.sum(cosmics)
msgs.info("{0:5d} remaining candidate pixels".format(nbcosmics))
# What follows is a special treatment for neighbors, with more relaxed constains.
msgs.info("Finding neighboring pixels affected by cosmic rays")
# We grow these cosmics a first time to determine the immediate neighborhod :
growcosmics = np.cast['bool'](signal.convolve2d(
np.cast['float32'](cosmics),
growkernel,
mode="same",
boundary="symm"))
# From this grown set, we keep those that have sp > sigmalim
# so obviously not requiring sp/f > objlim, otherwise it would be pointless
growcosmics = np.logical_and(sp > sigclip, growcosmics)
# Now we repeat this procedure, but lower the detection limit to sigmalimlow :
finalsel = np.cast['bool'](signal.convolve2d(
np.cast['float32'](growcosmics),
growkernel,
mode="same",
boundary="symm"))
finalsel = np.logical_and(sp > sigcliplow, finalsel)
# Unmask saturated pixels:
if satpix is not None:
msgs.info("Masking saturated stars")
finalsel = np.logical_and(np.logical_not(satpix), finalsel)
ncrp = np.sum(finalsel)
msgs.info("{0:5d} pixels detected as cosmics".format(ncrp))
# We find how many cosmics are not yet known :
newmask = np.logical_and(np.logical_not(crmask), finalsel)
nnew = np.sum(newmask)
# We update the mask with the cosmics we have found :
crmask = np.logical_or(crmask, finalsel)
msgs.info(
"Iteration {0:d} -- {1:d} pixels identified as cosmic rays ({2:d} new)"
.format(i, ncrp, nnew))
if ncrp == 0: break
# Additional algorithms (not traditionally implemented by LA cosmic) to remove some false positives.
msgs.work(
"The following algorithm would be better on the rectified, tilts-corrected image"
)
filt = ndimage.sobel(sciframe, axis=1, mode='constant')
filty = ndimage.sobel(filt / np.sqrt(np.abs(sciframe)),
axis=0,
mode='constant')
filty[np.where(np.isnan(filty))] = 0.0
sigimg = cr_screen(filty)
sigsmth = ndimage.filters.gaussian_filter(sigimg, 1.5)
sigsmth[np.where(np.isnan(sigsmth))] = 0.0
sigmask = np.cast['bool'](np.zeros(sciframe.shape))
sigmask[np.where(sigsmth > sigclip)] = True
crmask = np.logical_and(crmask, sigmask)
msgs.info("Growing cosmic ray mask by 1 pixel")
crmask = grow_masked(crmask.astype(np.float), grow, 1.0)
return crmask.astype(bool)
def apply_sens_tell_specobjs(specobjs, sens_meta, sens_table, airmass, exptime, extinct_correct=True, tell_correct=False,
longitude=None, latitude=None, debug=False, show=False):
# TODO This function should operate on a single object
func = sens_meta['FUNC'][0]
polyorder_vec = sens_meta['POLYORDER_VEC'][0]
nimgs = len(specobjs)
if show:
fig = plt.figure(figsize=(12, 8))
xmin, xmax = [], []
ymin, ymax = [], []
for ispec in range(nimgs):
# get the ECH_ORDER, ECH_ORDERINDX, WAVELENGTH from your science
sobj_ispec = specobjs[ispec]
## TODO Comment on the logich here. Hard to follow
try:
ech_order, ech_orderindx, idx = sobj_ispec.ech_order, sobj_ispec.ech_orderindx, sobj_ispec.idx
msgs.info('Applying sensfunc to Echelle data')
except:
ech_orderindx = 0
idx = sobj_ispec.idx
msgs.info('Applying sensfunc to Longslit/Multislit data')
for extract_type in ['boxcar', 'optimal']:
extract = getattr(sobj_ispec, extract_type)
if len(extract) == 0:
continue
msgs.info("Fluxing {:s} extraction for:".format(extract_type) + msgs.newline() + "{}".format(idx))
wave = extract['WAVE'].value.copy()
wave_mask = wave > 1.0
counts = extract['COUNTS'].copy()
counts_ivar = extract['COUNTS_IVAR'].copy()
mask = extract['MASK'].copy()
# get sensfunc from the sens_table
coeff = sens_table[ech_orderindx]['OBJ_THETA'][0:polyorder_vec[ech_orderindx] + 2]
wave_min = sens_table[ech_orderindx]['WAVE_MIN']
wave_max = sens_table[ech_orderindx]['WAVE_MAX']
sensfunc = np.zeros_like(wave)
sensfunc[wave_mask] = np.exp(utils.func_val(coeff, wave[wave_mask], func,
minx=wave_min, maxx=wave_max))
# get telluric from the sens_table
if tell_correct:
msgs.work('Evaluate telluric!')
telluric = None
else:
telluric = None
flam, flam_ivar, outmask = apply_sens_tell_spec(wave, counts, counts_ivar, sensfunc, airmass, exptime,
mask=mask, extinct_correct=extinct_correct, telluric=telluric,
longitude=longitude, latitude=latitude, debug=debug)
flam_sig = np.sqrt(utils.inverse(flam_ivar))
# The following will be changed directly in the specobjs, so do not need to return anything.
extract['MASK'] = outmask
extract['FLAM'] = flam
extract['FLAM_SIG'] = flam_sig
extract['FLAM_IVAR'] = flam_ivar
if show:
xmin_ispec = wave[wave_mask].min()
xmax_ispec = wave[wave_mask].max()
xmin.append(xmin_ispec)
xmax.append(xmax_ispec)
ymin_ispec, ymax_ispec = coadd1d.get_ylim(flam, flam_ivar, outmask)
ymin.append(ymin_ispec)
ymax.append(ymax_ispec)
med_width = (2.0 * np.ceil(0.1 / 10.0 * np.size(wave[outmask])) + 1).astype(int)
flam_med, flam_ivar_med = coadd1d.median_filt_spec(flam, flam_ivar, outmask, med_width)
if extract_type == 'boxcar':
plt.plot(wave[wave_mask], flam_med[wave_mask], color='black', drawstyle='steps-mid', zorder=1, alpha=0.8)
#plt.plot(wave[wave_mask], np.sqrt(utils.calc_ivar(flam_ivar_med[wave_mask])), zorder=2, color='m',
# alpha=0.7, drawstyle='steps-mid', linestyle=':')
else:
plt.plot(wave[wave_mask], flam_med[wave_mask], color='dodgerblue', drawstyle='steps-mid', zorder=1, alpha=0.8)
#plt.plot(wave[wave_mask], np.sqrt(utils.calc_ivar(flam_ivar_med[wave_mask])), zorder=2, color='red',
# alpha=0.7, drawstyle='steps-mid', linestyle=':')
if show:
xmin_final, xmax_final = np.min(xmin), np.max(xmax)
ymax_final = 1.3*np.median(ymax)
ymin_final = -0.15*ymax_final
plt.xlim([xmin_final, xmax_final])
plt.ylim([ymin_final, ymax_final])
plt.title('Blue is Optimal extraction and Black is Boxcar extraction',fontsize=16)
plt.xlabel('Wavelength (Angstrom)')
plt.ylabel('Flux')
plt.show()
def operations(self, key, axisID):
"""Canvas operations
Args:
key (str): Which key has been pressed
axisID (int): The index of the axis where the key has been pressed (see get_axisID)
"""
# Check if the user really wants to quit
if key == 'q' and self._qconf:
if self._changes:
self.update_infobox(
message=
"WARNING: There are unsaved changes!!\nPress q again to exit",
yesno=False)
self._qconf = True
else:
msgs.bug(
"Need to change this to kill and return the results to PypeIt"
)
plt.close()
elif self._qconf:
self.update_infobox(default=True)
self._qconf = False
# Manage responses from questions posed to the user.
if self._respreq[0]:
if key != "y" and key != "n":
return
else:
# Switch off the required response
self._respreq[0] = False
# Deal with the response
if self._respreq[1] == "write":
# First remove the old file, and save the new one
msgs.work("Not implemented yet!")
self.write()
else:
return
# Reset the info box
self.update_infobox(default=True)
return
if key == '?':
self.print_help()
elif key == 'a':
if self._fitdict['coeff'] is not None:
self.auto_id()
else:
msgs.info("You must identify a few lines first")
elif key == 'c':
wclr = np.where((self._lineflg == 2) | (self._lineflg == 3))
self._lineflg[wclr] = 0
self.replot()
elif key == 'd':
self._lineflg *= 0
self._lineids *= 0.0
self._fitdict['coeff'] = None
self.replot()
elif key == 'f':
self.fitsol_fit()
self.replot()
elif key == 'l':
self.load_IDs()
elif key == 'q':
if self._changes:
self.update_infobox(
message=
"WARNING: There are unsaved changes!!\nPress q again to exit",
yesno=False)
self._qconf = True
else:
plt.close()
elif key == 'r':
if self._detns_idx == -1:
msgs.info("You must select a line first")
elif self._fitr is None:
msgs.info("You must select a fitting region first")
else:
msgs.work("Feature not yet implemented")
elif key == 's':
self.save_IDs()
elif key == 'w':
self.toggle_wavepix(toggled=True)
self.replot()
elif key == 'z':
self.delete_line_id()
elif key == '+':
if self._fitdict["polyorder"] < 10:
self._fitdict["polyorder"] += 1
self.update_infobox(message="Polynomial order = {0:d}".format(
self._fitdict["polyorder"]),
yesno=False)
self.fitsol_fit()
self.replot()
else:
self.update_infobox(message="Polynomial order must be <= 10",
yesno=False)
elif key == '-':
if self._fitdict["polyorder"] > 1:
self._fitdict["polyorder"] -= 1
self.update_infobox(message="Polynomial order = {0:d}".format(
self._fitdict["polyorder"]),
yesno=False)
self.fitsol_fit()
self.replot()
else:
self.update_infobox(message="Polynomial order must be >= 1",
yesno=False)
self.canvas.draw()
def comb_frames(frames_arr, saturation=None,
maskvalue=1048577, method='weightmean', satpix='reject', cosmics=None,
n_lohi=[0,0], sig_lohi=[3.,3.], replace='maxnonsat'):
"""
Combine several frames
.. todo::
- Make better use of np.ma.MaskedArray objects throughout?
- More testing of replacement code necessary?
- Improve docstring...
Parameters
----------
frames_arr : ndarray (3D)
Array of frames to be combined
weights : str, or None (optional)
How should the frame combination by weighted (not currently
implemented)
maskvalue : int (optional)
What should the masked values be set to (should be greater than
the detector's saturation value -- Default = 1 + 2**20)
reject : dict, optional
Set the rejection parameters: cosmics, lowhigh, level, replace
Perhaps these should be called out separately
satpix : str, optional
Method for handling saturated pixels
saturation : float, optional
Saturation value; only required for some choices of reject['replace']
Returns
-------
comb_frame : ndarray
"""
###########
# FIRST DO SOME CHECKS ON THE INPUT
###########
# Was printtype specified
if frames_arr is None:
msgs.error("No frames were given to comb_frames to combine")
(sz_x, sz_y, num_frames) = np.shape(frames_arr)
if num_frames == 1:
msgs.info("Only one frame to combine!")
msgs.info("Returning input frame")
return frames_arr[:, :, 0]
else:
msgs.info("Combining {0:d} frames".format(num_frames))
# Check if the user has allowed the combination of long and short
# frames (e.g. different exposure times)
msgs.work("lscomb feature has not been included here yet...")
# Check the user hasn't requested to reject more frames than available
if n_lohi[0] > 0 and n_lohi[1] > 0 and n_lohi[0] + n_lohi[1] >= num_frames:
msgs.error('You cannot reject more frames than are available with \'n_lohi\'.'
+ msgs.newline() + 'There are {0:d} frames '.format(num_frames)
+ 'and n_lohi will reject {0:d} low and {1:d} high values.'.format(
n_lohi[0], n_lohi[1]))
# Calculate the values to be used if all frames are rejected in some pixels
if replace == 'min':
allrej_arr = np.amin(frames_arr, axis=2)
elif replace == 'max':
allrej_arr = np.amax(frames_arr, axis=2)
elif replace == 'mean':
allrej_arr = np.mean(frames_arr, axis=2)
elif replace == 'median':
allrej_arr = np.median(frames_arr, axis=2)
elif replace == 'weightmean':
msgs.work("No weights are implemented yet")
allrej_arr = frames_arr.copy()
allrej_arr = masked_weightmean(allrej_arr, maskvalue)
elif replace == 'maxnonsat':
allrej_arr = frames_arr.copy()
allrej_arr = maxnonsat(allrej_arr, saturation)
else:
msgs.error("You must specify what to do in case all pixels are rejected")
################
# Saturated Pixels
msgs.info("Finding saturated and non-linear pixels")
if satpix == 'force':
# If a saturated pixel is in one of the frames, force them to
# all have saturated pixels
# satw = np.zeros_like(frames_arr)
# satw[np.where(frames_arr > settings.spect['det']['saturation']*settings.spect['det']['nonlinear'])] = 1.0
# satw = np.any(satw,axis=2)
# del satw
setsat = np.zeros_like(frames_arr)
setsat[frames_arr > saturation] = 1
elif satpix == 'reject':
# Ignore saturated pixels in frames if possible
frames_arr[frames_arr > saturation] = maskvalue
elif satpix == 'nothing':
# Don't do anything special for saturated pixels (Hopefully the
# user has specified how to deal with them below!)
pass
else:
msgs.error('Option \'{0}\' '.format(satpix)
+ 'for dealing with saturated pixels was not recognised.')
################
# Cosmic Rays
if cosmics > 0.0:
msgs.info("Rejecting cosmic rays") # Use a robust statistic
masked_fa = np.ma.MaskedArray(frames_arr, mask=frames_arr==maskvalue)
medarr = np.ma.median(masked_fa, axis=2)
stdarr = 1.4826*np.ma.median(np.ma.absolute(masked_fa - medarr[:,:,None]), axis=2)
indx = (frames_arr != maskvalue) \
& (frames_arr > (medarr.data + cosmics * stdarr.data)[:,:,None])
frames_arr[indx] = maskvalue
# Delete unecessary arrays
del medarr, stdarr
else:
msgs.info("Not rejecting cosmic rays")
################
# Low and High pixel rejection --- Masks *additional* pixels
rejlo, rejhi = n_lohi
if n_lohi[0] > 0 or n_lohi[1] > 0:
# First reject low pixels
frames_arr = np.sort(frames_arr, axis=2)
if n_lohi[0] > 0:
msgs.info("Rejecting {0:d} deviant low pixels".format(n_lohi[0]))
while rejlo > 0:
xi, yi = np.indices(sz_x, sz_y)
frames_arr[xi, yi, np.argmin(frames_arr, axis=2)] = maskvalue
del xi, yi
rejlo -= 1
# Now reject high pixels
if n_lohi[1] > 0:
msgs.info("Rejecting {0:d} deviant high pixels".format(n_lohi[1]))
frames_arr[np.where(frames_arr == maskvalue)] *= -1
while rejhi > 0:
xi, yi = np.indices(sz_x, sz_y)
frames_arr[xi, yi, np.argmax(frames_arr, axis=2)] = -maskvalue
del xi, yi
rejhi -= 1
frames_arr[np.where(frames_arr) == -maskvalue] *= -1
# TODO: Do we need this?
# The following is an example of *not* masking additional pixels
# if reject['lowhigh'][1] > 0:
# msgs.info("Rejecting {0:d} deviant high pixels".format(reject['lowhigh'][1]))
# masktemp[:,:,-reject['lowhigh'][0]:] = True
else:
msgs.info("Not rejecting any low/high pixels")
################
# Deviant Pixels
# TODO: sig_lohi (what was level) is not actually used, instead this
# just selects if cosmics should be used. Is this intentional? Why
# not just do: `if cosmics > 0:`?
if sig_lohi[0] > 0.0 or sig_lohi[1] > 0.0:
msgs.info("Rejecting deviant pixels") # Use a robust statistic
masked_fa = np.ma.MaskedArray(frames_arr, mask=frames_arr==maskvalue)
medarr = np.ma.median(masked_fa, axis=2)
stdarr = 1.4826*np.ma.median(np.ma.absolute(masked_fa - medarr[:,:,None]), axis=2)
indx = (frames_arr != maskvalue) \
& ( (frames_arr > (medarr.data + cosmics*stdarr.data)[:,:,None])
| (frames_arr < (medarr.data - cosmics*stdarr.data)[:,:,None]))
frames_arr[indx] = maskvalue
# Delete unecessary arrays
del medarr, stdarr
else:
msgs.info("Not rejecting deviant pixels")
##############
# Combine the arrays
msgs.info("Combining frames with a {0:s} operation".format(method))
if method == 'mean':
comb_frame = np.ma.mean(np.ma.MaskedArray(frames_arr, mask=frames_arr==maskvalue), axis=2)
elif method == 'median':
comb_frame = np.ma.median(np.ma.MaskedArray(frames_arr, mask=frames_arr==maskvalue), axis=2)
elif method == 'weightmean':
comb_frame = frames_arr.copy()
comb_frame = masked_weightmean(comb_frame, maskvalue)
else:
msgs.error("Combination type '{0:s}' is unknown".format(method))
##############
# If any pixels are completely masked, apply user-specified function
msgs.info("Replacing completely masked pixels with the {0:s} value of the input frames".format(replace))
indx = comb_frame == maskvalue
comb_frame[indx] = allrej_arr[indx]
# Delete unecessary arrays
del allrej_arr
##############
# Apply the saturated pixels:
if satpix == 'force':
msgs.info("Applying saturated pixels to final combined image")
comb_frame[setsat] = saturation # settings.spect[dnum]['saturation']
##############
# And return a 2D numpy array
msgs.info("{0:d} frames combined successfully!".format(num_frames))
# Make sure the returned array is the correct type
comb_frame = np.array(comb_frame, dtype=np.float)
return comb_frame
def reduce_all(self):
"""
Main driver of the entire reduction
Calibration and extraction via a series of calls to reduce_exposure()
"""
# Validate the parameter set
self.par.validate_keys(required=[
'rdx', 'calibrations', 'scienceframe', 'reduce', 'flexure'
])
self.tstart = time.time()
# Find the standard frames
is_standard = self.fitstbl.find_frames('standard')
# Find the science frames
is_science = self.fitstbl.find_frames('science')
# Frame indices
frame_indx = np.arange(len(self.fitstbl))
# Iterate over each calibration group and reduce the standards
for i in range(self.fitstbl.n_calib_groups):
# Find all the frames in this calibration group
in_grp = self.fitstbl.find_calib_group(i)
# Find the indices of the standard frames in this calibration group:
grp_standards = frame_indx[is_standard & in_grp]
# Reduce all the standard frames, loop on unique comb_id
u_combid_std = np.unique(self.fitstbl['comb_id'][grp_standards])
for j, comb_id in enumerate(u_combid_std):
frames = np.where(self.fitstbl['comb_id'] == comb_id)[0]
bg_frames = np.where(self.fitstbl['bkg_id'] == comb_id)[0]
if not self.outfile_exists(frames[0]) or self.overwrite:
std_spec2d, std_sobjs = self.reduce_exposure(
frames, bg_frames=bg_frames)
# TODO come up with sensible naming convention for save_exposure for combined files
self.save_exposure(frames[0], std_spec2d, std_sobjs,
self.basename)
else:
msgs.info(
'Output file: {:s} already exists'.format(
self.fitstbl.construct_basename(frames[0])) +
'. Set overwrite=True to recreate and overwrite.')
# Iterate over each calibration group again and reduce the science frames
for i in range(self.fitstbl.n_calib_groups):
# Find all the frames in this calibration group
in_grp = self.fitstbl.find_calib_group(i)
# Find the indices of the science frames in this calibration group:
grp_science = frame_indx[is_science & in_grp]
# Associate standards (previously reduced above) for this setup
std_outfile = self.get_std_outfile(frame_indx[is_standard])
# Reduce all the science frames; keep the basenames of the science frames for use in flux calibration
science_basename = [None] * len(grp_science)
# Loop on unique comb_id
u_combid = np.unique(self.fitstbl['comb_id'][grp_science])
for j, comb_id in enumerate(u_combid):
frames = np.where(self.fitstbl['comb_id'] == comb_id)[0]
# Find all frames whose comb_id matches the current frames bkg_id.
bg_frames = np.where((self.fitstbl['comb_id'] ==
self.fitstbl['bkg_id'][frames][0])
& (self.fitstbl['comb_id'] >= 0))[0]
# JFH changed the syntax below to that above, which allows frames to be used more than once
# as a background image. The syntax below would require that we could somehow list multiple
# numbers for the bkg_id which is impossible without a comma separated list
# bg_frames = np.where(self.fitstbl['bkg_id'] == comb_id)[0]
if not self.outfile_exists(frames[0]) or self.overwrite:
# TODO -- Should we reset/regenerate self.slits.mask for a new exposure
sci_spec2d, sci_sobjs = self.reduce_exposure(
frames, bg_frames=bg_frames, std_outfile=std_outfile)
science_basename[j] = self.basename
# TODO come up with sensible naming convention for save_exposure for combined files
self.save_exposure(frames[0], sci_spec2d, sci_sobjs,
self.basename)
else:
msgs.warn(
'Output file: {:s} already exists'.format(
self.fitstbl.construct_basename(frames[0])) +
'. Set overwrite=True to recreate and overwrite.')
msgs.info('Finished calibration group {0}'.format(i))
# Check if this is an IFU reduction. If so, make a datacube
if self.spectrograph.pypeline == "IFU" and self.par['reduce']['cube'][
'make_cube']:
msgs.work("Generate datacube")
# Finish
self.print_end_time()
def flexure_obj(specobjs, maskslits, method, sky_file, mxshft=None):
"""Correct wavelengths for flexure, object by object
Parameters:
----------
method : str
'boxcar' -- Recommneded
'slitpix' --
sky_file: str
Returns:
----------
flex_list: list
list of dicts containing flexure results
Aligned with specobjs
Filled with a basically empty dict if the slit is skipped or there is no object
"""
sv_fdict = None
msgs.work("Consider doing 2 passes in flexure as in LowRedux")
# Load Archive
sky_spectrum = load_sky_spectrum(sky_file)
nslits = len(maskslits)
gdslits = np.where(~maskslits)[0]
# Loop on objects
flex_list = []
# Loop over slits, and then over objects here
for slit in range(nslits):
msgs.info("Working on flexure in slit (if an object was detected): {:d}".format(slit))
indx = specobjs.slitid == slit
this_specobjs = specobjs[indx]
# Reset
flex_dict = dict(polyfit=[], shift=[], subpix=[], corr=[],
corr_cen=[], spec_file=sky_file, smooth=[],
arx_spec=[], sky_spec=[])
# If no objects on this slit append an empty dictionary
if slit not in gdslits:
flex_list.append(flex_dict.copy())
continue
for specobj in this_specobjs:
if specobj is None:
continue
msgs.info("Working on flexure for object # {:d}".format(specobj.objid) + "in slit # {:d}".format(specobj.slitid))
# Using boxcar
if method in ['boxcar', 'slitcen']:
sky_wave = specobj.boxcar['WAVE'] #.to('AA').value
sky_flux = specobj.boxcar['COUNTS_SKY']
else:
msgs.error("Not ready for this flexure method: {}".format(method))
# Generate 1D spectrum for object
obj_sky = xspectrum1d.XSpectrum1D.from_tuple((sky_wave, sky_flux))
# Calculate the shift
fdict = flex_shift(obj_sky, sky_spectrum, mxshft=mxshft)
if fdict is None:
msgs.warn("Flexure shift calculation failed for this spectrum.")
if sv_fdict is not None:
msgs.warn("Will used saved estimate from a previous slit/object")
fdict = copy.deepcopy(sv_fdict)
else:
msgs.warn("No previous good solution. Punting on this object")
continue
else:
sv_fdict = copy.deepcopy(fdict)
# Simple interpolation to apply
npix = len(sky_wave)
x = np.linspace(0., 1., npix)
# Apply
for attr in ['boxcar', 'optimal']:
if not hasattr(specobj, attr):
continue
if 'WAVE' in getattr(specobj, attr).keys():
msgs.info("Applying flexure correction to {0:s} extraction for object:".format(attr) +
msgs.newline() + "{0:s}".format(str(specobj)))
f = interpolate.interp1d(x, sky_wave, bounds_error=False, fill_value="extrapolate")
getattr(specobj, attr)['WAVE'] = f(x+fdict['shift']/(npix-1))*units.AA
# Shift sky spec too
cut_sky = fdict['sky_spec']
x = np.linspace(0., 1., cut_sky.npix)
f = interpolate.interp1d(x, cut_sky.wavelength.value, bounds_error=False, fill_value="extrapolate")
twave = f(x + fdict['shift']/(cut_sky.npix-1))*units.AA
new_sky = xspectrum1d.XSpectrum1D.from_tuple((twave, cut_sky.flux))
# Update dict
for key in ['polyfit', 'shift', 'subpix', 'corr', 'corr_cen', 'smooth', 'arx_spec']:
flex_dict[key].append(fdict[key])
flex_dict['sky_spec'].append(new_sky)
flex_list.append(flex_dict.copy())
return flex_list
def spec_flexure_slit(slits,
slitord,
slit_bpm,
sky_file,
method="boxcar",
specobjs=None,
slit_specs=None,
mxshft=None):
"""Calculate the spectral flexure for every slit (global) or object (local)
Args:
slits (:class:`~pypeit.slittrace.SlitTraceSet`):
Slit trace set
slitord (`numpy.ndarray`_):
Array of slit/order numbers
slit_bpm (`numpy.ndarray`_):
True = masked slit
sky_file (str):
Sky file
method (:obj:`str`, optional):
Two methods are available:
- 'boxcar': Recommended for object extractions. This
method uses the boxcar extracted sky and wavelength
spectra from the input specobjs
- 'slitcen': Recommended when no objects are being
extracted. This method uses a spectrum (stored in
slitspecs) that is extracted from the center of
each slit.
specobjs (:class:`~pypeit.specobjs.Specobjs`, optional):
Spectral extractions
slit_specs (list, optional):
A list of linetools.xspectrum1d, one for each slit. The spectra stored in
this list are sky spectra, extracted from the center of each slit.
mxshft (int, optional):
Passed to flex_shift()
Returns:
:obj:`list`: A list of :obj:`dict` objects containing flexure
results of each slit. This is filled with a basically empty
dict if the slit is skipped.
"""
sv_fdict = None
msgs.work("Consider doing 2 passes in flexure as in LowRedux")
# Determine the method
slit_cen = True if (specobjs is None) or (method == "slitcen") else False
# Load Archive. Save the line information to avoid the performance hit from calling it on the archive sky spectrum
# multiple times
sky_spectrum = load_sky_spectrum(sky_file)
sky_lines = arc.detect_lines(sky_spectrum.flux.value)
nslits = slits.nslits
gpm = np.logical_not(slit_bpm)
gdslits = np.where(gpm)[0]
# Initialise the flexure list for each slit
flex_list = []
# Slit/objects to come back to
return_later_sobjs = []
# Loop over slits, and then over objects
for islit in range(nslits):
msgs.info("Working on spectral flexure of slit: {:d}".format(islit))
# Reset
flex_dict = dict(polyfit=[],
shift=[],
subpix=[],
corr=[],
corr_cen=[],
spec_file=sky_file,
smooth=[],
arx_spec=[],
sky_spec=[],
method=[])
# If no objects on this slit append an empty dictionary
if islit not in gdslits:
flex_list.append(flex_dict.copy())
continue
if slit_cen:
sky_wave = slit_specs[islit].wavelength.value
sky_flux = slit_specs[islit].flux.value
# Calculate the shift
fdict = spec_flex_shift(slit_specs[islit],
sky_spectrum,
sky_lines,
mxshft=mxshft)
# Failed?
if fdict is not None:
# Update dict
for key in [
'polyfit', 'shift', 'subpix', 'corr', 'corr_cen',
'smooth', 'arx_spec'
]:
flex_dict[key].append(fdict[key])
# Interpolate
sky_wave_new = flexure_interp(fdict['shift'], sky_wave)
flex_dict['sky_spec'].append(
xspectrum1d.XSpectrum1D.from_tuple(
(sky_wave_new, sky_flux)))
flex_dict['method'].append("slitcen")
else:
i_slitord = slitord[islit]
indx = specobjs.slitorder_indices(i_slitord)
this_specobjs = specobjs[indx]
# Loop through objects
for ss, sobj in enumerate(this_specobjs):
if sobj is None:
continue
if sobj['BOX_WAVE'] is None: #len(specobj._data.keys()) == 1: # Nothing extracted; only the trace exists
continue
msgs.info(
"Working on flexure for object # {:d}".format(sobj.OBJID) +
"in slit # {:d}".format(islit))
# Using boxcar
sky_wave = sobj.BOX_WAVE
sky_flux = sobj.BOX_COUNTS_SKY
# Generate 1D spectrum for object
obj_sky = xspectrum1d.XSpectrum1D.from_tuple(
(sky_wave, sky_flux))
# Calculate the shift
fdict = spec_flex_shift(obj_sky,
sky_spectrum,
sky_lines,
mxshft=mxshft)
punt = False
if fdict is None:
msgs.warn(
"Flexure shift calculation failed for this spectrum.")
if sv_fdict is not None:
msgs.warn(
"Will used saved estimate from a previous slit/object"
)
fdict = copy.deepcopy(sv_fdict)
else:
# One does not exist yet
# Save it for later
return_later_sobjs.append([islit, ss])
punt = True
else:
sv_fdict = copy.deepcopy(fdict)
# Punt?
if punt:
break
# Update dict
for key in [
'polyfit', 'shift', 'subpix', 'corr', 'corr_cen',
'smooth', 'arx_spec', 'sky_spec'
]:
flex_dict[key].append(fdict[key])
flex_dict['method'].append("boxcar")
# Check if we need to go back
# TODO :: This code just throws an error... probably need to delete or fix this "local" spectral flexure code
if not slit_cen:
# Do we need to go back?
for items in return_later_sobjs:
if sv_fdict is None:
msgs.info("No flexure corrections could be made")
break
# Setup
msgs.error("This probably needs to be updated")
slit, ss = items
flex_dict = flex_list[slit]
sobj = specobjs[ss]
# Copy me
fdict = copy.deepcopy(sv_fdict)
# Update dict
for key in [
'polyfit', 'shift', 'subpix', 'corr', 'corr_cen',
'smooth', 'arx_spec', 'sky_spec'
]:
flex_dict[key].append(fdict[key])
flex_dict['method'].append("boxcar")
# Append, this will be an empty dictionary if the flexure failed
flex_list.append(flex_dict.copy())
return flex_list
def spec_flex_shift(obj_skyspec, arx_skyspec, arx_lines, mxshft=20):
""" Calculate shift between object sky spectrum and archive sky spectrum
Args:
obj_skyspec (:class:`linetools.spectra.xspectrum1d.XSpectrum1d`):
Spectrum of the sky related to our object
arx_skyspec (:class:`linetools.spectra.xspectrum1d.XSpectrum1d`):
Archived sky spectrum
arx_lines (tuple): Line information returned by arc.detect_lines for
the Archived sky spectrum
mxshft (float, optional):
Maximum allowed shift from flexure; note there are cases that
have been known to exceed even 30 pixels..
Returns:
dict: Contains flexure info
"""
# TODO None of these routines should have dependencies on XSpectrum1d!
# Determine the brightest emission lines
msgs.warn("If we use Paranal, cut down on wavelength early on")
arx_amp, arx_amp_cont, arx_cent, arx_wid, _, arx_w, arx_yprep, nsig \
= arx_lines
obj_amp, obj_amp_cont, obj_cent, obj_wid, _, obj_w, obj_yprep, nsig_obj \
= arc.detect_lines(obj_skyspec.flux.value)
# Keep only 5 brightest amplitude lines (xxx_keep is array of
# indices within arx_w of the 5 brightest)
arx_keep = np.argsort(arx_amp[arx_w])[-5:]
obj_keep = np.argsort(obj_amp[obj_w])[-5:]
# Calculate wavelength (Angstrom per pixel)
arx_disp = np.append(
arx_skyspec.wavelength.value[1] - arx_skyspec.wavelength.value[0],
arx_skyspec.wavelength.value[1:] - arx_skyspec.wavelength.value[:-1])
obj_disp = np.append(
obj_skyspec.wavelength.value[1] - obj_skyspec.wavelength.value[0],
obj_skyspec.wavelength.value[1:] - obj_skyspec.wavelength.value[:-1])
# Calculate resolution (lambda/delta lambda_FWHM)..maybe don't need
# this? can just use sigmas
arx_idx = (arx_cent + 0.5).astype(
np.int)[arx_w][arx_keep] # The +0.5 is for rounding
arx_res = arx_skyspec.wavelength.value[arx_idx]/\
(arx_disp[arx_idx]*(2*np.sqrt(2*np.log(2)))*arx_wid[arx_w][arx_keep])
obj_idx = (obj_cent + 0.5).astype(
np.int)[obj_w][obj_keep] # The +0.5 is for rounding
obj_res = obj_skyspec.wavelength.value[obj_idx]/ \
(obj_disp[obj_idx]*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
if not np.all(np.isfinite(obj_res)):
msgs.warn(
'Failed to measure the resolution of the object spectrum, likely due to error '
'in the wavelength image.')
return None
msgs.info("Resolution of Archive={0} and Observation={1}".format(
np.median(arx_res), np.median(obj_res)))
# Determine sigma of gaussian for smoothing
arx_sig2 = np.power(arx_disp[arx_idx] * arx_wid[arx_w][arx_keep], 2)
obj_sig2 = np.power(obj_disp[obj_idx] * obj_wid[obj_w][obj_keep], 2)
arx_med_sig2 = np.median(arx_sig2)
obj_med_sig2 = np.median(obj_sig2)
if obj_med_sig2 >= arx_med_sig2:
smooth_sig = np.sqrt(obj_med_sig2 - arx_med_sig2) # Ang
smooth_sig_pix = smooth_sig / np.median(arx_disp[arx_idx])
arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix * 2 *
np.sqrt(2 * np.log(2)))
else:
msgs.warn("Prefer archival sky spectrum to have higher resolution")
smooth_sig_pix = 0.
msgs.warn("New Sky has higher resolution than Archive. Not smoothing")
#smooth_sig = np.sqrt(arx_med_sig**2-obj_med_sig**2)
#Determine region of wavelength overlap
min_wave = max(np.amin(arx_skyspec.wavelength.value),
np.amin(obj_skyspec.wavelength.value))
max_wave = min(np.amax(arx_skyspec.wavelength.value),
np.amax(obj_skyspec.wavelength.value))
#Smooth higher resolution spectrum by smooth_sig (flux is conserved!)
# if np.median(obj_res) >= np.median(arx_res):
# msgs.warn("New Sky has higher resolution than Archive. Not smoothing")
#obj_sky_newflux = ndimage.gaussian_filter(obj_sky.flux, smooth_sig)
# else:
#tmp = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
# arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix*2*np.sqrt(2*np.log(2)))
#arx_sky.flux = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
# Define wavelengths of overlapping spectra
keep_idx = np.where((obj_skyspec.wavelength.value >= min_wave)
& (obj_skyspec.wavelength.value <= max_wave))[0]
#keep_wave = [i for i in obj_sky.wavelength.value if i>=min_wave if i<=max_wave]
#Rebin both spectra onto overlapped wavelength range
if len(keep_idx) <= 50:
msgs.warn("Not enough overlap between sky spectra")
return None
# rebin onto object ALWAYS
keep_wave = obj_skyspec.wavelength[keep_idx]
arx_skyspec = arx_skyspec.rebin(keep_wave)
obj_skyspec = obj_skyspec.rebin(keep_wave)
# Trim edges (rebinning is junk there)
arx_skyspec.data['flux'][0, :2] = 0.
arx_skyspec.data['flux'][0, -2:] = 0.
obj_skyspec.data['flux'][0, :2] = 0.
obj_skyspec.data['flux'][0, -2:] = 0.
# Set minimum to 0. For bad rebinning and for pernicious extractions
obj_skyspec.data['flux'][0, :] = np.maximum(obj_skyspec.data['flux'][0, :],
0.)
arx_skyspec.data['flux'][0, :] = np.maximum(arx_skyspec.data['flux'][0, :],
0.)
# Normalize spectra to unit average sky count
norm = np.sum(obj_skyspec.flux.value) / obj_skyspec.npix
norm2 = np.sum(arx_skyspec.flux.value) / arx_skyspec.npix
if norm <= 0:
msgs.warn("Bad normalization of object in flexure algorithm")
msgs.warn("Will try the median")
norm = np.median(obj_skyspec.flux.value)
if norm <= 0:
msgs.warn("Improper sky spectrum for flexure. Is it too faint??")
return None
if norm2 <= 0:
msgs.warn(
'Bad normalization of archive in flexure. You are probably using wavelengths '
'well beyond the archive.')
return None
obj_skyspec.flux = obj_skyspec.flux / norm
arx_skyspec.flux = arx_skyspec.flux / norm2
# Deal with bad pixels
msgs.work("Need to mask bad pixels")
# Deal with underlying continuum
msgs.work("Consider taking median first [5 pixel]")
everyn = obj_skyspec.npix // 20
pypeitFit_obj, _ = fitting.iterfit(obj_skyspec.wavelength.value,
obj_skyspec.flux.value,
nord=3,
kwargs_bspline={'everyn': everyn},
kwargs_reject={
'groupbadpix': True,
'maxrej': 1
},
maxiter=15,
upper=3.0,
lower=3.0)
obj_sky_cont, _ = pypeitFit_obj.value(obj_skyspec.wavelength.value)
obj_sky_flux = obj_skyspec.flux.value - obj_sky_cont
pypeitFit_sky, _ = fitting.iterfit(arx_skyspec.wavelength.value,
arx_skyspec.flux.value,
nord=3,
kwargs_bspline={'everyn': everyn},
kwargs_reject={
'groupbadpix': True,
'maxrej': 1
},
maxiter=15,
upper=3.0,
lower=3.0)
arx_sky_cont, _ = pypeitFit_sky.value(arx_skyspec.wavelength.value)
arx_sky_flux = arx_skyspec.flux.value - arx_sky_cont
# Consider sharpness filtering (e.g. LowRedux)
msgs.work("Consider taking median first [5 pixel]")
#Cross correlation of spectra
#corr = np.correlate(arx_skyspec.flux, obj_skyspec.flux, "same")
corr = np.correlate(arx_sky_flux, obj_sky_flux, "same")
#Create array around the max of the correlation function for fitting for subpixel max
# Restrict to pixels within maxshift of zero lag
lag0 = corr.size // 2
#mxshft = settings.argflag['reduce']['flexure']['maxshift']
max_corr = np.argmax(corr[lag0 - mxshft:lag0 + mxshft]) + lag0 - mxshft
subpix_grid = np.linspace(max_corr - 3., max_corr + 3., 7)
#Fit a 2-degree polynomial to peak of correlation function. JFH added this if/else to not crash for bad slits
if np.any(np.isfinite(corr[subpix_grid.astype(np.int)])):
fit = fitting.PypeItFit(xval=subpix_grid,
yval=corr[subpix_grid.astype(np.int)],
func='polynomial',
order=np.atleast_1d(2))
fit.fit()
success = True
max_fit = -0.5 * fit.fitc[1] / fit.fitc[2]
else:
fit = fitting.PypeItFit(xval=subpix_grid,
yval=0.0 * subpix_grid,
func='polynomial',
order=np.atleast_1d(2))
fit.fit()
success = False
max_fit = 0.0
msgs.warn('Flexure compensation failed for one of your objects')
#Calculate and apply shift in wavelength
shift = float(max_fit) - lag0
msgs.info("Flexure correction of {:g} pixels".format(shift))
#model = (fit[2]*(subpix_grid**2.))+(fit[1]*subpix_grid)+fit[0]
return dict(polyfit=fit,
shift=shift,
subpix=subpix_grid,
corr=corr[subpix_grid.astype(np.int)],
sky_spec=obj_skyspec,
arx_spec=arx_skyspec,
corr_cen=corr.size / 2,
smooth=smooth_sig_pix,
success=success)
