def standard_sensfunc(wave, flux, ivar, mask_bad, flux_std, mask_balm=None, mask_tell=None,
maxiter=35, upper=2.0, lower=2.0, polyorder=5, balm_mask_wid=50., nresln=20., telluric=True,
resolution=2700., polycorrect=True, debug=False, polyfunc=False, show_QA=False):
"""
Generate a sensitivity function based on observed flux and standard spectrum.
Parameters
----------
wave : ndarray
wavelength as observed
flux : ndarray
counts/s as observed
ivar : ndarray
inverse variance
flux_std : Quantity array
standard star true flux (erg/s/cm^2/A)
msk_bad : ndarray
mask for bad pixels. True is good.
msk_star: ndarray
mask for hydrogen recombination lines. True is good.
msk_tell:ndarray
mask for telluric regions. True is good.
maxiter : integer
maximum number of iterations for polynomial fit
upper : integer
number of sigma for rejection in polynomial
lower : integer
number of sigma for rejection in polynomial
polyorder : integer
order of polynomial fit
balm_mask_wid: float
in units of angstrom
Mask parameter for Balmer absorption. A region equal to
balm_mask_wid is masked.
resolution: integer/float.
spectra resolution
This paramters should be removed in the future. The resolution should be estimated from spectra directly.
debug : bool
if True shows some dubugging plots
Returns
-------
sensfunc
"""
# Create copy of the arrays to avoid modification
wave_obs = wave.copy()
flux_obs = flux.copy()
ivar_obs = ivar.copy()
# preparing arrays
if np.any(np.invert(np.isfinite(ivar_obs))):
msgs.warn("NaN are present in the inverse variance")
# check masks
if mask_tell is None:
mask_tell = np.ones_like(wave_obs,dtype=bool)
if mask_balm is None:
mask_balm = np.ones_like(wave_obs, dtype=bool)
# Removing outliers
# Calculate log of flux_obs setting a floor at TINY
logflux_obs = 2.5 * np.log10(np.maximum(flux_obs, TINY))
# Set a fix value for the variance of logflux
logivar_obs = np.ones_like(logflux_obs) * (10.0 ** 2)
# Calculate log of flux_std model setting a floor at TINY
logflux_std = 2.5 * np.log10(np.maximum(flux_std, TINY))
# Calculate ratio setting a floor at MAGFUNC_MIN and a ceiling at
# MAGFUNC_MAX
magfunc = logflux_std - logflux_obs
magfunc = np.maximum(np.minimum(magfunc, MAGFUNC_MAX), MAGFUNC_MIN)
msk_magfunc = (magfunc < 0.99 * MAGFUNC_MAX) & (magfunc > 0.99 * MAGFUNC_MIN)
# Define two new masks, True is good and False is masked pixel
# mask for all bad pixels on sensfunc
masktot = mask_bad & msk_magfunc & np.isfinite(ivar_obs) & np.isfinite(logflux_obs) & np.isfinite(logflux_std)
logivar_obs[np.invert(masktot)] = 0.0
# mask used for polynomial fit
msk_fit_sens = masktot & mask_tell & mask_balm
# Polynomial fitting to derive a smooth sensfunc (i.e. without telluric)
pypeitFit = fitting.robust_fit(wave_obs[msk_fit_sens], magfunc[msk_fit_sens], polyorder,
function='polynomial', maxiter=maxiter,
lower=lower, upper=upper,
groupbadpix=False,
grow=0, sticky=True, use_mad=True)
magfunc_poly = pypeitFit.eval(wave_obs)
# Polynomial corrections on Hydrogen Recombination lines
if ((sum(msk_fit_sens) > 0.5 * len(msk_fit_sens)) & polycorrect):
## Only correct Hydrogen Recombination lines with polyfit in the telluric free region
balmer_clean = np.zeros_like(wave_obs, dtype=bool)
# Commented out the bluest recombination lines since they are weak for spectroscopic standard stars.
#836.4, 3969.6, 3890.1, 4102.8, 4102.8, 4341.6, 4862.7, \
lines_hydrogen = np.array([5407.0, 6564.6, 8224.8, 8239.2, 8203.6, 8440.3, 8469.6, 8504.8, 8547.7, 8600.8, \
8667.4, 8752.9, 8865.2, 9017.4, 9229.0, 10049.4, 10938.1, 12818.1, 21655.0])
for line_hydrogen in lines_hydrogen:
ihydrogen = np.abs(wave_obs - line_hydrogen) <= balm_mask_wid
balmer_clean[ihydrogen] = True
msk_clean = ((balmer_clean) | (magfunc == MAGFUNC_MAX) | (magfunc == MAGFUNC_MIN)) & \
(magfunc_poly > MAGFUNC_MIN) & (magfunc_poly < MAGFUNC_MAX)
magfunc[msk_clean] = magfunc_poly[msk_clean]
msk_badpix = np.isfinite(ivar_obs) & (ivar_obs > 0)
magfunc[np.invert(msk_badpix)] = magfunc_poly[np.invert(msk_badpix)]
else:
## if half more than half of your spectrum is masked (or polycorrect=False) then do not correct it with polyfit
msgs.warn('No polynomial corrections performed on Hydrogen Recombination line regions')
if not telluric:
# Apply mask to ivar
#logivar_obs[~msk_fit_sens] = 0.
# ToDo
# Compute an effective resolution for the standard. This could be improved
# to setup an array of breakpoints based on the resolution. At the
# moment we are using only one number
msgs.work("Should pull resolution from arc line analysis")
msgs.work("At the moment the resolution is taken as the PixelScale")
msgs.work("This needs to be changed!")
std_pix = np.median(np.abs(wave_obs - np.roll(wave_obs, 1)))
std_res = np.median(wave_obs/resolution) # median resolution in units of Angstrom.
#std_res = std_pix
#resln = std_res
if (nresln * std_res) < std_pix:
msgs.warn("Bspline breakpoints spacing shoud be larger than 1pixel")
msgs.warn("Changing input nresln to fix this")
nresln = std_res / std_pix
# Fit magfunc with bspline
kwargs_bspline = {'bkspace': std_res * nresln}
kwargs_reject = {'maxrej': 5}
msgs.info("Initialize bspline for flux calibration")
# init_bspline = pydl.bspline(wave_obs, bkspace=kwargs_bspline['bkspace'])
init_bspline = bspline.bspline(wave_obs, bkspace=kwargs_bspline['bkspace'])
fullbkpt = init_bspline.breakpoints
# TESTING turning off masking for now
# remove masked regions from breakpoints
msk_obs = np.ones_like(wave_obs).astype(bool)
msk_obs[np.invert(masktot)] = False
msk_bkpt = interpolate.interp1d(wave_obs, msk_obs, kind='nearest',
fill_value='extrapolate')(fullbkpt)
init_breakpoints = fullbkpt[msk_bkpt > 0.999]
# init_breakpoints = fullbkpt
msgs.info("Bspline fit on magfunc. ")
bset1, bmask = fitting.iterfit(wave_obs, magfunc, invvar=logivar_obs, inmask=msk_fit_sens, upper=upper, lower=lower,
fullbkpt=init_breakpoints, maxiter=maxiter, kwargs_bspline=kwargs_bspline,
kwargs_reject=kwargs_reject)
logfit1, _ = bset1.value(wave_obs)
logfit_bkpt, _ = bset1.value(init_breakpoints)
if debug:
# Check for calibration
plt.figure(1)
plt.plot(wave_obs, magfunc, drawstyle='steps-mid', color='black', label='magfunc')
plt.plot(wave_obs, logfit1, color='cornflowerblue', label='logfit1')
plt.plot(wave_obs[np.invert(msk_fit_sens)], magfunc[np.invert(msk_fit_sens)], '+', color='red', markersize=5.0,
label='masked magfunc')
plt.plot(wave_obs[np.invert(msk_fit_sens)], logfit1[np.invert(msk_fit_sens)], '+', color='red', markersize=5.0,
label='masked logfit1')
plt.plot(init_breakpoints, logfit_bkpt, '.', color='green', markersize=4.0, label='breakpoints')
plt.plot(init_breakpoints, np.interp(init_breakpoints, wave_obs, magfunc), '.', color='green',
markersize=4.0,
label='breakpoints')
plt.plot(wave_obs, 1.0 / np.sqrt(logivar_obs), color='orange', label='sigma')
plt.legend()
plt.xlabel('Wavelength [ang]')
plt.ylim(0.0, 1.2 * MAGFUNC_MAX)
plt.title('1st Bspline fit')
plt.show()
# Create sensitivity function
magfunc = np.maximum(np.minimum(logfit1, MAGFUNC_MAX), MAGFUNC_MIN)
if ((sum(msk_fit_sens) > 0.5 * len(msk_fit_sens)) & polycorrect):
msk_clean = ((magfunc == MAGFUNC_MAX) | (magfunc == MAGFUNC_MIN)) & \
(magfunc_poly > MAGFUNC_MIN) & (magfunc_poly<MAGFUNC_MAX)
magfunc[msk_clean] = magfunc_poly[msk_clean]
msk_badpix = np.isfinite(ivar_obs) & (ivar_obs>0)
magfunc[np.invert(msk_badpix)] = magfunc_poly[np.invert(msk_badpix)]
else:
## if half more than half of your spectrum is masked (or polycorrect=False) then do not correct it with polyfit
msgs.warn('No polynomial corrections performed on Hydrogen Recombination line regions')
# Calculate sensfunc
if polyfunc:
sensfunc = 10.0 ** (0.4 * magfunc_poly)
magfunc = magfunc_poly
else:
sensfunc = 10.0 ** (0.4 * magfunc)
if debug:
plt.figure()
magfunc_raw = logflux_std - logflux_obs
plt.plot(wave_obs[masktot],magfunc_raw[masktot] , 'k-',lw=3,label='Raw Magfunc')
plt.plot(wave_obs[masktot],magfunc_poly[masktot] , 'c-',lw=3,label='Polynomial Fit')
plt.plot(wave_obs[np.invert(mask_tell)], magfunc_raw[np.invert(mask_tell)], 's',
color='0.7',label='Telluric Region')
plt.plot(wave_obs[np.invert(mask_balm)], magfunc_raw[np.invert(mask_balm)], 'r+',label='Recombination Line region')
plt.plot(wave_obs[masktot], magfunc[masktot],'b-',label='Final Magfunc')
plt.legend(fancybox=True, shadow=True)
plt.xlim([0.995*np.min(wave_obs[masktot]),1.005*np.max(wave_obs[masktot])])
plt.ylim([0.,1.2*np.max(magfunc[masktot])])
plt.show()
plt.close()
return sensfunc, masktot
def sensfunc_eval(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, longitude, latitude,
mask_abs_lines=True, telluric=False, polyorder=4, balm_mask_wid=5., nresln=20., resolution=3000.,
trans_thresh=0.9, polycorrect=True, polyfunc=False, debug=False):
"""
Function to generate the sensitivity function. This function fits
a bspline to the 2.5*log10(flux_std/flux_counts). The break
points spacing, which determines the scale of variation of the
sensitivity function is determined by the nresln parameter. This
code can work in different regimes, but NOTE THAT TELLURIC MODE
IS DEPRECATED, use telluric.sensfunc_telluric instead.
- If telluric=False, a sensfunc is generated by fitting a
bspline to the using nresln=20.0 and masking out telluric
regions.
- If telluric=True, sensfunc is a pixelized sensfunc (not
smooth) for correcting both throughput and telluric lines.
if you set polycorrect=True, the sensfunc in the Hydrogen
recombination line region (often seen in star spectra) will
be replaced by a smoothed polynomial function.
Args:
wave (ndarray):
Wavelength of the star. Shape (nspec,)
counts (ndarray):
Flux (in counts) of the star. Shape (nspec,)
counts_ivar (ndarray):
Inverse variance of the star counts. Shape (nspec,)
counts_mask (ndarray):
Good pixel mask for the counts.
exptime (float):
Exposure time in seconds
airmass (float):
Airmass
std_dict (dict):
Dictionary containing information about the standard star returned by flux_calib.get_standard_spectrum
longitude (float):
Telescope longitude, used for extinction correction.
latitude (float):
Telescope latitude, used for extinction correction
telluric (bool):
If True attempts to fit telluric absorption. This feature is deprecated, as one should instead
use telluric.sensfunc_telluric. Default=False
mask_abs_lines (bool):
If True, mask stellar absorption lines before fitting sensitivity function. Default = True
balm_mask_wid (float):
Parameter describing the width of the mask for or stellar absorption lines (i.e. mask_abs_lines=True). A region
equal to balm_mask_wid*resln is masked where resln is the estimate for the spectral resolution in pixels
per resolution element.
polycorrect: bool
Whether you want to interpolate the sensfunc with polynomial in the stellar absortion line regions before
fitting with the bspline
nresln (float):
Parameter governing the spacing of the bspline breakpoints. default = 20.0
resolution (float):
Expected resolution of the standard star spectrum. This should probably be determined from the grating, but is
currently hard wired. default=3000.0
trans_thresh (float):
Parameter for selecting telluric regions which are masked. Locations below this transmission value are masked.
If you have significant telluric absorption you should be using telluric.sensnfunc_telluric. default = 0.9
Returns:
tuple: Returns:
- sensfunc (ndarray) -- Sensitivity function with same shape as wave (nspec,)
- mask_sens (ndarray, bool) -- Good pixel mask for sensitivity function with same shape as wave (nspec,)
"""
# Create copy of the arrays to avoid modification and convert to
# electrons / s
wave_star = wave.copy()
flux_star = counts.copy() / exptime
ivar_star = counts_ivar.copy() * exptime ** 2
# Extinction correction
msgs.info("Applying extinction correction")
extinct = load_extinction_data(longitude,latitude)
ext_corr = extinction_correction(wave * units.AA, airmass, extinct)
# Correct for extinction
flux_star = flux_star * ext_corr
ivar_star = ivar_star / ext_corr ** 2
mask_star = counts_mask
# Interpolate the standard star onto the current set of observed wavelengths
flux_true = interpolate.interp1d(std_dict['wave'], std_dict['flux'], bounds_error=False,
fill_value='extrapolate')(wave_star)
# Do we need to extrapolate? TODO Replace with a model or a grey body?
if np.min(flux_true) <= 0.:
msgs.warn('Your spectrum extends beyond calibrated standard star, extrapolating the spectra with polynomial.')
mask_model = flux_true <= 0
pypeitFit = fitting.robust_fit(std_dict['wave'].value, std_dict['flux'].value,8,function='polynomial',
maxiter=50, lower=3.0, upper=3.0, maxrej=3,
grow=0, sticky=True, use_mad=True)
star_poly = pypeitFit.eval(wave_star.value)
#flux_true[mask_model] = star_poly[mask_model]
flux_true = star_poly.copy()
if debug:
plt.plot(std_dict['wave'], std_dict['flux'],'bo',label='Raw Star Model')
plt.plot(std_dict['wave'], pypeitFit.eval(std_dict['wave'].value),
'k-',label='robust_poly_fit')
plt.plot(wave_star,flux_true,'r-',label='Your Final Star Model used for sensfunc')
plt.show()
# Get masks from observed star spectrum. True = Good pixels
mask_bad, mask_balm, mask_tell = get_mask(wave_star, flux_star, ivar_star, mask_star, mask_abs_lines=mask_abs_lines,
mask_telluric=True, balm_mask_wid=balm_mask_wid, trans_thresh=trans_thresh)
# Get sensfunc
#LBLRTM = False
#if LBLRTM:
# # sensfunc = lblrtm_sensfunc() ???
# msgs.develop('fluxing and telluric correction based on LBLRTM model is under developing.')
#else:
sensfunc, mask_sens = standard_sensfunc(
wave_star, flux_star, ivar_star, mask_bad, flux_true, mask_balm=mask_balm,
mask_tell=mask_tell, maxiter=35, upper=3.0, lower=3.0, polyorder=polyorder,
balm_mask_wid=balm_mask_wid, nresln=nresln,telluric=telluric, resolution=resolution,
polycorrect=polycorrect, polyfunc=polyfunc, debug=debug, show_QA=False)
if debug:
plt.plot(wave_star[mask_sens], flux_true[mask_sens], color='k',lw=2, label='Reference Star')
plt.plot(wave_star[mask_sens], flux_star[mask_sens]*sensfunc[mask_sens], color='r', label='Fluxed Observed Star')
plt.xlabel(r'Wavelength [$\AA$]')
plt.ylabel('Flux [erg/s/cm2/Ang.]')
plt.legend(fancybox=True, shadow=True)
plt.show()
return sensfunc, mask_sens
def fit_pca_coefficients(coeff,
order,
ivar=None,
weights=None,
function='legendre',
lower=3.0,
upper=3.0,
maxrej=1,
maxiter=25,
coo=None,
minx=None,
maxx=None,
debug=False):
r"""
Fit a parameterized function to a set of PCA coefficients,
primarily for the purpose of predicting coefficients at
intermediate locations.
The coefficients of each PCA component are fit by a low-order
polynomial, where the abscissa is set by the `coo` argument (see
:func:`pypeit.fitting.robust_fit`).
.. note::
This is a general function, not really specific to the PCA;
and is really just a wrapper for
:func:`pypeit.fitting.robust_fit`.
Args:
coeff (`numpy.ndarray`_):
PCA component coefficients. If the PCA decomposition used
:math:`N_{\rm comp}` components for :math:`N_{\rm vec}`
vectors, the shape of this array must be :math:`(N_{\rm
vec}, N_{\rm comp})`. The array can be 1D with shape
:math:`(N_{\rm vec},)` if there was only one PCA
component.
order (:obj:`int`, `numpy.ndarray`_):
The order, :math:`o`, of the function used to fit the PCA
coefficients. Can be a single number for all PCA
components, or an array with an order specific to each
component. If the latter, the shape must be
:math:`(N_{\rm comp},)`.
ivar (`numpy.ndarray`_, optional):
Inverse variance in the PCA coefficients to use during
the fit; see the `invvar` parameter of
:func:`pypeit.fitting.robust_fit`. If None, fit is
not error weighted. If a vector with shape :math:`(N_{\rm
vec},)`, the same error will be assumed for all PCA
components (i.e., `ivar` will be expanded to match the
shape of `coeff`). If a 2D array, the shape must match
`coeff`.
weights (`numpy.ndarray`_, optional):
Weights to apply to the PCA coefficients during the fit;
see the `weights` parameter of
:func:`pypeit.fitting.robust_fit`. If None, the
weights are uniform. If a vector with shape
:math:`(N_{\rm vec},)`, the same weights will be assumed
for all PCA components (i.e., `weights` will be expanded
to match the shape of `coeff`). If a 2D array, the shape
must match `coeff`.
function (:obj:`str`, optional):
Type of function used to fit the data.
lower (:obj:`float`, optional):
Number of standard deviations used for rejecting data
**below** the mean residual. If None, no rejection is
performed. See :func:`fitting.robust_fit`.
upper (:obj:`float`, optional):
Number of standard deviations used for rejecting data
**above** the mean residual. If None, no rejection is
performed. See :func:`fitting.robust_fit`.
maxrej (:obj:`int`, optional):
Maximum number of points to reject during fit iterations.
See :func:`fitting.robust_fit`.
maxiter (:obj:`int`, optional):
Maximum number of rejection iterations allows. To force
no rejection iterations, set to 0.
coo (`numpy.ndarray`_, optional):
Floating-point array with the independent coordinates to
use when fitting the PCA coefficients. If None, simply
uses a running number. Shape must be :math:`(N_{\rm
vec},)`.
minx, maxx (:obj:`float`, optional):
Minimum and maximum values used to rescale the
independent axis data. If None, the minimum and maximum
values of `coo` are used. See
:func:`fitting.robust_fit`.
debug (:obj:`bool`, optional):
Show plots useful for debugging.
Returns:
`numpy.ndarray`_: One or more
:class:`~pypeit.core.fitting.PypeItFit` instances, one per
PCA component, that models the PCA component coefficients as
a function of the reference coordinates. These can be used to
predict new vectors that follow the PCA model at a new
coordinate; see :func:`pca_predict`.
"""
# Check the input
# - Get the shape of the input data to fit
_coeff = np.asarray(coeff)
if _coeff.ndim == 1:
_coeff = np.expand_dims(_coeff, 1)
if _coeff.ndim != 2:
raise ValueError('Array with coefficiencts cannot be more than 2D')
nvec, npca = _coeff.shape
# - Check the inverse variance
_ivar = None if ivar is None else np.atleast_2d(ivar)
if _ivar is not None and _ivar.shape != _coeff.shape:
raise ValueError(
'Inverse variance array does not match input coefficients.')
# - Check the weights
_weights = np.ones(_coeff.shape,
dtype=float) if weights is None else np.asarray(weights)
if _weights.ndim == 1:
_weights = np.tile(_weights, (_coeff.shape[1], 1)).T
if _weights.shape != _coeff.shape:
raise ValueError('Weights array does not match input coefficients.')
# - Set the abscissa of the data if not provided and check its
# shape
if coo is None:
coo = np.arange(nvec, dtype=float)
if coo.size != nvec:
raise ValueError('Vector coordinates have incorrect shape.')
# - Check the order of the functions to fit
_order = np.atleast_1d(order)
if _order.size == 1:
_order = np.full(npca, order, dtype=int)
if _order.size != npca:
raise ValueError(
'Function order must be a single number or one number per PCA component.'
)
# - Force the values of minx and maxx if they're not provided directly
if minx is None:
minx = np.amin(coo)
if maxx is None:
maxx = np.amax(coo)
# Instantiate the output
# TODO: This fitting is fast. Maybe we should determine the best
# order for each PCA component, up to some maximum, by comparing
# reduction in chi-square vs added number of parameters?
# Fit the coefficients of each PCA component so that they can be
# interpolated to other coordinates.
inmask = np.ones_like(coo, dtype=bool)
model = np.empty(npca, dtype=fitting.PypeItFit)
for i in range(npca):
model[i] = fitting.robust_fit(
coo,
_coeff[:, i],
_order[i],
in_gpm=inmask,
invvar=None if _ivar is None else _ivar[:, i],
weights=_weights[:, i],
function=function,
maxiter=maxiter,
lower=lower,
upper=upper,
maxrej=maxrej,
sticky=False,
use_mad=_ivar is None,
minx=minx,
maxx=maxx)
if debug:
# Visually check the fits
xvec = np.linspace(np.amin(coo), np.amax(coo), num=100)
rejected = np.logical_not(model[i].gpm) & inmask
plt.scatter(coo[inmask],
_coeff[inmask, i],
marker='.',
color='k',
s=100,
facecolor='none',
label='pca coeff')
plt.scatter(coo[np.logical_not(inmask)],
_coeff[np.logical_not(inmask), i],
marker='.',
color='orange',
s=100,
facecolor='none',
label='pca coeff, masked from previous')
if np.any(rejected):
plt.scatter(coo[rejected],
_coeff[rejected, i],
marker='x',
color='C3',
s=80,
label='robust_polyfit_djs rejected')
plt.plot(xvec,
model[i].eval(xvec),
linestyle='--',
color='C0',
label='Polynomial fit of order={0}'.format(_order[i]))
plt.xlabel('Trace Coordinate', fontsize=14)
plt.ylabel('PCA Coefficient', fontsize=14)
plt.title('PCA Fit for Dimension #{0}/{1}'.format(i + 1, npca))
plt.legend()
plt.show()
# Propagate rejection of coeffs for this component to the next
# component.
# TODO: Can we put in a comment here or in the docstring
# explaining why we do this?
inmask = model[i].gpm.astype(bool)
# Return the fitted model
return model
def slit_match(x_det,
x_model,
step=1,
xlag_range=[-50, 50],
sigrej=3,
print_matches=False,
edge=None):
"""
Script that perform the slit edges matching.
This method uses :func:`discrete_correlate_match` to find the
indices of x_model that match x_det.
Taken from DEEP2/spec2d/pro/deimos_slit_match.pro
Parameters
----------
x_det: `numpy.ndarray`_
1D array of slit edge spatial positions found from image.
x_model: `numpy.ndarray`_
1D array of slit edge spatial positions predicted by the
optical model.
step: :obj:`int`, optional
Step size in pixels used to generate a list of possible
offsets within the `offsets_range`.
xlag_range: :obj:`list`, optional
Range of offsets in pixels allowed between the slit
positions predicted by the mask design and the traced
slit positions.
sigrej: :obj:`float`, optional
Reject slit matches larger than this number of sigma in
the match residuals.
print_matches: :obj:`bool`, optional
Print the result of the matching.
edge: :obj:`str`, optional
String that indicates which edges are being plotted,
i.e., left of right. Ignored if ``print_matches`` is
False.
Returns
-------
ind: `numpy.ndarray`_
1D array of indices for `x_model`, which defines the matches
to `x_det`, i.e., `x_det` matches `x_model[ind]`
dupl: `numpy.ndarray`_
1D array of `bool` that flags which `ind` are duplicates.
coeff: `numpy.ndarray`_
pypeitFit coefficients of the fitted relation between `x_det`
and `x_model[ind]`
sigres: :obj:`float`
RMS residual for the fitted relation between `x_det` and
`x_model[ind]`
"""
# Determine the indices of `x_model` that match `x_det`
ind = discrete_correlate_match(x_det,
np.ma.masked_equal(x_model, -1),
step=step,
xlag_range=xlag_range)
# Define the weights for the fitting
residual = (x_det - x_model[ind]) - np.median(x_det - x_model[ind])
weights = np.zeros(residual.size, dtype=int)
weights[np.abs(residual) < 100.] = 1
if weights.sum() == 0:
weights = np.ones(residual.size, dtype=int)
# Fit between `x_det` and `x_model[ind]`
pypeitFit = fitting.robust_fit(x_model[ind],
x_det,
1,
maxiter=100,
weights=weights,
lower=3,
upper=3)
coeff = pypeitFit.fitc
yfit = pypeitFit.eval(x_model[ind])
# compute residuals
res = yfit - x_det
sigres = sigma_clipped_stats(res, sigma=sigrej)[2] # RMS residuals
# flag the matches that have residuals > `sigrej` times the RMS, or if res>5
cut = 5 if res.size < 5 else sigrej * sigres
out = np.abs(res) > cut
# check for duplicate indices
dupl = np.ones(ind.size, dtype=bool)
# If there are duplicates of `ind`, for now we keep only the first one. We don't remove the others yet
dupl[np.unique(ind, return_index=True)[1]] = False
wdupl = np.where(dupl)[0]
# Iterate over the duplicates flagged as bad
if wdupl.size > 0:
for i in range(wdupl.size):
duplind = ind[wdupl[i]]
# Where are the other duplicates of this `ind`?
w = np.where(ind == duplind)[0]
# set those to be bad (for the moment)
dupl[w] = True
# Among the duplicates of this particular `ind`, which one has the smallest residual?
wdif = np.argmin(np.abs(res[w]))
# The one with the smallest residuals, is then set to not bad
dupl[w[wdif]] = False
# Both duplicates and matches with high RMS are considered bad
dupl = dupl | out
if edge is not None:
msgs.warn('{} duplicate match(es) for {} edges'.format(
dupl[dupl == 1].size, edge))
else:
msgs.warn('{} duplicate match(es)'.format(dupl[dupl == 1].size))
# I commented the 3 lines below because I don't really need to trim the duplicate matches. I just
# propagate the flag.
# good = dupl == 0
# ind = ind[good]
# x_det=x_det[good]
if print_matches:
if edge is not None:
msgs.info('-----------------------------------------------')
msgs.info(' {} slit edges '.format(edge))
msgs.info('-----------------------------------------------')
msgs.info('Index omodel_edge spat_edge ')
msgs.info('-----------------------------------------------')
for i in range(ind.size):
msgs.info('{} {} {}'.format(ind[i], x_model[ind][i], x_det[i]))
msgs.info('-----------------------------------------------')
return ind, dupl, coeff, sigres
def discrete_correlate_match(x_det, x_model, step=1, xlag_range=[-50, 50]):
"""
Script to find the the x_model values that match the traced edges.
This method uses :func:`best_offset` to determine the best offset between
slit edge predicted by the optical model and the one found in the image, given a range of
offsets. This is used iteratively.
Taken from in DEEP2/spec2d/pro/discrete_correlate_match.pro
x_det==x1, x_model==x2_in
Args:
x_det (`numpy.ndarray`_):
1D array of slit edge spatial positions found from image
x_model (`numpy.ndarray`_):
1D array of slit edge spatial positions predicted by the optical model
step (:obj:`int`):
step size in pixels used to generate a list of possible offsets within the `offsets_range`
xlag_range (:obj:`list`, optional):
range of offsets in pixels allowed between the slit positions predicted by
the mask design and the traced slit positions.
Returns:
`numpy.ndarray`_: array of indices for x_model, which defines the matches to x_det,
i.e., x_det matches x_model[ind]
"""
# -------- PASS 1: get offset between x1 and x2
# Determine the offset between x_det and x_model
best_off = best_offset(x_det, x_model, step=step, xlag_range=xlag_range)
# apply the offset to x_model
x_model_new = x_model - best_off
# for each traced edge (`x_det`) determine the value of x_model that gives the smallest offset
ind = np.ma.argmin(np.ma.absolute(x_det[:, None] - x_model_new[None, :]),
axis=1)
# -------- PASS 2: remove linear trend (i.e. adjust scale)
# fit the offsets to `x_det` to find the scale and apply it to x_model
dx = np.ma.compressed(x_det - x_model_new[ind])
pypeitFit = fitting.robust_fit(x_det, dx, 1, maxiter=100, lower=2, upper=2)
coeff = pypeitFit.fitc
scale = 1 + coeff[1] if x_det.size > 4 else 1
x_model_new *= scale
# Find again the best offset and apply it to x_model
new_best_off = best_offset(x_det,
x_model_new,
step=step,
xlag_range=xlag_range)
x_model_new -= new_best_off
# find again `ind`
ind = np.ma.argmin(np.ma.absolute(x_det[:, None] - x_model_new[None, :]),
axis=1)
# -------- PASS 3: tweak offset
dx = x_det - x_model_new[ind]
x_model_new += np.ma.median(dx)
# find again `ind`
ind = np.ma.argmin(np.ma.absolute(x_det[:, None] - x_model_new[None, :]),
axis=1)
return ind
def iterative_fitting(spec, tcent, ifit, IDs, llist, disp,
match_toler = 2.0, func = 'legendre', n_first=2, sigrej_first=2.0,
n_final=4, sigrej_final=3.0, input_only=False,
weights=None, plot_fil=None, verbose=False):
""" Routine for iteratively fitting wavelength solutions.
Parameters
----------
spec : ndarray, shape = (nspec,)
arcline spectrum
tcent : ndarray
Centroids in pixels of lines identified in spec
ifit : ndarray
Indices of the lines that will be fit
IDs: ndarray
wavelength IDs of the lines that will be fit (I think?)
llist: dict
Linelist dictionary
disp: float
dispersion
Optional Parameters
-------------------
match_toler: float, default = 3.0
Matching tolerance when searching for new lines. This is the difference in pixels between the wavlength assigned to
an arc line by an iteration of the wavelength solution to the wavelength in the line list.
func: str, default = 'legendre'
Name of function used for the wavelength solution
n_first: int, default = 2
Order of first guess to the wavelength solution.
sigrej_first: float, default = 2.0
Number of sigma for rejection for the first guess to the wavelength solution.
n_final: int, default = 4
Order of the final wavelength solution fit
sigrej_final: float, default = 3.0
Number of sigma for rejection for the final fit to the wavelength solution.
input_only: bool
If True, the routine will only perform a robust polyfit to the input IDs.
If False, the routine will fit the input IDs, and then include additional
lines in the linelist that are a satisfactory fit.
weights: ndarray
Weights to be used?
verbose : bool
If True, print out more information.
plot_fil:
Filename for plotting some QA?
Returns
-------
final_fit: :class:`pypeit.core.wavecal.wv_fitting.WaveFit`
"""
#TODO JFH add error checking here to ensure that IDs and ifit have the same size!
if weights is None:
weights = np.ones(tcent.size)
nspec = spec.size
xnspecmin1 = float(nspec-1)
# Setup for fitting
sv_ifit = list(ifit) # Keep the originals
all_ids = -999.*np.ones(len(tcent))
all_idsion = np.array(['UNKNWN']*len(tcent))
all_ids[ifit] = IDs
# Fit
n_order = n_first
flg_continue = True
flg_penultimate = False
fmin, fmax = 0.0, 1.0
# Note the number of parameters is actually n_order and not n_order+1
while flg_continue:
if flg_penultimate:
flg_continue = False
# Fit with rejection
xfit, yfit, wfit = tcent[ifit], all_ids[ifit], weights[ifit]
maxiter = xfit.size - n_order - 2
#
if xfit.size == 0:
msgs.warn("All points rejected !!")
return None
# Fit
pypeitFit = fitting.robust_fit(xfit/xnspecmin1, yfit, n_order, function=func, maxiter=maxiter,
lower=sigrej_first, upper=sigrej_first, maxrej=1, sticky=True,
minx=fmin, maxx=fmax, weights=wfit)
# Junk fit?
if pypeitFit is None:
msgs.warn("Bad fit!!")
return None
rms_ang = pypeitFit.calc_fit_rms(apply_mask=True)
rms_pix = rms_ang/disp
if verbose:
msgs.info('n_order = {:d}'.format(n_order) + ': RMS = {:g}'.format(rms_pix))
# Reject but keep originals (until final fit)
ifit = list(ifit[pypeitFit.gpm == 1]) + sv_ifit
if not input_only:
# Find new points from the linelist (should we allow removal of the originals?)
twave = pypeitFit.eval(tcent/xnspecmin1)#, func, minx=fmin, maxx=fmax)
for ss, iwave in enumerate(twave):
mn = np.min(np.abs(iwave-llist['wave']))
if mn/disp < match_toler:
imn = np.argmin(np.abs(iwave-llist['wave']))
#if verbose:
# print('Adding {:g} at {:g}'.format(llist['wave'][imn],tcent[ss]))
# Update and append
all_ids[ss] = llist['wave'][imn]
all_idsion[ss] = llist['ion'][imn]
ifit.append(ss)
# Keep unique ones
ifit = np.unique(np.array(ifit, dtype=int))
# Increment order?
if n_order < n_final:
n_order += 1
else:
flg_penultimate = True
# Final fit (originals can now be rejected)
xfit, yfit, wfit = tcent[ifit], all_ids[ifit], weights[ifit]
pypeitFit = fitting.robust_fit(xfit/xnspecmin1, yfit, n_order, function=func,
lower=sigrej_final, upper=sigrej_final, maxrej=1, sticky=True,
minx=fmin, maxx=fmax, weights=wfit)#, debug=True)
irej = np.where(np.logical_not(pypeitFit.bool_gpm))[0]
if len(irej) > 0:
xrej = xfit[irej]
yrej = yfit[irej]
if verbose:
for kk, imask in enumerate(irej):
wave = pypeitFit.eval(xrej[kk]/xnspecmin1)#, func, minx=fmin, maxx=fmax)
msgs.info('Rejecting arc line {:g}; {:g}'.format(yfit[imask], wave))
else:
xrej = []
yrej = []
ions = all_idsion[ifit]
# Final RMS
rms_ang = pypeitFit.calc_fit_rms(apply_mask=True)
rms_pix = rms_ang/disp
# Pack up fit
spec_vec = np.arange(nspec)
wave_soln = pypeitFit.eval(spec_vec/xnspecmin1)
cen_wave = pypeitFit.eval(float(nspec)/2/xnspecmin1)
cen_wave_min1 = pypeitFit.eval((float(nspec)/2 - 1.0)/xnspecmin1)
cen_disp = cen_wave - cen_wave_min1
# Ions bit
ion_bits = np.zeros(len(ions), dtype=WaveFit.bitmask.minimum_dtype())
for kk,ion in enumerate(ions):
ion_bits[kk] = WaveFit.bitmask.turn_on(ion_bits[kk], ion.replace(' ', ''))
# DataContainer time
# spat_id is set to an arbitrary -1 here and is updated in wavecalib.py
final_fit = WaveFit(-1, pypeitfit=pypeitFit, pixel_fit=xfit, wave_fit=yfit,
ion_bits=ion_bits, xnorm=xnspecmin1,
cen_wave=cen_wave, cen_disp=cen_disp,
spec=spec, wave_soln = wave_soln, sigrej=sigrej_final,
shift=0., tcent=tcent, rms=rms_pix)
# QA
if plot_fil is not None:
autoid.arc_fit_qa(final_fit, plot_fil)
# Return
return final_fit