def standard_sensfunc(wave, flux, ivar, mask_bad, flux_std, mask_balm=None, mask_tell=None, maxiter=35, upper=2.0, lower=2.0, polyorder=5, balm_mask_wid=50., nresln=20., telluric=True, resolution=2700., polycorrect=True, debug=False, polyfunc=False, show_QA=False): """ Generate a sensitivity function based on observed flux and standard spectrum. Parameters ---------- wave : ndarray wavelength as observed flux : ndarray counts/s as observed ivar : ndarray inverse variance flux_std : Quantity array standard star true flux (erg/s/cm^2/A) msk_bad : ndarray mask for bad pixels. True is good. msk_star: ndarray mask for hydrogen recombination lines. True is good. msk_tell:ndarray mask for telluric regions. True is good. maxiter : integer maximum number of iterations for polynomial fit upper : integer number of sigma for rejection in polynomial lower : integer number of sigma for rejection in polynomial polyorder : integer order of polynomial fit balm_mask_wid: float in units of angstrom Mask parameter for Balmer absorption. A region equal to balm_mask_wid is masked. resolution: integer/float. spectra resolution This paramters should be removed in the future. The resolution should be estimated from spectra directly. debug : bool if True shows some dubugging plots Returns ------- sensfunc """ # Create copy of the arrays to avoid modification wave_obs = wave.copy() flux_obs = flux.copy() ivar_obs = ivar.copy() # preparing arrays if np.any(np.invert(np.isfinite(ivar_obs))): msgs.warn("NaN are present in the inverse variance") # check masks if mask_tell is None: mask_tell = np.ones_like(wave_obs,dtype=bool) if mask_balm is None: mask_balm = np.ones_like(wave_obs, dtype=bool) # Removing outliers # Calculate log of flux_obs setting a floor at TINY logflux_obs = 2.5 * np.log10(np.maximum(flux_obs, TINY)) # Set a fix value for the variance of logflux logivar_obs = np.ones_like(logflux_obs) * (10.0 ** 2) # Calculate log of flux_std model setting a floor at TINY logflux_std = 2.5 * np.log10(np.maximum(flux_std, TINY)) # Calculate ratio setting a floor at MAGFUNC_MIN and a ceiling at # MAGFUNC_MAX magfunc = logflux_std - logflux_obs magfunc = np.maximum(np.minimum(magfunc, MAGFUNC_MAX), MAGFUNC_MIN) msk_magfunc = (magfunc < 0.99 * MAGFUNC_MAX) & (magfunc > 0.99 * MAGFUNC_MIN) # Define two new masks, True is good and False is masked pixel # mask for all bad pixels on sensfunc masktot = mask_bad & msk_magfunc & np.isfinite(ivar_obs) & np.isfinite(logflux_obs) & np.isfinite(logflux_std) logivar_obs[np.invert(masktot)] = 0.0 # mask used for polynomial fit msk_fit_sens = masktot & mask_tell & mask_balm # Polynomial fitting to derive a smooth sensfunc (i.e. without telluric) pypeitFit = fitting.robust_fit(wave_obs[msk_fit_sens], magfunc[msk_fit_sens], polyorder, function='polynomial', maxiter=maxiter, lower=lower, upper=upper, groupbadpix=False, grow=0, sticky=True, use_mad=True) magfunc_poly = pypeitFit.eval(wave_obs) # Polynomial corrections on Hydrogen Recombination lines if ((sum(msk_fit_sens) > 0.5 * len(msk_fit_sens)) & polycorrect): ## Only correct Hydrogen Recombination lines with polyfit in the telluric free region balmer_clean = np.zeros_like(wave_obs, dtype=bool) # Commented out the bluest recombination lines since they are weak for spectroscopic standard stars. #836.4, 3969.6, 3890.1, 4102.8, 4102.8, 4341.6, 4862.7, \ lines_hydrogen = np.array([5407.0, 6564.6, 8224.8, 8239.2, 8203.6, 8440.3, 8469.6, 8504.8, 8547.7, 8600.8, \ 8667.4, 8752.9, 8865.2, 9017.4, 9229.0, 10049.4, 10938.1, 12818.1, 21655.0]) for line_hydrogen in lines_hydrogen: ihydrogen = np.abs(wave_obs - line_hydrogen) <= balm_mask_wid balmer_clean[ihydrogen] = True msk_clean = ((balmer_clean) | (magfunc == MAGFUNC_MAX) | (magfunc == MAGFUNC_MIN)) & \ (magfunc_poly > MAGFUNC_MIN) & (magfunc_poly < MAGFUNC_MAX) magfunc[msk_clean] = magfunc_poly[msk_clean] msk_badpix = np.isfinite(ivar_obs) & (ivar_obs > 0) magfunc[np.invert(msk_badpix)] = magfunc_poly[np.invert(msk_badpix)] else: ## if half more than half of your spectrum is masked (or polycorrect=False) then do not correct it with polyfit msgs.warn('No polynomial corrections performed on Hydrogen Recombination line regions') if not telluric: # Apply mask to ivar #logivar_obs[~msk_fit_sens] = 0. # ToDo # Compute an effective resolution for the standard. This could be improved # to setup an array of breakpoints based on the resolution. At the # moment we are using only one number msgs.work("Should pull resolution from arc line analysis") msgs.work("At the moment the resolution is taken as the PixelScale") msgs.work("This needs to be changed!") std_pix = np.median(np.abs(wave_obs - np.roll(wave_obs, 1))) std_res = np.median(wave_obs/resolution) # median resolution in units of Angstrom. #std_res = std_pix #resln = std_res if (nresln * std_res) < std_pix: msgs.warn("Bspline breakpoints spacing shoud be larger than 1pixel") msgs.warn("Changing input nresln to fix this") nresln = std_res / std_pix # Fit magfunc with bspline kwargs_bspline = {'bkspace': std_res * nresln} kwargs_reject = {'maxrej': 5} msgs.info("Initialize bspline for flux calibration") # init_bspline = pydl.bspline(wave_obs, bkspace=kwargs_bspline['bkspace']) init_bspline = bspline.bspline(wave_obs, bkspace=kwargs_bspline['bkspace']) fullbkpt = init_bspline.breakpoints # TESTING turning off masking for now # remove masked regions from breakpoints msk_obs = np.ones_like(wave_obs).astype(bool) msk_obs[np.invert(masktot)] = False msk_bkpt = interpolate.interp1d(wave_obs, msk_obs, kind='nearest', fill_value='extrapolate')(fullbkpt) init_breakpoints = fullbkpt[msk_bkpt > 0.999] # init_breakpoints = fullbkpt msgs.info("Bspline fit on magfunc. ") bset1, bmask = fitting.iterfit(wave_obs, magfunc, invvar=logivar_obs, inmask=msk_fit_sens, upper=upper, lower=lower, fullbkpt=init_breakpoints, maxiter=maxiter, kwargs_bspline=kwargs_bspline, kwargs_reject=kwargs_reject) logfit1, _ = bset1.value(wave_obs) logfit_bkpt, _ = bset1.value(init_breakpoints) if debug: # Check for calibration plt.figure(1) plt.plot(wave_obs, magfunc, drawstyle='steps-mid', color='black', label='magfunc') plt.plot(wave_obs, logfit1, color='cornflowerblue', label='logfit1') plt.plot(wave_obs[np.invert(msk_fit_sens)], magfunc[np.invert(msk_fit_sens)], '+', color='red', markersize=5.0, label='masked magfunc') plt.plot(wave_obs[np.invert(msk_fit_sens)], logfit1[np.invert(msk_fit_sens)], '+', color='red', markersize=5.0, label='masked logfit1') plt.plot(init_breakpoints, logfit_bkpt, '.', color='green', markersize=4.0, label='breakpoints') plt.plot(init_breakpoints, np.interp(init_breakpoints, wave_obs, magfunc), '.', color='green', markersize=4.0, label='breakpoints') plt.plot(wave_obs, 1.0 / np.sqrt(logivar_obs), color='orange', label='sigma') plt.legend() plt.xlabel('Wavelength [ang]') plt.ylim(0.0, 1.2 * MAGFUNC_MAX) plt.title('1st Bspline fit') plt.show() # Create sensitivity function magfunc = np.maximum(np.minimum(logfit1, MAGFUNC_MAX), MAGFUNC_MIN) if ((sum(msk_fit_sens) > 0.5 * len(msk_fit_sens)) & polycorrect): msk_clean = ((magfunc == MAGFUNC_MAX) | (magfunc == MAGFUNC_MIN)) & \ (magfunc_poly > MAGFUNC_MIN) & (magfunc_poly<MAGFUNC_MAX) magfunc[msk_clean] = magfunc_poly[msk_clean] msk_badpix = np.isfinite(ivar_obs) & (ivar_obs>0) magfunc[np.invert(msk_badpix)] = magfunc_poly[np.invert(msk_badpix)] else: ## if half more than half of your spectrum is masked (or polycorrect=False) then do not correct it with polyfit msgs.warn('No polynomial corrections performed on Hydrogen Recombination line regions') # Calculate sensfunc if polyfunc: sensfunc = 10.0 ** (0.4 * magfunc_poly) magfunc = magfunc_poly else: sensfunc = 10.0 ** (0.4 * magfunc) if debug: plt.figure() magfunc_raw = logflux_std - logflux_obs plt.plot(wave_obs[masktot],magfunc_raw[masktot] , 'k-',lw=3,label='Raw Magfunc') plt.plot(wave_obs[masktot],magfunc_poly[masktot] , 'c-',lw=3,label='Polynomial Fit') plt.plot(wave_obs[np.invert(mask_tell)], magfunc_raw[np.invert(mask_tell)], 's', color='0.7',label='Telluric Region') plt.plot(wave_obs[np.invert(mask_balm)], magfunc_raw[np.invert(mask_balm)], 'r+',label='Recombination Line region') plt.plot(wave_obs[masktot], magfunc[masktot],'b-',label='Final Magfunc') plt.legend(fancybox=True, shadow=True) plt.xlim([0.995*np.min(wave_obs[masktot]),1.005*np.max(wave_obs[masktot])]) plt.ylim([0.,1.2*np.max(magfunc[masktot])]) plt.show() plt.close() return sensfunc, masktot
def sensfunc_eval(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, longitude, latitude, mask_abs_lines=True, telluric=False, polyorder=4, balm_mask_wid=5., nresln=20., resolution=3000., trans_thresh=0.9, polycorrect=True, polyfunc=False, debug=False): """ Function to generate the sensitivity function. This function fits a bspline to the 2.5*log10(flux_std/flux_counts). The break points spacing, which determines the scale of variation of the sensitivity function is determined by the nresln parameter. This code can work in different regimes, but NOTE THAT TELLURIC MODE IS DEPRECATED, use telluric.sensfunc_telluric instead. - If telluric=False, a sensfunc is generated by fitting a bspline to the using nresln=20.0 and masking out telluric regions. - If telluric=True, sensfunc is a pixelized sensfunc (not smooth) for correcting both throughput and telluric lines. if you set polycorrect=True, the sensfunc in the Hydrogen recombination line region (often seen in star spectra) will be replaced by a smoothed polynomial function. Args: wave (ndarray): Wavelength of the star. Shape (nspec,) counts (ndarray): Flux (in counts) of the star. Shape (nspec,) counts_ivar (ndarray): Inverse variance of the star counts. Shape (nspec,) counts_mask (ndarray): Good pixel mask for the counts. exptime (float): Exposure time in seconds airmass (float): Airmass std_dict (dict): Dictionary containing information about the standard star returned by flux_calib.get_standard_spectrum longitude (float): Telescope longitude, used for extinction correction. latitude (float): Telescope latitude, used for extinction correction telluric (bool): If True attempts to fit telluric absorption. This feature is deprecated, as one should instead use telluric.sensfunc_telluric. Default=False mask_abs_lines (bool): If True, mask stellar absorption lines before fitting sensitivity function. Default = True balm_mask_wid (float): Parameter describing the width of the mask for or stellar absorption lines (i.e. mask_abs_lines=True). A region equal to balm_mask_wid*resln is masked where resln is the estimate for the spectral resolution in pixels per resolution element. polycorrect: bool Whether you want to interpolate the sensfunc with polynomial in the stellar absortion line regions before fitting with the bspline nresln (float): Parameter governing the spacing of the bspline breakpoints. default = 20.0 resolution (float): Expected resolution of the standard star spectrum. This should probably be determined from the grating, but is currently hard wired. default=3000.0 trans_thresh (float): Parameter for selecting telluric regions which are masked. Locations below this transmission value are masked. If you have significant telluric absorption you should be using telluric.sensnfunc_telluric. default = 0.9 Returns: tuple: Returns: - sensfunc (ndarray) -- Sensitivity function with same shape as wave (nspec,) - mask_sens (ndarray, bool) -- Good pixel mask for sensitivity function with same shape as wave (nspec,) """ # Create copy of the arrays to avoid modification and convert to # electrons / s wave_star = wave.copy() flux_star = counts.copy() / exptime ivar_star = counts_ivar.copy() * exptime ** 2 # Extinction correction msgs.info("Applying extinction correction") extinct = load_extinction_data(longitude,latitude) ext_corr = extinction_correction(wave * units.AA, airmass, extinct) # Correct for extinction flux_star = flux_star * ext_corr ivar_star = ivar_star / ext_corr ** 2 mask_star = counts_mask # Interpolate the standard star onto the current set of observed wavelengths flux_true = interpolate.interp1d(std_dict['wave'], std_dict['flux'], bounds_error=False, fill_value='extrapolate')(wave_star) # Do we need to extrapolate? TODO Replace with a model or a grey body? if np.min(flux_true) <= 0.: msgs.warn('Your spectrum extends beyond calibrated standard star, extrapolating the spectra with polynomial.') mask_model = flux_true <= 0 pypeitFit = fitting.robust_fit(std_dict['wave'].value, std_dict['flux'].value,8,function='polynomial', maxiter=50, lower=3.0, upper=3.0, maxrej=3, grow=0, sticky=True, use_mad=True) star_poly = pypeitFit.eval(wave_star.value) #flux_true[mask_model] = star_poly[mask_model] flux_true = star_poly.copy() if debug: plt.plot(std_dict['wave'], std_dict['flux'],'bo',label='Raw Star Model') plt.plot(std_dict['wave'], pypeitFit.eval(std_dict['wave'].value), 'k-',label='robust_poly_fit') plt.plot(wave_star,flux_true,'r-',label='Your Final Star Model used for sensfunc') plt.show() # Get masks from observed star spectrum. True = Good pixels mask_bad, mask_balm, mask_tell = get_mask(wave_star, flux_star, ivar_star, mask_star, mask_abs_lines=mask_abs_lines, mask_telluric=True, balm_mask_wid=balm_mask_wid, trans_thresh=trans_thresh) # Get sensfunc #LBLRTM = False #if LBLRTM: # # sensfunc = lblrtm_sensfunc() ??? # msgs.develop('fluxing and telluric correction based on LBLRTM model is under developing.') #else: sensfunc, mask_sens = standard_sensfunc( wave_star, flux_star, ivar_star, mask_bad, flux_true, mask_balm=mask_balm, mask_tell=mask_tell, maxiter=35, upper=3.0, lower=3.0, polyorder=polyorder, balm_mask_wid=balm_mask_wid, nresln=nresln,telluric=telluric, resolution=resolution, polycorrect=polycorrect, polyfunc=polyfunc, debug=debug, show_QA=False) if debug: plt.plot(wave_star[mask_sens], flux_true[mask_sens], color='k',lw=2, label='Reference Star') plt.plot(wave_star[mask_sens], flux_star[mask_sens]*sensfunc[mask_sens], color='r', label='Fluxed Observed Star') plt.xlabel(r'Wavelength [$\AA$]') plt.ylabel('Flux [erg/s/cm2/Ang.]') plt.legend(fancybox=True, shadow=True) plt.show() return sensfunc, mask_sens
def fit_pca_coefficients(coeff, order, ivar=None, weights=None, function='legendre', lower=3.0, upper=3.0, maxrej=1, maxiter=25, coo=None, minx=None, maxx=None, debug=False): r""" Fit a parameterized function to a set of PCA coefficients, primarily for the purpose of predicting coefficients at intermediate locations. The coefficients of each PCA component are fit by a low-order polynomial, where the abscissa is set by the `coo` argument (see :func:`pypeit.fitting.robust_fit`). .. note:: This is a general function, not really specific to the PCA; and is really just a wrapper for :func:`pypeit.fitting.robust_fit`. Args: coeff (`numpy.ndarray`_): PCA component coefficients. If the PCA decomposition used :math:`N_{\rm comp}` components for :math:`N_{\rm vec}` vectors, the shape of this array must be :math:`(N_{\rm vec}, N_{\rm comp})`. The array can be 1D with shape :math:`(N_{\rm vec},)` if there was only one PCA component. order (:obj:`int`, `numpy.ndarray`_): The order, :math:`o`, of the function used to fit the PCA coefficients. Can be a single number for all PCA components, or an array with an order specific to each component. If the latter, the shape must be :math:`(N_{\rm comp},)`. ivar (`numpy.ndarray`_, optional): Inverse variance in the PCA coefficients to use during the fit; see the `invvar` parameter of :func:`pypeit.fitting.robust_fit`. If None, fit is not error weighted. If a vector with shape :math:`(N_{\rm vec},)`, the same error will be assumed for all PCA components (i.e., `ivar` will be expanded to match the shape of `coeff`). If a 2D array, the shape must match `coeff`. weights (`numpy.ndarray`_, optional): Weights to apply to the PCA coefficients during the fit; see the `weights` parameter of :func:`pypeit.fitting.robust_fit`. If None, the weights are uniform. If a vector with shape :math:`(N_{\rm vec},)`, the same weights will be assumed for all PCA components (i.e., `weights` will be expanded to match the shape of `coeff`). If a 2D array, the shape must match `coeff`. function (:obj:`str`, optional): Type of function used to fit the data. lower (:obj:`float`, optional): Number of standard deviations used for rejecting data **below** the mean residual. If None, no rejection is performed. See :func:`fitting.robust_fit`. upper (:obj:`float`, optional): Number of standard deviations used for rejecting data **above** the mean residual. If None, no rejection is performed. See :func:`fitting.robust_fit`. maxrej (:obj:`int`, optional): Maximum number of points to reject during fit iterations. See :func:`fitting.robust_fit`. maxiter (:obj:`int`, optional): Maximum number of rejection iterations allows. To force no rejection iterations, set to 0. coo (`numpy.ndarray`_, optional): Floating-point array with the independent coordinates to use when fitting the PCA coefficients. If None, simply uses a running number. Shape must be :math:`(N_{\rm vec},)`. minx, maxx (:obj:`float`, optional): Minimum and maximum values used to rescale the independent axis data. If None, the minimum and maximum values of `coo` are used. See :func:`fitting.robust_fit`. debug (:obj:`bool`, optional): Show plots useful for debugging. Returns: `numpy.ndarray`_: One or more :class:`~pypeit.core.fitting.PypeItFit` instances, one per PCA component, that models the PCA component coefficients as a function of the reference coordinates. These can be used to predict new vectors that follow the PCA model at a new coordinate; see :func:`pca_predict`. """ # Check the input # - Get the shape of the input data to fit _coeff = np.asarray(coeff) if _coeff.ndim == 1: _coeff = np.expand_dims(_coeff, 1) if _coeff.ndim != 2: raise ValueError('Array with coefficiencts cannot be more than 2D') nvec, npca = _coeff.shape # - Check the inverse variance _ivar = None if ivar is None else np.atleast_2d(ivar) if _ivar is not None and _ivar.shape != _coeff.shape: raise ValueError( 'Inverse variance array does not match input coefficients.') # - Check the weights _weights = np.ones(_coeff.shape, dtype=float) if weights is None else np.asarray(weights) if _weights.ndim == 1: _weights = np.tile(_weights, (_coeff.shape[1], 1)).T if _weights.shape != _coeff.shape: raise ValueError('Weights array does not match input coefficients.') # - Set the abscissa of the data if not provided and check its # shape if coo is None: coo = np.arange(nvec, dtype=float) if coo.size != nvec: raise ValueError('Vector coordinates have incorrect shape.') # - Check the order of the functions to fit _order = np.atleast_1d(order) if _order.size == 1: _order = np.full(npca, order, dtype=int) if _order.size != npca: raise ValueError( 'Function order must be a single number or one number per PCA component.' ) # - Force the values of minx and maxx if they're not provided directly if minx is None: minx = np.amin(coo) if maxx is None: maxx = np.amax(coo) # Instantiate the output # TODO: This fitting is fast. Maybe we should determine the best # order for each PCA component, up to some maximum, by comparing # reduction in chi-square vs added number of parameters? # Fit the coefficients of each PCA component so that they can be # interpolated to other coordinates. inmask = np.ones_like(coo, dtype=bool) model = np.empty(npca, dtype=fitting.PypeItFit) for i in range(npca): model[i] = fitting.robust_fit( coo, _coeff[:, i], _order[i], in_gpm=inmask, invvar=None if _ivar is None else _ivar[:, i], weights=_weights[:, i], function=function, maxiter=maxiter, lower=lower, upper=upper, maxrej=maxrej, sticky=False, use_mad=_ivar is None, minx=minx, maxx=maxx) if debug: # Visually check the fits xvec = np.linspace(np.amin(coo), np.amax(coo), num=100) rejected = np.logical_not(model[i].gpm) & inmask plt.scatter(coo[inmask], _coeff[inmask, i], marker='.', color='k', s=100, facecolor='none', label='pca coeff') plt.scatter(coo[np.logical_not(inmask)], _coeff[np.logical_not(inmask), i], marker='.', color='orange', s=100, facecolor='none', label='pca coeff, masked from previous') if np.any(rejected): plt.scatter(coo[rejected], _coeff[rejected, i], marker='x', color='C3', s=80, label='robust_polyfit_djs rejected') plt.plot(xvec, model[i].eval(xvec), linestyle='--', color='C0', label='Polynomial fit of order={0}'.format(_order[i])) plt.xlabel('Trace Coordinate', fontsize=14) plt.ylabel('PCA Coefficient', fontsize=14) plt.title('PCA Fit for Dimension #{0}/{1}'.format(i + 1, npca)) plt.legend() plt.show() # Propagate rejection of coeffs for this component to the next # component. # TODO: Can we put in a comment here or in the docstring # explaining why we do this? inmask = model[i].gpm.astype(bool) # Return the fitted model return model
def slit_match(x_det, x_model, step=1, xlag_range=[-50, 50], sigrej=3, print_matches=False, edge=None): """ Script that perform the slit edges matching. This method uses :func:`discrete_correlate_match` to find the indices of x_model that match x_det. Taken from DEEP2/spec2d/pro/deimos_slit_match.pro Parameters ---------- x_det: `numpy.ndarray`_ 1D array of slit edge spatial positions found from image. x_model: `numpy.ndarray`_ 1D array of slit edge spatial positions predicted by the optical model. step: :obj:`int`, optional Step size in pixels used to generate a list of possible offsets within the `offsets_range`. xlag_range: :obj:`list`, optional Range of offsets in pixels allowed between the slit positions predicted by the mask design and the traced slit positions. sigrej: :obj:`float`, optional Reject slit matches larger than this number of sigma in the match residuals. print_matches: :obj:`bool`, optional Print the result of the matching. edge: :obj:`str`, optional String that indicates which edges are being plotted, i.e., left of right. Ignored if ``print_matches`` is False. Returns ------- ind: `numpy.ndarray`_ 1D array of indices for `x_model`, which defines the matches to `x_det`, i.e., `x_det` matches `x_model[ind]` dupl: `numpy.ndarray`_ 1D array of `bool` that flags which `ind` are duplicates. coeff: `numpy.ndarray`_ pypeitFit coefficients of the fitted relation between `x_det` and `x_model[ind]` sigres: :obj:`float` RMS residual for the fitted relation between `x_det` and `x_model[ind]` """ # Determine the indices of `x_model` that match `x_det` ind = discrete_correlate_match(x_det, np.ma.masked_equal(x_model, -1), step=step, xlag_range=xlag_range) # Define the weights for the fitting residual = (x_det - x_model[ind]) - np.median(x_det - x_model[ind]) weights = np.zeros(residual.size, dtype=int) weights[np.abs(residual) < 100.] = 1 if weights.sum() == 0: weights = np.ones(residual.size, dtype=int) # Fit between `x_det` and `x_model[ind]` pypeitFit = fitting.robust_fit(x_model[ind], x_det, 1, maxiter=100, weights=weights, lower=3, upper=3) coeff = pypeitFit.fitc yfit = pypeitFit.eval(x_model[ind]) # compute residuals res = yfit - x_det sigres = sigma_clipped_stats(res, sigma=sigrej)[2] # RMS residuals # flag the matches that have residuals > `sigrej` times the RMS, or if res>5 cut = 5 if res.size < 5 else sigrej * sigres out = np.abs(res) > cut # check for duplicate indices dupl = np.ones(ind.size, dtype=bool) # If there are duplicates of `ind`, for now we keep only the first one. We don't remove the others yet dupl[np.unique(ind, return_index=True)[1]] = False wdupl = np.where(dupl)[0] # Iterate over the duplicates flagged as bad if wdupl.size > 0: for i in range(wdupl.size): duplind = ind[wdupl[i]] # Where are the other duplicates of this `ind`? w = np.where(ind == duplind)[0] # set those to be bad (for the moment) dupl[w] = True # Among the duplicates of this particular `ind`, which one has the smallest residual? wdif = np.argmin(np.abs(res[w])) # The one with the smallest residuals, is then set to not bad dupl[w[wdif]] = False # Both duplicates and matches with high RMS are considered bad dupl = dupl | out if edge is not None: msgs.warn('{} duplicate match(es) for {} edges'.format( dupl[dupl == 1].size, edge)) else: msgs.warn('{} duplicate match(es)'.format(dupl[dupl == 1].size)) # I commented the 3 lines below because I don't really need to trim the duplicate matches. I just # propagate the flag. # good = dupl == 0 # ind = ind[good] # x_det=x_det[good] if print_matches: if edge is not None: msgs.info('-----------------------------------------------') msgs.info(' {} slit edges '.format(edge)) msgs.info('-----------------------------------------------') msgs.info('Index omodel_edge spat_edge ') msgs.info('-----------------------------------------------') for i in range(ind.size): msgs.info('{} {} {}'.format(ind[i], x_model[ind][i], x_det[i])) msgs.info('-----------------------------------------------') return ind, dupl, coeff, sigres
def discrete_correlate_match(x_det, x_model, step=1, xlag_range=[-50, 50]): """ Script to find the the x_model values that match the traced edges. This method uses :func:`best_offset` to determine the best offset between slit edge predicted by the optical model and the one found in the image, given a range of offsets. This is used iteratively. Taken from in DEEP2/spec2d/pro/discrete_correlate_match.pro x_det==x1, x_model==x2_in Args: x_det (`numpy.ndarray`_): 1D array of slit edge spatial positions found from image x_model (`numpy.ndarray`_): 1D array of slit edge spatial positions predicted by the optical model step (:obj:`int`): step size in pixels used to generate a list of possible offsets within the `offsets_range` xlag_range (:obj:`list`, optional): range of offsets in pixels allowed between the slit positions predicted by the mask design and the traced slit positions. Returns: `numpy.ndarray`_: array of indices for x_model, which defines the matches to x_det, i.e., x_det matches x_model[ind] """ # -------- PASS 1: get offset between x1 and x2 # Determine the offset between x_det and x_model best_off = best_offset(x_det, x_model, step=step, xlag_range=xlag_range) # apply the offset to x_model x_model_new = x_model - best_off # for each traced edge (`x_det`) determine the value of x_model that gives the smallest offset ind = np.ma.argmin(np.ma.absolute(x_det[:, None] - x_model_new[None, :]), axis=1) # -------- PASS 2: remove linear trend (i.e. adjust scale) # fit the offsets to `x_det` to find the scale and apply it to x_model dx = np.ma.compressed(x_det - x_model_new[ind]) pypeitFit = fitting.robust_fit(x_det, dx, 1, maxiter=100, lower=2, upper=2) coeff = pypeitFit.fitc scale = 1 + coeff[1] if x_det.size > 4 else 1 x_model_new *= scale # Find again the best offset and apply it to x_model new_best_off = best_offset(x_det, x_model_new, step=step, xlag_range=xlag_range) x_model_new -= new_best_off # find again `ind` ind = np.ma.argmin(np.ma.absolute(x_det[:, None] - x_model_new[None, :]), axis=1) # -------- PASS 3: tweak offset dx = x_det - x_model_new[ind] x_model_new += np.ma.median(dx) # find again `ind` ind = np.ma.argmin(np.ma.absolute(x_det[:, None] - x_model_new[None, :]), axis=1) return ind
def iterative_fitting(spec, tcent, ifit, IDs, llist, disp, match_toler = 2.0, func = 'legendre', n_first=2, sigrej_first=2.0, n_final=4, sigrej_final=3.0, input_only=False, weights=None, plot_fil=None, verbose=False): """ Routine for iteratively fitting wavelength solutions. Parameters ---------- spec : ndarray, shape = (nspec,) arcline spectrum tcent : ndarray Centroids in pixels of lines identified in spec ifit : ndarray Indices of the lines that will be fit IDs: ndarray wavelength IDs of the lines that will be fit (I think?) llist: dict Linelist dictionary disp: float dispersion Optional Parameters ------------------- match_toler: float, default = 3.0 Matching tolerance when searching for new lines. This is the difference in pixels between the wavlength assigned to an arc line by an iteration of the wavelength solution to the wavelength in the line list. func: str, default = 'legendre' Name of function used for the wavelength solution n_first: int, default = 2 Order of first guess to the wavelength solution. sigrej_first: float, default = 2.0 Number of sigma for rejection for the first guess to the wavelength solution. n_final: int, default = 4 Order of the final wavelength solution fit sigrej_final: float, default = 3.0 Number of sigma for rejection for the final fit to the wavelength solution. input_only: bool If True, the routine will only perform a robust polyfit to the input IDs. If False, the routine will fit the input IDs, and then include additional lines in the linelist that are a satisfactory fit. weights: ndarray Weights to be used? verbose : bool If True, print out more information. plot_fil: Filename for plotting some QA? Returns ------- final_fit: :class:`pypeit.core.wavecal.wv_fitting.WaveFit` """ #TODO JFH add error checking here to ensure that IDs and ifit have the same size! if weights is None: weights = np.ones(tcent.size) nspec = spec.size xnspecmin1 = float(nspec-1) # Setup for fitting sv_ifit = list(ifit) # Keep the originals all_ids = -999.*np.ones(len(tcent)) all_idsion = np.array(['UNKNWN']*len(tcent)) all_ids[ifit] = IDs # Fit n_order = n_first flg_continue = True flg_penultimate = False fmin, fmax = 0.0, 1.0 # Note the number of parameters is actually n_order and not n_order+1 while flg_continue: if flg_penultimate: flg_continue = False # Fit with rejection xfit, yfit, wfit = tcent[ifit], all_ids[ifit], weights[ifit] maxiter = xfit.size - n_order - 2 # if xfit.size == 0: msgs.warn("All points rejected !!") return None # Fit pypeitFit = fitting.robust_fit(xfit/xnspecmin1, yfit, n_order, function=func, maxiter=maxiter, lower=sigrej_first, upper=sigrej_first, maxrej=1, sticky=True, minx=fmin, maxx=fmax, weights=wfit) # Junk fit? if pypeitFit is None: msgs.warn("Bad fit!!") return None rms_ang = pypeitFit.calc_fit_rms(apply_mask=True) rms_pix = rms_ang/disp if verbose: msgs.info('n_order = {:d}'.format(n_order) + ': RMS = {:g}'.format(rms_pix)) # Reject but keep originals (until final fit) ifit = list(ifit[pypeitFit.gpm == 1]) + sv_ifit if not input_only: # Find new points from the linelist (should we allow removal of the originals?) twave = pypeitFit.eval(tcent/xnspecmin1)#, func, minx=fmin, maxx=fmax) for ss, iwave in enumerate(twave): mn = np.min(np.abs(iwave-llist['wave'])) if mn/disp < match_toler: imn = np.argmin(np.abs(iwave-llist['wave'])) #if verbose: # print('Adding {:g} at {:g}'.format(llist['wave'][imn],tcent[ss])) # Update and append all_ids[ss] = llist['wave'][imn] all_idsion[ss] = llist['ion'][imn] ifit.append(ss) # Keep unique ones ifit = np.unique(np.array(ifit, dtype=int)) # Increment order? if n_order < n_final: n_order += 1 else: flg_penultimate = True # Final fit (originals can now be rejected) xfit, yfit, wfit = tcent[ifit], all_ids[ifit], weights[ifit] pypeitFit = fitting.robust_fit(xfit/xnspecmin1, yfit, n_order, function=func, lower=sigrej_final, upper=sigrej_final, maxrej=1, sticky=True, minx=fmin, maxx=fmax, weights=wfit)#, debug=True) irej = np.where(np.logical_not(pypeitFit.bool_gpm))[0] if len(irej) > 0: xrej = xfit[irej] yrej = yfit[irej] if verbose: for kk, imask in enumerate(irej): wave = pypeitFit.eval(xrej[kk]/xnspecmin1)#, func, minx=fmin, maxx=fmax) msgs.info('Rejecting arc line {:g}; {:g}'.format(yfit[imask], wave)) else: xrej = [] yrej = [] ions = all_idsion[ifit] # Final RMS rms_ang = pypeitFit.calc_fit_rms(apply_mask=True) rms_pix = rms_ang/disp # Pack up fit spec_vec = np.arange(nspec) wave_soln = pypeitFit.eval(spec_vec/xnspecmin1) cen_wave = pypeitFit.eval(float(nspec)/2/xnspecmin1) cen_wave_min1 = pypeitFit.eval((float(nspec)/2 - 1.0)/xnspecmin1) cen_disp = cen_wave - cen_wave_min1 # Ions bit ion_bits = np.zeros(len(ions), dtype=WaveFit.bitmask.minimum_dtype()) for kk,ion in enumerate(ions): ion_bits[kk] = WaveFit.bitmask.turn_on(ion_bits[kk], ion.replace(' ', '')) # DataContainer time # spat_id is set to an arbitrary -1 here and is updated in wavecalib.py final_fit = WaveFit(-1, pypeitfit=pypeitFit, pixel_fit=xfit, wave_fit=yfit, ion_bits=ion_bits, xnorm=xnspecmin1, cen_wave=cen_wave, cen_disp=cen_disp, spec=spec, wave_soln = wave_soln, sigrej=sigrej_final, shift=0., tcent=tcent, rms=rms_pix) # QA if plot_fil is not None: autoid.arc_fit_qa(final_fit, plot_fil) # Return return final_fit