def _residual(self, paramgroup=None, data_only=False, **kws): """return the residual for this data set residual = self.transform.apply(data_chi - model_chi) where model_chi is the result of ff2chi(pathlist) """ if (paramgroup is not None and self._larch.symtable.isgroup(paramgroup)): self._larch.symtable._sys.paramGroup = paramgroup if not isNamedClass(self.transform, TransformGroup): return if not self.__prepared: self.prepare_fit() _ff2chi(self.pathlist, k=self.model.k, _larch=self._larch, group=self.model) eps_k = self.epsilon_k if isinstance(eps_k, np.ndarray): eps_k[np.where(eps_k<1.e-12)[0]] = 1.e-12 else: eps_k = max(1.e-12, eps_k) diff = (self.__chi - self.model.chi) if data_only: # for extracting transformed data separately from residual diff = self.__chi trans = self.transform k = trans.k_[:len(diff)] all_kweights = isinstance(trans.kweight, Iterable) if trans.fitspace == 'k': iqmin = max(0, int(0.01 + trans.kmin/trans.kstep)) iqmax = min(trans.nfft/2, int(0.01 + trans.kmax/trans.kstep)) if all_kweights: out = [] for i, kw in enumerate(trans.kweight): out.append(((diff/eps_k[i])*k**kw)[iqmin:iqmax]) return np.concatenate(out) else: return ((diff/eps_k) * k**trans.kweight)[iqmin:iqmax] else: out = [] if all_kweights: chir = [trans.fftf(diff, kweight=kw) for kw in trans.kweight] eps_r = self.epsilon_r else: chir = [trans.fftf(diff)] eps_r = [self.epsilon_r] if trans.fitspace == 'r': irmin = max(0, int(0.01 + trans.rmin/trans.rstep)) irmax = min(trans.nfft/2, int(0.01 + trans.rmax/trans.rstep)) for i, chir_ in enumerate(chir): chir_ = chir_ / (eps_r[i]) out.append(realimag(chir_[irmin:irmax])) else: chiq = [trans.fftr(c)/eps for c, eps in zip(chir, eps_r)] iqmin = max(0, int(0.01 + trans.kmin/trans.kstep)) iqmax = min(trans.nfft/2, int(0.01 + trans.kmax/trans.kstep)) for chiq_ in chiq: out.append( realimag(chiq_[iqmin:iqmax])[::2]) return np.concatenate(out)
def __resid(pars, ncoefs=1, knots=None, order=3, irbkg=1, nfft=2048, kraw=None, mu=None, kout=None, ftwin=1, kweight=1, chi_std=None, nclamp=0, clamp_lo=1, clamp_hi=1, **kws): coefs = [getattr(pars, FMT_COEF % i) for i in range(ncoefs)] bkg, chi = spline_eval(kraw, mu, knots, coefs, order, kout) if chi_std is not None: chi = chi - chi_std out = realimag(xftf_fast(chi * ftwin, nfft=nfft)[:irbkg]) if nclamp == 0: return out # spline clamps: scale = (1.0 + 100 * (out * out).sum()) / (len(out) * nclamp) scaled_chik = scale * chi * kout**kweight return np.concatenate((out, abs(clamp_lo) * scaled_chik[:nclamp], abs(clamp_hi) * scaled_chik[-nclamp:]))
def estimate_noise(self, chi=None, rmin=15.0, rmax=30.0, all_kweights=True): """estimage noise in a chi spectrum from its high r components""" trans = self.transform trans.make_karrays() if chi is None: chi = self.__chi save = trans.rmin, trans.rmax, trans.fitspace all_kweights = all_kweights and isinstance(trans.kweight, Iterable) if all_kweights: chir = [trans.fftf(chi, kweight=kw) for kw in trans.kweight] else: chir = [trans.fftf(chi)] irmin = int(0.01 + rmin / trans.rstep) irmax = min(trans.nfft / 2, int(1.01 + rmax / trans.rstep)) highr = [realimag(chir_[irmin:irmax]) for chir_ in chir] # get average of window function value, we will scale eps_r scale by this kwin_ave = trans.kwin.sum() * trans.kstep / (trans.kmax - trans.kmin) eps_r = [(sqrt((chi * chi).sum() / len(chi)) / kwin_ave) for chi in highr] eps_k = [] # use Parseval's theorem to convert epsilon_r to epsilon_k, # compensating for kweight if all_kweights: kweights = trans.kweight[:] else: kweights = [trans.kweight] for i, kw in enumerate(kweights): w = 2 * kw + 1 scale = sqrt( (2 * pi * w) / (trans.kstep * (trans.kmax**w - trans.kmin**w))) eps_k.append(scale * eps_r[i]) trans.rmin, trans.rmax, trans.fitspace = save ## self.n_idp = 1 + 2*(trans.rmax-trans.rmin)*(trans.kmax-trans.kmin)/pi self.epsilon_k = eps_k self.epsilon_r = eps_r if len(eps_r) == 1: self.epsilon_k = eps_k[0] self.epsilon_r = eps_r[0] if isinstance(eps_r, np.ndarray): self.epsilon_r = eps_r.mean()
def __resid(pars, ncoefs=1, knots=None, order=3, irbkg=1, nfft=2048, kraw=None, mu=None, kout=None, ftwin=1, kweight=1, chi_std=None, nclamp=0, clamp_lo=1, clamp_hi=1, **kws): coefs = [getattr(pars, FMT_COEF % i) for i in range(ncoefs)] bkg, chi = spline_eval(kraw, mu, knots, coefs, order, kout) if chi_std is not None: chi = chi - chi_std out = realimag(xftf_fast(chi*ftwin, nfft=nfft)[:irbkg]) if nclamp == 0: return out # spline clamps: scale = (1.0 + 100*(out*out).sum())/(len(out)*nclamp) scaled_chik = scale * chi * kout**kweight return np.concatenate((out, abs(clamp_lo)*scaled_chik[:nclamp], abs(clamp_hi)*scaled_chik[-nclamp:]))
def estimate_noise(self, chi=None, rmin=15.0, rmax=30.0, all_kweights=True): """estimage noise in a chi spectrum from its high r components""" trans = self.transform trans.make_karrays() if chi is None: chi = self.__chi save = trans.rmin, trans.rmax, trans.fitspace all_kweights = all_kweights and isinstance(trans.kweight, Iterable) if all_kweights: chir = [trans.fftf(chi, kweight=kw) for kw in trans.kweight] else: chir = [trans.fftf(chi)] irmin = int(0.01 + rmin/trans.rstep) irmax = min(trans.nfft/2, int(1.01 + rmax/trans.rstep)) highr = [realimag(chir_[irmin:irmax]) for chir_ in chir] # get average of window function value, we will scale eps_r scale by this kwin_ave = trans.kwin.sum()*trans.kstep/(trans.kmax-trans.kmin) eps_r = [(sqrt((chi*chi).sum() / len(chi)) / kwin_ave) for chi in highr] eps_k = [] # use Parseval's theorem to convert epsilon_r to epsilon_k, # compensating for kweight if all_kweights: kweights = trans.kweight[:] else: kweights = [trans.kweight] for i, kw in enumerate(kweights): w = 2 * kw + 1 scale = sqrt((2*pi*w)/(trans.kstep*(trans.kmax**w - trans.kmin**w))) eps_k.append(scale*eps_r[i]) trans.rmin, trans.rmax, trans.fitspace = save ## self.n_idp = 1 + 2*(trans.rmax-trans.rmin)*(trans.kmax-trans.kmin)/pi self.epsilon_k = eps_k self.epsilon_r = eps_r if len(eps_r) == 1: self.epsilon_k = eps_k[0] self.epsilon_r = eps_r[0] if isinstance(eps_r, np.ndarray): self.epsilon_r = eps_r.mean()
def _residual(self, paramgroup=None, data_only=False, **kws): """return the residual for this data set residual = self.transform.apply(data_chi - model_chi) where model_chi is the result of ff2chi(pathlist) """ if (paramgroup is not None and self._larch.symtable.isgroup(paramgroup)): self._larch.symtable._sys.paramGroup = paramgroup if not isNamedClass(self.transform, TransformGroup): return if not self.__prepared: self.prepare_fit() _ff2chi(self.pathlist, k=self.model.k, _larch=self._larch, group=self.model) eps_k = self.epsilon_k if isinstance(eps_k, np.ndarray): eps_k[np.where(eps_k < 1.e-12)[0]] = 1.e-12 else: eps_k = max(1.e-12, eps_k) diff = (self.__chi - self.model.chi) if data_only: # for extracting transformed data separately from residual diff = self.__chi trans = self.transform k = trans.k_[:len(diff)] all_kweights = isinstance(trans.kweight, Iterable) if trans.fitspace == 'k': iqmin = max(0, int(0.01 + trans.kmin / trans.kstep)) iqmax = min(trans.nfft / 2, int(0.01 + trans.kmax / trans.kstep)) if all_kweights: out = [] for i, kw in enumerate(trans.kweight): out.append(((diff / eps_k[i]) * k**kw)[iqmin:iqmax]) return np.concatenate(out) else: return ((diff / eps_k) * k**trans.kweight)[iqmin:iqmax] else: out = [] if all_kweights: chir = [trans.fftf(diff, kweight=kw) for kw in trans.kweight] eps_r = self.epsilon_r else: chir = [trans.fftf(diff)] eps_r = [self.epsilon_r] if trans.fitspace == 'r': irmin = max(0, int(0.01 + trans.rmin / trans.rstep)) irmax = min(trans.nfft / 2, int(0.01 + trans.rmax / trans.rstep)) for i, chir_ in enumerate(chir): chir_ = chir_ / (eps_r[i]) out.append(realimag(chir_[irmin:irmax])) else: chiq = [trans.fftr(c) / eps for c, eps in zip(chir, eps_r)] iqmin = max(0, int(0.01 + trans.kmin / trans.kstep)) iqmax = min(trans.nfft / 2, int(0.01 + trans.kmax / trans.kstep)) for chiq_ in chiq: out.append(realimag(chiq_[iqmin:iqmax])[::2]) return np.concatenate(out)
def estimate_noise(k, chi=None, group=None, rmin=15.0, rmax=30.0, kweight=1, kmin=0, kmax=20, dk=4, dk2=None, kstep=0.05, kwindow='kaiser', nfft=2048, _larch=None, **kws): """ estimate noise levels in EXAFS spectrum and estimate highest k where data is above the noise level Parameters: ----------- k: 1-d array of photo-electron wavenumber in Ang^-1 (or group) chi: 1-d array of chi group: output Group [see Note below] rmin: minimum R value for high-R region of chi(R) rmax: maximum R value for high-R region of chi(R) kweight: exponent for weighting spectra by k**kweight [1] kmin: starting k for FT Window [0] kmax: ending k for FT Window [20] dk: tapering parameter for FT Window [4] dk2: second tapering parameter for FT Window [None] kstep: value to use for delta_k ( Ang^-1) [0.05] window: name of window type ['kaiser'] nfft: value to use for N_fft [2048]. Returns: --------- None -- outputs are written to supplied group. Values (scalars) written to output group: epsilon_k estimated noise in chi(k) epsilon_r estimated noise in chi(R) kmax_suggest highest estimated k value where |chi(k)| > epsilon_k Notes: ------- 1. This method uses the high-R portion of chi(R) as a measure of the noise level in the chi(R) data and uses Parseval's theorem to convert this noise level to that in chi(k). This method implicitly assumes that there is no signal in the high-R portion of the spectrum, and that the noise in the spectrum s "white" (independent of R) . Each of these assumptions can be questioned. 2. The estimate for 'kmax_suggest' has a tendency to be fair but pessimistic in how far out the chi(k) data goes before being dominated by noise. 3. Follows the 'First Argument Group' convention, so that you can either specifiy all of (an array for 'k', an array for 'chi', option output Group) OR pass a group with 'k' and 'chi' as the first argument """ k, chi, group = parse_group_args(k, members=('k', 'chi'), defaults=(chi, ), group=group, fcn_name='esitmate_noise') # save _sys.xafsGroup -- we want to NOT write to it here! savgroup = set_xafsGroup(None, _larch=_larch) tmpgroup = Group() rmax_out = min(10 * pi, rmax + 2) xftf(k, chi, kmin=kmin, kmax=kmax, rmax_out=rmax_out, kweight=kweight, dk=dk, dk2=dk2, kwindow=kwindow, nfft=nfft, kstep=kstep, group=tmpgroup, _larch=_larch) chir = tmpgroup.chir rstep = tmpgroup.r[1] - tmpgroup.r[0] irmin = int(0.01 + rmin / rstep) irmax = min(nfft / 2, int(1.01 + rmax / rstep)) highr = realimag(chir[irmin:irmax]) # get average of window function value, scale eps_r scale by this # this is imperfect, but improves the result. kwin_ave = tmpgroup.kwin.sum() * kstep / (kmax - kmin) eps_r = sqrt((highr * highr).sum() / len(highr)) / kwin_ave # use Parseval's theorem to convert epsilon_r to epsilon_k, # compensating for kweight w = 2 * kweight + 1 scale = sqrt((2 * pi * w) / (kstep * (kmax**w - kmin**w))) eps_k = scale * eps_r # do reverse FT to get chiq array xftr(tmpgroup.r, tmpgroup.chir, group=tmpgroup, rmin=0.5, rmax=9.5, dr=1.0, window='parzen', nfft=nfft, kstep=kstep, _larch=_larch) # sets kmax_suggest to the largest k value for which # | chi(q) / k**kweight| > epsilon_k iq0 = index_of(tmpgroup.q, (kmax + kmin) / 2.0) tst = tmpgroup.chiq_mag[iq0:] / (tmpgroup.q[iq0:])**kweight kmax_suggest = tmpgroup.q[iq0 + where(tst < eps_k)[0][0]] # restore original _sys.xafsGroup, set output variables _larch.symtable._sys.xafsGroup = savgroup group = set_xafsGroup(group, _larch=_larch) group.epsilon_k = eps_k group.epsilon_r = eps_r group.kmax_suggest = kmax_suggest
def estimate_noise(k, chi=None, group=None, rmin=15.0, rmax=30.0, kweight=1, kmin=0, kmax=20, dk=4, dk2=None, kstep=0.05, kwindow='kaiser', nfft=2048, _larch=None, **kws): """ estimate noise levels in EXAFS spectrum and estimate highest k where data is above the noise level Parameters: ----------- k: 1-d array of photo-electron wavenumber in Ang^-1 (or group) chi: 1-d array of chi group: output Group [see Note below] rmin: minimum R value for high-R region of chi(R) rmax: maximum R value for high-R region of chi(R) kweight: exponent for weighting spectra by k**kweight [1] kmin: starting k for FT Window [0] kmax: ending k for FT Window [20] dk: tapering parameter for FT Window [4] dk2: second tapering parameter for FT Window [None] kstep: value to use for delta_k ( Ang^-1) [0.05] window: name of window type ['kaiser'] nfft: value to use for N_fft [2048]. Returns: --------- None -- outputs are written to supplied group. Values (scalars) written to output group: epsilon_k estimated noise in chi(k) epsilon_r estimated noise in chi(R) kmax_suggest highest estimated k value where |chi(k)| > epsilon_k Notes: ------- 1. This method uses the high-R portion of chi(R) as a measure of the noise level in the chi(R) data and uses Parseval's theorem to convert this noise level to that in chi(k). This method implicitly assumes that there is no signal in the high-R portion of the spectrum, and that the noise in the spectrum s "white" (independent of R) . Each of these assumptions can be questioned. 2. The estimate for 'kmax_suggest' has a tendency to be fair but pessimistic in how far out the chi(k) data goes before being dominated by noise. 3. Follows the 'First Argument Group' convention, so that you can either specifiy all of (an array for 'k', an array for 'chi', option output Group) OR pass a group with 'k' and 'chi' as the first argument """ k, chi, group = parse_group_args(k, members=('k', 'chi'), defaults=(chi,), group=group, fcn_name='esitmate_noise') # save _sys.xafsGroup -- we want to NOT write to it here! savgroup = set_xafsGroup(None, _larch=_larch) tmpgroup = Group() rmax_out = min(10*pi, rmax+2) xftf(k, chi, kmin=kmin, kmax=kmax, rmax_out=rmax_out, kweight=kweight, dk=dk, dk2=dk2, kwindow=kwindow, nfft=nfft, kstep=kstep, group=tmpgroup, _larch=_larch) chir = tmpgroup.chir rstep = tmpgroup.r[1] - tmpgroup.r[0] irmin = int(0.01 + rmin/rstep) irmax = min(nfft/2, int(1.01 + rmax/rstep)) highr = realimag(chir[irmin:irmax]) # get average of window function value, scale eps_r scale by this # this is imperfect, but improves the result. kwin_ave = tmpgroup.kwin.sum()*kstep/(kmax-kmin) eps_r = sqrt((highr*highr).sum() / len(highr)) / kwin_ave # use Parseval's theorem to convert epsilon_r to epsilon_k, # compensating for kweight w = 2 * kweight + 1 scale = sqrt((2*pi*w)/(kstep*(kmax**w - kmin**w))) eps_k = scale*eps_r # do reverse FT to get chiq array xftr(tmpgroup.r, tmpgroup.chir, group=tmpgroup, rmin=0.5, rmax=9.5, dr=1.0, window='parzen', nfft=nfft, kstep=kstep, _larch=_larch) # sets kmax_suggest to the largest k value for which # | chi(q) / k**kweight| > epsilon_k iq0 = index_of(tmpgroup.q, (kmax+kmin)/2.0) tst = tmpgroup.chiq_mag[iq0:] / ( tmpgroup.q[iq0:])**kweight kmax_suggest = tmpgroup.q[iq0 + where(tst < eps_k)[0][0]] # restore original _sys.xafsGroup, set output variables _larch.symtable._sys.xafsGroup = savgroup group = set_xafsGroup(group, _larch=_larch) group.epsilon_k = eps_k group.epsilon_r = eps_r group.kmax_suggest = kmax_suggest