def _residual(self, paramgroup=None, data_only=False, **kws):
"""return the residual for this data set
residual = self.transform.apply(data_chi - model_chi)
where model_chi is the result of ff2chi(pathlist)
"""
if (paramgroup is not None and
self._larch.symtable.isgroup(paramgroup)):
self._larch.symtable._sys.paramGroup = paramgroup
if not isNamedClass(self.transform, TransformGroup):
return
if not self.__prepared:
self.prepare_fit()
_ff2chi(self.pathlist, k=self.model.k,
_larch=self._larch, group=self.model)
eps_k = self.epsilon_k
if isinstance(eps_k, np.ndarray):
eps_k[np.where(eps_k<1.e-12)[0]] = 1.e-12
else:
eps_k = max(1.e-12, eps_k)
diff = (self.__chi - self.model.chi)
if data_only: # for extracting transformed data separately from residual
diff = self.__chi
trans = self.transform
k = trans.k_[:len(diff)]
all_kweights = isinstance(trans.kweight, Iterable)
if trans.fitspace == 'k':
iqmin = max(0, int(0.01 + trans.kmin/trans.kstep))
iqmax = min(trans.nfft/2, int(0.01 + trans.kmax/trans.kstep))
if all_kweights:
out = []
for i, kw in enumerate(trans.kweight):
out.append(((diff/eps_k[i])*k**kw)[iqmin:iqmax])
return np.concatenate(out)
else:
return ((diff/eps_k) * k**trans.kweight)[iqmin:iqmax]
else:
out = []
if all_kweights:
chir = [trans.fftf(diff, kweight=kw) for kw in trans.kweight]
eps_r = self.epsilon_r
else:
chir = [trans.fftf(diff)]
eps_r = [self.epsilon_r]
if trans.fitspace == 'r':
irmin = max(0, int(0.01 + trans.rmin/trans.rstep))
irmax = min(trans.nfft/2, int(0.01 + trans.rmax/trans.rstep))
for i, chir_ in enumerate(chir):
chir_ = chir_ / (eps_r[i])
out.append(realimag(chir_[irmin:irmax]))
else:
chiq = [trans.fftr(c)/eps for c, eps in zip(chir, eps_r)]
iqmin = max(0, int(0.01 + trans.kmin/trans.kstep))
iqmax = min(trans.nfft/2, int(0.01 + trans.kmax/trans.kstep))
for chiq_ in chiq:
out.append( realimag(chiq_[iqmin:iqmax])[::2])
return np.concatenate(out)
def __resid(pars,
ncoefs=1,
knots=None,
order=3,
irbkg=1,
nfft=2048,
kraw=None,
mu=None,
kout=None,
ftwin=1,
kweight=1,
chi_std=None,
nclamp=0,
clamp_lo=1,
clamp_hi=1,
**kws):
coefs = [getattr(pars, FMT_COEF % i) for i in range(ncoefs)]
bkg, chi = spline_eval(kraw, mu, knots, coefs, order, kout)
if chi_std is not None:
chi = chi - chi_std
out = realimag(xftf_fast(chi * ftwin, nfft=nfft)[:irbkg])
if nclamp == 0:
return out
# spline clamps:
scale = (1.0 + 100 * (out * out).sum()) / (len(out) * nclamp)
scaled_chik = scale * chi * kout**kweight
return np.concatenate((out, abs(clamp_lo) * scaled_chik[:nclamp],
abs(clamp_hi) * scaled_chik[-nclamp:]))
def estimate_noise(self,
chi=None,
rmin=15.0,
rmax=30.0,
all_kweights=True):
"""estimage noise in a chi spectrum from its high r components"""
trans = self.transform
trans.make_karrays()
if chi is None: chi = self.__chi
save = trans.rmin, trans.rmax, trans.fitspace
all_kweights = all_kweights and isinstance(trans.kweight, Iterable)
if all_kweights:
chir = [trans.fftf(chi, kweight=kw) for kw in trans.kweight]
else:
chir = [trans.fftf(chi)]
irmin = int(0.01 + rmin / trans.rstep)
irmax = min(trans.nfft / 2, int(1.01 + rmax / trans.rstep))
highr = [realimag(chir_[irmin:irmax]) for chir_ in chir]
# get average of window function value, we will scale eps_r scale by this
kwin_ave = trans.kwin.sum() * trans.kstep / (trans.kmax - trans.kmin)
eps_r = [(sqrt((chi * chi).sum() / len(chi)) / kwin_ave)
for chi in highr]
eps_k = []
# use Parseval's theorem to convert epsilon_r to epsilon_k,
# compensating for kweight
if all_kweights:
kweights = trans.kweight[:]
else:
kweights = [trans.kweight]
for i, kw in enumerate(kweights):
w = 2 * kw + 1
scale = sqrt(
(2 * pi * w) / (trans.kstep * (trans.kmax**w - trans.kmin**w)))
eps_k.append(scale * eps_r[i])
trans.rmin, trans.rmax, trans.fitspace = save
## self.n_idp = 1 + 2*(trans.rmax-trans.rmin)*(trans.kmax-trans.kmin)/pi
self.epsilon_k = eps_k
self.epsilon_r = eps_r
if len(eps_r) == 1:
self.epsilon_k = eps_k[0]
self.epsilon_r = eps_r[0]
if isinstance(eps_r, np.ndarray):
self.epsilon_r = eps_r.mean()
def __resid(pars, ncoefs=1, knots=None, order=3, irbkg=1, nfft=2048,
kraw=None, mu=None, kout=None, ftwin=1, kweight=1, chi_std=None,
nclamp=0, clamp_lo=1, clamp_hi=1, **kws):
coefs = [getattr(pars, FMT_COEF % i) for i in range(ncoefs)]
bkg, chi = spline_eval(kraw, mu, knots, coefs, order, kout)
if chi_std is not None:
chi = chi - chi_std
out = realimag(xftf_fast(chi*ftwin, nfft=nfft)[:irbkg])
if nclamp == 0:
return out
# spline clamps:
scale = (1.0 + 100*(out*out).sum())/(len(out)*nclamp)
scaled_chik = scale * chi * kout**kweight
return np.concatenate((out,
abs(clamp_lo)*scaled_chik[:nclamp],
abs(clamp_hi)*scaled_chik[-nclamp:]))
def estimate_noise(self, chi=None, rmin=15.0, rmax=30.0, all_kweights=True):
"""estimage noise in a chi spectrum from its high r components"""
trans = self.transform
trans.make_karrays()
if chi is None: chi = self.__chi
save = trans.rmin, trans.rmax, trans.fitspace
all_kweights = all_kweights and isinstance(trans.kweight, Iterable)
if all_kweights:
chir = [trans.fftf(chi, kweight=kw) for kw in trans.kweight]
else:
chir = [trans.fftf(chi)]
irmin = int(0.01 + rmin/trans.rstep)
irmax = min(trans.nfft/2, int(1.01 + rmax/trans.rstep))
highr = [realimag(chir_[irmin:irmax]) for chir_ in chir]
# get average of window function value, we will scale eps_r scale by this
kwin_ave = trans.kwin.sum()*trans.kstep/(trans.kmax-trans.kmin)
eps_r = [(sqrt((chi*chi).sum() / len(chi)) / kwin_ave) for chi in highr]
eps_k = []
# use Parseval's theorem to convert epsilon_r to epsilon_k,
# compensating for kweight
if all_kweights:
kweights = trans.kweight[:]
else:
kweights = [trans.kweight]
for i, kw in enumerate(kweights):
w = 2 * kw + 1
scale = sqrt((2*pi*w)/(trans.kstep*(trans.kmax**w - trans.kmin**w)))
eps_k.append(scale*eps_r[i])
trans.rmin, trans.rmax, trans.fitspace = save
## self.n_idp = 1 + 2*(trans.rmax-trans.rmin)*(trans.kmax-trans.kmin)/pi
self.epsilon_k = eps_k
self.epsilon_r = eps_r
if len(eps_r) == 1:
self.epsilon_k = eps_k[0]
self.epsilon_r = eps_r[0]
if isinstance(eps_r, np.ndarray):
self.epsilon_r = eps_r.mean()
def _residual(self, paramgroup=None, data_only=False, **kws):
"""return the residual for this data set
residual = self.transform.apply(data_chi - model_chi)
where model_chi is the result of ff2chi(pathlist)
"""
if (paramgroup is not None
and self._larch.symtable.isgroup(paramgroup)):
self._larch.symtable._sys.paramGroup = paramgroup
if not isNamedClass(self.transform, TransformGroup):
return
if not self.__prepared:
self.prepare_fit()
_ff2chi(self.pathlist,
k=self.model.k,
_larch=self._larch,
group=self.model)
eps_k = self.epsilon_k
if isinstance(eps_k, np.ndarray):
eps_k[np.where(eps_k < 1.e-12)[0]] = 1.e-12
else:
eps_k = max(1.e-12, eps_k)
diff = (self.__chi - self.model.chi)
if data_only: # for extracting transformed data separately from residual
diff = self.__chi
trans = self.transform
k = trans.k_[:len(diff)]
all_kweights = isinstance(trans.kweight, Iterable)
if trans.fitspace == 'k':
iqmin = max(0, int(0.01 + trans.kmin / trans.kstep))
iqmax = min(trans.nfft / 2, int(0.01 + trans.kmax / trans.kstep))
if all_kweights:
out = []
for i, kw in enumerate(trans.kweight):
out.append(((diff / eps_k[i]) * k**kw)[iqmin:iqmax])
return np.concatenate(out)
else:
return ((diff / eps_k) * k**trans.kweight)[iqmin:iqmax]
else:
out = []
if all_kweights:
chir = [trans.fftf(diff, kweight=kw) for kw in trans.kweight]
eps_r = self.epsilon_r
else:
chir = [trans.fftf(diff)]
eps_r = [self.epsilon_r]
if trans.fitspace == 'r':
irmin = max(0, int(0.01 + trans.rmin / trans.rstep))
irmax = min(trans.nfft / 2,
int(0.01 + trans.rmax / trans.rstep))
for i, chir_ in enumerate(chir):
chir_ = chir_ / (eps_r[i])
out.append(realimag(chir_[irmin:irmax]))
else:
chiq = [trans.fftr(c) / eps for c, eps in zip(chir, eps_r)]
iqmin = max(0, int(0.01 + trans.kmin / trans.kstep))
iqmax = min(trans.nfft / 2,
int(0.01 + trans.kmax / trans.kstep))
for chiq_ in chiq:
out.append(realimag(chiq_[iqmin:iqmax])[::2])
return np.concatenate(out)
def estimate_noise(k,
chi=None,
group=None,
rmin=15.0,
rmax=30.0,
kweight=1,
kmin=0,
kmax=20,
dk=4,
dk2=None,
kstep=0.05,
kwindow='kaiser',
nfft=2048,
_larch=None,
**kws):
"""
estimate noise levels in EXAFS spectrum and estimate highest k
where data is above the noise level
Parameters:
-----------
k: 1-d array of photo-electron wavenumber in Ang^-1 (or group)
chi: 1-d array of chi
group: output Group [see Note below]
rmin: minimum R value for high-R region of chi(R)
rmax: maximum R value for high-R region of chi(R)
kweight: exponent for weighting spectra by k**kweight [1]
kmin: starting k for FT Window [0]
kmax: ending k for FT Window [20]
dk: tapering parameter for FT Window [4]
dk2: second tapering parameter for FT Window [None]
kstep: value to use for delta_k ( Ang^-1) [0.05]
window: name of window type ['kaiser']
nfft: value to use for N_fft [2048].
Returns:
---------
None -- outputs are written to supplied group. Values (scalars) written
to output group:
epsilon_k estimated noise in chi(k)
epsilon_r estimated noise in chi(R)
kmax_suggest highest estimated k value where |chi(k)| > epsilon_k
Notes:
-------
1. This method uses the high-R portion of chi(R) as a measure of the noise
level in the chi(R) data and uses Parseval's theorem to convert this noise
level to that in chi(k). This method implicitly assumes that there is no
signal in the high-R portion of the spectrum, and that the noise in the
spectrum s "white" (independent of R) . Each of these assumptions can be
questioned.
2. The estimate for 'kmax_suggest' has a tendency to be fair but pessimistic
in how far out the chi(k) data goes before being dominated by noise.
3. Follows the 'First Argument Group' convention, so that you can either
specifiy all of (an array for 'k', an array for 'chi', option output Group)
OR pass a group with 'k' and 'chi' as the first argument
"""
k, chi, group = parse_group_args(k,
members=('k', 'chi'),
defaults=(chi, ),
group=group,
fcn_name='esitmate_noise')
# save _sys.xafsGroup -- we want to NOT write to it here!
savgroup = set_xafsGroup(None, _larch=_larch)
tmpgroup = Group()
rmax_out = min(10 * pi, rmax + 2)
xftf(k,
chi,
kmin=kmin,
kmax=kmax,
rmax_out=rmax_out,
kweight=kweight,
dk=dk,
dk2=dk2,
kwindow=kwindow,
nfft=nfft,
kstep=kstep,
group=tmpgroup,
_larch=_larch)
chir = tmpgroup.chir
rstep = tmpgroup.r[1] - tmpgroup.r[0]
irmin = int(0.01 + rmin / rstep)
irmax = min(nfft / 2, int(1.01 + rmax / rstep))
highr = realimag(chir[irmin:irmax])
# get average of window function value, scale eps_r scale by this
# this is imperfect, but improves the result.
kwin_ave = tmpgroup.kwin.sum() * kstep / (kmax - kmin)
eps_r = sqrt((highr * highr).sum() / len(highr)) / kwin_ave
# use Parseval's theorem to convert epsilon_r to epsilon_k,
# compensating for kweight
w = 2 * kweight + 1
scale = sqrt((2 * pi * w) / (kstep * (kmax**w - kmin**w)))
eps_k = scale * eps_r
# do reverse FT to get chiq array
xftr(tmpgroup.r,
tmpgroup.chir,
group=tmpgroup,
rmin=0.5,
rmax=9.5,
dr=1.0,
window='parzen',
nfft=nfft,
kstep=kstep,
_larch=_larch)
# sets kmax_suggest to the largest k value for which
# | chi(q) / k**kweight| > epsilon_k
iq0 = index_of(tmpgroup.q, (kmax + kmin) / 2.0)
tst = tmpgroup.chiq_mag[iq0:] / (tmpgroup.q[iq0:])**kweight
kmax_suggest = tmpgroup.q[iq0 + where(tst < eps_k)[0][0]]
# restore original _sys.xafsGroup, set output variables
_larch.symtable._sys.xafsGroup = savgroup
group = set_xafsGroup(group, _larch=_larch)
group.epsilon_k = eps_k
group.epsilon_r = eps_r
group.kmax_suggest = kmax_suggest
def estimate_noise(k, chi=None, group=None, rmin=15.0, rmax=30.0,
kweight=1, kmin=0, kmax=20, dk=4, dk2=None, kstep=0.05,
kwindow='kaiser', nfft=2048, _larch=None, **kws):
"""
estimate noise levels in EXAFS spectrum and estimate highest k
where data is above the noise level
Parameters:
-----------
k: 1-d array of photo-electron wavenumber in Ang^-1 (or group)
chi: 1-d array of chi
group: output Group [see Note below]
rmin: minimum R value for high-R region of chi(R)
rmax: maximum R value for high-R region of chi(R)
kweight: exponent for weighting spectra by k**kweight [1]
kmin: starting k for FT Window [0]
kmax: ending k for FT Window [20]
dk: tapering parameter for FT Window [4]
dk2: second tapering parameter for FT Window [None]
kstep: value to use for delta_k ( Ang^-1) [0.05]
window: name of window type ['kaiser']
nfft: value to use for N_fft [2048].
Returns:
---------
None -- outputs are written to supplied group. Values (scalars) written
to output group:
epsilon_k estimated noise in chi(k)
epsilon_r estimated noise in chi(R)
kmax_suggest highest estimated k value where |chi(k)| > epsilon_k
Notes:
-------
1. This method uses the high-R portion of chi(R) as a measure of the noise
level in the chi(R) data and uses Parseval's theorem to convert this noise
level to that in chi(k). This method implicitly assumes that there is no
signal in the high-R portion of the spectrum, and that the noise in the
spectrum s "white" (independent of R) . Each of these assumptions can be
questioned.
2. The estimate for 'kmax_suggest' has a tendency to be fair but pessimistic
in how far out the chi(k) data goes before being dominated by noise.
3. Follows the 'First Argument Group' convention, so that you can either
specifiy all of (an array for 'k', an array for 'chi', option output Group)
OR pass a group with 'k' and 'chi' as the first argument
"""
k, chi, group = parse_group_args(k, members=('k', 'chi'),
defaults=(chi,), group=group,
fcn_name='esitmate_noise')
# save _sys.xafsGroup -- we want to NOT write to it here!
savgroup = set_xafsGroup(None, _larch=_larch)
tmpgroup = Group()
rmax_out = min(10*pi, rmax+2)
xftf(k, chi, kmin=kmin, kmax=kmax, rmax_out=rmax_out,
kweight=kweight, dk=dk, dk2=dk2, kwindow=kwindow,
nfft=nfft, kstep=kstep, group=tmpgroup, _larch=_larch)
chir = tmpgroup.chir
rstep = tmpgroup.r[1] - tmpgroup.r[0]
irmin = int(0.01 + rmin/rstep)
irmax = min(nfft/2, int(1.01 + rmax/rstep))
highr = realimag(chir[irmin:irmax])
# get average of window function value, scale eps_r scale by this
# this is imperfect, but improves the result.
kwin_ave = tmpgroup.kwin.sum()*kstep/(kmax-kmin)
eps_r = sqrt((highr*highr).sum() / len(highr)) / kwin_ave
# use Parseval's theorem to convert epsilon_r to epsilon_k,
# compensating for kweight
w = 2 * kweight + 1
scale = sqrt((2*pi*w)/(kstep*(kmax**w - kmin**w)))
eps_k = scale*eps_r
# do reverse FT to get chiq array
xftr(tmpgroup.r, tmpgroup.chir, group=tmpgroup, rmin=0.5, rmax=9.5,
dr=1.0, window='parzen', nfft=nfft, kstep=kstep, _larch=_larch)
# sets kmax_suggest to the largest k value for which
# | chi(q) / k**kweight| > epsilon_k
iq0 = index_of(tmpgroup.q, (kmax+kmin)/2.0)
tst = tmpgroup.chiq_mag[iq0:] / ( tmpgroup.q[iq0:])**kweight
kmax_suggest = tmpgroup.q[iq0 + where(tst < eps_k)[0][0]]
# restore original _sys.xafsGroup, set output variables
_larch.symtable._sys.xafsGroup = savgroup
group = set_xafsGroup(group, _larch=_larch)
group.epsilon_k = eps_k
group.epsilon_r = eps_r
group.kmax_suggest = kmax_suggest