def test_eval_group1(self): out = deval(self.group1) assert(isgroup(out)) assert(getattr(out, 'label') == 'a string') assert(out.opts['a'] == 1) assert_allclose(out.a, 1.03, rtol=1.e-2) assert_allclose(out.x, np.linspace(0, 10, 21), rtol=1.e-2)
def spline_eval(x, group, name='spl1', _larch=None): """evaluate spline at specified x values arguments: ------------ x input 1-d array for absicca group Group containing spline representation, as defined by spline_rep() name name for spline params and subgroups ['spl1'] returns: -------- 1-d array with interpolated values """ sgroup = getattr(group, "{:s}_details".format(name), None) if sgroup is None or not isgroup(sgroup): raise Warning("spline_eval: subgroup '{:s}' not found".format(name)) knots = getattr(sgroup, 'knots') order = getattr(sgroup, 'order') coefs = getattr(sgroup, 'coefs') for i, val in enumerate(coefs[2:-2]): pname = "{:s}_c{:d}".format(name, i) cval = getattr(group, pname, None) if cval is None: raise Warning("spline_eval: param'{:s}' not found".format(pname)) if isParameter(cval): cval = cval.value coefs[2 + i] = cval setattr(sgroup, 'coefs', coefs) return splev(x, [knots, coefs, order])
def spline_eval(x, group, name='spl1', _larch=None): """evaluate spline at specified x values arguments: ------------ x input 1-d array for absicca group Group containing spline representation, as defined by spline_rep() name name for spline params and subgroups ['spl1'] returns: -------- 1-d array with interpolated values """ sgroup = getattr(group, "{:s}_details".format(name), None) if sgroup is None or not isgroup(sgroup): raise Warning("spline_eval: subgroup '{:s}' not found".format(name)) knots = getattr(sgroup, 'knots') order = getattr(sgroup, 'order') coefs = getattr(sgroup, 'coefs') for i, val in enumerate(coefs[2:-2]): pname = "{:s}_c{:d}".format(name, i) cval = getattr(group, pname, None) if cval is None: raise Warning("spline_eval: param'{:s}' not found".format(pname)) if isParameter(cval): cval = cval.value coefs[2+i] = cval setattr(sgroup, 'coefs', coefs) return splev(x, [knots, coefs, order])
def test_eval_group2(self): out = deval(self.group2) assert(isgroup(out)) assert(out.sub.label == 'a label') assert(isParameter(out.par1)) assert(isParameter(out.par2)) assert(out.par1.name == 'p1') assert(out.par2.name == 'p2') assert(out.par1.vary == False) assert(out.par2.vary == True) assert(out.par1.value == 3.0) assert(out.par2.value == 1.0) assert(out.par1.min == 0.0)
def encode4js(obj): """return an object ready for json encoding. has special handling for many Python types numpy array complex numbers Larch Groups Larch Parameters """ if isinstance(obj, np.ndarray): out = {'__class__': 'Array', '__shape__': obj.shape, '__dtype__': obj.dtype.name} out['value'] = obj.flatten().tolist() if 'complex' in obj.dtype.name: out['value'] = [(obj.real).tolist(), (obj.imag).tolist()] return out elif isinstance(obj, (np.float, np.int)): return float(obj) elif isinstance(obj, six.string_types): return str(obj) elif isinstance(obj, np.complex): return {'__class__': 'Complex', 'value': (obj.real, obj.imag)} elif isgroup(obj): out = {'__class__': 'Group'} for item in dir(obj): out[item] = encode4js(getattr(obj, item)) return out elif isParameter(obj): out = {'__class__': 'Parameter'} for attr in ('value', 'name', 'vary', 'min', 'max', 'expr', 'stderr', 'correl'): val = getattr(obj, attr, None) if val is not None: out[attr] = val return out elif isinstance(obj, (tuple, list)): ctype = 'List' if isinstance(obj, tuple): ctype = 'Tuple' val = [encode4js(item) for item in obj] return {'__class__': ctype, 'value': val} elif isinstance(obj, dict): out = {'__class__': 'Dict'} for key, val in obj.items(): out[encode4js(key)] = encode4js(val) return out return obj
def autobk(energy, mu=None, group=None, rbkg=1, nknots=None, e0=None, edge_step=None, kmin=0, kmax=None, kweight=1, dk=0, win='hanning', k_std=None, chi_std=None, nfft=2048, kstep=0.05, pre_edge_kws=None, nclamp=4, clamp_lo=1, clamp_hi=1, calc_uncertainties=True, err_sigma=1, _larch=None, **kws): """Use Autobk algorithm to remove XAFS background Parameters: ----------- energy: 1-d array of x-ray energies, in eV, or group mu: 1-d array of mu(E) group: output group (and input group for e0 and edge_step). rbkg: distance (in Ang) for chi(R) above which the signal is ignored. Default = 1. e0: edge energy, in eV. If None, it will be determined. edge_step: edge step. If None, it will be determined. pre_edge_kws: keyword arguments to pass to pre_edge() nknots: number of knots in spline. If None, it will be determined. kmin: minimum k value [0] kmax: maximum k value [full data range]. kweight: k weight for FFT. [1] dk: FFT window window parameter. [0] win: FFT window function name. ['hanning'] nfft: array size to use for FFT [2048] kstep: k step size to use for FFT [0.05] k_std: optional k array for standard chi(k). chi_std: optional chi array for standard chi(k). nclamp: number of energy end-points for clamp [2] clamp_lo: weight of low-energy clamp [1] clamp_hi: weight of high-energy clamp [1] calc_uncertaintites: Flag to calculate uncertainties in mu_0(E) and chi(k) [True] err_sigma: sigma level for uncertainties in mu_0(E) and chi(k) [1] Output arrays are written to the provided group. Follows the 'First Argument Group' convention. """ msg = _larch.writer.write if 'kw' in kws: kweight = kws.pop('kw') if len(kws) > 0: msg('Unrecognized a:rguments for autobk():\n') msg(' %s\n' % (', '.join(kws.keys()))) return energy, mu, group = parse_group_args(energy, members=('energy', 'mu'), defaults=(mu,), group=group, fcn_name='autobk') if len(energy.shape) > 1: energy = energy.squeeze() if len(mu.shape) > 1: mu = mu.squeeze() energy = remove_dups(energy) # if e0 or edge_step are not specified, get them, either from the # passed-in group or from running pre_edge() group = set_xafsGroup(group, _larch=_larch) if edge_step is None and isgroup(group, 'edge_step'): edge_step = group.edge_step if e0 is None and isgroup(group, 'e0'): e0 = group.e0 if e0 is None or edge_step is None: # need to run pre_edge: pre_kws = dict(nnorm=3, nvict=0, pre1=None, pre2=-50., norm1=100., norm2=None) if pre_edge_kws is not None: pre_kws.update(pre_edge_kws) pre_edge(energy, mu, group=group, _larch=_larch, **pre_kws) if e0 is None: e0 = group.e0 if edge_step is None: edge_step = group.edge_step if e0 is None or edge_step is None: msg('autobk() could not determine e0 or edge_step!: trying running pre_edge first\n') return # get array indices for rkbg and e0: irbkg, ie0 ie0 = index_of(energy, e0) rgrid = np.pi/(kstep*nfft) if rbkg < 2*rgrid: rbkg = 2*rgrid irbkg = int(1.01 + rbkg/rgrid) # save ungridded k (kraw) and grided k (kout) # and ftwin (*k-weighting) for FT in residual enpe = energy[ie0:] - e0 kraw = np.sign(enpe)*np.sqrt(ETOK*abs(enpe)) if kmax is None: kmax = max(kraw) else: kmax = max(0, min(max(kraw), kmax)) kout = kstep * np.arange(int(1.01+kmax/kstep), dtype='float64') iemax = min(len(energy), 2+index_of(energy, e0+kmax*kmax/ETOK)) - 1 # interpolate provided chi(k) onto the kout grid if chi_std is not None and k_std is not None: chi_std = np.interp(kout, k_std, chi_std) # pre-load FT window ftwin = kout**kweight * ftwindow(kout, xmin=kmin, xmax=kmax, window=win, dx=dk) # calc k-value and initial guess for y-values of spline params nspl = max(4, min(128, 2*int(rbkg*(kmax-kmin)/np.pi) + 1)) spl_y, spl_k, spl_e = np.zeros(nspl), np.zeros(nspl), np.zeros(nspl) for i in range(nspl): q = kmin + i*(kmax-kmin)/(nspl - 1) ik = index_nearest(kraw, q) i1 = min(len(kraw)-1, ik + 5) i2 = max(0, ik - 5) spl_k[i] = kraw[ik] spl_e[i] = energy[ik+ie0] spl_y[i] = (2*mu[ik+ie0] + mu[i1+ie0] + mu[i2+ie0] ) / 4.0 # get spline represention: knots, coefs, order=3 # coefs will be varied in fit. knots, coefs, order = splrep(spl_k, spl_y) # set fit parameters from initial coefficients params = Parameters() for i in range(len(coefs)): params.add(name = FMT_COEF % i, value=coefs[i], vary=i<len(spl_y)) initbkg, initchi = spline_eval(kraw[:iemax-ie0+1], mu[ie0:iemax+1], knots, coefs, order, kout) # do fit result = minimize(__resid, params, method='leastsq', gtol=1.e-5, ftol=1.e-5, xtol=1.e-5, epsfcn=1.e-5, kws = dict(ncoefs=len(coefs), chi_std=chi_std, knots=knots, order=order, kraw=kraw[:iemax-ie0+1], mu=mu[ie0:iemax+1], irbkg=irbkg, kout=kout, ftwin=ftwin, kweight=kweight, nfft=nfft, nclamp=nclamp, clamp_lo=clamp_lo, clamp_hi=clamp_hi)) # write final results coefs = [result.params[FMT_COEF % i].value for i in range(len(coefs))] bkg, chi = spline_eval(kraw[:iemax-ie0+1], mu[ie0:iemax+1], knots, coefs, order, kout) obkg = np.copy(mu) obkg[ie0:ie0+len(bkg)] = bkg # outputs to group group = set_xafsGroup(group, _larch=_larch) group.bkg = obkg group.chie = (mu-obkg)/edge_step group.k = kout group.chi = chi/edge_step # now fill in 'autobk_details' group details = Group(params=result.params) details.init_bkg = np.copy(mu) details.init_bkg[ie0:ie0+len(bkg)] = initbkg details.init_chi = initchi/edge_step details.knots_e = spl_e details.knots_y = np.array([coefs[i] for i in range(nspl)]) details.init_knots_y = spl_y details.nfev = result.nfev details.kmin = kmin details.kmax = kmax group.autobk_details = details # uncertainties in mu0 and chi: can be fairly slow. if calc_uncertainties: nchi = len(chi) nmue = iemax-ie0 + 1 redchi = result.redchi covar = result.covar / redchi jac_chi = np.zeros(nchi*nspl).reshape((nspl, nchi)) jac_bkg = np.zeros(nmue*nspl).reshape((nspl, nmue)) cvals, cerrs = [], [] for i in range(len(coefs)): par = result.params[FMT_COEF % i] cvals.append(getattr(par, 'value', 0.0)) cdel = getattr(par, 'stderr', 0.0) if cdel is None: cdel = 0.0 cerrs.append(cdel/2.0) cvals = np.array(cvals) cerrs = np.array(cerrs) # find derivatives by hand! _k = kraw[:nmue] _m = mu[ie0:iemax+1] for i in range(nspl): cval0 = cvals[i] cvals[i] = cval0 + cerrs[i] bkg1, chi1 = spline_eval(_k, _m, knots, cvals, order, kout) cvals[i] = cval0 - cerrs[i] bkg2, chi2 = spline_eval(_k, _m, knots, cvals, order, kout) cvals[i] = cval0 jac_chi[i] = (chi1 - chi2) / (2*cerrs[i]) jac_bkg[i] = (bkg1 - bkg2) / (2*cerrs[i]) dfchi = np.zeros(nchi) dfbkg = np.zeros(nmue) for i in range(nspl): for j in range(nspl): dfchi += jac_chi[i]*jac_chi[j]*covar[i,j] dfbkg += jac_bkg[i]*jac_bkg[j]*covar[i,j] prob = 0.5*(1.0 + erf(err_sigma/np.sqrt(2.0))) dchi = t.ppf(prob, nchi-nspl) * np.sqrt(dfchi*redchi) dbkg = t.ppf(prob, nmue-nspl) * np.sqrt(dfbkg*redchi) group.delta_chi = dchi group.delta_bkg = 0.0*mu group.delta_bkg[ie0:ie0+len(dbkg)] = dbkg
def isLarchMCAGroup(grp): """tests whether variable holds a valid Larch MCAGroup""" return isgroup(grp, 'energy', 'counts', 'rois')
def autobk(energy, mu=None, group=None, rbkg=1, nknots=None, e0=None, edge_step=None, kmin=0, kmax=None, kweight=1, dk=0, win='hanning', k_std=None, chi_std=None, nfft=2048, kstep=0.05, pre_edge_kws=None, nclamp=4, clamp_lo=1, clamp_hi=1, calc_uncertainties=False, _larch=None, **kws): """Use Autobk algorithm to remove XAFS background Parameters: ----------- energy: 1-d array of x-ray energies, in eV, or group mu: 1-d array of mu(E) group: output group (and input group for e0 and edge_step). rbkg: distance (in Ang) for chi(R) above which the signal is ignored. Default = 1. e0: edge energy, in eV. If None, it will be determined. edge_step: edge step. If None, it will be determined. pre_edge_kws: keyword arguments to pass to pre_edge() nknots: number of knots in spline. If None, it will be determined. kmin: minimum k value [0] kmax: maximum k value [full data range]. kweight: k weight for FFT. [1] dk: FFT window window parameter. [0] win: FFT window function name. ['hanning'] nfft: array size to use for FFT [2048] kstep: k step size to use for FFT [0.05] k_std: optional k array for standard chi(k). chi_std: optional chi array for standard chi(k). nclamp: number of energy end-points for clamp [2] clamp_lo: weight of low-energy clamp [1] clamp_hi: weight of high-energy clamp [1] calc_uncertaintites: Flag to calculate uncertainties in mu_0(E) and chi(k) [False] Output arrays are written to the provided group. Follows the 'First Argument Group' convention. """ msg = _larch.writer.write if 'kw' in kws: kweight = kws.pop('kw') if len(kws) > 0: msg('Unrecognized a:rguments for autobk():\n') msg(' %s\n' % (', '.join(kws.keys()))) return energy, mu, group = parse_group_args(energy, members=('energy', 'mu'), defaults=(mu,), group=group, fcn_name='autobk') energy = remove_dups(energy) # if e0 or edge_step are not specified, get them, either from the # passed-in group or from running pre_edge() group = set_xafsGroup(group, _larch=_larch) if edge_step is None and isgroup(group, 'edge_step'): edge_step = group.edge_step if e0 is None and isgroup(group, 'e0'): e0 = group.e0 if e0 is None or edge_step is None: # need to run pre_edge: pre_kws = dict(nnorm=3, nvict=0, pre1=None, pre2=-50., norm1=100., norm2=None) if pre_edge_kws is not None: pre_kws.update(pre_edge_kws) pre_edge(energy, mu, group=group, _larch=_larch, **pre_kws) if e0 is None: e0 = group.e0 if edge_step is None: edge_step = group.edge_step if e0 is None or edge_step is None: msg('autobk() could not determine e0 or edge_step!: trying running pre_edge first\n') return # get array indices for rkbg and e0: irbkg, ie0 ie0 = index_of(energy, e0) rgrid = np.pi/(kstep*nfft) if rbkg < 2*rgrid: rbkg = 2*rgrid irbkg = int(1.01 + rbkg/rgrid) # save ungridded k (kraw) and grided k (kout) # and ftwin (*k-weighting) for FT in residual enpe = energy[ie0:] - e0 kraw = np.sign(enpe)*np.sqrt(ETOK*abs(enpe)) if kmax is None: kmax = max(kraw) else: kmax = max(0, min(max(kraw), kmax)) kout = kstep * np.arange(int(1.01+kmax/kstep), dtype='float64') iemax = min(len(energy), 2+index_of(energy, e0+kmax*kmax/ETOK)) - 1 # interpolate provided chi(k) onto the kout grid if chi_std is not None and k_std is not None: chi_std = np.interp(kout, k_std, chi_std) # pre-load FT window ftwin = kout**kweight * ftwindow(kout, xmin=kmin, xmax=kmax, window=win, dx=dk) # calc k-value and initial guess for y-values of spline params nspl = max(4, min(128, 2*int(rbkg*(kmax-kmin)/np.pi) + 1)) spl_y, spl_k, spl_e = np.zeros(nspl), np.zeros(nspl), np.zeros(nspl) for i in range(nspl): q = kmin + i*(kmax-kmin)/(nspl - 1) ik = index_nearest(kraw, q) i1 = min(len(kraw)-1, ik + 5) i2 = max(0, ik - 5) spl_k[i] = kraw[ik] spl_e[i] = energy[ik+ie0] spl_y[i] = (2*mu[ik+ie0] + mu[i1+ie0] + mu[i2+ie0] ) / 4.0 # get spline represention: knots, coefs, order=3 # coefs will be varied in fit. knots, coefs, order = splrep(spl_k, spl_y) # set fit parameters from initial coefficients params = Group() for i in range(len(coefs)): name = FMT_COEF % i p = Parameter(coefs[i], name=name, vary=i<len(spl_y)) p._getval() setattr(params, name, p) initbkg, initchi = spline_eval(kraw[:iemax-ie0+1], mu[ie0:iemax+1], knots, coefs, order, kout) # do fit fit = Minimizer(__resid, params, _larch=_larch, toler=1.e-4, fcn_kws = dict(ncoefs=len(coefs), chi_std=chi_std, knots=knots, order=order, kraw=kraw[:iemax-ie0+1], mu=mu[ie0:iemax+1], irbkg=irbkg, kout=kout, ftwin=ftwin, kweight=kweight, nfft=nfft, nclamp=nclamp, clamp_lo=clamp_lo, clamp_hi=clamp_hi)) fit.leastsq() # write final results coefs = [getattr(params, FMT_COEF % i) for i in range(len(coefs))] bkg, chi = spline_eval(kraw[:iemax-ie0+1], mu[ie0:iemax+1], knots, coefs, order, kout) obkg = np.copy(mu) obkg[ie0:ie0+len(bkg)] = bkg # outputs to group group = set_xafsGroup(group, _larch=_larch) group.bkg = obkg group.chie = (mu-obkg)/edge_step group.k = kout group.chi = chi/edge_step # now fill in 'autobk_details' group params.init_bkg = np.copy(mu) params.init_bkg[ie0:ie0+len(bkg)] = initbkg params.init_chi = initchi/edge_step params.knots_e = spl_e params.knots_y = np.array([coefs[i] for i in range(nspl)]) params.init_knots_y = spl_y params.nfev = params.fit_details.nfev params.kmin = kmin params.kmax = kmax group.autobk_details = params # uncertainties in mu0 and chi: fairly slow!! if HAS_UNCERTAIN and calc_uncertainties: vbest, vstd = [], [] for n in fit.var_names: par = getattr(params, n) vbest.append(par.value) vstd.append(par.stderr) uvars = uncertainties.correlated_values(vbest, params.covar) # uncertainty in bkg (aka mu0) # note that much of this is working around # limitations in the uncertainty package that make it # 1. take an argument list (not array) # 2. work on returned scalars (but not arrays) # 3. not handle kw args and *args well (so use # of global "index" is important here) nkx = iemax-ie0 + 1 def my_dsplev(*args): coefs = np.array(args) return splev(kraw[:nkx], [knots, coefs, order])[index] fdbkg = uncertainties.wrap(my_dsplev) dmu0 = [fdbkg(*uvars).std_dev() for index in range(len(bkg))] group.delta_bkg = np.zeros(len(mu)) group.delta_bkg[ie0:ie0+len(bkg)] = np.array(dmu0) # uncertainty in chi (see notes above) def my_dchi(*args): coefs = np.array(args) b,chi = spline_eval(kraw[:nkx], mu[ie0:iemax+1], knots, coefs, order, kout) return chi[index] fdchi = uncertainties.wrap(my_dchi) dchi = [fdchi(*uvars).std_dev() for index in range(len(kout))] group.delta_chi = np.array(dchi)/edge_step
def parse_group_args(arg0, members=None, group=None, defaults=None, fcn_name=None, check_outputs=True): """parse arguments for functions supporting First Argument Group convention That is, if the first argument is a Larch Group and contains members named in 'members', this will return data extracted from that group. Arguments ---------- arg0: first argument for function call. members: list/tuple of names of required members (in order) defaults: tuple of default values for remaining required arguments past the first (in order) group: group sent to parent function, used for outputs fcn_name: name of parent function, used for error messages check_output: True/False (default True) setting whether a Warning should be raised in any of the outputs (except for the final group) are None. This effectively checks that all expected inputs have been specified Returns ------- tuple of output values in the order listed by members, followed by the output group (which could be None). Notes ----- This implements the First Argument Group convention, used for many Larch functions. As an example, the function _xafs.find_e0 is defined like this: find_e0(energy, mu=None, group=None, ...) and uses this function as energy, mu, group = parse_group_arg(energy, members=('energy', 'mu'), defaults=(mu,), group=group, fcn_name='find_e0', check_output=True) This allows the caller to use find_e0(grp) as a shorthand for find_e0(grp.energy, grp.mu, group=grp) as long as the Group grp has member 'energy', and 'mu'. With 'check_output=True', the value for 'mu' is not actually allowed to be None. The defaults tuple should be passed so that correct values are assigned if the caller actually specifies arrays as for the full call signature. """ if members is None: members = [] if isgroup(arg0, *members): if group is None: group = arg0 out = [getattr(arg0, attr) for attr in members] else: out = [arg0] + list(defaults) # test that all outputs are non-None if check_outputs: _errmsg = """%s: needs First Argument Group or valid arguments for %s""" if fcn_name is None: fcn_name ='unknown function' for i, nam in enumerate(members): if out[i] is None: raise Warning(_errmsg % (fcn_name, ', '.join(members))) out.append(group) return out
def test_eval_group3(self): out = deval(self.group3) assert(isgroup(out)) assert(out.dx['a'] == 1.3 + 0.2j) assert(out.dx['tup'] == (0, None)) assert(out.dx['blist'] == [1,2,3.0])
def autobk(energy, mu=None, group=None, rbkg=1, nknots=None, e0=None, edge_step=None, kmin=0, kmax=None, kweight=1, dk=0, win='hanning', k_std=None, chi_std=None, nfft=2048, kstep=0.05, pre_edge_kws=None, nclamp=4, clamp_lo=1, clamp_hi=1, calc_uncertainties=False, _larch=None, **kws): """Use Autobk algorithm to remove XAFS background Parameters: ----------- energy: 1-d array of x-ray energies, in eV, or group mu: 1-d array of mu(E) group: output group (and input group for e0 and edge_step). rbkg: distance (in Ang) for chi(R) above which the signal is ignored. Default = 1. e0: edge energy, in eV. If None, it will be determined. edge_step: edge step. If None, it will be determined. pre_edge_kws: keyword arguments to pass to pre_edge() nknots: number of knots in spline. If None, it will be determined. kmin: minimum k value [0] kmax: maximum k value [full data range]. kweight: k weight for FFT. [1] dk: FFT window window parameter. [0] win: FFT window function name. ['hanning'] nfft: array size to use for FFT [2048] kstep: k step size to use for FFT [0.05] k_std: optional k array for standard chi(k). chi_std: optional chi array for standard chi(k). nclamp: number of energy end-points for clamp [2] clamp_lo: weight of low-energy clamp [1] clamp_hi: weight of high-energy clamp [1] calc_uncertaintites: Flag to calculate uncertainties in mu_0(E) and chi(k) [False] Output arrays are written to the provided group. Follows the 'First Argument Group' convention. """ msg = _larch.writer.write if 'kw' in kws: kweight = kws.pop('kw') if len(kws) > 0: msg('Unrecognized a:rguments for autobk():\n') msg(' %s\n' % (', '.join(kws.keys()))) return energy, mu, group = parse_group_args(energy, members=('energy', 'mu'), defaults=(mu, ), group=group, fcn_name='autobk') energy = remove_dups(energy) # if e0 or edge_step are not specified, get them, either from the # passed-in group or from running pre_edge() group = set_xafsGroup(group, _larch=_larch) if edge_step is None and isgroup(group, 'edge_step'): edge_step = group.edge_step if e0 is None and isgroup(group, 'e0'): e0 = group.e0 if e0 is None or edge_step is None: # need to run pre_edge: pre_kws = dict(nnorm=3, nvict=0, pre1=None, pre2=-50., norm1=100., norm2=None) if pre_edge_kws is not None: pre_kws.update(pre_edge_kws) pre_edge(energy, mu, group=group, _larch=_larch, **pre_kws) if e0 is None: e0 = group.e0 if edge_step is None: edge_step = group.edge_step if e0 is None or edge_step is None: msg('autobk() could not determine e0 or edge_step!: trying running pre_edge first\n' ) return # get array indices for rkbg and e0: irbkg, ie0 ie0 = index_of(energy, e0) rgrid = np.pi / (kstep * nfft) if rbkg < 2 * rgrid: rbkg = 2 * rgrid irbkg = int(1.01 + rbkg / rgrid) # save ungridded k (kraw) and grided k (kout) # and ftwin (*k-weighting) for FT in residual enpe = energy[ie0:] - e0 kraw = np.sign(enpe) * np.sqrt(ETOK * abs(enpe)) if kmax is None: kmax = max(kraw) else: kmax = max(0, min(max(kraw), kmax)) kout = kstep * np.arange(int(1.01 + kmax / kstep), dtype='float64') iemax = min(len(energy), 2 + index_of(energy, e0 + kmax * kmax / ETOK)) - 1 # interpolate provided chi(k) onto the kout grid if chi_std is not None and k_std is not None: chi_std = np.interp(kout, k_std, chi_std) # pre-load FT window ftwin = kout**kweight * ftwindow( kout, xmin=kmin, xmax=kmax, window=win, dx=dk) # calc k-value and initial guess for y-values of spline params nspl = max(4, min(128, 2 * int(rbkg * (kmax - kmin) / np.pi) + 1)) spl_y, spl_k, spl_e = np.zeros(nspl), np.zeros(nspl), np.zeros(nspl) for i in range(nspl): q = kmin + i * (kmax - kmin) / (nspl - 1) ik = index_nearest(kraw, q) i1 = min(len(kraw) - 1, ik + 5) i2 = max(0, ik - 5) spl_k[i] = kraw[ik] spl_e[i] = energy[ik + ie0] spl_y[i] = (2 * mu[ik + ie0] + mu[i1 + ie0] + mu[i2 + ie0]) / 4.0 # get spline represention: knots, coefs, order=3 # coefs will be varied in fit. knots, coefs, order = splrep(spl_k, spl_y) # set fit parameters from initial coefficients params = Group() for i in range(len(coefs)): name = FMT_COEF % i p = Parameter(coefs[i], name=name, vary=i < len(spl_y)) p._getval() setattr(params, name, p) initbkg, initchi = spline_eval(kraw[:iemax - ie0 + 1], mu[ie0:iemax + 1], knots, coefs, order, kout) # do fit fit = Minimizer(__resid, params, _larch=_larch, toler=1.e-4, fcn_kws=dict(ncoefs=len(coefs), chi_std=chi_std, knots=knots, order=order, kraw=kraw[:iemax - ie0 + 1], mu=mu[ie0:iemax + 1], irbkg=irbkg, kout=kout, ftwin=ftwin, kweight=kweight, nfft=nfft, nclamp=nclamp, clamp_lo=clamp_lo, clamp_hi=clamp_hi)) fit.leastsq() # write final results coefs = [getattr(params, FMT_COEF % i) for i in range(len(coefs))] bkg, chi = spline_eval(kraw[:iemax - ie0 + 1], mu[ie0:iemax + 1], knots, coefs, order, kout) obkg = np.copy(mu) obkg[ie0:ie0 + len(bkg)] = bkg # outputs to group group = set_xafsGroup(group, _larch=_larch) group.bkg = obkg group.chie = (mu - obkg) / edge_step group.k = kout group.chi = chi / edge_step # now fill in 'autobk_details' group params.init_bkg = np.copy(mu) params.init_bkg[ie0:ie0 + len(bkg)] = initbkg params.init_chi = initchi / edge_step params.knots_e = spl_e params.knots_y = np.array([coefs[i] for i in range(nspl)]) params.init_knots_y = spl_y params.nfev = params.fit_details.nfev params.kmin = kmin params.kmax = kmax group.autobk_details = params # uncertainties in mu0 and chi: fairly slow!! if HAS_UNCERTAIN and calc_uncertainties: vbest, vstd = [], [] for n in fit.var_names: par = getattr(params, n) vbest.append(par.value) vstd.append(par.stderr) uvars = uncertainties.correlated_values(vbest, params.covar) # uncertainty in bkg (aka mu0) # note that much of this is working around # limitations in the uncertainty package that make it # 1. take an argument list (not array) # 2. work on returned scalars (but not arrays) # 3. not handle kw args and *args well (so use # of global "index" is important here) nkx = iemax - ie0 + 1 def my_dsplev(*args): coefs = np.array(args) return splev(kraw[:nkx], [knots, coefs, order])[index] fdbkg = uncertainties.wrap(my_dsplev) dmu0 = [fdbkg(*uvars).std_dev() for index in range(len(bkg))] group.delta_bkg = np.zeros(len(mu)) group.delta_bkg[ie0:ie0 + len(bkg)] = np.array(dmu0) # uncertainty in chi (see notes above) def my_dchi(*args): coefs = np.array(args) b, chi = spline_eval(kraw[:nkx], mu[ie0:iemax + 1], knots, coefs, order, kout) return chi[index] fdchi = uncertainties.wrap(my_dchi) dchi = [fdchi(*uvars).std_dev() for index in range(len(kout))] group.delta_chi = np.array(dchi) / edge_step
def autobk(energy, mu=None, group=None, rbkg=1, nknots=None, e0=None, edge_step=None, kmin=0, kmax=None, kweight=1, dk=0.1, win='hanning', k_std=None, chi_std=None, nfft=2048, kstep=0.05, pre_edge_kws=None, nclamp=4, clamp_lo=1, clamp_hi=1, calc_uncertainties=True, err_sigma=1, _larch=None, **kws): """Use Autobk algorithm to remove XAFS background Parameters: ----------- energy: 1-d array of x-ray energies, in eV, or group mu: 1-d array of mu(E) group: output group (and input group for e0 and edge_step). rbkg: distance (in Ang) for chi(R) above which the signal is ignored. Default = 1. e0: edge energy, in eV. If None, it will be determined. edge_step: edge step. If None, it will be determined. pre_edge_kws: keyword arguments to pass to pre_edge() nknots: number of knots in spline. If None, it will be determined. kmin: minimum k value [0] kmax: maximum k value [full data range]. kweight: k weight for FFT. [1] dk: FFT window window parameter. [0.1] win: FFT window function name. ['hanning'] nfft: array size to use for FFT [2048] kstep: k step size to use for FFT [0.05] k_std: optional k array for standard chi(k). chi_std: optional chi array for standard chi(k). nclamp: number of energy end-points for clamp [2] clamp_lo: weight of low-energy clamp [1] clamp_hi: weight of high-energy clamp [1] calc_uncertaintites: Flag to calculate uncertainties in mu_0(E) and chi(k) [True] err_sigma: sigma level for uncertainties in mu_0(E) and chi(k) [1] Output arrays are written to the provided group. Follows the 'First Argument Group' convention. """ msg = sys.stdout if _larch is not None: msg = _larch.writer.write if 'kw' in kws: kweight = kws.pop('kw') if len(kws) > 0: msg('Unrecognized a:rguments for autobk():\n') msg(' %s\n' % (', '.join(kws.keys()))) return energy, mu, group = parse_group_args(energy, members=('energy', 'mu'), defaults=(mu,), group=group, fcn_name='autobk') if len(energy.shape) > 1: energy = energy.squeeze() if len(mu.shape) > 1: mu = mu.squeeze() energy = remove_dups(energy) # if e0 or edge_step are not specified, get them, either from the # passed-in group or from running pre_edge() group = set_xafsGroup(group, _larch=_larch) if edge_step is None and isgroup(group, 'edge_step'): edge_step = group.edge_step if e0 is None and isgroup(group, 'e0'): e0 = group.e0 if e0 is None or edge_step is None: # need to run pre_edge: pre_kws = dict(nnorm=3, nvict=0, pre1=None, pre2=-50., norm1=100., norm2=None) if pre_edge_kws is not None: pre_kws.update(pre_edge_kws) pre_edge(energy, mu, group=group, _larch=_larch, **pre_kws) if e0 is None: e0 = group.e0 if edge_step is None: edge_step = group.edge_step if e0 is None or edge_step is None: msg('autobk() could not determine e0 or edge_step!: trying running pre_edge first\n') return # get array indices for rkbg and e0: irbkg, ie0 ie0 = index_of(energy, e0) rgrid = np.pi/(kstep*nfft) if rbkg < 2*rgrid: rbkg = 2*rgrid irbkg = int(1.01 + rbkg/rgrid) # save ungridded k (kraw) and grided k (kout) # and ftwin (*k-weighting) for FT in residual enpe = energy[ie0:] - e0 kraw = np.sign(enpe)*np.sqrt(ETOK*abs(enpe)) if kmax is None: kmax = max(kraw) else: kmax = max(0, min(max(kraw), kmax)) kout = kstep * np.arange(int(1.01+kmax/kstep), dtype='float64') iemax = min(len(energy), 2+index_of(energy, e0+kmax*kmax/ETOK)) - 1 # interpolate provided chi(k) onto the kout grid if chi_std is not None and k_std is not None: chi_std = np.interp(kout, k_std, chi_std) # pre-load FT window ftwin = kout**kweight * ftwindow(kout, xmin=kmin, xmax=kmax, window=win, dx=dk, dx2=dk) # calc k-value and initial guess for y-values of spline params nspl = max(5, min(64, int(2*rbkg*(kmax-kmin)/np.pi) + 2)) spl_y, spl_k, spl_e = np.zeros(nspl), np.zeros(nspl), np.zeros(nspl) for i in range(nspl): q = kmin + i*(kmax-kmin)/(nspl - 1) ik = index_nearest(kraw, q) i1 = min(len(kraw)-1, ik + 5) i2 = max(0, ik - 5) spl_k[i] = kraw[ik] spl_e[i] = energy[ik+ie0] spl_y[i] = (2*mu[ik+ie0] + mu[i1+ie0] + mu[i2+ie0] ) / 4.0 # get spline represention: knots, coefs, order=3 # coefs will be varied in fit. knots, coefs, order = splrep(spl_k, spl_y) # set fit parameters from initial coefficients params = Parameters() for i in range(len(coefs)): params.add(name = FMT_COEF % i, value=coefs[i], vary=i<len(spl_y)) initbkg, initchi = spline_eval(kraw[:iemax-ie0+1], mu[ie0:iemax+1], knots, coefs, order, kout) # do fit result = minimize(__resid, params, method='leastsq', gtol=1.e-5, ftol=1.e-5, xtol=1.e-5, epsfcn=1.e-5, kws = dict(ncoefs=len(coefs), chi_std=chi_std, knots=knots, order=order, kraw=kraw[:iemax-ie0+1], mu=mu[ie0:iemax+1], irbkg=irbkg, kout=kout, ftwin=ftwin, kweight=kweight, nfft=nfft, nclamp=nclamp, clamp_lo=clamp_lo, clamp_hi=clamp_hi)) # write final results coefs = [result.params[FMT_COEF % i].value for i in range(len(coefs))] bkg, chi = spline_eval(kraw[:iemax-ie0+1], mu[ie0:iemax+1], knots, coefs, order, kout) obkg = np.copy(mu) obkg[ie0:ie0+len(bkg)] = bkg # outputs to group group = set_xafsGroup(group, _larch=_larch) group.bkg = obkg group.chie = (mu-obkg)/edge_step group.k = kout group.chi = chi/edge_step group.e0 = e0 # now fill in 'autobk_details' group details = Group(params=result.params) details.init_bkg = np.copy(mu) details.init_bkg[ie0:ie0+len(bkg)] = initbkg details.init_chi = initchi/edge_step details.knots_e = spl_e details.knots_y = np.array([coefs[i] for i in range(nspl)]) details.init_knots_y = spl_y details.nfev = result.nfev details.kmin = kmin details.kmax = kmax group.autobk_details = details # uncertainties in mu0 and chi: can be fairly slow. if calc_uncertainties: nchi = len(chi) nmue = iemax-ie0 + 1 redchi = result.redchi covar = result.covar / redchi jac_chi = np.zeros(nchi*nspl).reshape((nspl, nchi)) jac_bkg = np.zeros(nmue*nspl).reshape((nspl, nmue)) cvals, cerrs = [], [] for i in range(len(coefs)): par = result.params[FMT_COEF % i] cvals.append(getattr(par, 'value', 0.0)) cdel = getattr(par, 'stderr', 0.0) if cdel is None: cdel = 0.0 cerrs.append(cdel/2.0) cvals = np.array(cvals) cerrs = np.array(cerrs) # find derivatives by hand! _k = kraw[:nmue] _m = mu[ie0:iemax+1] for i in range(nspl): cval0 = cvals[i] cvals[i] = cval0 + cerrs[i] bkg1, chi1 = spline_eval(_k, _m, knots, cvals, order, kout) cvals[i] = cval0 - cerrs[i] bkg2, chi2 = spline_eval(_k, _m, knots, cvals, order, kout) cvals[i] = cval0 jac_chi[i] = (chi1 - chi2) / (2*cerrs[i]) jac_bkg[i] = (bkg1 - bkg2) / (2*cerrs[i]) dfchi = np.zeros(nchi) dfbkg = np.zeros(nmue) for i in range(nspl): for j in range(nspl): dfchi += jac_chi[i]*jac_chi[j]*covar[i,j] dfbkg += jac_bkg[i]*jac_bkg[j]*covar[i,j] prob = 0.5*(1.0 + erf(err_sigma/np.sqrt(2.0))) dchi = t.ppf(prob, nchi-nspl) * np.sqrt(dfchi*redchi) dbkg = t.ppf(prob, nmue-nspl) * np.sqrt(dfbkg*redchi) group.delta_chi = dchi group.delta_bkg = 0.0*mu group.delta_bkg[ie0:ie0+len(dbkg)] = dbkg