def _path2chi(path, paramgroup=None, _larch=None, **kws): """calculate chi(k) for a Feff Path, optionally setting path parameter values output chi array will be written to path group Parameters: ------------ path: a FeffPath Group paramgroup: a Parameter Group for calculating Path Parameters [None] kmax: maximum k value for chi calculation [20]. kstep: step in k value for chi calculation [0.05]. k: explicit array of k values to calculate chi. Returns: --------- None - outputs are written to path group """ if not isNamedClass(path, FeffPathGroup): msg('%s is not a valid Feff Path' % path) return if _larch is not None: if (paramgroup is not None and _larch.symtable.isgroup(paramgroup)): _larch.symtable._sys.paramGroup = paramgroup elif not hasattr(_larch.symtable._sys, 'paramGroup'): _larch.symtable._sys.paramGroup = Group() path._calc_chi(**kws)
def _residual(self, paramgroup, data_only=False, **kws): """return the residual for this data set residual = self.transform.apply(data_chi - model_chi) where model_chi is the result of ff2chi(pathlist) """ if not isNamedClass(self.transform, TransformGroup): return if not self.__prepared: self.prepare_fit() _ff2chi(self.pathlist, paramgroup=paramgroup, k=self.model.k, _larch=self._larch, group=self.model) eps_k = self.epsilon_k if isinstance(eps_k, np.ndarray): eps_k[np.where(eps_k < 1.e-12)[0]] = 1.e-12 diff = (self.__chi - self.model.chi) if data_only: # for extracting transformed data separately from residual diff = self.__chi trans = self.transform k = trans.k_[:len(diff)] all_kweights = isinstance(trans.kweight, Iterable) if trans.fitspace == 'k': iqmin = max(0, int(0.01 + trans.kmin / trans.kstep)) iqmax = min(trans.nfft / 2, int(0.01 + trans.kmax / trans.kstep)) if all_kweights: out = [] for i, kw in enumerate(trans.kweight): out.append(((diff / eps_k[i]) * k**kw)[iqmin:iqmax]) return np.concatenate(out) else: return ((diff / eps_k) * k**trans.kweight)[iqmin:iqmax] else: out = [] if all_kweights: chir = [trans.fftf(diff, kweight=kw) for kw in trans.kweight] eps_r = self.epsilon_r else: chir = [trans.fftf(diff)] eps_r = [self.epsilon_r] if trans.fitspace == 'r': irmin = max(0, int(0.01 + trans.rmin / trans.rstep)) irmax = min(trans.nfft / 2, int(0.01 + trans.rmax / trans.rstep)) for i, chir_ in enumerate(chir): chir_ = chir_ / (eps_r[i]) out.append(realimag(chir_[irmin:irmax])) else: chiq = [trans.fftr(c) / eps for c, eps in zip(chir, eps_r)] iqmin = max(0, int(0.01 + trans.kmin / trans.kstep)) iqmax = min(trans.nfft / 2, int(0.01 + trans.kmax / trans.kstep)) for chiq_ in chiq: out.append(realimag(chiq_[iqmin:iqmax])[::2]) return np.concatenate(out)
def _residual(self, paramgroup=None, data_only=False, **kws): """return the residual for this data set residual = self.transform.apply(data_chi - model_chi) where model_chi is the result of ff2chi(pathlist) """ if (paramgroup is not None and self._larch.symtable.isgroup(paramgroup)): self._larch.symtable._sys.paramGroup = paramgroup if not isNamedClass(self.transform, TransformGroup): return if not self.__prepared: self.prepare_fit() _ff2chi(self.pathlist, k=self.model.k, _larch=self._larch, group=self.model) eps_k = self.epsilon_k if isinstance(eps_k, np.ndarray): eps_k[np.where(eps_k<1.e-12)[0]] = 1.e-12 else: eps_k = max(1.e-12, eps_k) diff = (self.__chi - self.model.chi) if data_only: # for extracting transformed data separately from residual diff = self.__chi trans = self.transform k = trans.k_[:len(diff)] all_kweights = isinstance(trans.kweight, Iterable) if trans.fitspace == 'k': iqmin = max(0, int(0.01 + trans.kmin/trans.kstep)) iqmax = min(trans.nfft/2, int(0.01 + trans.kmax/trans.kstep)) if all_kweights: out = [] for i, kw in enumerate(trans.kweight): out.append(((diff/eps_k[i])*k**kw)[iqmin:iqmax]) return np.concatenate(out) else: return ((diff/eps_k) * k**trans.kweight)[iqmin:iqmax] else: out = [] if all_kweights: chir = [trans.fftf(diff, kweight=kw) for kw in trans.kweight] eps_r = self.epsilon_r else: chir = [trans.fftf(diff)] eps_r = [self.epsilon_r] if trans.fitspace == 'r': irmin = max(0, int(0.01 + trans.rmin/trans.rstep)) irmax = min(trans.nfft/2, int(0.01 + trans.rmax/trans.rstep)) for i, chir_ in enumerate(chir): chir_ = chir_ / (eps_r[i]) out.append(realimag(chir_[irmin:irmax])) else: chiq = [trans.fftr(c)/eps for c, eps in zip(chir, eps_r)] iqmin = max(0, int(0.01 + trans.kmin/trans.kstep)) iqmax = min(trans.nfft/2, int(0.01 + trans.kmax/trans.kstep)) for chiq_ in chiq: out.append( realimag(chiq_[iqmin:iqmax])[::2]) return np.concatenate(out)
def _ff2chi(pathlist, group=None, paramgroup=None, _larch=None, k=None, kmax=None, kstep=0.05, **kws): """sum chi(k) for a list of FeffPath Groups. Parameters: ------------ pathlist: a list of FeffPath Groups paramgroup: a Parameter Group for calculating Path Parameters [None] kmax: maximum k value for chi calculation [20]. kstep: step in k value for chi calculation [0.05]. k: explicit array of k values to calculate chi. Returns: --------- group contain arrays for k and chi This essentially calls path2chi() for each of the paths in the pathlist and writes the resulting arrays to group.k and group.chi. """ msg = _larch.writer.write if (paramgroup is not None and _larch is not None and _larch.symtable.isgroup(paramgroup)): _larch.symtable._sys.paramGroup = paramgroup for path in pathlist: if not isNamedClass(path, FeffPathGroup): msg('%s is not a valid Feff Path' % path) return path._calc_chi(k=k, kstep=kstep, kmax=kmax) k = pathlist[0].k[:] out = np.zeros_like(k) for path in pathlist: out += path.chi if group is None: group = Group() else: group = set_xafsGroup(group, _larch=_larch) group.k = k group.chi = out return group
def _path2chi(path, paramgroup=None, _larch=None, **kws): """calculate chi(k) for a Feff Path, optionally setting path parameter values output chi array will be written to path group Parameters: ------------ path: a FeffPath Group kmax: maximum k value for chi calculation [20]. kstep: step in k value for chi calculation [0.05]. k: explicit array of k values to calculate chi. Returns: --------- None - outputs are written to path group """ params = group2params(paramgroup, _larch=_larch) if not isNamedClass(path, FeffPathGroup): msg('%s is not a valid Feff Path' % path) return path.create_path_params() path._calc_chi(**kws)
def path2chi(path, paramgroup=None, **kws): """calculate chi(k) for a Feff Path, optionally setting path parameter values output chi array will be written to path group Parameters: ------------ path: a FeffPath Group kmax: maximum k value for chi calculation [20]. kstep: step in k value for chi calculation [0.05]. k: explicit array of k values to calculate chi. Returns: --------- None - outputs are written to path group """ params = group2params(paramgroup) if not isNamedClass(path, FeffPathGroup): msg('%s is not a valid Feff Path' % path) return path.create_path_params(params=params) path._calc_chi(**kws)
def feffit(paramgroup, datasets, rmax_out=10, path_outputs=True, _larch=None, **kws): """execute a Feffit fit: a fit of feff paths to a list of datasets Parameters: ------------ paramgroup: group containing parameters for fit datasets: Feffit Dataset group or list of Feffit Dataset group. rmax_out: maximum R value to calculate output arrays. path_output: Flag to set whether all Path outputs should be written. Returns: --------- a fit results group. This will contain subgroups of: datasets: an array of FeffitDataSet groups used in the fit. params: This will be identical to the input parameter group. fit: an object which points to the low-level fit. Statistical parameters will be put into the params group. Each dataset will have a 'data' and 'model' subgroup, each with arrays: k wavenumber array of k chi chi(k). kwin window Omega(k) (length of input chi(k)). r uniform array of R, out to rmax_out. chir complex array of chi(R). chir_mag magnitude of chi(R). chir_pha phase of chi(R). chir_re real part of chi(R). chir_im imaginary part of chi(R). """ def _resid(params, datasets=None, paramgroup=None, **kwargs): """ this is the residual function""" params2group(params, paramgroup) return concatenate([d._residual(paramgroup) for d in datasets]) if isNamedClass(datasets, FeffitDataSet): datasets = [datasets] params = group2params(paramgroup, _larch=_larch) for ds in datasets: if not isNamedClass(ds, FeffitDataSet): print( "feffit needs a list of FeffitDataSets") return ds.prepare_fit() fit = Minimizer(_resid, params, fcn_kws=dict(datasets=datasets, paramgroup=paramgroup), scale_covar=True, **kws) result = fit.leastsq() params2group(result.params, paramgroup) dat = concatenate([d._residual(paramgroup, data_only=True) for d in datasets]) n_idp = 0 for ds in datasets: n_idp += ds.n_idp # here we rescale chi-square and reduced chi-square to n_idp npts = len(result.residual) chi_square = result.chisqr * n_idp*1.0 / npts chi_reduced = chi_square/(n_idp*1.0 - result.nvarys) rfactor = (result.residual**2).sum() / (dat**2).sum() # calculate 'aic', 'bic' rescaled to n_idp # note that neg2_loglikel is -2*log(likelihood) neg2_loglikel = n_idp * np.log(chi_square / n_idp) aic = neg2_loglikel + 2 * result.nvarys bic = neg2_loglikel + np.log(n_idp) * result.nvarys # With scale_covar = True, Minimizer() scales the uncertainties # by reduced chi-square assuming params.nfree is the correct value # for degrees-of-freedom. But n_idp-params.nvarys is a better measure, # so we rescale uncertainties here. covar = getattr(result, 'covar', None) # print("COVAR " , covar) if covar is not None: err_scale = (result.nfree / (n_idp - result.nvarys)) for name in result.var_names: p = result.params[name] if isParameter(p) and p.vary: p.stderr *= sqrt(err_scale) # next, propagate uncertainties to constraints and path parameters. result.covar *= err_scale vsave, vbest = {}, [] # 1. save current params for vname in result.var_names: par = result.params[vname] vsave[vname] = par vbest.append(par.value) # 2. get correlated uncertainties, set params accordingly uvars = correlated_values(vbest, result.covar) # 3. evaluate constrained params, save stderr for nam, obj in result.params.items(): eval_stderr(obj, uvars, result.var_names, result.params) # 3. evaluate path params, save stderr for ds in datasets: for p in ds.pathlist: p.store_feffdat() for pname in ('degen', 's02', 'e0', 'ei', 'deltar', 'sigma2', 'third', 'fourth'): obj = p.params[PATHPAR_FMT % (pname, p.label)] eval_stderr(obj, uvars, result.var_names, result.params) # restore saved parameters again for vname in result.var_names: # setattr(params, vname, vsave[vname]) params[vname] = vsave[vname] # clear any errors evaluting uncertainties if _larch is not None and (len(_larch.error) > 0): _larch.error = [] # reset the parameters group with the newly updated uncertainties params2group(result.params, paramgroup) # here we create outputs arrays for chi(k), chi(r): for ds in datasets: ds.save_ffts(rmax_out=rmax_out, path_outputs=path_outputs) out = Group(name='feffit results', datasets=datasets, fitter=fit, fit_details=result, chi_square=chi_square, n_independent=n_idp, chi_reduced=chi_reduced, rfactor=rfactor, aic=aic, bic=bic, covar=covar) for attr in ('params', 'nvarys', 'nfree', 'ndata', 'var_names', 'nfev', 'success', 'errorbars', 'message', 'lmdif_message'): setattr(out, attr, getattr(result, attr, None)) return out
def feffit(params, datasets, _larch=None, rmax_out=10, path_outputs=True, **kws): """execute a Feffit fit: a fit of feff paths to a list of datasets Parameters: ------------ paramgroup: group containing parameters for fit datasets: Feffit Dataset group or list of Feffit Dataset group. rmax_out: maximum R value to calculate output arrays. path_output: Flag to set whether all Path outputs should be written. Returns: --------- a fit results group. This will contain subgroups of: datasets: an array of FeffitDataSet groups used in the fit. params: This will be identical to the input parameter group. fit: an object which points to the low-level fit. Statistical parameters will be put into the params group. Each dataset will have a 'data' and 'model' subgroup, each with arrays: k wavenumber array of k chi chi(k). kwin window Omega(k) (length of input chi(k)). r uniform array of R, out to rmax_out. chir complex array of chi(R). chir_mag magnitude of chi(R). chir_pha phase of chi(R). chir_re real part of chi(R). chir_im imaginary part of chi(R). """ def _resid(params, datasets=None, _larch=None, **kwargs): """ this is the residual function""" return concatenate([d._residual() for d in datasets]) if isNamedClass(datasets, FeffitDataSet): datasets = [datasets] for ds in datasets: if not isNamedClass(ds, FeffitDataSet): print("feffit needs a list of FeffitDataSets") return fitkws = dict(datasets=datasets) fit = Minimizer(_resid, params, fcn_kws=fitkws, scale_covar=True, _larch=_larch, **kws) fit.leastsq() dat = concatenate([d._residual(data_only=True) for d in datasets]) params.rfactor = (params.fit_details.fvec**2).sum() / (dat**2).sum() # remove temporary parameters for _feffdat and reff # that had been placed by _pathparams() #for pname in ('_feffdat', 'reff'): # if hasattr(params, pname): # delattr(params, pname) n_idp = 0 for ds in datasets: n_idp += ds.n_idp # here we rescale chi-square and reduced chi-square to n_idp npts = len(params.residual) params.chi_square *= n_idp * 1.0 / npts params.chi_reduced = params.chi_square / (n_idp * 1.0 - params.nvarys) # With scale_covar = True, Minimizer() scales the uncertainties # by reduced chi-square assuming params.nfree is the correct value # for degrees-of-freedom. But n_idp-params.nvarys is a better measure, # so we rescale uncertainties here. covar = getattr(params, 'covar', None) if covar is not None: err_scale = (params.nfree / (n_idp - params.nvarys)) for name in dir(params): p = getattr(params, name) if isParameter(p) and p.vary: p.stderr *= sqrt(err_scale) # next, propagate uncertainties to constraints and path parameters. params.covar *= err_scale vsave, vbest = {}, [] # 1. save current params for vname in params.covar_vars: par = getattr(params, vname) vsave[vname] = par vbest.append(par.value) # 2. get correlated uncertainties, set params accordingly uvars = correlated_values(vbest, params.covar) # 3. evaluate constrained params, save stderr for nam in dir(params): obj = getattr(params, nam) eval_stderr(obj, uvars, params.covar_vars, vsave, _larch) # 3. evaluate path params, save stderr for ds in datasets: for p in ds.pathlist: _larch.symtable._sys.paramGroup._feffdat = copy(p._feffdat) _larch.symtable._sys.paramGroup.reff = p._feffdat.reff for param in ('degen', 's02', 'e0', 'ei', 'deltar', 'sigma2', 'third', 'fourth'): obj = getattr(p, param) eval_stderr(obj, uvars, params.covar_vars, vsave, _larch) # restore saved parameters again for vname in params.covar_vars: setattr(params, vname, vsave[vname]) # clear any errors evaluting uncertainties if len(_larch.error) > 0: _larch.error = [] # here we create outputs arrays for chi(k), chi(r): for ds in datasets: ds.save_ffts(rmax_out=rmax_out, path_outputs=path_outputs) return Group(name='feffit fit results', fit=fit, params=params, datasets=datasets)
def feffit(params, datasets, _larch=None, rmax_out=10, path_outputs=True, **kws): """execute a Feffit fit: a fit of feff paths to a list of datasets Parameters: ------------ paramgroup: group containing parameters for fit datasets: Feffit Dataset group or list of Feffit Dataset group. rmax_out: maximum R value to calculate output arrays. path_output: Flag to set whether all Path outputs should be written. Returns: --------- a fit results group. This will contain subgroups of: datasets: an array of FeffitDataSet groups used in the fit. params: This will be identical to the input parameter group. fit: an object which points to the low-level fit. Statistical parameters will be put into the params group. Each dataset will have a 'data' and 'model' subgroup, each with arrays: k wavenumber array of k chi chi(k). kwin window Omega(k) (length of input chi(k)). r uniform array of R, out to rmax_out. chir complex array of chi(R). chir_mag magnitude of chi(R). chir_pha phase of chi(R). chir_re real part of chi(R). chir_im imaginary part of chi(R). """ def _resid(params, datasets=None, _larch=None, **kwargs): """ this is the residual function""" return concatenate([d._residual() for d in datasets]) if isNamedClass(datasets, FeffitDataSet): datasets = [datasets] for ds in datasets: if not isNamedClass(ds, FeffitDataSet): print( "feffit needs a list of FeffitDataSets") return fitkws = dict(datasets=datasets) fit = Minimizer(_resid, params, fcn_kws=fitkws, scale_covar=True, _larch=_larch, **kws) fit.leastsq() dat = concatenate([d._residual(data_only=True) for d in datasets]) params.rfactor = (params.fit_details.fvec**2).sum() / (dat**2).sum() # remove temporary parameters for _feffdat and reff # that had been placed by _pathparams() #for pname in ('_feffdat', 'reff'): # if hasattr(params, pname): # delattr(params, pname) n_idp = 0 for ds in datasets: n_idp += ds.n_idp # here we rescale chi-square and reduced chi-square to n_idp npts = len(params.residual) params.chi_square *= n_idp*1.0 / npts params.chi_reduced = params.chi_square/(n_idp*1.0 - params.nvarys) # With scale_covar = True, Minimizer() scales the uncertainties # by reduced chi-square assuming params.nfree is the correct value # for degrees-of-freedom. But n_idp-params.nvarys is a better measure, # so we rescale uncertainties here. covar = getattr(params, 'covar', None) if covar is not None: err_scale = (params.nfree / (n_idp - params.nvarys)) for name in dir(params): p = getattr(params, name) if isParameter(p) and p.vary: p.stderr *= sqrt(err_scale) # next, propagate uncertainties to constraints and path parameters. params.covar *= err_scale vsave, vbest = {}, [] # 1. save current params for vname in params.covar_vars: par = getattr(params, vname) vsave[vname] = par vbest.append(par.value) # 2. get correlated uncertainties, set params accordingly uvars = correlated_values(vbest, params.covar) # 3. evaluate constrained params, save stderr for nam in dir(params): obj = getattr(params, nam) eval_stderr(obj, uvars, params.covar_vars, vsave, _larch) # 3. evaluate path params, save stderr for ds in datasets: for p in ds.pathlist: _larch.symtable._sys.paramGroup._feffdat = copy(p._feffdat) _larch.symtable._sys.paramGroup.reff = p._feffdat.reff for param in ('degen', 's02', 'e0', 'ei', 'deltar', 'sigma2', 'third', 'fourth'): obj = getattr(p, param) eval_stderr(obj, uvars, params.covar_vars, vsave, _larch) # restore saved parameters again for vname in params.covar_vars: setattr(params, vname, vsave[vname]) # clear any errors evaluting uncertainties if len(_larch.error) > 0: _larch.error = [] # here we create outputs arrays for chi(k), chi(r): for ds in datasets: ds.save_ffts(rmax_out=rmax_out, path_outputs=path_outputs) return Group(name='feffit fit results', fit=fit, params=params, datasets=datasets)