def normalize_force(self):
pre_edge(self.larch, group=self.larch, _larch=self._larch, e0=self.e0, pre1=self.pre1, pre2=self.pre2,
nnorm=self.nnorm, norm1=self.norm1, norm2=self.norm2)
self.norm = self.larch.norm
self.e0 = self.larch.e0
self.pre_edge=self.larch.pre_edge
self.post_edge = self.larch.post_edge
self.edge_step = self.larch.edge_step
self.flatten()
def process(self, dgroup, proc_opts=None):
if not hasattr(dgroup, 'proc_opts'):
dgroup.proc_opts = {}
if 'xscale' not in dgroup.proc_opts:
dgroup.proc_opts.update(self.get_proc_opts(dgroup))
if proc_opts is not None:
dgroup.proc_opts.update(proc_opts)
opts = dgroup.proc_opts
# scaling
dgroup.x = opts['xscale']*(dgroup.xdat - opts['xshift'])
dgroup.y = opts['yscale']*(dgroup.ydat - opts['yshift'])
# smoothing
smop = opts['smooth_op'].lower()
cform = str(opts['smooth_conv'].lower())
if smop.startswith('box'):
dgroup.y = boxcar(dgroup.y, opts['smooth_c0'])
elif smop.startswith('savit'):
winsize = 2*opts['smooth_c0'] + 1
dgroup.y = savitzky_golay(dgroup.y, winsize, opts['smooth_c1'])
elif smop.startswith('conv'):
dgroup.y = smooth(dgroup.x, dgroup.y,
sigma=opts['smooth_sig'], form=cform)
# xas
if dgroup.datatype.startswith('xas'):
dgroup.energy = dgroup.x
dgroup.mu = dgroup.y
preopts = {'e0': None, 'step': None, 'make_flat':False,
'_larch':self.larch}
if not opts['auto_e0']:
_e0 = opts['e0']
if _e0 < max(dgroup.energy) and _e0 > min(dgroup.energy):
preopts['e0'] = float(_e0)
if not opts['auto_step']:
preopts['step'] = opts['step']
for attr in ('pre1', 'pre2', 'nvict', 'nnorm', 'norm1', 'norm2'):
preopts[attr] = opts[attr]
pre_edge(dgroup, **preopts)
for attr in ('e0', 'edge_step'):
opts[attr] = getattr(dgroup, attr)
for attr in ('pre1', 'pre2', 'norm1', 'norm2'):
opts[attr] = getattr(dgroup.pre_edge_details, attr)
dgroup.proc_opts.update(opts)
def pre_edge(self):
for scan in self.scans:
pre_edge(scan.Energy,
scan.I_norm,
pre1=-10,
pre2=-2,
group=scan,
_larch=mylarch)
# print 'Automatic edge position found as', scan.e0, 'eV'
scan.I_norm = scan.I_norm - scan.pre_edge # Subtract fitted pre-edge curve
def XANES_Norm(self, e0=None, step=None, nnorm=3, nvict=0,
pre1=None, pre2=-50, norm1=100, norm2=None, make_flat=True, **kwd):
"""pre edge subtraction, normalization for XAFS from larch
This performs a number of steps:
1. determine E0 (if not supplied) from max of deriv(mu)
2. fit a line of polymonial to the region below the edge
3. fit a polymonial to the region above the edge
4. extrapolae the two curves to E0 to determine the edge jump
Arguments
----------
e0: edge energy, in eV. If None, it will be determined here.
step: edge jump. If None, it will be determined here.
pre1: low E range (relative to E0) for pre-edge fit
pre2: high E range (relative to E0) for pre-edge fit
nvict: energy exponent to use for pre-edg fit. See Note
norm1: low E range (relative to E0) for post-edge fit
norm2: high E range (relative to E0) for post-edge fit
nnorm: degree of polynomial (ie, nnorm+1 coefficients will be found) for
post-edge normalization curve. Default=3 (quadratic), max=5
make_flat: boolean (Default True) to calculate flattened output.
_larch : Interpreter class (please not touch)
Returns
-------
None
The following attributes will be written to the group:
-------
e0 energy origin
edge_step edge step
norm normalized mu(E)
flat flattened, normalized mu(E)
pre_edge determined pre-edge curve
post_edge determined post-edge, normalization curve
dmude derivative of mu(E)
Notes
-----
1 nvict gives an exponent to the energy term for the fits to the pre-edge
and the post-edge region. For the pre-edge, a line (m * energy + b) is
fit to mu(energy)*energy**nvict over the pre-edge region,
energy=[e0+pre1, e0+pre2]. For the post-edge, a polynomial of order
nnorm will be fit to mu(energy)*energy**nvict of the post-edge region
energy=[e0+norm1, e0+norm2]."""
xafs.pre_edge(self, _larch=self._larch, e0=e0, step=step, nnorm=nnorm,
nvict=nvict, pre1=pre1, pre2=pre2, norm1=norm1,
norm2=norm2, make_flat=True)
return
def larch_baseline(data, plot = False): '''Conventional baseline correction/normalization approach as implemented in Larch. Alternatively, MBACK algorithm implemented in Larch can be used''' pre_edge(data, _larch = _larch) print(data.edge_step) #mback(data, z=29, edge='K', order=4, _larch = _larch) if plot == True: plt.plot(data.energy, data.mu) plt.plot(data.energy, data.norm) plt.plot(data.energy, data.flat) return np.c_[data.energy, data.mu]
def XANES_Norm(self, e0=None, step=None, nnorm=3, nvict=0,
pre1=None, pre2=-50, norm1=100, norm2=None, make_flat=True, **kwd):
"""pre edge subtraction, normalization for XAFS from larch
This performs a number of steps:
1. determine E0 (if not supplied) from max of deriv(mu)
2. fit a line of polymonial to the region below the edge
3. fit a polymonial to the region above the edge
4. extrapolae the two curves to E0 to determine the edge jump
Arguments
----------
e0: edge energy, in eV. If None, it will be determined here.
step: edge jump. If None, it will be determined here.
pre1: low E range (relative to E0) for pre-edge fit
pre2: high E range (relative to E0) for pre-edge fit
nvict: energy exponent to use for pre-edg fit. See Note
norm1: low E range (relative to E0) for post-edge fit
norm2: high E range (relative to E0) for post-edge fit
nnorm: degree of polynomial (ie, nnorm+1 coefficients will be found) for
post-edge normalization curve. Default=3 (quadratic), max=5
make_flat: boolean (Default True) to calculate flattened output.
_larch : Interpreter class (please not touch)
Returns
-------
None
The following attributes will be written to the group:
-------
e0 energy origin
edge_step edge step
norm normalized mu(E)
flat flattened, normalized mu(E)
pre_edge determined pre-edge curve
post_edge determined post-edge, normalization curve
dmude derivative of mu(E)
Notes
-----
1 nvict gives an exponent to the energy term for the fits to the pre-edge
and the post-edge region. For the pre-edge, a line (m * energy + b) is
fit to mu(energy)*energy**nvict over the pre-edge region,
energy=[e0+pre1, e0+pre2]. For the post-edge, a polynomial of order
nnorm will be fit to mu(energy)*energy**nvict of the post-edge region
energy=[e0+norm1, e0+norm2]."""
xafs.pre_edge(self, _larch=self._larch, e0=e0, step=step, nnorm=nnorm,
nvict=nvict, pre1=pre1, pre2=pre2, norm1=norm1,
norm2=norm2, make_flat=True)
return
def normalize(self):
pre_edge(self.larch, group=self.larch, _larch=self._larch)
self.energy = self.larch.energy
self.mu = self.larch.mu
self.norm = self.larch.norm
self.new_ds = False
self.pre1 = self.larch.pre_edge_details.pre1
self.pre2 = self.larch.pre_edge_details.pre2
self.norm1 = self.larch.pre_edge_details.norm1
self.norm2 = self.larch.pre_edge_details.norm2
self.e0 = self.larch.e0
self.pre_edge = self.larch.pre_edge
self.post_edge = self.larch.post_edge
self.edge_step = self.larch.edge_step
self.flatten()
def add_group(self, group, label=None, signal=None):
"""add Larch group (presumably XAFS data) to Athena project"""
from larch_plugins.xafs import pre_edge
from larch_plugins.xray.xraydb_plugin import guess_edge
x = athena_array(group, 'energy')
yname = None
for _name in ('mu', 'mutrans', 'mufluor'):
if hasattr(group, _name):
yname = _name
break
if x is None or yname is None:
raise ValueError("can only add XAFS data to Athena project")
y = athena_array(group, yname)
i0 = athena_array(group, 'i0')
if signal is not None:
signal = athena_array(group, signal)
elif yname in ('mu', 'mutrans'):
sname = None
for _name in ('i1', 'itrans'):
if hasattr(group, _name):
sname = _name
break
if sname is not None:
signal = athena_array(group, sname)
hashkey = make_hashkey()
while hashkey in self.groups:
hashkey = make_hashkey()
# fill in data from pre-edge subtraction
if not (hasattr(group, 'e0') and hasattr(group, 'edge_step')):
pre_edge(group, _larch=self._larch)
args = make_athena_args(group, hashkey)
_elem, _edge = guess_edge(group.e0, _larch=self._larch)
args['bkg_z'] = _elem
self.groups[hashkey] = Group(args=args, x=x, y=y, i0=i0, signal=signal)
def add_group(self, group, signal=None):
"""add Larch group (presumably XAFS data) to Athena project"""
from larch_plugins.xafs import pre_edge
from larch_plugins.xray.xraydb_plugin import guess_edge
x = athena_array(group, 'energy')
yname = None
for _name in ('mu', 'mutrans', 'mufluor'):
if hasattr(group, _name):
yname = _name
break
if x is None or yname is None:
raise ValueError("can only add XAFS data to Athena project")
y = athena_array(group, yname)
i0 = athena_array(group, 'i0')
if signal is not None:
signal = athena_array(group, signal)
elif yname in ('mu', 'mutrans'):
sname = None
for _name in ('i1', 'itrans'):
if hasattr(group, _name):
sname = _name
break
if sname is not None:
signal = athena_array(group, sname)
hashkey = make_hashkey()
while hashkey in self.groups:
hashkey = make_hashkey()
# fill in data from pre-edge subtraction
if not (hasattr(group, 'e0') and hasattr(group, 'edge_step')):
pre_edge(group, _larch=self._larch)
args = make_athena_args(group, hashkey)
_elem, _edge = guess_edge(group.e0, _larch=self._larch)
args['bkg_z'] = _elem
self.groups[hashkey] = Group(args=args, x=x, y=y, i0=i0, signal=signal)
def autobk(energy, mu=None, group=None, rbkg=1, nknots=None, e0=None,
edge_step=None, kmin=0, kmax=None, kweight=1, dk=0,
win='hanning', k_std=None, chi_std=None, nfft=2048, kstep=0.05,
pre_edge_kws=None, nclamp=4, clamp_lo=1, clamp_hi=1,
calc_uncertainties=False, _larch=None, **kws):
"""Use Autobk algorithm to remove XAFS background
Parameters:
-----------
energy: 1-d array of x-ray energies, in eV, or group
mu: 1-d array of mu(E)
group: output group (and input group for e0 and edge_step).
rbkg: distance (in Ang) for chi(R) above
which the signal is ignored. Default = 1.
e0: edge energy, in eV. If None, it will be determined.
edge_step: edge step. If None, it will be determined.
pre_edge_kws: keyword arguments to pass to pre_edge()
nknots: number of knots in spline. If None, it will be determined.
kmin: minimum k value [0]
kmax: maximum k value [full data range].
kweight: k weight for FFT. [1]
dk: FFT window window parameter. [0]
win: FFT window function name. ['hanning']
nfft: array size to use for FFT [2048]
kstep: k step size to use for FFT [0.05]
k_std: optional k array for standard chi(k).
chi_std: optional chi array for standard chi(k).
nclamp: number of energy end-points for clamp [2]
clamp_lo: weight of low-energy clamp [1]
clamp_hi: weight of high-energy clamp [1]
calc_uncertaintites: Flag to calculate uncertainties in
mu_0(E) and chi(k) [False]
Output arrays are written to the provided group.
Follows the 'First Argument Group' convention.
"""
msg = _larch.writer.write
if 'kw' in kws:
kweight = kws.pop('kw')
if len(kws) > 0:
msg('Unrecognized a:rguments for autobk():\n')
msg(' %s\n' % (', '.join(kws.keys())))
return
energy, mu, group = parse_group_args(energy, members=('energy', 'mu'),
defaults=(mu,), group=group,
fcn_name='autobk')
energy = remove_dups(energy)
# if e0 or edge_step are not specified, get them, either from the
# passed-in group or from running pre_edge()
group = set_xafsGroup(group, _larch=_larch)
if edge_step is None and isgroup(group, 'edge_step'):
edge_step = group.edge_step
if e0 is None and isgroup(group, 'e0'):
e0 = group.e0
if e0 is None or edge_step is None:
# need to run pre_edge:
pre_kws = dict(nnorm=3, nvict=0, pre1=None,
pre2=-50., norm1=100., norm2=None)
if pre_edge_kws is not None:
pre_kws.update(pre_edge_kws)
pre_edge(energy, mu, group=group, _larch=_larch, **pre_kws)
if e0 is None:
e0 = group.e0
if edge_step is None:
edge_step = group.edge_step
if e0 is None or edge_step is None:
msg('autobk() could not determine e0 or edge_step!: trying running pre_edge first\n')
return
# get array indices for rkbg and e0: irbkg, ie0
ie0 = index_of(energy, e0)
rgrid = np.pi/(kstep*nfft)
if rbkg < 2*rgrid: rbkg = 2*rgrid
irbkg = int(1.01 + rbkg/rgrid)
# save ungridded k (kraw) and grided k (kout)
# and ftwin (*k-weighting) for FT in residual
enpe = energy[ie0:] - e0
kraw = np.sign(enpe)*np.sqrt(ETOK*abs(enpe))
if kmax is None:
kmax = max(kraw)
else:
kmax = max(0, min(max(kraw), kmax))
kout = kstep * np.arange(int(1.01+kmax/kstep), dtype='float64')
iemax = min(len(energy), 2+index_of(energy, e0+kmax*kmax/ETOK)) - 1
# interpolate provided chi(k) onto the kout grid
if chi_std is not None and k_std is not None:
chi_std = np.interp(kout, k_std, chi_std)
# pre-load FT window
ftwin = kout**kweight * ftwindow(kout, xmin=kmin, xmax=kmax,
window=win, dx=dk)
# calc k-value and initial guess for y-values of spline params
nspl = max(4, min(128, 2*int(rbkg*(kmax-kmin)/np.pi) + 1))
spl_y, spl_k, spl_e = np.zeros(nspl), np.zeros(nspl), np.zeros(nspl)
for i in range(nspl):
q = kmin + i*(kmax-kmin)/(nspl - 1)
ik = index_nearest(kraw, q)
i1 = min(len(kraw)-1, ik + 5)
i2 = max(0, ik - 5)
spl_k[i] = kraw[ik]
spl_e[i] = energy[ik+ie0]
spl_y[i] = (2*mu[ik+ie0] + mu[i1+ie0] + mu[i2+ie0] ) / 4.0
# get spline represention: knots, coefs, order=3
# coefs will be varied in fit.
knots, coefs, order = splrep(spl_k, spl_y)
# set fit parameters from initial coefficients
params = Group()
for i in range(len(coefs)):
name = FMT_COEF % i
p = Parameter(coefs[i], name=name, vary=i<len(spl_y))
p._getval()
setattr(params, name, p)
initbkg, initchi = spline_eval(kraw[:iemax-ie0+1], mu[ie0:iemax+1],
knots, coefs, order, kout)
# do fit
fit = Minimizer(__resid, params, _larch=_larch, toler=1.e-4,
fcn_kws = dict(ncoefs=len(coefs), chi_std=chi_std,
knots=knots, order=order,
kraw=kraw[:iemax-ie0+1],
mu=mu[ie0:iemax+1], irbkg=irbkg, kout=kout,
ftwin=ftwin, kweight=kweight,
nfft=nfft, nclamp=nclamp,
clamp_lo=clamp_lo, clamp_hi=clamp_hi))
fit.leastsq()
# write final results
coefs = [getattr(params, FMT_COEF % i) for i in range(len(coefs))]
bkg, chi = spline_eval(kraw[:iemax-ie0+1], mu[ie0:iemax+1],
knots, coefs, order, kout)
obkg = np.copy(mu)
obkg[ie0:ie0+len(bkg)] = bkg
# outputs to group
group = set_xafsGroup(group, _larch=_larch)
group.bkg = obkg
group.chie = (mu-obkg)/edge_step
group.k = kout
group.chi = chi/edge_step
# now fill in 'autobk_details' group
params.init_bkg = np.copy(mu)
params.init_bkg[ie0:ie0+len(bkg)] = initbkg
params.init_chi = initchi/edge_step
params.knots_e = spl_e
params.knots_y = np.array([coefs[i] for i in range(nspl)])
params.init_knots_y = spl_y
params.nfev = params.fit_details.nfev
params.kmin = kmin
params.kmax = kmax
group.autobk_details = params
# uncertainties in mu0 and chi: fairly slow!!
if HAS_UNCERTAIN and calc_uncertainties:
vbest, vstd = [], []
for n in fit.var_names:
par = getattr(params, n)
vbest.append(par.value)
vstd.append(par.stderr)
uvars = uncertainties.correlated_values(vbest, params.covar)
# uncertainty in bkg (aka mu0)
# note that much of this is working around
# limitations in the uncertainty package that make it
# 1. take an argument list (not array)
# 2. work on returned scalars (but not arrays)
# 3. not handle kw args and *args well (so use
# of global "index" is important here)
nkx = iemax-ie0 + 1
def my_dsplev(*args):
coefs = np.array(args)
return splev(kraw[:nkx], [knots, coefs, order])[index]
fdbkg = uncertainties.wrap(my_dsplev)
dmu0 = [fdbkg(*uvars).std_dev() for index in range(len(bkg))]
group.delta_bkg = np.zeros(len(mu))
group.delta_bkg[ie0:ie0+len(bkg)] = np.array(dmu0)
# uncertainty in chi (see notes above)
def my_dchi(*args):
coefs = np.array(args)
b,chi = spline_eval(kraw[:nkx], mu[ie0:iemax+1],
knots, coefs, order, kout)
return chi[index]
fdchi = uncertainties.wrap(my_dchi)
dchi = [fdchi(*uvars).std_dev() for index in range(len(kout))]
group.delta_chi = np.array(dchi)/edge_step
def read_athena(filename,
match=None,
do_preedge=True,
do_bkg=True,
do_fft=True,
use_hashkey=False,
_larch=None):
"""read athena project file
returns a Group of Groups, one for each Athena Group in the project file
Arguments:
filename (string): name of Athena Project file
match (string): pattern to use to limit imported groups (see Note 1)
do_preedge (bool): whether to do pre-edge subtraction [True]
do_bkg (bool): whether to do XAFS background subtraction [True]
do_fft (bool): whether to do XAFS Fast Fourier transform [True]
use_hashkey (bool): whether to use Athena's hash key as the
group name instead of the Athena label [False]
Returns:
group of groups each named according the label used by Athena.
Notes:
1. To limit the imported groups, use the pattern in `match`,
using '*' to match 'all' '?' to match any single character,
or [sequence] to match any of a sequence of letters. The match
will always be insensitive to case.
3. do_preedge, do_bkg, and do_fft will attempt to reproduce the
pre-edge, background subtraction, and FFT from Athena by using
the parameters saved in the project file.
2. use_hashkey=True will name groups from the internal 5 character
string used by Athena, instead of the group label.
Example:
1. read in all groups from a project file:
cr_data = read_athena('My Cr Project.prj')
2. read in only the "merged" data from a Project, and don't do FFT:
zn_data = read_athena('Zn on Stuff.prj', match='*merge*', do_fft=False)
"""
from larch_plugins.xafs import pre_edge, autobk, xftf
if not os.path.exists(filename):
raise IOError("%s '%s': cannot find file" % (ERR_MSG, filename))
try:
fh = GzipFile(filename)
lines = [bytes2str(t) for t in fh.readlines()]
fh.close()
except:
raise ValueError("%s '%s': invalid gzip file" % (ERR_MSG, filename))
athenagroups = []
dat = {'name': ''}
Athena_version = None
vline = lines.pop(0)
if "Athena project file -- Demeter version" not in vline:
raise ValueError("%s '%s': invalid Athena File" % (ERR_MSG, filename))
major, minor, fix = '0', '0', '0'
try:
vs = vline.split("Athena project file -- Demeter version")[1]
major, minor, fix = vs.split('.')
except:
raise ValueError("%s '%s': cannot read version" % (ERR_MSG, filename))
if int(minor) < 9 or int(fix[:2]) < 21:
raise ValueError("%s '%s': file is too old to read" %
(ERR_MSG, filename))
for t in lines:
if t.startswith('#') or len(t) < 2 or 'undef' in t:
continue
key = t.split(' ')[0].strip()
key = key.replace('$', '').replace('@', '')
if key == 'old_group':
dat['name'] = perl2json(t)
elif key == '[record]':
athenagroups.append(dat)
dat = {'name': ''}
elif key == 'args':
dat['args'] = perl2json(t)
elif key in ('x', 'y', 'i0', 'signal'):
dat[key] = np.array([float(x) for x in perl2json(t)])
if match is not None:
match = match.lower()
out = Group()
out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename)
for dat in athenagroups:
label = dat.get('name', 'unknown')
this = Group(athena_id=label,
energy=dat['x'],
mu=dat['y'],
bkg_params=Group(),
fft_params=Group(),
athena_params=Group())
if 'i0' in dat:
this.i0 = dat['i0']
if 'args' in dat:
for i in range(len(dat['args']) // 2):
key = dat['args'][2 * i]
val = dat['args'][2 * i + 1]
if key.startswith('bkg_'):
setattr(this.bkg_params, key[4:], val)
elif key.startswith('fft_'):
setattr(this.fft_params, key[4:], val)
elif key == 'label':
this.label = val
if not use_hashkey:
label = this.label
else:
setattr(this.athena_params, key, val)
this.__doc__ = """Athena Group Name %s (key='%s')""" % (label,
dat['name'])
olabel = fix_varname(label)
if match is not None:
if not fnmatch(olabel.lower(), match):
continue
if do_preedge or do_bkg:
pars = this.bkg_params
pre_edge(this,
_larch=_larch,
e0=float(pars.e0),
pre1=float(pars.pre1),
pre2=float(pars.pre2),
norm1=float(pars.nor1),
norm2=float(pars.nor2),
nnorm=float(pars.nnorm) - 1,
make_flat=bool(pars.flatten))
if do_bkg and hasattr(pars, 'rbkg'):
autobk(this,
_larch=_larch,
e0=float(pars.e0),
rbkg=float(pars.rbkg),
kmin=float(pars.spl1),
kmax=float(pars.spl2),
kweight=float(pars.kw),
dk=float(pars.dk),
clamp_lo=float(pars.clamp1),
clamp_hi=float(pars.clamp2))
if do_fft:
pars = this.fft_params
kweight = 2
if hasattr(pars, 'kw'):
kweight = float(pars.kw)
xftf(this,
_larch=_larch,
kmin=float(pars.kmin),
kmax=float(pars.kmax),
kweight=kweight,
window=pars.kwindow,
dk=float(pars.dk))
setattr(out, olabel, this)
return out
10067.28 120488.7 89110.0998902 0.30168329
10073.72 118833.7 88265.1000656 0.29738025
10080.18 118434.7 88372.1004302 0.29280544
10086.66 117995.7 88449.0998063 0.28822094
10093.17 118435.7 89180.0997098 0.28371228
10099.69 117303.7 88720.0998253 0.27927983
10106.22 117929.7 89581.1003571 0.27494432
10112.78 116857.7 89144.1003332 0.27070279
10119.36 115791.7 88718.1000129 0.26632896
10125.96 111467.7 85797.099695 0.26174966
10132.57 110079.7 85128.099834 0.25704747
10139.21 104190.7 80953.0999403 0.2523529
10145.86 93726.7 73074.0996945 0.24890911'''
raw_data_lines = raw_data.split('\n')
raw_data_table = []
for line in raw_data_lines:
raw_data_table.append(list(map(lambda x: float(x), line.strip().split())))
table = np.array(raw_data_table)
group.energy = table[:, 0]
group.mu = table[:, 3]
e0 = find_e0(group, _larch=mylarch)
pre_edge(group, _larch=mylarch)
autobk(group, _larch=mylarch)
xftf(group, _larch=mylarch)
xftr(group, _larch=mylarch)
def autobk(energy,
mu=None,
group=None,
rbkg=1,
nknots=None,
e0=None,
edge_step=None,
kmin=0,
kmax=None,
kweight=1,
dk=0,
win='hanning',
k_std=None,
chi_std=None,
nfft=2048,
kstep=0.05,
pre_edge_kws=None,
nclamp=4,
clamp_lo=1,
clamp_hi=1,
calc_uncertainties=False,
_larch=None,
**kws):
"""Use Autobk algorithm to remove XAFS background
Parameters:
-----------
energy: 1-d array of x-ray energies, in eV, or group
mu: 1-d array of mu(E)
group: output group (and input group for e0 and edge_step).
rbkg: distance (in Ang) for chi(R) above
which the signal is ignored. Default = 1.
e0: edge energy, in eV. If None, it will be determined.
edge_step: edge step. If None, it will be determined.
pre_edge_kws: keyword arguments to pass to pre_edge()
nknots: number of knots in spline. If None, it will be determined.
kmin: minimum k value [0]
kmax: maximum k value [full data range].
kweight: k weight for FFT. [1]
dk: FFT window window parameter. [0]
win: FFT window function name. ['hanning']
nfft: array size to use for FFT [2048]
kstep: k step size to use for FFT [0.05]
k_std: optional k array for standard chi(k).
chi_std: optional chi array for standard chi(k).
nclamp: number of energy end-points for clamp [2]
clamp_lo: weight of low-energy clamp [1]
clamp_hi: weight of high-energy clamp [1]
calc_uncertaintites: Flag to calculate uncertainties in
mu_0(E) and chi(k) [False]
Output arrays are written to the provided group.
Follows the 'First Argument Group' convention.
"""
msg = _larch.writer.write
if 'kw' in kws:
kweight = kws.pop('kw')
if len(kws) > 0:
msg('Unrecognized a:rguments for autobk():\n')
msg(' %s\n' % (', '.join(kws.keys())))
return
energy, mu, group = parse_group_args(energy,
members=('energy', 'mu'),
defaults=(mu, ),
group=group,
fcn_name='autobk')
energy = remove_dups(energy)
# if e0 or edge_step are not specified, get them, either from the
# passed-in group or from running pre_edge()
group = set_xafsGroup(group, _larch=_larch)
if edge_step is None and isgroup(group, 'edge_step'):
edge_step = group.edge_step
if e0 is None and isgroup(group, 'e0'):
e0 = group.e0
if e0 is None or edge_step is None:
# need to run pre_edge:
pre_kws = dict(nnorm=3,
nvict=0,
pre1=None,
pre2=-50.,
norm1=100.,
norm2=None)
if pre_edge_kws is not None:
pre_kws.update(pre_edge_kws)
pre_edge(energy, mu, group=group, _larch=_larch, **pre_kws)
if e0 is None:
e0 = group.e0
if edge_step is None:
edge_step = group.edge_step
if e0 is None or edge_step is None:
msg('autobk() could not determine e0 or edge_step!: trying running pre_edge first\n'
)
return
# get array indices for rkbg and e0: irbkg, ie0
ie0 = index_of(energy, e0)
rgrid = np.pi / (kstep * nfft)
if rbkg < 2 * rgrid: rbkg = 2 * rgrid
irbkg = int(1.01 + rbkg / rgrid)
# save ungridded k (kraw) and grided k (kout)
# and ftwin (*k-weighting) for FT in residual
enpe = energy[ie0:] - e0
kraw = np.sign(enpe) * np.sqrt(ETOK * abs(enpe))
if kmax is None:
kmax = max(kraw)
else:
kmax = max(0, min(max(kraw), kmax))
kout = kstep * np.arange(int(1.01 + kmax / kstep), dtype='float64')
iemax = min(len(energy), 2 + index_of(energy, e0 + kmax * kmax / ETOK)) - 1
# interpolate provided chi(k) onto the kout grid
if chi_std is not None and k_std is not None:
chi_std = np.interp(kout, k_std, chi_std)
# pre-load FT window
ftwin = kout**kweight * ftwindow(
kout, xmin=kmin, xmax=kmax, window=win, dx=dk)
# calc k-value and initial guess for y-values of spline params
nspl = max(4, min(128, 2 * int(rbkg * (kmax - kmin) / np.pi) + 1))
spl_y, spl_k, spl_e = np.zeros(nspl), np.zeros(nspl), np.zeros(nspl)
for i in range(nspl):
q = kmin + i * (kmax - kmin) / (nspl - 1)
ik = index_nearest(kraw, q)
i1 = min(len(kraw) - 1, ik + 5)
i2 = max(0, ik - 5)
spl_k[i] = kraw[ik]
spl_e[i] = energy[ik + ie0]
spl_y[i] = (2 * mu[ik + ie0] + mu[i1 + ie0] + mu[i2 + ie0]) / 4.0
# get spline represention: knots, coefs, order=3
# coefs will be varied in fit.
knots, coefs, order = splrep(spl_k, spl_y)
# set fit parameters from initial coefficients
params = Group()
for i in range(len(coefs)):
name = FMT_COEF % i
p = Parameter(coefs[i], name=name, vary=i < len(spl_y))
p._getval()
setattr(params, name, p)
initbkg, initchi = spline_eval(kraw[:iemax - ie0 + 1], mu[ie0:iemax + 1],
knots, coefs, order, kout)
# do fit
fit = Minimizer(__resid,
params,
_larch=_larch,
toler=1.e-4,
fcn_kws=dict(ncoefs=len(coefs),
chi_std=chi_std,
knots=knots,
order=order,
kraw=kraw[:iemax - ie0 + 1],
mu=mu[ie0:iemax + 1],
irbkg=irbkg,
kout=kout,
ftwin=ftwin,
kweight=kweight,
nfft=nfft,
nclamp=nclamp,
clamp_lo=clamp_lo,
clamp_hi=clamp_hi))
fit.leastsq()
# write final results
coefs = [getattr(params, FMT_COEF % i) for i in range(len(coefs))]
bkg, chi = spline_eval(kraw[:iemax - ie0 + 1], mu[ie0:iemax + 1], knots,
coefs, order, kout)
obkg = np.copy(mu)
obkg[ie0:ie0 + len(bkg)] = bkg
# outputs to group
group = set_xafsGroup(group, _larch=_larch)
group.bkg = obkg
group.chie = (mu - obkg) / edge_step
group.k = kout
group.chi = chi / edge_step
# now fill in 'autobk_details' group
params.init_bkg = np.copy(mu)
params.init_bkg[ie0:ie0 + len(bkg)] = initbkg
params.init_chi = initchi / edge_step
params.knots_e = spl_e
params.knots_y = np.array([coefs[i] for i in range(nspl)])
params.init_knots_y = spl_y
params.nfev = params.fit_details.nfev
params.kmin = kmin
params.kmax = kmax
group.autobk_details = params
# uncertainties in mu0 and chi: fairly slow!!
if HAS_UNCERTAIN and calc_uncertainties:
vbest, vstd = [], []
for n in fit.var_names:
par = getattr(params, n)
vbest.append(par.value)
vstd.append(par.stderr)
uvars = uncertainties.correlated_values(vbest, params.covar)
# uncertainty in bkg (aka mu0)
# note that much of this is working around
# limitations in the uncertainty package that make it
# 1. take an argument list (not array)
# 2. work on returned scalars (but not arrays)
# 3. not handle kw args and *args well (so use
# of global "index" is important here)
nkx = iemax - ie0 + 1
def my_dsplev(*args):
coefs = np.array(args)
return splev(kraw[:nkx], [knots, coefs, order])[index]
fdbkg = uncertainties.wrap(my_dsplev)
dmu0 = [fdbkg(*uvars).std_dev() for index in range(len(bkg))]
group.delta_bkg = np.zeros(len(mu))
group.delta_bkg[ie0:ie0 + len(bkg)] = np.array(dmu0)
# uncertainty in chi (see notes above)
def my_dchi(*args):
coefs = np.array(args)
b, chi = spline_eval(kraw[:nkx], mu[ie0:iemax + 1], knots, coefs,
order, kout)
return chi[index]
fdchi = uncertainties.wrap(my_dchi)
dchi = [fdchi(*uvars).std_dev() for index in range(len(kout))]
group.delta_chi = np.array(dchi) / edge_step
def xas_process(self, gname, new_mu=False, **kws):
""" process (pre-edge/normalize) XAS data from XAS form, overwriting
larch group '_y1_' attribute to be plotted
"""
dgroup = getattr(self.larch.symtable, gname)
if not hasattr(dgroup, 'energy'):
dgroup.energy = dgroup._xdat
if not hasattr(dgroup, 'mu'):
dgroup.mu = dgroup._ydat
e0 = None
if not self.xas_autoe0.IsChecked():
_e0 = self.xas_e0.GetValue()
if _e0 < max(dgroup.energy) and _e0 > min(dgroup.energy):
e0 = float(_e0)
preopts = {'e0': e0}
if not self.xas_autostep.IsChecked():
preopts['step'] = self.xas_step.GetValue()
preopts['pre1'] = self.xas_pre1.GetValue()
preopts['pre2'] = self.xas_pre2.GetValue()
preopts['norm1'] = self.xas_nor1.GetValue()
preopts['norm2'] = self.xas_nor2.GetValue()
preopts['nvict'] = self.xas_vict.GetSelection()
preopts['nnorm'] = self.xas_nnor.GetSelection()
preopts['make_flat'] = False
preopts['_larch'] = self.larch
pre_edge(dgroup, **preopts)
dgroup.pre_edge_details.e0 = dgroup.e0
dgroup.pre_edge_details.edge_step = dgroup.edge_step
dgroup.pre_edge_details.auto_e0 = self.xas_autoe0.IsChecked()
dgroup.pre_edge_details.show_e0 = self.xas_showe0.IsChecked()
dgroup.pre_edge_details.auto_step = self.xas_autostep.IsChecked()
if self.xas_autoe0.IsChecked():
self.xas_e0.SetValue(dgroup.e0)
if self.xas_autostep.IsChecked():
self.xas_step.SetValue(dgroup.edge_step)
details_group = dgroup.pre_edge_details
self.xas_pre1.SetValue(details_group.pre1)
self.xas_pre2.SetValue(details_group.pre2)
self.xas_nor1.SetValue(details_group.norm1)
self.xas_nor2.SetValue(details_group.norm2)
dgroup.orig_ylabel = dgroup.plot_ylabel
dgroup.plot_ylabel = '$\mu$'
dgroup.plot_y2label = None
dgroup.plot_xlabel = '$E \,\mathrm{(eV)}$'
dgroup.plot_yarrays = [(dgroup.mu, PLOTOPTS_1, dgroup.plot_ylabel)]
y4e0 = dgroup.mu
out = self.xas_op.GetStringSelection().lower() # raw, pre, norm, flat
if out.startswith('raw data + pre'):
dgroup.plot_yarrays = [(dgroup.mu, PLOTOPTS_1, '$\mu$'),
(dgroup.pre_edge, PLOTOPTS_2, 'pre edge'),
(dgroup.post_edge, PLOTOPTS_2, 'post edge')]
elif out.startswith('pre'):
dgroup.pre_edge_sub = dgroup.norm * dgroup.edge_step
dgroup.plot_yarrays = [(dgroup.pre_edge_sub, PLOTOPTS_1,
'pre-edge subtracted $\mu$')]
y4e0 = dgroup.pre_edge_sub
dgroup.plot_ylabel = 'pre-edge subtracted $\mu$'
elif 'norm' in out and 'deriv' in out:
dgroup.plot_yarrays = [(dgroup.norm, PLOTOPTS_1,
'normalized $\mu$'),
(dgroup.dmude, PLOTOPTS_D, '$d\mu/dE$')]
y4e0 = dgroup.norm
dgroup.plot_ylabel = 'normalized $\mu$'
dgroup.plot_y2label = '$d\mu/dE$'
elif out.startswith('norm'):
dgroup.plot_yarrays = [(dgroup.norm, PLOTOPTS_1,
'normalized $\mu$')]
y4e0 = dgroup.norm
dgroup.plot_ylabel = 'normalized $\mu$'
elif out.startswith('deriv'):
dgroup.plot_yarrays = [(dgroup.dmude, PLOTOPTS_1, '$d\mu/dE$')]
y4e0 = dgroup.dmude
dgroup.plot_ylabel = '$d\mu/dE$'
dgroup.plot_ymarkers = []
if self.xas_showe0.IsChecked():
ie0 = index_of(dgroup._xdat, dgroup.e0)
dgroup.plot_ymarkers = [(dgroup.e0, y4e0[ie0], {'label': 'e0'})]
return
def autobk(energy, mu=None, group=None, rbkg=1, nknots=None, e0=None,
edge_step=None, kmin=0, kmax=None, kweight=1, dk=0,
win='hanning', k_std=None, chi_std=None, nfft=2048, kstep=0.05,
pre_edge_kws=None, nclamp=4, clamp_lo=1, clamp_hi=1,
calc_uncertainties=True, err_sigma=1, _larch=None, **kws):
"""Use Autobk algorithm to remove XAFS background
Parameters:
-----------
energy: 1-d array of x-ray energies, in eV, or group
mu: 1-d array of mu(E)
group: output group (and input group for e0 and edge_step).
rbkg: distance (in Ang) for chi(R) above
which the signal is ignored. Default = 1.
e0: edge energy, in eV. If None, it will be determined.
edge_step: edge step. If None, it will be determined.
pre_edge_kws: keyword arguments to pass to pre_edge()
nknots: number of knots in spline. If None, it will be determined.
kmin: minimum k value [0]
kmax: maximum k value [full data range].
kweight: k weight for FFT. [1]
dk: FFT window window parameter. [0]
win: FFT window function name. ['hanning']
nfft: array size to use for FFT [2048]
kstep: k step size to use for FFT [0.05]
k_std: optional k array for standard chi(k).
chi_std: optional chi array for standard chi(k).
nclamp: number of energy end-points for clamp [2]
clamp_lo: weight of low-energy clamp [1]
clamp_hi: weight of high-energy clamp [1]
calc_uncertaintites: Flag to calculate uncertainties in
mu_0(E) and chi(k) [True]
err_sigma: sigma level for uncertainties in mu_0(E) and chi(k) [1]
Output arrays are written to the provided group.
Follows the 'First Argument Group' convention.
"""
msg = _larch.writer.write
if 'kw' in kws:
kweight = kws.pop('kw')
if len(kws) > 0:
msg('Unrecognized a:rguments for autobk():\n')
msg(' %s\n' % (', '.join(kws.keys())))
return
energy, mu, group = parse_group_args(energy, members=('energy', 'mu'),
defaults=(mu,), group=group,
fcn_name='autobk')
if len(energy.shape) > 1:
energy = energy.squeeze()
if len(mu.shape) > 1:
mu = mu.squeeze()
energy = remove_dups(energy)
# if e0 or edge_step are not specified, get them, either from the
# passed-in group or from running pre_edge()
group = set_xafsGroup(group, _larch=_larch)
if edge_step is None and isgroup(group, 'edge_step'):
edge_step = group.edge_step
if e0 is None and isgroup(group, 'e0'):
e0 = group.e0
if e0 is None or edge_step is None:
# need to run pre_edge:
pre_kws = dict(nnorm=3, nvict=0, pre1=None,
pre2=-50., norm1=100., norm2=None)
if pre_edge_kws is not None:
pre_kws.update(pre_edge_kws)
pre_edge(energy, mu, group=group, _larch=_larch, **pre_kws)
if e0 is None:
e0 = group.e0
if edge_step is None:
edge_step = group.edge_step
if e0 is None or edge_step is None:
msg('autobk() could not determine e0 or edge_step!: trying running pre_edge first\n')
return
# get array indices for rkbg and e0: irbkg, ie0
ie0 = index_of(energy, e0)
rgrid = np.pi/(kstep*nfft)
if rbkg < 2*rgrid: rbkg = 2*rgrid
irbkg = int(1.01 + rbkg/rgrid)
# save ungridded k (kraw) and grided k (kout)
# and ftwin (*k-weighting) for FT in residual
enpe = energy[ie0:] - e0
kraw = np.sign(enpe)*np.sqrt(ETOK*abs(enpe))
if kmax is None:
kmax = max(kraw)
else:
kmax = max(0, min(max(kraw), kmax))
kout = kstep * np.arange(int(1.01+kmax/kstep), dtype='float64')
iemax = min(len(energy), 2+index_of(energy, e0+kmax*kmax/ETOK)) - 1
# interpolate provided chi(k) onto the kout grid
if chi_std is not None and k_std is not None:
chi_std = np.interp(kout, k_std, chi_std)
# pre-load FT window
ftwin = kout**kweight * ftwindow(kout, xmin=kmin, xmax=kmax,
window=win, dx=dk)
# calc k-value and initial guess for y-values of spline params
nspl = max(4, min(128, 2*int(rbkg*(kmax-kmin)/np.pi) + 1))
spl_y, spl_k, spl_e = np.zeros(nspl), np.zeros(nspl), np.zeros(nspl)
for i in range(nspl):
q = kmin + i*(kmax-kmin)/(nspl - 1)
ik = index_nearest(kraw, q)
i1 = min(len(kraw)-1, ik + 5)
i2 = max(0, ik - 5)
spl_k[i] = kraw[ik]
spl_e[i] = energy[ik+ie0]
spl_y[i] = (2*mu[ik+ie0] + mu[i1+ie0] + mu[i2+ie0] ) / 4.0
# get spline represention: knots, coefs, order=3
# coefs will be varied in fit.
knots, coefs, order = splrep(spl_k, spl_y)
# set fit parameters from initial coefficients
params = Parameters()
for i in range(len(coefs)):
params.add(name = FMT_COEF % i, value=coefs[i], vary=i<len(spl_y))
initbkg, initchi = spline_eval(kraw[:iemax-ie0+1], mu[ie0:iemax+1],
knots, coefs, order, kout)
# do fit
result = minimize(__resid, params, method='leastsq',
gtol=1.e-5, ftol=1.e-5, xtol=1.e-5, epsfcn=1.e-5,
kws = dict(ncoefs=len(coefs), chi_std=chi_std,
knots=knots, order=order,
kraw=kraw[:iemax-ie0+1],
mu=mu[ie0:iemax+1], irbkg=irbkg, kout=kout,
ftwin=ftwin, kweight=kweight,
nfft=nfft, nclamp=nclamp,
clamp_lo=clamp_lo, clamp_hi=clamp_hi))
# write final results
coefs = [result.params[FMT_COEF % i].value for i in range(len(coefs))]
bkg, chi = spline_eval(kraw[:iemax-ie0+1], mu[ie0:iemax+1],
knots, coefs, order, kout)
obkg = np.copy(mu)
obkg[ie0:ie0+len(bkg)] = bkg
# outputs to group
group = set_xafsGroup(group, _larch=_larch)
group.bkg = obkg
group.chie = (mu-obkg)/edge_step
group.k = kout
group.chi = chi/edge_step
# now fill in 'autobk_details' group
details = Group(params=result.params)
details.init_bkg = np.copy(mu)
details.init_bkg[ie0:ie0+len(bkg)] = initbkg
details.init_chi = initchi/edge_step
details.knots_e = spl_e
details.knots_y = np.array([coefs[i] for i in range(nspl)])
details.init_knots_y = spl_y
details.nfev = result.nfev
details.kmin = kmin
details.kmax = kmax
group.autobk_details = details
# uncertainties in mu0 and chi: can be fairly slow.
if calc_uncertainties:
nchi = len(chi)
nmue = iemax-ie0 + 1
redchi = result.redchi
covar = result.covar / redchi
jac_chi = np.zeros(nchi*nspl).reshape((nspl, nchi))
jac_bkg = np.zeros(nmue*nspl).reshape((nspl, nmue))
cvals, cerrs = [], []
for i in range(len(coefs)):
par = result.params[FMT_COEF % i]
cvals.append(getattr(par, 'value', 0.0))
cdel = getattr(par, 'stderr', 0.0)
if cdel is None:
cdel = 0.0
cerrs.append(cdel/2.0)
cvals = np.array(cvals)
cerrs = np.array(cerrs)
# find derivatives by hand!
_k = kraw[:nmue]
_m = mu[ie0:iemax+1]
for i in range(nspl):
cval0 = cvals[i]
cvals[i] = cval0 + cerrs[i]
bkg1, chi1 = spline_eval(_k, _m, knots, cvals, order, kout)
cvals[i] = cval0 - cerrs[i]
bkg2, chi2 = spline_eval(_k, _m, knots, cvals, order, kout)
cvals[i] = cval0
jac_chi[i] = (chi1 - chi2) / (2*cerrs[i])
jac_bkg[i] = (bkg1 - bkg2) / (2*cerrs[i])
dfchi = np.zeros(nchi)
dfbkg = np.zeros(nmue)
for i in range(nspl):
for j in range(nspl):
dfchi += jac_chi[i]*jac_chi[j]*covar[i,j]
dfbkg += jac_bkg[i]*jac_bkg[j]*covar[i,j]
prob = 0.5*(1.0 + erf(err_sigma/np.sqrt(2.0)))
dchi = t.ppf(prob, nchi-nspl) * np.sqrt(dfchi*redchi)
dbkg = t.ppf(prob, nmue-nspl) * np.sqrt(dfbkg*redchi)
group.delta_chi = dchi
group.delta_bkg = 0.0*mu
group.delta_bkg[ie0:ie0+len(dbkg)] = dbkg
def read(self,
filename=None,
match=None,
do_preedge=True,
do_bkg=True,
do_fft=True,
use_hashkey=False):
"""
read Athena project to group of groups, one for each Athena dataset
in the project file. This supports both gzipped and unzipped files
and old-style perl-like project files and new-style JSON project files
Arguments:
filename (string): name of Athena Project file
match (string): pattern to use to limit imported groups (see Note 1)
do_preedge (bool): whether to do pre-edge subtraction [True]
do_bkg (bool): whether to do XAFS background subtraction [True]
do_fft (bool): whether to do XAFS Fast Fourier transform [True]
use_hashkey (bool): whether to use Athena's hash key as the
group name instead of the Athena label [False]
Returns:
None, fills in attributes `header`, `journal`, `filename`, `groups`
Notes:
1. To limit the imported groups, use the pattern in `match`,
using '*' to match 'all', '?' to match any single character,
or [sequence] to match any of a sequence of letters. The match
will always be insensitive to case.
3. do_preedge, do_bkg, and do_fft will attempt to reproduce the
pre-edge, background subtraction, and FFT from Athena by using
the parameters saved in the project file.
2. use_hashkey=True will name groups from the internal 5 character
string used by Athena, instead of the group label.
Example:
1. read in all groups from a project file:
cr_data = read_athena('My Cr Project.prj')
2. read in only the "merged" data from a Project, and don't do FFT:
zn_data = read_athena('Zn on Stuff.prj', match='*merge*', do_fft=False)
"""
if filename is not None:
self.filename = filename
if not os.path.exists(self.filename):
raise IOError("%s '%s': cannot find file" %
(ERR_MSG, self.filename))
from larch_plugins.xafs import pre_edge, autobk, xftf
if not os.path.exists(filename):
raise IOError("file '%s' not found" % filename)
text = _read_raw_athena(filename)
# failed to read:
if text is None:
raise OSError(errval)
if not _test_athena_text(text):
raise ValueError("%s '%s': invalid Athena File" %
(ERR_MSG, filename))
# decode JSON or Perl format
data = None
try:
data = parse_jsonathena(text, self.filename)
except ValueError:
# try as perl format
# print("Not json-athena ", sys.exc_info())
try:
data = parse_perlathena(text, self.filename)
except:
# print("Not perl-athena ", sys.exc_info())
pass
if data is None:
raise ValueError("cannot read file '%s' as Athena Project File" %
(self.filename))
self.header = data.header
self.journal = data.journal
self.group_names = data.group_names
for gname in data.group_names:
oname = gname
if match is not None:
if not fnmatch(gname.lower(), match):
continue
this = getattr(data, gname)
if use_hashkey:
oname = this.athena_id
if (do_preedge or do_bkg) and (self._larch is not None):
pars = this.bkg_params
pre_edge(this,
e0=float(pars.e0),
pre1=float(pars.pre1),
pre2=float(pars.pre2),
norm1=float(pars.nor1),
norm2=float(pars.nor2),
nnorm=float(pars.nnorm),
make_flat=bool(pars.flatten),
_larch=self._larch)
if do_bkg and hasattr(pars, 'rbkg'):
autobk(this,
_larch=self._larch,
e0=float(pars.e0),
rbkg=float(pars.rbkg),
kmin=float(pars.spl1),
kmax=float(pars.spl2),
kweight=float(pars.kw),
dk=float(pars.dk),
clamp_lo=float(pars.clamp1),
clamp_hi=float(pars.clamp2))
if do_fft:
pars = this.fft_params
kweight = 2
if hasattr(pars, 'kw'):
kweight = float(pars.kw)
xftf(this,
_larch=self._larch,
kmin=float(pars.kmin),
kmax=float(pars.kmax),
kweight=kweight,
window=pars.kwindow,
dk=float(pars.dk))
self.groups[oname] = this
#!/usr/bin/env python ## Autobk (XAFS background subtraction) in pure Python, ## using Python code from Lxsarch. from larch import Interpreter from larch_plugins.xafs import pre_edge, autobk from larch_plugins.io import read_ascii # create plain interpreter, don't load all the plugins _larch = Interpreter(with_plugins=False) fname = '../xafsdata/cu_rt01.xmu' cu = read_ascii(fname, labels='energy mu i0', _larch=_larch) print( 'Read ASCII File:', cu) print( dir(cu)) pre_edge(cu, _larch=_larch) print( 'After pre-edge:') print( dir(cu)) autobk(cu, rbkg=1.0, kweight=1, _larch=_larch) print( 'After autobk:') print( dir(cu))
def read(self, filename=None, match=None, do_preedge=True, do_bkg=True,
do_fft=True, use_hashkey=False):
"""
read Athena project to group of groups, one for each Athena dataset
in the project file. This supports both gzipped and unzipped files
and old-style perl-like project files and new-style JSON project files
Arguments:
filename (string): name of Athena Project file
match (string): pattern to use to limit imported groups (see Note 1)
do_preedge (bool): whether to do pre-edge subtraction [True]
do_bkg (bool): whether to do XAFS background subtraction [True]
do_fft (bool): whether to do XAFS Fast Fourier transform [True]
use_hashkey (bool): whether to use Athena's hash key as the
group name instead of the Athena label [False]
Returns:
None, fills in attributes `header`, `journal`, `filename`, `groups`
Notes:
1. To limit the imported groups, use the pattern in `match`,
using '*' to match 'all', '?' to match any single character,
or [sequence] to match any of a sequence of letters. The match
will always be insensitive to case.
3. do_preedge, do_bkg, and do_fft will attempt to reproduce the
pre-edge, background subtraction, and FFT from Athena by using
the parameters saved in the project file.
2. use_hashkey=True will name groups from the internal 5 character
string used by Athena, instead of the group label.
Example:
1. read in all groups from a project file:
cr_data = read_athena('My Cr Project.prj')
2. read in only the "merged" data from a Project, and don't do FFT:
zn_data = read_athena('Zn on Stuff.prj', match='*merge*', do_fft=False)
"""
if filename is not None:
self.filename = filename
if not os.path.exists(self.filename):
raise IOError("%s '%s': cannot find file" % (ERR_MSG, self.filename))
from larch_plugins.xafs import pre_edge, autobk, xftf
if not os.path.exists(filename):
raise IOError("file '%s' not found" % filename)
text = _read_raw_athena(filename)
# failed to read:
if text is None:
raise OSError(errval)
if not _test_athena_text(text):
raise ValueError("%s '%s': invalid Athena File" % (ERR_MSG, filename))
# decode JSON or Perl format
data = None
try:
data = parse_jsonathena(text, self.filename)
except ValueError:
# try as perl format
# print("Not json-athena ", sys.exc_info())
try:
data = parse_perlathena(text, self.filename)
except:
# print("Not perl-athena ", sys.exc_info())
pass
if data is None:
raise ValueError("cannot read file '%s' as Athena Project File" % (self.filename))
self.header = data.header
self.journal = data.journal
self.group_names = data.group_names
for gname in data.group_names:
oname = gname
if match is not None:
if not fnmatch(gname.lower(), match):
continue
this = getattr(data, gname)
if use_hashkey:
oname = this.athena_id
if (do_preedge or do_bkg) and (self._larch is not None):
pars = this.bkg_params
pre_edge(this, e0=float(pars.e0),
pre1=float(pars.pre1), pre2=float(pars.pre2),
norm1=float(pars.nor1), norm2=float(pars.nor2),
nnorm=float(pars.nnorm),
make_flat=bool(pars.flatten), _larch=self._larch)
if do_bkg and hasattr(pars, 'rbkg'):
autobk(this, _larch=self._larch, e0=float(pars.e0),
rbkg=float(pars.rbkg), kmin=float(pars.spl1),
kmax=float(pars.spl2), kweight=float(pars.kw),
dk=float(pars.dk), clamp_lo=float(pars.clamp1),
clamp_hi=float(pars.clamp2))
if do_fft:
pars = this.fft_params
kweight=2
if hasattr(pars, 'kw'):
kweight = float(pars.kw)
xftf(this, _larch=self._larch, kmin=float(pars.kmin),
kmax=float(pars.kmax), kweight=kweight,
window=pars.kwindow, dk=float(pars.dk))
self.groups[oname] = this
def read_athena(filename, match=None, do_preedge=True, do_bkg=True, do_fft=True, use_hashkey=False, _larch=None):
"""read athena project file
returns a Group of Groups, one for each Athena Group in the project file
Arguments:
filename (string): name of Athena Project file
match (sring): pattern to use to limit imported groups (see Note 1)
do_preedge (bool): whether to do pre-edge subtraction [True]
do_bkg (bool): whether to do XAFS background subtraction [True]
do_fft (bool): whether to do XAFS Fast Fourier transform [True]
use_hashkey (bool): whether to use Athena's hash key as the
group name instead of the Athena label [False]
Returns:
group of groups each named according the label used by Athena.
Notes:
1. To limit the imported groups, use the pattern in `match`,
using '*' to match 'all' '?' to match any single character,
or [sequence] to match any of a sequence of letters. The match
will always be insensitive to case.
3. do_preedge, do_bkg, and do_fft will attempt to reproduce the
pre-edge, background subtraction, and FFT from Athena by using
the parameters saved in the project file.
2. use_hashkey=True will name groups from the internal 5 character
string used by Athena, instead of the group label.
Example:
1. read in all groups from a project file:
cr_data = read_athena('My Cr Project.prj')
2. read in only the "merged" data from a Project, and don't do FFT:
zn_data = read_athena('Zn on Stuff.prj', match='*merge*', do_fft=False)
"""
from larch_plugins.xafs import pre_edge, autobk, xftf
if not os.path.exists(filename):
raise IOError("%s '%s': cannot find file" % (ERR_MSG, filename))
try:
fh = GzipFile(filename)
lines = [bytes2str(t) for t in fh.readlines()]
fh.close()
except:
raise ValueError("%s '%s': invalid gzip file" % (ERR_MSG, filename))
athenagroups = []
dat = {"name": ""}
Athena_version = None
vline = lines.pop(0)
if "Athena project file -- Demeter version" not in vline:
raise ValueError("%s '%s': invalid Athena File" % (ERR_MSG, filename))
major, minor, fix = "0", "0", "0"
try:
vs = vline.split("Athena project file -- Demeter version")[1]
major, minor, fix = vs.split(".")
except:
raise ValueError("%s '%s': cannot read version" % (ERR_MSG, filename))
if int(minor) < 9 or int(fix[:2]) < 21:
raise ValueError("%s '%s': file is too old to read" % (ERR_MSG, filename))
for t in lines:
if t.startswith("#") or len(t) < 2:
continue
key = t.split(" ")[0].strip()
key = key.replace("$", "").replace("@", "")
if key == "old_group":
dat["name"] = perl2json(t)
elif key == "[record]":
athenagroups.append(dat)
dat = {"name": ""}
elif key == "args":
dat["args"] = perl2json(t)
elif key in ("x", "y", "i0"):
dat[key] = np.array([float(x) for x in perl2json(t)])
if match is not None:
match = match.lower()
out = Group()
out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename)
for dat in athenagroups:
label = dat["name"]
this = Group(
athena_id=label, energy=dat["x"], mu=dat["y"], bkg_params=Group(), fft_params=Group(), athena_params=Group()
)
if "i0" in dat:
this.i0 = dat["i0"]
if "args" in dat:
for i in range(len(dat["args"]) // 2):
key = dat["args"][2 * i]
val = dat["args"][2 * i + 1]
if key.startswith("bkg_"):
setattr(this.bkg_params, key[4:], val)
elif key.startswith("fft_"):
setattr(this.fft_params, key[4:], val)
elif key == "label":
this.label = val
if not use_hashkey:
label = this.label
else:
setattr(this.athena_params, key, val)
this.__doc__ = """Athena Group Name %s (key='%s')""" % (label, dat["name"])
olabel = fix_varname(label)
if match is not None:
if not fnmatch(olabel.lower(), match):
continue
if do_preedge or do_bkg:
pars = this.bkg_params
pre_edge(
this,
_larch=_larch,
e0=float(pars.e0),
pre1=float(pars.pre1),
pre2=float(pars.pre2),
norm1=float(pars.nor1),
norm2=float(pars.nor2),
nnorm=float(pars.nnorm) - 1,
make_flat=bool(pars.flatten),
)
if do_bkg and hasattr(pars, "rbkg"):
autobk(
this,
_larch=_larch,
e0=float(pars.e0),
rbkg=float(pars.rbkg),
kmin=float(pars.spl1),
kmax=float(pars.spl2),
kweight=float(pars.kw),
dk=float(pars.dk),
clamp_lo=float(pars.clamp1),
clamp_hi=float(pars.clamp2),
)
if do_fft:
pars = this.fft_params
kweight = 2
if hasattr(pars, "kw"):
kweight = float(pars.kw)
xftf(
this,
_larch=_larch,
kmin=float(pars.kmin),
kmax=float(pars.kmax),
kweight=kweight,
window=pars.kwindow,
dk=float(pars.dk),
)
setattr(out, olabel, this)
return out
def autobk(energy, mu=None, group=None, rbkg=1, nknots=None, e0=None,
edge_step=None, kmin=0, kmax=None, kweight=1, dk=0.1,
win='hanning', k_std=None, chi_std=None, nfft=2048, kstep=0.05,
pre_edge_kws=None, nclamp=4, clamp_lo=1, clamp_hi=1,
calc_uncertainties=True, err_sigma=1, _larch=None, **kws):
"""Use Autobk algorithm to remove XAFS background
Parameters:
-----------
energy: 1-d array of x-ray energies, in eV, or group
mu: 1-d array of mu(E)
group: output group (and input group for e0 and edge_step).
rbkg: distance (in Ang) for chi(R) above
which the signal is ignored. Default = 1.
e0: edge energy, in eV. If None, it will be determined.
edge_step: edge step. If None, it will be determined.
pre_edge_kws: keyword arguments to pass to pre_edge()
nknots: number of knots in spline. If None, it will be determined.
kmin: minimum k value [0]
kmax: maximum k value [full data range].
kweight: k weight for FFT. [1]
dk: FFT window window parameter. [0.1]
win: FFT window function name. ['hanning']
nfft: array size to use for FFT [2048]
kstep: k step size to use for FFT [0.05]
k_std: optional k array for standard chi(k).
chi_std: optional chi array for standard chi(k).
nclamp: number of energy end-points for clamp [2]
clamp_lo: weight of low-energy clamp [1]
clamp_hi: weight of high-energy clamp [1]
calc_uncertaintites: Flag to calculate uncertainties in
mu_0(E) and chi(k) [True]
err_sigma: sigma level for uncertainties in mu_0(E) and chi(k) [1]
Output arrays are written to the provided group.
Follows the 'First Argument Group' convention.
"""
msg = sys.stdout
if _larch is not None:
msg = _larch.writer.write
if 'kw' in kws:
kweight = kws.pop('kw')
if len(kws) > 0:
msg('Unrecognized a:rguments for autobk():\n')
msg(' %s\n' % (', '.join(kws.keys())))
return
energy, mu, group = parse_group_args(energy, members=('energy', 'mu'),
defaults=(mu,), group=group,
fcn_name='autobk')
if len(energy.shape) > 1:
energy = energy.squeeze()
if len(mu.shape) > 1:
mu = mu.squeeze()
energy = remove_dups(energy)
# if e0 or edge_step are not specified, get them, either from the
# passed-in group or from running pre_edge()
group = set_xafsGroup(group, _larch=_larch)
if edge_step is None and isgroup(group, 'edge_step'):
edge_step = group.edge_step
if e0 is None and isgroup(group, 'e0'):
e0 = group.e0
if e0 is None or edge_step is None:
# need to run pre_edge:
pre_kws = dict(nnorm=3, nvict=0, pre1=None,
pre2=-50., norm1=100., norm2=None)
if pre_edge_kws is not None:
pre_kws.update(pre_edge_kws)
pre_edge(energy, mu, group=group, _larch=_larch, **pre_kws)
if e0 is None:
e0 = group.e0
if edge_step is None:
edge_step = group.edge_step
if e0 is None or edge_step is None:
msg('autobk() could not determine e0 or edge_step!: trying running pre_edge first\n')
return
# get array indices for rkbg and e0: irbkg, ie0
ie0 = index_of(energy, e0)
rgrid = np.pi/(kstep*nfft)
if rbkg < 2*rgrid: rbkg = 2*rgrid
irbkg = int(1.01 + rbkg/rgrid)
# save ungridded k (kraw) and grided k (kout)
# and ftwin (*k-weighting) for FT in residual
enpe = energy[ie0:] - e0
kraw = np.sign(enpe)*np.sqrt(ETOK*abs(enpe))
if kmax is None:
kmax = max(kraw)
else:
kmax = max(0, min(max(kraw), kmax))
kout = kstep * np.arange(int(1.01+kmax/kstep), dtype='float64')
iemax = min(len(energy), 2+index_of(energy, e0+kmax*kmax/ETOK)) - 1
# interpolate provided chi(k) onto the kout grid
if chi_std is not None and k_std is not None:
chi_std = np.interp(kout, k_std, chi_std)
# pre-load FT window
ftwin = kout**kweight * ftwindow(kout, xmin=kmin, xmax=kmax,
window=win, dx=dk, dx2=dk)
# calc k-value and initial guess for y-values of spline params
nspl = max(5, min(64, int(2*rbkg*(kmax-kmin)/np.pi) + 2))
spl_y, spl_k, spl_e = np.zeros(nspl), np.zeros(nspl), np.zeros(nspl)
for i in range(nspl):
q = kmin + i*(kmax-kmin)/(nspl - 1)
ik = index_nearest(kraw, q)
i1 = min(len(kraw)-1, ik + 5)
i2 = max(0, ik - 5)
spl_k[i] = kraw[ik]
spl_e[i] = energy[ik+ie0]
spl_y[i] = (2*mu[ik+ie0] + mu[i1+ie0] + mu[i2+ie0] ) / 4.0
# get spline represention: knots, coefs, order=3
# coefs will be varied in fit.
knots, coefs, order = splrep(spl_k, spl_y)
# set fit parameters from initial coefficients
params = Parameters()
for i in range(len(coefs)):
params.add(name = FMT_COEF % i, value=coefs[i], vary=i<len(spl_y))
initbkg, initchi = spline_eval(kraw[:iemax-ie0+1], mu[ie0:iemax+1],
knots, coefs, order, kout)
# do fit
result = minimize(__resid, params, method='leastsq',
gtol=1.e-5, ftol=1.e-5, xtol=1.e-5, epsfcn=1.e-5,
kws = dict(ncoefs=len(coefs), chi_std=chi_std,
knots=knots, order=order,
kraw=kraw[:iemax-ie0+1],
mu=mu[ie0:iemax+1], irbkg=irbkg, kout=kout,
ftwin=ftwin, kweight=kweight,
nfft=nfft, nclamp=nclamp,
clamp_lo=clamp_lo, clamp_hi=clamp_hi))
# write final results
coefs = [result.params[FMT_COEF % i].value for i in range(len(coefs))]
bkg, chi = spline_eval(kraw[:iemax-ie0+1], mu[ie0:iemax+1],
knots, coefs, order, kout)
obkg = np.copy(mu)
obkg[ie0:ie0+len(bkg)] = bkg
# outputs to group
group = set_xafsGroup(group, _larch=_larch)
group.bkg = obkg
group.chie = (mu-obkg)/edge_step
group.k = kout
group.chi = chi/edge_step
group.e0 = e0
# now fill in 'autobk_details' group
details = Group(params=result.params)
details.init_bkg = np.copy(mu)
details.init_bkg[ie0:ie0+len(bkg)] = initbkg
details.init_chi = initchi/edge_step
details.knots_e = spl_e
details.knots_y = np.array([coefs[i] for i in range(nspl)])
details.init_knots_y = spl_y
details.nfev = result.nfev
details.kmin = kmin
details.kmax = kmax
group.autobk_details = details
# uncertainties in mu0 and chi: can be fairly slow.
if calc_uncertainties:
nchi = len(chi)
nmue = iemax-ie0 + 1
redchi = result.redchi
covar = result.covar / redchi
jac_chi = np.zeros(nchi*nspl).reshape((nspl, nchi))
jac_bkg = np.zeros(nmue*nspl).reshape((nspl, nmue))
cvals, cerrs = [], []
for i in range(len(coefs)):
par = result.params[FMT_COEF % i]
cvals.append(getattr(par, 'value', 0.0))
cdel = getattr(par, 'stderr', 0.0)
if cdel is None:
cdel = 0.0
cerrs.append(cdel/2.0)
cvals = np.array(cvals)
cerrs = np.array(cerrs)
# find derivatives by hand!
_k = kraw[:nmue]
_m = mu[ie0:iemax+1]
for i in range(nspl):
cval0 = cvals[i]
cvals[i] = cval0 + cerrs[i]
bkg1, chi1 = spline_eval(_k, _m, knots, cvals, order, kout)
cvals[i] = cval0 - cerrs[i]
bkg2, chi2 = spline_eval(_k, _m, knots, cvals, order, kout)
cvals[i] = cval0
jac_chi[i] = (chi1 - chi2) / (2*cerrs[i])
jac_bkg[i] = (bkg1 - bkg2) / (2*cerrs[i])
dfchi = np.zeros(nchi)
dfbkg = np.zeros(nmue)
for i in range(nspl):
for j in range(nspl):
dfchi += jac_chi[i]*jac_chi[j]*covar[i,j]
dfbkg += jac_bkg[i]*jac_bkg[j]*covar[i,j]
prob = 0.5*(1.0 + erf(err_sigma/np.sqrt(2.0)))
dchi = t.ppf(prob, nchi-nspl) * np.sqrt(dfchi*redchi)
dbkg = t.ppf(prob, nmue-nspl) * np.sqrt(dfbkg*redchi)
group.delta_chi = dchi
group.delta_bkg = 0.0*mu
group.delta_bkg[ie0:ie0+len(dbkg)] = dbkg
