def parse_jsonathena(text, filename):
"""parse a JSON-style athena file"""
jsdict = json.loads(text)
out = Group()
out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename)
header = []
athena_names = []
for key, val in jsdict.items():
if key.startswith('_____head'):
header.append(val)
elif key.startswith('_____journ'):
journal = val
elif key.startswith('_____order'):
athena_names = val
out.journal = journal
out.header = '\n'.join(header)
out.group_names = []
for name in athena_names:
label = name
dat = jsdict[name]
x = np.array(dat['x'], dtype='float64')
y = np.array(dat['y'], dtype='float64')
this = Group(athena_id=name,
energy=x,
mu=y,
bkg_params=Group(),
fft_params=Group(),
athena_params=Group())
if 'i0' in dat:
this.i0 = np.array(dat['i0'], dtype='float64')
if 'signal' in dat:
this.signal = np.array(dat['signal'], dtype='float64')
if 'stddev' in dat:
this.stddev = np.array(dat['stddev'], dtype='float64')
if 'args' in dat:
for key, val in dat['args'].items():
if key.startswith('bkg_'):
setattr(this.bkg_params, key[4:], asfloat(val))
elif key.startswith('fft_'):
setattr(this.fft_params, key[4:], asfloat(val))
elif key == 'label':
label = this.label = val
else:
setattr(this.athena_params, key, asfloat(val))
this.__doc__ = """Athena Group Name %s (key='%s')""" % (label, name)
name = fix_varname(label)
if name.startswith('_'):
name = 'd' + name
setattr(out, name, this)
out.group_names.append(name)
return out
def parse_jsonathena(text, filename):
"""parse a JSON-style athena file"""
jsdict = json.loads(text)
out = Group()
out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename)
header = []
athena_names = []
for key, val in jsdict.items():
if key.startswith('_____head'):
header.append(val)
elif key.startswith('_____journ'):
journal = val
elif key.startswith('_____order'):
athena_names = val
out.journal = journal
out.header = '\n'.join(header)
out.group_names = []
for name in athena_names:
label = name
dat = jsdict[name]
x = np.array(dat['x'], dtype='float64')
y = np.array(dat['y'], dtype='float64')
this = Group(athena_id=name, energy=x, mu=y,
bkg_params=Group(), fft_params=Group(),
athena_params=Group())
if 'i0' in dat:
this.i0 = np.array(dat['i0'], dtype='float64')
if 'signal' in dat:
this.signal = np.array(dat['signal'], dtype='float64')
if 'stddev' in dat:
this.stddev = np.array(dat['stddev'], dtype='float64')
if 'args' in dat:
for key, val in dat['args'].items():
if key.startswith('bkg_'):
setattr(this.bkg_params, key[4:], asfloat(val))
elif key.startswith('fft_'):
setattr(this.fft_params, key[4:], asfloat(val))
elif key == 'label':
label = this.label = val
else:
setattr(this.athena_params, key, asfloat(val))
this.__doc__ = """Athena Group Name %s (key='%s')""" % (label, name)
name = fix_varname(label)
if name.startswith('_'):
name = 'd' + name
setattr(out, name, this)
out.group_names.append(name)
return out
def as_group(self):
"""convert AthenaProect to Larch group"""
out = Group()
out.__doc__ = """XAFS Data from Athena Project File %s""" % (self.filename)
out._athena_journal = self.journal
out._athena_header = self.header
for name, group in self.groups.items():
setattr(out, name, group)
return out
def as_group(self):
"""convert AthenaProect to Larch group"""
out = Group()
out.__doc__ = """XAFS Data from Athena Project File %s""" % (self.filename)
out._athena_journal = self.journal
out._athena_header = self.header
out._athena_groups = self.groups
for name, group in self.groups.items():
setattr(out, name, group)
return out
def read_xdi(filename, labels=None, _larch=None):
"""simple mapping of XDI file to larch groups"""
xdif = XDIFile(filename, labels=labels)
group = Group()
for key, val in xdif.__dict__.items():
if not key.startswith('_'):
if six.PY3 and key in string_attrs:
val = tostr(val)
setattr(group, key, val)
group.__name__ ='XDI file %s' % filename
doc = ['%i arrays, %i npts' % (xdif.narrays, xdif.npts)]
arr_labels = getattr(xdif, 'array_labels', None)
if arr_labels is not None:
doc.append("Array Labels: %s" % repr(arr_labels))
group.__doc__ = '\n'.join(doc)
group.path = filename
path, fname = os.path.split(filename)
group.filename = fname
return group
def parse_perlathena(text, filename):
"""
parse old athena file format to Group of Groups
"""
lines = text.split('\n')
athenagroups = []
raw = {'name':''}
vline = lines.pop(0)
if "Athena project file -- Demeter version" not in vline:
raise ValueError("%s '%s': invalid Athena File" % (ERR_MSG, filename))
major, minor, fix = '0', '0', '0'
try:
vs = vline.split("Athena project file -- Demeter version")[1]
major, minor, fix = vs.split('.')
except:
raise ValueError("%s '%s': cannot read version" % (ERR_MSG, filename))
if int(minor) < 9 or int(fix[:2]) < 21:
raise ValueError("%s '%s': file is too old to read" % (ERR_MSG, filename))
header = [vline]
journal = {}
is_header = True
for t in lines:
if t.startswith('#') or len(t) < 2 or 'undef' in t:
if is_header:
header.append(t)
continue
is_header = False
key = t.split(' ')[0].strip()
key = key.replace('$', '').replace('@', '')
if key == 'old_group':
raw['name'] = plarray2json(t)
elif key == '[record]':
athenagroups.append(raw)
raw = {'name':''}
elif key == 'journal':
journal = plarray2json(t)
elif key == 'args':
raw['args'] = plarray2json(t)
elif key == 'xdi':
raw['xdi'] = t
elif key in ('x', 'y', 'i0', 'signal', 'stddev'):
raw[key] = np.array([float(x) for x in plarray2json(t)])
elif key == '1;': # end of list
pass
else:
print(" do not know what to do with key ", key, raw['name'])
out = Group()
out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename)
out.journal = journal
out.group_names = []
out.header = '\n'.join(header)
for dat in athenagroups:
label = dat.get('name', 'unknown')
this = Group(athena_id=label, energy=dat['x'], mu=dat['y'],
bkg_params=Group(), fft_params=Group(),
athena_params=Group())
if 'i0' in dat:
this.i0 = dat['i0']
if 'signal' in dat:
this.signal = dat['signal']
if 'stddev' in dat:
this.stddev = dat['stddev']
if 'args' in dat:
for i in range(len(dat['args'])//2):
key = dat['args'][2*i]
val = dat['args'][2*i+1]
if key.startswith('bkg_'):
setattr(this.bkg_params, key[4:], asfloat(val))
elif key.startswith('fft_'):
setattr(this.fft_params, key[4:], asfloat(val))
elif key == 'label':
label = this.label = val
elif key in ('valence', 'lasso_yvalue',
'epsk', 'epsr', 'importance'):
setattr(this, key, asfloat(val))
elif key in ('atsym', 'edge', 'provenance'):
setattr(this, key, val)
else:
setattr(this.athena_params, key, asfloat(val))
this.__doc__ = """Athena Group Name %s (key='%s')""" % (label, dat['name'])
name = fix_varname(label)
if name.startswith('_'):
name = 'd' + name
setattr(out, name, this)
out.group_names.append(name)
return out
def initializeLarchPlugin(_larch=None):
"""initialize _scan"""
if not _larch.symtable.has_group(MODNAME):
g = Group()
g.__doc__ = MODDOC
_larch.symtable.set_symbol(MODNAME, g)
def read_athena(filename, match=None, do_preedge=True, do_bkg=True, do_fft=True, use_hashkey=False, _larch=None):
"""read athena project file
returns a Group of Groups, one for each Athena Group in the project file
Arguments:
filename (string): name of Athena Project file
match (sring): pattern to use to limit imported groups (see Note 1)
do_preedge (bool): whether to do pre-edge subtraction [True]
do_bkg (bool): whether to do XAFS background subtraction [True]
do_fft (bool): whether to do XAFS Fast Fourier transform [True]
use_hashkey (bool): whether to use Athena's hash key as the
group name instead of the Athena label [False]
Returns:
group of groups each named according the label used by Athena.
Notes:
1. To limit the imported groups, use the pattern in `match`,
using '*' to match 'all' '?' to match any single character,
or [sequence] to match any of a sequence of letters. The match
will always be insensitive to case.
3. do_preedge, do_bkg, and do_fft will attempt to reproduce the
pre-edge, background subtraction, and FFT from Athena by using
the parameters saved in the project file.
2. use_hashkey=True will name groups from the internal 5 character
string used by Athena, instead of the group label.
Example:
1. read in all groups from a project file:
cr_data = read_athena('My Cr Project.prj')
2. read in only the "merged" data from a Project, and don't do FFT:
zn_data = read_athena('Zn on Stuff.prj', match='*merge*', do_fft=False)
"""
from larch_plugins.xafs import pre_edge, autobk, xftf
if not os.path.exists(filename):
raise IOError("%s '%s': cannot find file" % (ERR_MSG, filename))
try:
fh = GzipFile(filename)
lines = [bytes2str(t) for t in fh.readlines()]
fh.close()
except:
raise ValueError("%s '%s': invalid gzip file" % (ERR_MSG, filename))
athenagroups = []
dat = {"name": ""}
Athena_version = None
vline = lines.pop(0)
if "Athena project file -- Demeter version" not in vline:
raise ValueError("%s '%s': invalid Athena File" % (ERR_MSG, filename))
major, minor, fix = "0", "0", "0"
try:
vs = vline.split("Athena project file -- Demeter version")[1]
major, minor, fix = vs.split(".")
except:
raise ValueError("%s '%s': cannot read version" % (ERR_MSG, filename))
if int(minor) < 9 or int(fix[:2]) < 21:
raise ValueError("%s '%s': file is too old to read" % (ERR_MSG, filename))
for t in lines:
if t.startswith("#") or len(t) < 2:
continue
key = t.split(" ")[0].strip()
key = key.replace("$", "").replace("@", "")
if key == "old_group":
dat["name"] = perl2json(t)
elif key == "[record]":
athenagroups.append(dat)
dat = {"name": ""}
elif key == "args":
dat["args"] = perl2json(t)
elif key in ("x", "y", "i0"):
dat[key] = np.array([float(x) for x in perl2json(t)])
if match is not None:
match = match.lower()
out = Group()
out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename)
for dat in athenagroups:
label = dat["name"]
this = Group(
athena_id=label, energy=dat["x"], mu=dat["y"], bkg_params=Group(), fft_params=Group(), athena_params=Group()
)
if "i0" in dat:
this.i0 = dat["i0"]
if "args" in dat:
for i in range(len(dat["args"]) // 2):
key = dat["args"][2 * i]
val = dat["args"][2 * i + 1]
if key.startswith("bkg_"):
setattr(this.bkg_params, key[4:], val)
elif key.startswith("fft_"):
setattr(this.fft_params, key[4:], val)
elif key == "label":
this.label = val
if not use_hashkey:
label = this.label
else:
setattr(this.athena_params, key, val)
this.__doc__ = """Athena Group Name %s (key='%s')""" % (label, dat["name"])
olabel = fix_varname(label)
if match is not None:
if not fnmatch(olabel.lower(), match):
continue
if do_preedge or do_bkg:
pars = this.bkg_params
pre_edge(
this,
_larch=_larch,
e0=float(pars.e0),
pre1=float(pars.pre1),
pre2=float(pars.pre2),
norm1=float(pars.nor1),
norm2=float(pars.nor2),
nnorm=float(pars.nnorm) - 1,
make_flat=bool(pars.flatten),
)
if do_bkg and hasattr(pars, "rbkg"):
autobk(
this,
_larch=_larch,
e0=float(pars.e0),
rbkg=float(pars.rbkg),
kmin=float(pars.spl1),
kmax=float(pars.spl2),
kweight=float(pars.kw),
dk=float(pars.dk),
clamp_lo=float(pars.clamp1),
clamp_hi=float(pars.clamp2),
)
if do_fft:
pars = this.fft_params
kweight = 2
if hasattr(pars, "kw"):
kweight = float(pars.kw)
xftf(
this,
_larch=_larch,
kmin=float(pars.kmin),
kmax=float(pars.kmax),
kweight=kweight,
window=pars.kwindow,
dk=float(pars.dk),
)
setattr(out, olabel, this)
return out
def initializeLarchPlugin(_larch=None):
"""initialize _scan"""
if not _larch.symtable.has_group(MODNAME):
g = Group()
g.__doc__ = MODDOC
_larch.symtable.set_symbol(MODNAME, g)
def read_athena(filename,
match=None,
do_preedge=True,
do_bkg=True,
do_fft=True,
use_hashkey=False,
_larch=None):
"""read athena project file
returns a Group of Groups, one for each Athena Group in the project file
Arguments:
filename (string): name of Athena Project file
match (string): pattern to use to limit imported groups (see Note 1)
do_preedge (bool): whether to do pre-edge subtraction [True]
do_bkg (bool): whether to do XAFS background subtraction [True]
do_fft (bool): whether to do XAFS Fast Fourier transform [True]
use_hashkey (bool): whether to use Athena's hash key as the
group name instead of the Athena label [False]
Returns:
group of groups each named according the label used by Athena.
Notes:
1. To limit the imported groups, use the pattern in `match`,
using '*' to match 'all' '?' to match any single character,
or [sequence] to match any of a sequence of letters. The match
will always be insensitive to case.
3. do_preedge, do_bkg, and do_fft will attempt to reproduce the
pre-edge, background subtraction, and FFT from Athena by using
the parameters saved in the project file.
2. use_hashkey=True will name groups from the internal 5 character
string used by Athena, instead of the group label.
Example:
1. read in all groups from a project file:
cr_data = read_athena('My Cr Project.prj')
2. read in only the "merged" data from a Project, and don't do FFT:
zn_data = read_athena('Zn on Stuff.prj', match='*merge*', do_fft=False)
"""
from larch_plugins.xafs import pre_edge, autobk, xftf
if not os.path.exists(filename):
raise IOError("%s '%s': cannot find file" % (ERR_MSG, filename))
try:
fh = GzipFile(filename)
lines = [bytes2str(t) for t in fh.readlines()]
fh.close()
except:
raise ValueError("%s '%s': invalid gzip file" % (ERR_MSG, filename))
athenagroups = []
dat = {'name': ''}
Athena_version = None
vline = lines.pop(0)
if "Athena project file -- Demeter version" not in vline:
raise ValueError("%s '%s': invalid Athena File" % (ERR_MSG, filename))
major, minor, fix = '0', '0', '0'
try:
vs = vline.split("Athena project file -- Demeter version")[1]
major, minor, fix = vs.split('.')
except:
raise ValueError("%s '%s': cannot read version" % (ERR_MSG, filename))
if int(minor) < 9 or int(fix[:2]) < 21:
raise ValueError("%s '%s': file is too old to read" %
(ERR_MSG, filename))
for t in lines:
if t.startswith('#') or len(t) < 2 or 'undef' in t:
continue
key = t.split(' ')[0].strip()
key = key.replace('$', '').replace('@', '')
if key == 'old_group':
dat['name'] = perl2json(t)
elif key == '[record]':
athenagroups.append(dat)
dat = {'name': ''}
elif key == 'args':
dat['args'] = perl2json(t)
elif key in ('x', 'y', 'i0', 'signal'):
dat[key] = np.array([float(x) for x in perl2json(t)])
if match is not None:
match = match.lower()
out = Group()
out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename)
for dat in athenagroups:
label = dat.get('name', 'unknown')
this = Group(athena_id=label,
energy=dat['x'],
mu=dat['y'],
bkg_params=Group(),
fft_params=Group(),
athena_params=Group())
if 'i0' in dat:
this.i0 = dat['i0']
if 'args' in dat:
for i in range(len(dat['args']) // 2):
key = dat['args'][2 * i]
val = dat['args'][2 * i + 1]
if key.startswith('bkg_'):
setattr(this.bkg_params, key[4:], val)
elif key.startswith('fft_'):
setattr(this.fft_params, key[4:], val)
elif key == 'label':
this.label = val
if not use_hashkey:
label = this.label
else:
setattr(this.athena_params, key, val)
this.__doc__ = """Athena Group Name %s (key='%s')""" % (label,
dat['name'])
olabel = fix_varname(label)
if match is not None:
if not fnmatch(olabel.lower(), match):
continue
if do_preedge or do_bkg:
pars = this.bkg_params
pre_edge(this,
_larch=_larch,
e0=float(pars.e0),
pre1=float(pars.pre1),
pre2=float(pars.pre2),
norm1=float(pars.nor1),
norm2=float(pars.nor2),
nnorm=float(pars.nnorm) - 1,
make_flat=bool(pars.flatten))
if do_bkg and hasattr(pars, 'rbkg'):
autobk(this,
_larch=_larch,
e0=float(pars.e0),
rbkg=float(pars.rbkg),
kmin=float(pars.spl1),
kmax=float(pars.spl2),
kweight=float(pars.kw),
dk=float(pars.dk),
clamp_lo=float(pars.clamp1),
clamp_hi=float(pars.clamp2))
if do_fft:
pars = this.fft_params
kweight = 2
if hasattr(pars, 'kw'):
kweight = float(pars.kw)
xftf(this,
_larch=_larch,
kmin=float(pars.kmin),
kmax=float(pars.kmax),
kweight=kweight,
window=pars.kwindow,
dk=float(pars.dk))
setattr(out, olabel, this)
return out
def parse_perlathena(text, filename):
"""
parse old athena file format to Group of Groups
"""
lines = text.split('\n')
athenagroups = []
raw = {'name':''}
vline = lines.pop(0)
if "Athena project file -- " not in vline:
raise ValueError("%s '%s': invalid Athena File" % (ERR_MSG, filename))
major, minor, fix = '0', '0', '0'
if 'Demeter' in vline:
try:
vs = vline.split("Athena project file -- Demeter version")[1]
major, minor, fix = vs.split('.')
except:
raise ValueError("%s '%s': cannot read version" % (ERR_MSG, filename))
else:
try:
vs = vline.split("Athena project file -- Athena version")[1]
major, minor, fix = vs.split('.')
except:
raise ValueError("%s '%s': cannot read version" % (ERR_MSG, filename))
header = [vline]
journal = ['']
is_header = True
for t in lines:
if t.startswith('#') or len(t) < 2 or 'undef' in t:
if is_header:
header.append(t)
continue
is_header = False
key = t.split(' ')[0].strip()
key = key.replace('$', '').replace('@', '').replace('%', '').strip()
if key == 'old_group':
raw['name'] = plarray2json(t)
elif key == '[record]':
athenagroups.append(raw)
raw = {'name':''}
elif key == 'journal':
journal = parse_arglist(t)
elif key == 'args':
raw['args'] = parse_arglist(t)
elif key == 'xdi':
raw['xdi'] = t
elif key in ('x', 'y', 'i0', 'signal', 'stddev'):
raw[key] = np.array([float(x) for x in plarray2json(t)])
elif key in ('1;', 'indicator', 'lcf_data', 'plot_features'):
pass
else:
print(" do not know what to do with key '%s' at '%s'" % (key, raw['name']))
out = Group()
out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename)
out.journal = '\n'.join(journal)
out.group_names = []
out.header = '\n'.join(header)
for dat in athenagroups:
label = dat.get('name', 'unknown')
this = Group(athena_id=label, energy=dat['x'], mu=dat['y'],
bkg_params=Group(),
fft_params=Group(),
athena_params=Group())
if 'i0' in dat:
this.i0 = dat['i0']
if 'signal' in dat:
this.signal = dat['signal']
if 'stddev' in dat:
this.stddev = dat['stddev']
if 'args' in dat:
for i in range(len(dat['args'])//2):
key = dat['args'][2*i]
val = dat['args'][2*i+1]
if key.startswith('bkg_'):
setattr(this.bkg_params, key[4:], asfloat(val))
elif key.startswith('fft_'):
setattr(this.fft_params, key[4:], asfloat(val))
elif key == 'label':
label = this.label = val
elif key in ('valence', 'lasso_yvalue',
'epsk', 'epsr', 'importance'):
setattr(this, key, asfloat(val))
elif key in ('atsym', 'edge', 'provenance'):
setattr(this, key, val)
else:
setattr(this.athena_params, key, asfloat(val))
this.__doc__ = """Athena Group Name %s (key='%s')""" % (label, dat['name'])
name = fix_varname(label)
if name.startswith('_'):
name = 'd' + name
setattr(out, name, this)
out.group_names.append(name)
return out
