def parse_jsonathena(text, filename): """parse a JSON-style athena file""" jsdict = json.loads(text) out = Group() out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename) header = [] athena_names = [] for key, val in jsdict.items(): if key.startswith('_____head'): header.append(val) elif key.startswith('_____journ'): journal = val elif key.startswith('_____order'): athena_names = val out.journal = journal out.header = '\n'.join(header) out.group_names = [] for name in athena_names: label = name dat = jsdict[name] x = np.array(dat['x'], dtype='float64') y = np.array(dat['y'], dtype='float64') this = Group(athena_id=name, energy=x, mu=y, bkg_params=Group(), fft_params=Group(), athena_params=Group()) if 'i0' in dat: this.i0 = np.array(dat['i0'], dtype='float64') if 'signal' in dat: this.signal = np.array(dat['signal'], dtype='float64') if 'stddev' in dat: this.stddev = np.array(dat['stddev'], dtype='float64') if 'args' in dat: for key, val in dat['args'].items(): if key.startswith('bkg_'): setattr(this.bkg_params, key[4:], asfloat(val)) elif key.startswith('fft_'): setattr(this.fft_params, key[4:], asfloat(val)) elif key == 'label': label = this.label = val else: setattr(this.athena_params, key, asfloat(val)) this.__doc__ = """Athena Group Name %s (key='%s')""" % (label, name) name = fix_varname(label) if name.startswith('_'): name = 'd' + name setattr(out, name, this) out.group_names.append(name) return out
def import_xdi(name): if name[-3:] == 'txt': data_raw = np.loadtxt(name) data = Group() data.energy = data_raw[:,2] data.mu = data_raw[:,8]/data_raw[:,5] else: data_raw = np.loadtxt(name) data = Group() data.energy = data_raw[:,0] data.mu = data_raw[:,1] data.i0 = data_raw[:,2] return data
def parse_perlathena(text, filename): """ parse old athena file format to Group of Groups """ lines = text.split('\n') athenagroups = [] raw = {'name':''} vline = lines.pop(0) if "Athena project file -- Demeter version" not in vline: raise ValueError("%s '%s': invalid Athena File" % (ERR_MSG, filename)) major, minor, fix = '0', '0', '0' try: vs = vline.split("Athena project file -- Demeter version")[1] major, minor, fix = vs.split('.') except: raise ValueError("%s '%s': cannot read version" % (ERR_MSG, filename)) if int(minor) < 9 or int(fix[:2]) < 21: raise ValueError("%s '%s': file is too old to read" % (ERR_MSG, filename)) header = [vline] journal = {} is_header = True for t in lines: if t.startswith('#') or len(t) < 2 or 'undef' in t: if is_header: header.append(t) continue is_header = False key = t.split(' ')[0].strip() key = key.replace('$', '').replace('@', '') if key == 'old_group': raw['name'] = plarray2json(t) elif key == '[record]': athenagroups.append(raw) raw = {'name':''} elif key == 'journal': journal = plarray2json(t) elif key == 'args': raw['args'] = plarray2json(t) elif key == 'xdi': raw['xdi'] = t elif key in ('x', 'y', 'i0', 'signal', 'stddev'): raw[key] = np.array([float(x) for x in plarray2json(t)]) elif key == '1;': # end of list pass else: print(" do not know what to do with key ", key, raw['name']) out = Group() out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename) out.journal = journal out.group_names = [] out.header = '\n'.join(header) for dat in athenagroups: label = dat.get('name', 'unknown') this = Group(athena_id=label, energy=dat['x'], mu=dat['y'], bkg_params=Group(), fft_params=Group(), athena_params=Group()) if 'i0' in dat: this.i0 = dat['i0'] if 'signal' in dat: this.signal = dat['signal'] if 'stddev' in dat: this.stddev = dat['stddev'] if 'args' in dat: for i in range(len(dat['args'])//2): key = dat['args'][2*i] val = dat['args'][2*i+1] if key.startswith('bkg_'): setattr(this.bkg_params, key[4:], asfloat(val)) elif key.startswith('fft_'): setattr(this.fft_params, key[4:], asfloat(val)) elif key == 'label': label = this.label = val elif key in ('valence', 'lasso_yvalue', 'epsk', 'epsr', 'importance'): setattr(this, key, asfloat(val)) elif key in ('atsym', 'edge', 'provenance'): setattr(this, key, val) else: setattr(this.athena_params, key, asfloat(val)) this.__doc__ = """Athena Group Name %s (key='%s')""" % (label, dat['name']) name = fix_varname(label) if name.startswith('_'): name = 'd' + name setattr(out, name, this) out.group_names.append(name) return out
def read_athena(filename, match=None, do_preedge=True, do_bkg=True, do_fft=True, use_hashkey=False, _larch=None): """read athena project file returns a Group of Groups, one for each Athena Group in the project file Arguments: filename (string): name of Athena Project file match (sring): pattern to use to limit imported groups (see Note 1) do_preedge (bool): whether to do pre-edge subtraction [True] do_bkg (bool): whether to do XAFS background subtraction [True] do_fft (bool): whether to do XAFS Fast Fourier transform [True] use_hashkey (bool): whether to use Athena's hash key as the group name instead of the Athena label [False] Returns: group of groups each named according the label used by Athena. Notes: 1. To limit the imported groups, use the pattern in `match`, using '*' to match 'all' '?' to match any single character, or [sequence] to match any of a sequence of letters. The match will always be insensitive to case. 3. do_preedge, do_bkg, and do_fft will attempt to reproduce the pre-edge, background subtraction, and FFT from Athena by using the parameters saved in the project file. 2. use_hashkey=True will name groups from the internal 5 character string used by Athena, instead of the group label. Example: 1. read in all groups from a project file: cr_data = read_athena('My Cr Project.prj') 2. read in only the "merged" data from a Project, and don't do FFT: zn_data = read_athena('Zn on Stuff.prj', match='*merge*', do_fft=False) """ from larch_plugins.xafs import pre_edge, autobk, xftf if not os.path.exists(filename): raise IOError("%s '%s': cannot find file" % (ERR_MSG, filename)) try: fh = GzipFile(filename) lines = [bytes2str(t) for t in fh.readlines()] fh.close() except: raise ValueError("%s '%s': invalid gzip file" % (ERR_MSG, filename)) athenagroups = [] dat = {"name": ""} Athena_version = None vline = lines.pop(0) if "Athena project file -- Demeter version" not in vline: raise ValueError("%s '%s': invalid Athena File" % (ERR_MSG, filename)) major, minor, fix = "0", "0", "0" try: vs = vline.split("Athena project file -- Demeter version")[1] major, minor, fix = vs.split(".") except: raise ValueError("%s '%s': cannot read version" % (ERR_MSG, filename)) if int(minor) < 9 or int(fix[:2]) < 21: raise ValueError("%s '%s': file is too old to read" % (ERR_MSG, filename)) for t in lines: if t.startswith("#") or len(t) < 2: continue key = t.split(" ")[0].strip() key = key.replace("$", "").replace("@", "") if key == "old_group": dat["name"] = perl2json(t) elif key == "[record]": athenagroups.append(dat) dat = {"name": ""} elif key == "args": dat["args"] = perl2json(t) elif key in ("x", "y", "i0"): dat[key] = np.array([float(x) for x in perl2json(t)]) if match is not None: match = match.lower() out = Group() out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename) for dat in athenagroups: label = dat["name"] this = Group( athena_id=label, energy=dat["x"], mu=dat["y"], bkg_params=Group(), fft_params=Group(), athena_params=Group() ) if "i0" in dat: this.i0 = dat["i0"] if "args" in dat: for i in range(len(dat["args"]) // 2): key = dat["args"][2 * i] val = dat["args"][2 * i + 1] if key.startswith("bkg_"): setattr(this.bkg_params, key[4:], val) elif key.startswith("fft_"): setattr(this.fft_params, key[4:], val) elif key == "label": this.label = val if not use_hashkey: label = this.label else: setattr(this.athena_params, key, val) this.__doc__ = """Athena Group Name %s (key='%s')""" % (label, dat["name"]) olabel = fix_varname(label) if match is not None: if not fnmatch(olabel.lower(), match): continue if do_preedge or do_bkg: pars = this.bkg_params pre_edge( this, _larch=_larch, e0=float(pars.e0), pre1=float(pars.pre1), pre2=float(pars.pre2), norm1=float(pars.nor1), norm2=float(pars.nor2), nnorm=float(pars.nnorm) - 1, make_flat=bool(pars.flatten), ) if do_bkg and hasattr(pars, "rbkg"): autobk( this, _larch=_larch, e0=float(pars.e0), rbkg=float(pars.rbkg), kmin=float(pars.spl1), kmax=float(pars.spl2), kweight=float(pars.kw), dk=float(pars.dk), clamp_lo=float(pars.clamp1), clamp_hi=float(pars.clamp2), ) if do_fft: pars = this.fft_params kweight = 2 if hasattr(pars, "kw"): kweight = float(pars.kw) xftf( this, _larch=_larch, kmin=float(pars.kmin), kmax=float(pars.kmax), kweight=kweight, window=pars.kwindow, dk=float(pars.dk), ) setattr(out, olabel, this) return out
def read_athena(filename, match=None, do_preedge=True, do_bkg=True, do_fft=True, use_hashkey=False, _larch=None): """read athena project file returns a Group of Groups, one for each Athena Group in the project file Arguments: filename (string): name of Athena Project file match (string): pattern to use to limit imported groups (see Note 1) do_preedge (bool): whether to do pre-edge subtraction [True] do_bkg (bool): whether to do XAFS background subtraction [True] do_fft (bool): whether to do XAFS Fast Fourier transform [True] use_hashkey (bool): whether to use Athena's hash key as the group name instead of the Athena label [False] Returns: group of groups each named according the label used by Athena. Notes: 1. To limit the imported groups, use the pattern in `match`, using '*' to match 'all' '?' to match any single character, or [sequence] to match any of a sequence of letters. The match will always be insensitive to case. 3. do_preedge, do_bkg, and do_fft will attempt to reproduce the pre-edge, background subtraction, and FFT from Athena by using the parameters saved in the project file. 2. use_hashkey=True will name groups from the internal 5 character string used by Athena, instead of the group label. Example: 1. read in all groups from a project file: cr_data = read_athena('My Cr Project.prj') 2. read in only the "merged" data from a Project, and don't do FFT: zn_data = read_athena('Zn on Stuff.prj', match='*merge*', do_fft=False) """ from larch_plugins.xafs import pre_edge, autobk, xftf if not os.path.exists(filename): raise IOError("%s '%s': cannot find file" % (ERR_MSG, filename)) try: fh = GzipFile(filename) lines = [bytes2str(t) for t in fh.readlines()] fh.close() except: raise ValueError("%s '%s': invalid gzip file" % (ERR_MSG, filename)) athenagroups = [] dat = {'name': ''} Athena_version = None vline = lines.pop(0) if "Athena project file -- Demeter version" not in vline: raise ValueError("%s '%s': invalid Athena File" % (ERR_MSG, filename)) major, minor, fix = '0', '0', '0' try: vs = vline.split("Athena project file -- Demeter version")[1] major, minor, fix = vs.split('.') except: raise ValueError("%s '%s': cannot read version" % (ERR_MSG, filename)) if int(minor) < 9 or int(fix[:2]) < 21: raise ValueError("%s '%s': file is too old to read" % (ERR_MSG, filename)) for t in lines: if t.startswith('#') or len(t) < 2 or 'undef' in t: continue key = t.split(' ')[0].strip() key = key.replace('$', '').replace('@', '') if key == 'old_group': dat['name'] = perl2json(t) elif key == '[record]': athenagroups.append(dat) dat = {'name': ''} elif key == 'args': dat['args'] = perl2json(t) elif key in ('x', 'y', 'i0', 'signal'): dat[key] = np.array([float(x) for x in perl2json(t)]) if match is not None: match = match.lower() out = Group() out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename) for dat in athenagroups: label = dat.get('name', 'unknown') this = Group(athena_id=label, energy=dat['x'], mu=dat['y'], bkg_params=Group(), fft_params=Group(), athena_params=Group()) if 'i0' in dat: this.i0 = dat['i0'] if 'args' in dat: for i in range(len(dat['args']) // 2): key = dat['args'][2 * i] val = dat['args'][2 * i + 1] if key.startswith('bkg_'): setattr(this.bkg_params, key[4:], val) elif key.startswith('fft_'): setattr(this.fft_params, key[4:], val) elif key == 'label': this.label = val if not use_hashkey: label = this.label else: setattr(this.athena_params, key, val) this.__doc__ = """Athena Group Name %s (key='%s')""" % (label, dat['name']) olabel = fix_varname(label) if match is not None: if not fnmatch(olabel.lower(), match): continue if do_preedge or do_bkg: pars = this.bkg_params pre_edge(this, _larch=_larch, e0=float(pars.e0), pre1=float(pars.pre1), pre2=float(pars.pre2), norm1=float(pars.nor1), norm2=float(pars.nor2), nnorm=float(pars.nnorm) - 1, make_flat=bool(pars.flatten)) if do_bkg and hasattr(pars, 'rbkg'): autobk(this, _larch=_larch, e0=float(pars.e0), rbkg=float(pars.rbkg), kmin=float(pars.spl1), kmax=float(pars.spl2), kweight=float(pars.kw), dk=float(pars.dk), clamp_lo=float(pars.clamp1), clamp_hi=float(pars.clamp2)) if do_fft: pars = this.fft_params kweight = 2 if hasattr(pars, 'kw'): kweight = float(pars.kw) xftf(this, _larch=_larch, kmin=float(pars.kmin), kmax=float(pars.kmax), kweight=kweight, window=pars.kwindow, dk=float(pars.dk)) setattr(out, olabel, this) return out
# now import larch-specific Python code from larch_plugins.xafs import autobk, xftf # create a larch interpreter, passed to most functions my_larch = Interpreter(with_plugins=False) # create an empty Group xafsdat = Group() # read data (here using np.loadtxt), stick into Group # using "expected names" for XAFS data rawdata = np.loadtxt('../xafsdata/fe2o3_rt1.xmu') xafsdat.energy = rawdata[:, 0] xafsdat.mu = rawdata[:, 1] xafsdat.i0 = rawdata[:, 1] # run autobk on the xafsdat Group, including a larch Interpreter.... # note that this expects 'energy' and 'mu' to be in xafsdat, and will # write data for 'k', 'chi', 'kwin', 'e0', ... into xafsdat autobk(xafsdat, rbkg=1.0, kweight=2, _larch=my_larch) # Fourier transform to R space, again passing in a Group (here, # 'k' and 'chi' are expected, and writitng out 'r', 'chir_mag', # and so on xftf(xafsdat, kmin=2, kmax=15, dk=3, kweight=2, _larch=my_larch) # # plot grid of results: # mu + bkg pylab.subplot(2, 2, 1)
def parse_perlathena(text, filename): """ parse old athena file format to Group of Groups """ lines = text.split('\n') athenagroups = [] raw = {'name':''} vline = lines.pop(0) if "Athena project file -- " not in vline: raise ValueError("%s '%s': invalid Athena File" % (ERR_MSG, filename)) major, minor, fix = '0', '0', '0' if 'Demeter' in vline: try: vs = vline.split("Athena project file -- Demeter version")[1] major, minor, fix = vs.split('.') except: raise ValueError("%s '%s': cannot read version" % (ERR_MSG, filename)) else: try: vs = vline.split("Athena project file -- Athena version")[1] major, minor, fix = vs.split('.') except: raise ValueError("%s '%s': cannot read version" % (ERR_MSG, filename)) header = [vline] journal = [''] is_header = True for t in lines: if t.startswith('#') or len(t) < 2 or 'undef' in t: if is_header: header.append(t) continue is_header = False key = t.split(' ')[0].strip() key = key.replace('$', '').replace('@', '').replace('%', '').strip() if key == 'old_group': raw['name'] = plarray2json(t) elif key == '[record]': athenagroups.append(raw) raw = {'name':''} elif key == 'journal': journal = parse_arglist(t) elif key == 'args': raw['args'] = parse_arglist(t) elif key == 'xdi': raw['xdi'] = t elif key in ('x', 'y', 'i0', 'signal', 'stddev'): raw[key] = np.array([float(x) for x in plarray2json(t)]) elif key in ('1;', 'indicator', 'lcf_data', 'plot_features'): pass else: print(" do not know what to do with key '%s' at '%s'" % (key, raw['name'])) out = Group() out.__doc__ = """XAFS Data from Athena Project File %s""" % (filename) out.journal = '\n'.join(journal) out.group_names = [] out.header = '\n'.join(header) for dat in athenagroups: label = dat.get('name', 'unknown') this = Group(athena_id=label, energy=dat['x'], mu=dat['y'], bkg_params=Group(), fft_params=Group(), athena_params=Group()) if 'i0' in dat: this.i0 = dat['i0'] if 'signal' in dat: this.signal = dat['signal'] if 'stddev' in dat: this.stddev = dat['stddev'] if 'args' in dat: for i in range(len(dat['args'])//2): key = dat['args'][2*i] val = dat['args'][2*i+1] if key.startswith('bkg_'): setattr(this.bkg_params, key[4:], asfloat(val)) elif key.startswith('fft_'): setattr(this.fft_params, key[4:], asfloat(val)) elif key == 'label': label = this.label = val elif key in ('valence', 'lasso_yvalue', 'epsk', 'epsr', 'importance'): setattr(this, key, asfloat(val)) elif key in ('atsym', 'edge', 'provenance'): setattr(this, key, val) else: setattr(this.athena_params, key, asfloat(val)) this.__doc__ = """Athena Group Name %s (key='%s')""" % (label, dat['name']) name = fix_varname(label) if name.startswith('_'): name = 'd' + name setattr(out, name, this) out.group_names.append(name) return out