def prepareDataStack(fileList):
if (not os.path.exists(fileList[0])) and \
os.path.exists(fileList[0].split("::")[0]):
# odo convention to get a dataset form an HDF5
fname, dataPath = fileList[0].split("::")
# compared to the ROI imaging tool, this way of reading puts data
# into memory while with the ROI imaging tool, there is a check.
if 0:
import h5py
h5 = h5py.File(fname, "r")
dataStack = h5[dataPath][:]
h5.close()
else:
from PyMca5.PyMcaIO import HDF5Stack1D
# this way reads information associated to the dataset (if present)
if dataPath.startswith("/"):
pathItems = dataPath[1:].split("/")
else:
pathItems = dataPath.split("/")
if len(pathItems) > 1:
scanlist = ["/" + pathItems[0]]
selection = {"y": "/" + "/".join(pathItems[1:])}
else:
selection = {"y": dataPath}
scanlist = None
print(selection)
print("scanlist = ", scanlist)
dataStack = HDF5Stack1D.HDF5Stack1D([fname],
selection,
scanlist=scanlist)
else:
from PyMca5.PyMca import EDFStack
dataStack = EDFStack.EDFStack(fileList, dtype=numpy.float32)
return dataStack
def PerformRoi(filelist, rois, norm=None):
"""ROI XRF spectra in batch with changing primary beam energy.
Args:
filelist(list(str)|np.array): spectra to fit
rois(dict(2-tuple)): ROIs
norm(Optional(np.array)): normalization array
Returns:
dict: {label:nenergies x nfiles,...}
"""
# Load data
# Each spectrum (each row) in 1 edf file is acquired at a different energy
if isinstance(filelist, list):
dataStack = EDFStack.EDFStack(filelist, dtype=np.float32).data
else:
dataStack = filelist
nfiles, nenergies, nchannels = dataStack.shape
# Normalization
if norm is None:
norm = [1] * nenergies
else:
if hasattr(norm, "__iter__"):
if len(norm) == 1:
norm = [norm[0]] * nenergies
elif len(norm) != nenergies:
raise ValueError(
"Expected {} normalization values ({} given)".format(
nenergies, len(norm)
)
)
else:
norm = [norm] * nenergies
# ROI
ret = {}
for k in rois:
ret[k] = np.zeros((nenergies, nfiles), dtype=type(dataStack))
for i in range(nfiles):
for k, roi in rois.items():
ret[k][:, i] = np.sum(dataStack[i, :, roi[0] : roi[1]], axis=1) / norm
return ret
pathItems = dataPath[1:].split("/")
else:
pathItems = dataPath.split("/")
if len(pathItems) > 1:
scanlist = ["/" + pathItems[0]]
selection = {"y":"/" + "/".join(pathItems[1:])}
else:
selection = {"y":dataPath}
scanlist = None
print(selection)
print("scanlist = ", scanlist)
dataStack = HDF5Stack1D.HDF5Stack1D([fname],
selection,
scanlist=scanlist)
else:
dataStack = EDFStack.EDFStack(fileList, dtype=numpy.float32)
else:
print("OPTIONS:", longoptions)
sys.exit(0)
if outputDir is None:
print("RESULTS WILL NOT BE SAVED: No output directory specified")
t0 = time.time()
fastFit = FastXRFLinearFit()
fastFit.setFitConfigurationFile(configurationFile)
print("Main configuring Elapsed = % s " % (time.time() - t0))
result = fastFit.fitMultipleSpectra(y=dataStack,
weight=weight,
refit=refit,
concentrations=concentrations)
print("Total Elapsed = % s " % (time.time() - t0))
if outputDir is not None:
def _generateData(self, fast=False, typ='hdf5'):
# Generate data (in memory + save in requested format)
nDet = 1 # TODO: currently only works with 1 detector
nRows = 5
nColumns = 4
nTimes = 3
filename = os.path.join(self.path, 'Map')
if typ == 'edf':
genFunc = XrfData.generateEdfMap
filename += '.edf'
elif typ == 'specmesh':
genFunc = XrfData.generateSpecMesh
filename += '.dat'
elif typ == 'hdf5':
genFunc = XrfData.generateHdf5Map
filename += '.h5'
else:
raise ValueError('Unknown data type {} for XRF map'.format(
repr(typ)))
# TODO: cannot provide live time when fitting .edf list of files
liveTimeIsProvided = fast or typ == 'hdf5'
def modfunc(configuration):
configuration["concentrations"]["usematrix"] = 0
configuration["concentrations"]["useautotime"] = int(
liveTimeIsProvided)
if fast:
configuration['fit']['stripalgorithm'] = 1
else:
configuration['fit']['linearfitflag'] = 1
info = genFunc(filename,
nDet=nDet,
nRows=nRows,
nColumns=nColumns,
nTimes=nTimes,
modfunc=modfunc)
# Concentrations are multiplied by this factor to
# normalize live time to preset time
# TODO: currently only works with 1 detector
info['liveTime'] = info['liveTime'][0, ...]
if liveTimeIsProvided:
info['liveTimeCorrection'] = float(
info['presetTime']) / info['liveTime']
else:
info['liveTimeCorrection'] = numpy.ones_like(info['liveTime'])
if typ == 'specmesh':
# REMARK: spec file data is flattened by the spec loaders
nRows, nColumns = info['liveTimeCorrection'].shape
info['liveTimeCorrection'] = info['liveTimeCorrection'].reshape(
(1, nRows * nColumns))
# Batch fit input (list of strings or stack object)
filelist = info['filelist']
if typ == 'edf':
if fast:
from PyMca5.PyMca import EDFStack
info['input'] = EDFStack.EDFStack(filelist,
dtype=numpy.float32)
else:
info['input'] = filelist
info['selection'] = None
elif typ == 'specmesh':
if fast:
from PyMca5.PyMcaIO import SpecFileStack
info['input'] = SpecFileStack.SpecFileStack(filelist)
else:
info['input'] = filelist
info['selection'] = None
elif typ == 'hdf5':
datasets = ['/xrf/mca{:02d}/data'.format(k) for k in range(nDet)]
if fast:
from PyMca5.PyMcaIO import HDF5Stack1D
info['selection'] = selection = {'y': datasets[0]}
info['input'] = HDF5Stack1D.HDF5Stack1D(filelist, selection)
else:
info['selection'] = {'x': [], 'm': [], 'y': [datasets[0]]}
info['input'] = filelist
# Batch fit configuration
info['cfgname'] = os.path.join(self.path, 'Map.cfg')
return info
def PerformBatchFitOld(
filelist,
outdir,
outname,
cfg,
energy,
mlines=None,
quant=None,
fast=False,
addhigh=0,
):
"""Fit XRF spectra in batch with one primary beam energy.
Least-square fitting. If you intend a linear fit, modify the configuration:
- Get current energy calibration with "Load From Fit"
- Enable: Perform a Linear Fit
- Disable: Stripping
- Strip iterations = 0
Fast linear least squares:
- Use SNIP instead of STRIP
Args:
filelist(list(str)): spectra to fit
outdir(str): directory for results
outname(str): output radix
cfg(str or ConfigDict): configuration file to use
energy(num): primary beam energy
mlines(Optional(dict)): elements (keys) which M line group must be replaced by some M subgroups (values)
fast(Optional(bool)): fast fitting (linear)
quant(Optional(dict)):
addhigh(Optional(int))
Returns:
files(list(str)): files produced by pymca
labels(list(str)): corresponding HDF5 labels
"""
outdir = localfs.Path(outdir).mkdir()
if instance.isstring(cfg):
cfg = ConfigDict.ConfigDict(filelist=cfg)
with outdir.temp(name=outname + ".cfg", force=True) as cfgfile:
AdaptPyMcaConfig(
cfg, energy, mlines=mlines, quant=quant, fast=fast, addhigh=addhigh
)
cfg.write(cfgfile.path)
buncertainties = False
bconcentrations = bool(quant)
if fast:
# Prepare fit
fastFit = FastXRFLinearFit.FastXRFLinearFit()
fastFit.setFitConfiguration(cfg)
dataStack = EDFStack.EDFStack(filelist, dtype=np.float32)
# Fit
result = fastFit.fitMultipleSpectra(
y=dataStack, refit=1, concentrations=bconcentrations
)
# Save result and keep filenames + labels
names = result["names"]
if bconcentrations:
names = names[: -len(result["concentrations"])]
parse = re.compile("^(?P<Z>.+)[_ -](?P<line>.+)$")
def filename(x):
return outdir["{}_{}.edf".format(outname, x)].path
labels = []
files = []
j = 0
for i, name in enumerate(names):
m = parse.match(name)
if not m:
continue
m = m.groupdict()
Z, line = m["Z"], m["line"]
# Peak area
label = "{}_{}".format(Z, line)
f = filename(label)
edf.saveedf(
f, result["parameters"][i], {"Title": label}, overwrite=True
)
labels.append(label)
files.append(f)
# Error on peak area
if buncertainties:
label = "s{}_{}".format(Z, line)
f = filename(label)
edf.saveedf(
f, result["uncertainties"][i], {"Title": label}, overwrite=True
)
labels.append(label)
files.append(f)
# Mass fraction
if bconcentrations and Z.lower() != "scatter":
label = "w{}_{}".format(Z, line)
f = filename(label)
edf.saveedf(
f, result["concentrations"][j], {"Title": label}, overwrite=True
)
labels.append(label)
files.append(f)
j += 1
else:
b = McaAdvancedFitBatch.McaAdvancedFitBatch(
cfgfile.path,
filelist=filelist,
outputdir=outdir.path,
fitfiles=0,
concentrations=bconcentrations,
)
b.processList()
filemask = os.path.join(outdir.path, "IMAGES", "*.dat")
def basename(x):
return os.path.splitext(os.path.basename(x))[0]
nbase = len(basename(glob.glob(filemask)[0])) + 1
filemask = os.path.join(outdir.path, "IMAGES", "*.edf")
labels = []
files = []
for name in sorted(glob.glob(filemask)):
label = basename(name)[nbase:]
if label.endswith("mass_fraction"):
label = "w" + label[:-14]
if label == "chisq":
label = "calc_chisq"
labels.append(label)
files.append(name)
return files, labels
def PerformFit(
filelist,
cfgfile,
energies,
mlines={},
norm=None,
fast=False,
addhigh=0,
prog=None,
plot=False,
):
"""Fit XRF spectra in batch with changing primary beam energy.
Args:
filelist(list(str)|np.array): spectra to fit
cfgfile(str): configuration file to use
energies(np.array): primary beam energies
mlines(Optional(dict)): elements (keys) which M line group must be replaced by some M subgroups (values)
norm(Optional(np.array)): normalization array
fast(Optional(bool)): fast fitting (linear)
addhigh(Optional(number)): add higher energy
prog(Optional(timing.ProgessLogger)): progress object
plot(Optional(bool))
Returns:
dict: {label:nenergies x nfiles,...}
"""
# Load data
# Each spectrum (each row) in 1 edf file is acquired at a different energy
if isinstance(filelist, list):
dataStack = EDFStack.EDFStack(filelist, dtype=np.float32).data
else:
dataStack = filelist
nfiles, nenergies, nchannels = dataStack.shape
# MCA channels
xmin = 0
xmax = nchannels - 1
x = np.arange(nchannels, dtype=np.float32)
# Energies
if hasattr(energies, "__iter__"):
if len(energies) == 1:
energies = [energies[0]] * nenergies
elif len(energies) != nenergies:
raise ValueError(
"Expected {} energies ({} given)".format(nenergies, len(energies))
)
else:
energies = [energies] * nenergies
# Normalization
if norm is None:
norm = [1] * nenergies
else:
if hasattr(norm, "__iter__"):
if len(norm) == 1:
norm = [norm[0]] * nenergies
elif len(norm) != nenergies:
raise ValueError(
"Expected {} normalization values ({} given)".format(
nenergies, len(norm)
)
)
else:
norm = [norm] * nenergies
# Prepare plot
if plot:
fig, ax = plt.subplots()
# Prepare fit
# ClassMcaTheory.DEBUG = 1
mcafit = ClassMcaTheory.McaTheory()
try:
mcafit.useFisxEscape(True)
except:
pass
if fast:
mcafit.enableOptimizedLinearFit()
else:
mcafit.disableOptimizedLinearFit()
cfg = mcafit.configure(ReadPyMcaConfigFile(cfgfile))
# Fit at each energy
if prog is not None:
prog.setnfine(nenergies * nfiles)
ret = {}
for j in range(nenergies):
# Prepare fit with this energy
AdaptPyMcaConfig(cfg, energies[j], mlines=mlines, fast=fast, addhigh=addhigh)
mcafit.configure(cfg)
# Fit all spectra with this energy
for i in range(nfiles):
# Data to fit
y = dataStack[i, j, :].flatten()
mcafit.setData(x, y, xmin=xmin, xmax=xmax)
# Initial parameter estimates
mcafit.estimate()
# Fit
fitresult = mcafit.startfit(digest=0)
# Extract result
if plot:
mcafitresult = mcafit.digestresult()
ax.cla()
if (
plot == 2
or not any(np.isfinite(np.log(mcafitresult["ydata"])))
or not any(mcafitresult["ydata"] > 0)
):
ax.plot(mcafitresult["energy"], mcafitresult["ydata"])
ax.plot(mcafitresult["energy"], mcafitresult["yfit"], color="red")
else:
ax.semilogy(mcafitresult["energy"], mcafitresult["ydata"])
ax.semilogy(
mcafitresult["energy"], mcafitresult["yfit"], color="red"
)
ax.set_ylim(
ymin=np.nanmin(
mcafitresult["ydata"][np.nonzero(mcafitresult["ydata"])]
)
)
ax.set_title("Primary energy: {} keV".format(energies[j]))
ax.set_xlabel("Energy (keV)")
ax.set_ylabel("Intensity (cts)")
plt.pause(0.0001)
else:
mcafitresult = mcafit.imagingDigestResult()
# Store result
for k in mcafitresult["groups"]:
if k not in ret:
ret[k] = np.zeros(
(nenergies, nfiles), dtype=type(mcafitresult[k]["fitarea"])
)
ret[k][j, i] = mcafitresult[k]["fitarea"] / norm[j]
if "chisq" not in ret:
ret["chisq"] = np.zeros((nenergies, nfiles), dtype=type(mcafit.chisq))
ret["chisq"][j, i] = mcafit.chisq
# Print progress
if prog is not None:
prog.ndonefine(nfiles)
prog.printprogress()
return ret