def getFileHandle(self, inputfile):
try:
self._HDF5 = False
if type(inputfile) == numpy.ndarray:
return NumpyStack.NumpyStack(inputfile)
if HDF5SUPPORT:
if h5py.is_hdf5(inputfile):
self._HDF5 = True
try:
# if (len(self._filelist) == 1) && (self.mcaStep > 1)
# it should attempt to avoid loading many times
# the stack into memory in case of multiple processes
return HDF5Stack1D.HDF5Stack1D(self._filelist,
self.selection)
except:
raise
ffile = self.__tryEdf(inputfile)
if ffile is None:
ffile = self.__tryLucia(inputfile)
if ffile is None:
if inputfile[-3:] == "DAT":
ffile = self.__tryAifira(inputfile)
if ffile is None:
if LispixMap.isLispixMapFile(inputfile):
ffile = LispixMap.LispixMap(inputfile, native=False)
if ffile is None:
del ffile
ffile = SpecFileLayer.SpecFileLayer()
ffile.SetSource(inputfile)
return ffile
except:
raise IOError("I do not know what to do with file %s" % inputfile)
def prepareDataStack(fileList):
if (not os.path.exists(fileList[0])) and \
os.path.exists(fileList[0].split("::")[0]):
# odo convention to get a dataset form an HDF5
fname, dataPath = fileList[0].split("::")
# compared to the ROI imaging tool, this way of reading puts data
# into memory while with the ROI imaging tool, there is a check.
if 0:
import h5py
h5 = h5py.File(fname, "r")
dataStack = h5[dataPath][:]
h5.close()
else:
from PyMca5.PyMcaIO import HDF5Stack1D
# this way reads information associated to the dataset (if present)
if dataPath.startswith("/"):
pathItems = dataPath[1:].split("/")
else:
pathItems = dataPath.split("/")
if len(pathItems) > 1:
scanlist = ["/" + pathItems[0]]
selection = {"y": "/" + "/".join(pathItems[1:])}
else:
selection = {"y": dataPath}
scanlist = None
print(selection)
print("scanlist = ", scanlist)
dataStack = HDF5Stack1D.HDF5Stack1D([fname],
selection,
scanlist=scanlist)
else:
from PyMca5.PyMca import EDFStack
dataStack = EDFStack.EDFStack(fileList, dtype=numpy.float32)
return dataStack
def getFileHandle(self, inputfile):
try:
self._HDF5 = False
if HDF5SUPPORT:
if h5py.is_hdf5(inputfile):
self._HDF5 = True
try:
return HDF5Stack1D.HDF5Stack1D(self._filelist,
self.selection)
except:
raise
ffile = self.__tryEdf(inputfile)
if ffile is None:
ffile = self.__tryLucia(inputfile)
if ffile is None:
if inputfile[-3:] == "DAT":
ffile = self.__tryAifira(inputfile)
if ffile is None:
if LispixMap.isLispixMapFile(inputfile):
ffile = LispixMap.LispixMap(inputfile, native=False)
if (ffile is None):
del ffile
ffile = SpecFileLayer.SpecFileLayer()
ffile.SetSource(inputfile)
return ffile
except:
raise IOError("I do not know what to do with file %s" % inputfile)
def testSingleArrayExport(self):
from PyMca5.PyMcaCore import StackBase
from PyMca5.PyMcaCore import McaStackExport
tmpDir = tempfile.gettempdir()
self._h5File = os.path.join(tmpDir, "Array.h5")
data = numpy.arange(3 * 1024).reshape(3, 1024)
McaStackExport.exportStackList([data], self._h5File)
# read back the stack
from PyMca5.PyMcaIO import HDF5Stack1D
stackRead = HDF5Stack1D.HDF5Stack1D([self._h5File],
{"y": "/measurement/detector_00"})
# let's play
sb = StackBase.StackBase()
sb.setStack(stackRead)
# check the data
self.assertTrue(numpy.allclose(data, stackRead.data),
"Incorrect data readout")
def testHdf5Map(self):
from PyMca5.PyMcaIO import HDF5Stack1D
filename = os.path.join(self.path, 'xrfmap.h5')
# TODO: only works for 1 detector
nDet = 1
info = XrfData.generateHdf5Map(filename, nDet=nDet, same=False)
nDet0, nRows0, nColumns0, nChannels = info['data'].shape
datasets = ['/xrf/mca{:02d}/data'.format(k) for k in range(nDet)]
selection = {'y': datasets[0]}
stack = HDF5Stack1D.HDF5Stack1D([filename], selection).data
self.assertEqual(stack.shape, (nRows0, nColumns0, nChannels))
for i in range(nRows0):
for j in range(nColumns0):
for k in range(nDet):
numpy.testing.assert_array_equal(stack[i, j],
info['data'][k, i, j])
from PyMca5.PyMcaIO import HDF5Stack1D
# this way reads information associated to the dataset (if present)
if dataPath.startswith("/"):
pathItems = dataPath[1:].split("/")
else:
pathItems = dataPath.split("/")
if len(pathItems) > 1:
scanlist = ["/" + pathItems[0]]
selection = {"y":"/" + "/".join(pathItems[1:])}
else:
selection = {"y":dataPath}
scanlist = None
print(selection)
print("scanlist = ", scanlist)
dataStack = HDF5Stack1D.HDF5Stack1D([fname],
selection,
scanlist=scanlist)
else:
dataStack = EDFStack.EDFStack(fileList, dtype=numpy.float32)
else:
print("OPTIONS:", longoptions)
sys.exit(0)
if outputDir is None:
print("RESULTS WILL NOT BE SAVED: No output directory specified")
t0 = time.time()
fastFit = FastXRFLinearFit()
fastFit.setFitConfigurationFile(configurationFile)
print("Main configuring Elapsed = % s " % (time.time() - t0))
result = fastFit.fitMultipleSpectra(y=dataStack,
weight=weight,
refit=refit,
def _generateData(self, fast=False, typ='hdf5'):
# Generate data (in memory + save in requested format)
nDet = 1 # TODO: currently only works with 1 detector
nRows = 5
nColumns = 4
nTimes = 3
filename = os.path.join(self.path, 'Map')
if typ == 'edf':
genFunc = XrfData.generateEdfMap
filename += '.edf'
elif typ == 'specmesh':
genFunc = XrfData.generateSpecMesh
filename += '.dat'
elif typ == 'hdf5':
genFunc = XrfData.generateHdf5Map
filename += '.h5'
else:
raise ValueError('Unknown data type {} for XRF map'.format(
repr(typ)))
# TODO: cannot provide live time when fitting .edf list of files
liveTimeIsProvided = fast or typ == 'hdf5'
def modfunc(configuration):
configuration["concentrations"]["usematrix"] = 0
configuration["concentrations"]["useautotime"] = int(
liveTimeIsProvided)
if fast:
configuration['fit']['stripalgorithm'] = 1
else:
configuration['fit']['linearfitflag'] = 1
info = genFunc(filename,
nDet=nDet,
nRows=nRows,
nColumns=nColumns,
nTimes=nTimes,
modfunc=modfunc)
# Concentrations are multiplied by this factor to
# normalize live time to preset time
# TODO: currently only works with 1 detector
info['liveTime'] = info['liveTime'][0, ...]
if liveTimeIsProvided:
info['liveTimeCorrection'] = float(
info['presetTime']) / info['liveTime']
else:
info['liveTimeCorrection'] = numpy.ones_like(info['liveTime'])
if typ == 'specmesh':
# REMARK: spec file data is flattened by the spec loaders
nRows, nColumns = info['liveTimeCorrection'].shape
info['liveTimeCorrection'] = info['liveTimeCorrection'].reshape(
(1, nRows * nColumns))
# Batch fit input (list of strings or stack object)
filelist = info['filelist']
if typ == 'edf':
if fast:
from PyMca5.PyMca import EDFStack
info['input'] = EDFStack.EDFStack(filelist,
dtype=numpy.float32)
else:
info['input'] = filelist
info['selection'] = None
elif typ == 'specmesh':
if fast:
from PyMca5.PyMcaIO import SpecFileStack
info['input'] = SpecFileStack.SpecFileStack(filelist)
else:
info['input'] = filelist
info['selection'] = None
elif typ == 'hdf5':
datasets = ['/xrf/mca{:02d}/data'.format(k) for k in range(nDet)]
if fast:
from PyMca5.PyMcaIO import HDF5Stack1D
info['selection'] = selection = {'y': datasets[0]}
info['input'] = HDF5Stack1D.HDF5Stack1D(filelist, selection)
else:
info['selection'] = {'x': [], 'm': [], 'y': [datasets[0]]}
info['input'] = filelist
# Batch fit configuration
info['cfgname'] = os.path.join(self.path, 'Map.cfg')
return info
def testFitHdf5Stack(self):
import tempfile
from PyMca5.PyMcaIO import specfilewrapper as specfile
from PyMca5.PyMcaIO import ConfigDict
from PyMca5.PyMcaIO import HDF5Stack1D
from PyMca5.PyMcaPhysics.xrf import McaAdvancedFitBatch
spe = os.path.join(self.dataDir, "Steel.spe")
cfg = os.path.join(self.dataDir, "Steel.cfg")
sf = specfile.Specfile(spe)
self.assertTrue(len(sf) == 1, "File %s cannot be read" % spe)
self.assertTrue(sf[0].nbmca() == 1, "Spe file should contain MCA data")
y = counts = sf[0].mca(1)
x = channels = numpy.arange(y.size).astype(numpy.float)
sf = None
configuration = ConfigDict.ConfigDict()
configuration.read(cfg)
calibration = configuration["detector"]["zero"], \
configuration["detector"]["gain"], 0.0
initialTime = configuration["concentrations"]["time"]
# create the data
nRows = 5
nColumns = 10
nTimes = 3
data = numpy.zeros((nRows, nColumns, counts.size), dtype=numpy.float)
live_time = numpy.zeros((nRows * nColumns), dtype=numpy.float)
mcaIndex = 0
for i in range(nRows):
for j in range(nColumns):
data[i, j] = counts
live_time[i * nColumns + j] = initialTime * \
(1 + mcaIndex % nTimes)
mcaIndex += 1
self._h5File = os.path.join(tempfile.gettempdir(), "Steel.h5")
# write the stack to an HDF5 file
if os.path.exists(self._h5File):
os.remove(self._h5File)
h5 = h5py.File(self._h5File, "w")
h5["/entry/instrument/detector/calibration"] = calibration
h5["/entry/instrument/detector/channels"] = channels
h5["/entry/instrument/detector/data"] = data
h5["/entry/instrument/detector/live_time"] = live_time
# add nexus conventions
h5["/entry"].attrs["NX_class"] = u"NXentry"
h5["/entry/instrument"].attrs["NX_class"] = u"NXinstrument"
h5["/entry/instrument/detector/"].attrs["NX_class"] = u"NXdetector"
h5["/entry/instrument/detector/data"].attrs["interpretation"] = \
u"spectrum"
# case with softlink
h5["/entry/measurement/mca_soft/data"] = \
h5py.SoftLink("/entry/instrument/detector/data")
# case with info
h5["/entry/measurement/mca_with_info/data"] = \
h5["/entry/instrument/detector/data"]
h5["/entry/measurement/mca_with_info/info"] = \
h5["/entry/instrument/detector"]
h5.flush()
h5.close()
h5 = None
# check that the data can be read as a stack as
# single top level dataset (issue #226)
external = self._h5File + "external.h5"
if os.path.exists(external):
os.remove(external)
h5 = h5py.File(external, "w")
h5["/data_at_top"] = h5py.ExternalLink(
self._h5File, "/entry/measurement/mca_soft/data")
h5.flush()
h5.close()
h5 = None
stack = HDF5Stack1D.HDF5Stack1D([external], {"y": "/data_at_top"})
# check that the data can be read as a stack through a external link
external = self._h5File + "external.h5"
if os.path.exists(external):
os.remove(external)
h5 = h5py.File(external, "w")
h5["/data_at_top"] = h5py.ExternalLink(
self._h5File, "/entry/measurement/mca_soft/data")
h5["/entry/data"] = h5py.ExternalLink(
self._h5File, "/entry/measurement/mca_soft/data")
h5.flush()
h5.close()
h5 = None
fileList = [external]
for selection in [
{
"y": "/data_at_top"
}, # dataset at top level
{
"y": "/data"
}, # GOOD: selection inside /entry
{
"y": "/entry/data"
}
]: # WRONG: complete path
stack = HDF5Stack1D.HDF5Stack1D(fileList, selection)
info = stack.info
for key in ["McaCalib", "McaLiveTime"]:
self.assertTrue(
key in info,
"Key <%s> not present but it should be there")
readCalib = info["McaCalib"]
readLiveTime = info["McaLiveTime"]
self.assertTrue(abs(readCalib[0] - calibration[0]) < 1.0e-10,
"Calibration zero. Expected %f got %f" % \
(calibration[0], readCalib[0]))
self.assertTrue(abs(readCalib[1] - calibration[1]) < 1.0e-10,
"Calibration gain. Expected %f got %f" % \
(calibration[1], readCalib[0]))
self.assertTrue(abs(readCalib[2] - calibration[2]) < 1.0e-10,
"Calibration 2nd order. Expected %f got %f" % \
(calibration[2], readCalib[2]))
self.assertTrue(live_time.size == readLiveTime.size,
"Incorrect size of live time data")
self.assertTrue(numpy.allclose(live_time, readLiveTime),
"Incorrect live time read")
self.assertTrue(numpy.allclose(stack.x, channels),
"Incorrect channels read")
self.assertTrue(numpy.allclose(stack.data, data),
"Incorrect data read")
# check that the data can be read as a stack
fileList = [self._h5File]
for selection in [{
"y": "/measurement/mca_with_info/data"
}, {
"y": "/measurement/mca_soft/data"
}, {
"y": "/instrument/detector/data"
}]:
stack = HDF5Stack1D.HDF5Stack1D(fileList, selection)
info = stack.info
for key in ["McaCalib", "McaLiveTime"]:
self.assertTrue(
key in info,
"Key <%s> not present but it should be there")
readCalib = info["McaCalib"]
readLiveTime = info["McaLiveTime"]
self.assertTrue(abs(readCalib[0] - calibration[0]) < 1.0e-10,
"Calibration zero. Expected %f got %f" % \
(calibration[0], readCalib[0]))
self.assertTrue(abs(readCalib[1] - calibration[1]) < 1.0e-10,
"Calibration gain. Expected %f got %f" % \
(calibration[1], readCalib[0]))
self.assertTrue(abs(readCalib[2] - calibration[2]) < 1.0e-10,
"Calibration 2nd order. Expected %f got %f" % \
(calibration[2], readCalib[2]))
self.assertTrue(live_time.size == readLiveTime.size,
"Incorrect size of live time data")
self.assertTrue(numpy.allclose(live_time, readLiveTime),
"Incorrect live time read")
self.assertTrue(numpy.allclose(stack.x, channels),
"Incorrect channels read")
self.assertTrue(numpy.allclose(stack.data, data),
"Incorrect data read")
# perform the batch fit
self._outputDir = os.path.join(tempfile.gettempdir(), "SteelTestDir")
if not os.path.exists(self._outputDir):
os.mkdir(self._outputDir)
cfgFile = os.path.join(tempfile.gettempdir(), "SteelNew.cfg")
if os.path.exists(cfgFile):
try:
os.remove(cfgFile)
except:
print("Cannot remove file %s" % cfgFile)
# we need to make sure we use fundamental parameters and
# the time read from the file
configuration["concentrations"]["usematrix"] = 0
configuration["concentrations"]["useautotime"] = 1
if not os.path.exists(cfgFile):
configuration.write(cfgFile)
os.chmod(cfgFile, 0o777)
batch = McaAdvancedFitBatch.McaAdvancedFitBatch(
cfgFile,
filelist=[self._h5File],
outputdir=self._outputDir,
concentrations=True,
selection=selection,
quiet=True)
batch.processList()
# recover the results
imageFile = os.path.join(self._outputDir, "IMAGES", "Steel.dat")
self.assertTrue(os.path.isfile(imageFile),
"Batch fit result file <%s> not present" % imageFile)
sf = specfile.Specfile(imageFile)
labels = sf[0].alllabels()
scanData = sf[0].data()
sf = None
self.assertTrue(
scanData.shape[-1] == (nRows * nColumns),
"Expected %d values got %d" %
(nRows * nColumns, scanData.shape[-1]))
referenceResult = {}
for point in range(scanData.shape[-1]):
for label in labels:
idx = labels.index(label)
if label in ["Point", "row", "column"]:
continue
elif point == 0:
referenceResult[label] = scanData[idx, point]
elif label.endswith("-mass-fraction"):
#print("label = ", label)
#print("reference = ", referenceResult[label])
#print("current = ", scanData[idx, point])
reference = referenceResult[label]
current = scanData[idx, point]
#print("ratio = ", current / reference)
#print("time ratio = ", readLiveTime[point] / readLiveTime[0])
if point % nTimes:
if abs(reference) > 1.0e-10:
self.assertTrue(
reference != current,
"Incorrect concentration for point %d" % point)
corrected = current * \
(readLiveTime[point] / readLiveTime[0])
if abs(reference) > 1.0e-10:
delta = \
100 * abs((reference - corrected) / reference)
self.assertTrue(
delta < 0.01,
"Incorrect concentration(t) for point %d" %
point)
else:
self.assertTrue(
abs(reference - corrected) < 1.0e-5,
"Incorrect concentration(t) for point %d" %
point)
else:
self.assertTrue(
reference == current,
"Incorrect concentration for point %d" % point)
elif label not in ["Point", "row", "column"]:
reference = referenceResult[label]
current = scanData[idx, point]
self.assertTrue(reference == current,
"Incorrect value for point %d" % point)
# Batch fitting went well
# Test the fast XRF
from PyMca5.PyMcaPhysics.xrf import FastXRFLinearFit
ffit = FastXRFLinearFit.FastXRFLinearFit()
configuration["concentrations"]["usematrix"] = 0
configuration["concentrations"]["useautotime"] = 1
configuration['fit']['stripalgorithm'] = 1
outputDict = ffit.fitMultipleSpectra(y=stack,
weight=0,
configuration=configuration,
concentrations=True,
refit=0)
names = outputDict["names"]
parameters = outputDict["parameters"]
uncertainties = outputDict["uncertainties"]
concentrations = outputDict["concentrations"]
cCounter = 0
for i in range(len(names)):
name = names[i]
if name.startswith("C(") and name.endswith(")"):
# it is a concentrations parameter
# verify that concentrations took into account the time
reference = concentrations[cCounter][0, 0]
cTime = configuration['concentrations']['time']
values = concentrations[cCounter][:]
values.shape = -1
for point in range(live_time.size):
current = values[point]
if DEBUG:
print(name, point, reference, current, point % nTimes)
if (point % nTimes) and (abs(reference) > 1.0e-10):
self.assertTrue(
reference != current,
"Incorrect concentration for point %d" % point)
corrected = current * live_time[point] / cTime
if abs(reference) > 1.0e-10:
delta = 100 * abs((reference - corrected) / reference)
self.assertTrue(
delta < 0.01,
"Incorrect concentration(t) for point %d" % point)
else:
self.assertTrue(
abs(reference - corrected) < 1.0e-5,
"Incorrect concentration(t) for point %d" % point)
cCounter += 1
else:
if DEBUG:
print(name, parameters[i][0, 0])
delta = (parameters[i] - parameters[i][0, 0])
self.assertTrue(delta.max() == 0,
"Different fit value for parameter %s delta %f" % \
(name, delta.max()))
self.assertTrue(delta.min() == 0,
"Different fit value for parameter %s delta %f" % \
(name, delta.min()))
delta = (uncertainties[i] - uncertainties[i][0, 0])
self.assertTrue(delta.max() == 0,
"Different sigma value for parameter %s delta %f" % \
(name, delta.max()))
self.assertTrue(delta.min() == 0,
"Different sigma value for parameter %s delta %f" % \
(name, delta.min()))
outputDict = ffit.fitMultipleSpectra(y=stack,
weight=0,
configuration=configuration,
concentrations=True,
refit=1)
names = outputDict["names"]
parameters = outputDict["parameters"]
uncertainties = outputDict["uncertainties"]
concentrations = outputDict["concentrations"]
cCounter = 0
for i in range(len(names)):
name = names[i]
if name.startswith("C(") and name.endswith(")"):
# it is a concentrations parameter
# verify that concentrations took into account the time
reference = concentrations[cCounter][0, 0]
cTime = configuration['concentrations']['time']
values = concentrations[cCounter][:]
values.shape = -1
for point in range(live_time.size):
current = values[point]
if DEBUG:
print(name, point, reference, current, point % nTimes)
if (point % nTimes) and (abs(reference) > 1.0e-10):
self.assertTrue(
reference != current,
"Incorrect concentration for point %d" % point)
corrected = current * live_time[point] / cTime
if abs(reference) > 1.0e-10:
delta = 100 * abs((reference - corrected) / reference)
self.assertTrue(
delta < 0.01,
"Incorrect concentration(t) for point %d" % point)
else:
self.assertTrue(
abs(reference - corrected) < 1.0e-5,
"Incorrect concentration(t) for point %d" % point)
cCounter += 1
else:
if DEBUG:
print(name, parameters[i][0, 0])
delta = (parameters[i] - parameters[i][0, 0])
self.assertTrue(delta.max() == 0,
"Different fit value for parameter %s delta %f" % \
(name, delta.max()))
self.assertTrue(delta.min() == 0,
"Different fit value for parameter %s delta %f" % \
(name, delta.min()))
delta = (uncertainties[i] - uncertainties[i][0, 0])
self.assertTrue(delta.max() == 0,
"Different sigma value for parameter %s delta %f" % \
(name, delta.max()))
self.assertTrue(delta.min() == 0,
"Different sigma value for parameter %s delta %f" % \
(name, delta.min()))
def testSingleStackExport(self):
from PyMca5 import PyMcaDataDir
from PyMca5.PyMcaIO import specfilewrapper as specfile
from PyMca5.PyMcaIO import ConfigDict
from PyMca5.PyMcaCore import DataObject
from PyMca5.PyMcaCore import StackBase
from PyMca5.PyMcaCore import McaStackExport
spe = os.path.join(self.dataDir, "Steel.spe")
cfg = os.path.join(self.dataDir, "Steel.cfg")
sf = specfile.Specfile(spe)
self.assertTrue(len(sf) == 1, "File %s cannot be read" % spe)
self.assertTrue(sf[0].nbmca() == 1, "Spe file should contain MCA data")
y = counts = sf[0].mca(1)
x = channels = numpy.arange(y.size).astype(numpy.float)
sf = None
configuration = ConfigDict.ConfigDict()
configuration.read(cfg)
calibration = configuration["detector"]["zero"], \
configuration["detector"]["gain"], 0.0
initialTime = configuration["concentrations"]["time"]
# create the data
nRows = 5
nColumns = 10
nTimes = 3
data = numpy.zeros((nRows, nColumns, counts.size), dtype=numpy.float)
live_time = numpy.zeros((nRows * nColumns), dtype=numpy.float)
xpos = 10 + numpy.zeros((nRows * nColumns), dtype=numpy.float)
ypos = 100 + numpy.zeros((nRows * nColumns), dtype=numpy.float)
mcaIndex = 0
for i in range(nRows):
for j in range(nColumns):
data[i, j] = counts
live_time[i * nColumns + j] = initialTime * \
(1 + mcaIndex % nTimes)
xpos[mcaIndex] += j
ypos[mcaIndex] += i
mcaIndex += 1
# create the stack data object
stack = DataObject.DataObject()
stack.data = data
stack.info = {}
stack.info["McaCalib"] = calibration
stack.info["McaLiveTime"] = live_time
stack.x = [channels]
stack.info["positioners"] = {"x": xpos, "y": ypos}
tmpDir = tempfile.gettempdir()
self._h5File = os.path.join(tmpDir, "SteelStack.h5")
if os.path.exists(self._h5File):
os.remove(self._h5File)
McaStackExport.exportStackList(stack, self._h5File)
# read back the stack
from PyMca5.PyMcaIO import HDF5Stack1D
stackRead = HDF5Stack1D.HDF5Stack1D([self._h5File],
{"y": "/measurement/detector_00"})
# let's play
sb = StackBase.StackBase()
sb.setStack(stackRead)
# positioners
data = stackRead.info["positioners"]["x"]
self.assertTrue(numpy.allclose(data, xpos),
"Incorrect readout of x positions")
data = stackRead.info["positioners"]["y"]
self.assertTrue(numpy.allclose(data, ypos),
"Incorrect readout of y positions")
# calibration and live time
x, y, legend, info = sb.getStackOriginalCurve()
readCalib = info["McaCalib"]
readLiveTime = info["McaLiveTime"]
self.assertTrue(abs(readCalib[0] - calibration[0]) < 1.0e-10,
"Calibration zero. Expected %f got %f" % \
(calibration[0], readCalib[0]))
self.assertTrue(abs(readCalib[1] - calibration[1]) < 1.0e-10,
"Calibration gain. Expected %f got %f" % \
(calibration[1], readCalib[0]))
self.assertTrue(abs(readCalib[2] - calibration[2]) < 1.0e-10,
"Calibration 2nd order. Expected %f got %f" % \
(calibration[2], readCalib[2]))
self.assertTrue(
abs(live_time.sum() - readLiveTime) < 1.0e-5,
"Incorrect sum of live time data")
def testFitHdf5Stack(self):
import tempfile
from PyMca5.PyMcaIO import specfilewrapper as specfile
from PyMca5.PyMcaIO import ConfigDict
from PyMca5.PyMcaIO import HDF5Stack1D
from PyMca5.PyMcaPhysics.xrf import McaAdvancedFitBatch
from PyMca5.PyMcaPhysics.xrf import LegacyMcaAdvancedFitBatch
spe = os.path.join(self.dataDir, "Steel.spe")
cfg = os.path.join(self.dataDir, "Steel.cfg")
sf = specfile.Specfile(spe)
self.assertTrue(len(sf) == 1, "File %s cannot be read" % spe)
self.assertTrue(sf[0].nbmca() == 1, "Spe file should contain MCA data")
y = counts = sf[0].mca(1)
x = channels = numpy.arange(y.size).astype(numpy.float)
sf = None
configuration = ConfigDict.ConfigDict()
configuration.read(cfg)
calibration = configuration["detector"]["zero"], \
configuration["detector"]["gain"], 0.0
initialTime = configuration["concentrations"]["time"]
# create the data
nRows = 5
nColumns = 10
nTimes = 3
data = numpy.zeros((nRows, nColumns, counts.size), dtype=numpy.float)
live_time = numpy.zeros((nRows * nColumns), dtype=numpy.float)
mcaIndex = 0
for i in range(nRows):
for j in range(nColumns):
data[i, j] = counts
live_time[i * nColumns + j] = initialTime * \
(1 + mcaIndex % nTimes)
mcaIndex += 1
self._h5File = os.path.join(tempfile.gettempdir(), "Steel.h5")
# write the stack to an HDF5 file
if os.path.exists(self._h5File):
os.remove(self._h5File)
h5 = h5py.File(self._h5File, "w")
h5["/entry/instrument/detector/calibration"] = calibration
h5["/entry/instrument/detector/channels"] = channels
h5["/entry/instrument/detector/data"] = data
h5["/entry/instrument/detector/live_time"] = live_time
# add nexus conventions
h5["/entry"].attrs["NX_class"] = u"NXentry"
h5["/entry/instrument"].attrs["NX_class"] = u"NXinstrument"
h5["/entry/instrument/detector/"].attrs["NX_class"] = u"NXdetector"
h5["/entry/instrument/detector/data"].attrs["interpretation"] = \
u"spectrum"
# case with softlink
h5["/entry/measurement/mca_soft/data"] = \
h5py.SoftLink("/entry/instrument/detector/data")
# case with info
h5["/entry/measurement/mca_with_info/data"] = \
h5["/entry/instrument/detector/data"]
h5["/entry/measurement/mca_with_info/info"] = \
h5["/entry/instrument/detector"]
h5.flush()
h5.close()
h5 = None
# check that the data can be read as a stack as
# single top level dataset (issue #226)
external = self._h5File + "external.h5"
if os.path.exists(external):
os.remove(external)
h5 = h5py.File(external, "w")
h5["/data_at_top"] = h5py.ExternalLink(
self._h5File, "/entry/measurement/mca_soft/data")
h5.flush()
h5.close()
h5 = None
stack = HDF5Stack1D.HDF5Stack1D([external], {"y": "/data_at_top"})
# check that the data can be read as a stack through a external link
external = self._h5File + "external.h5"
if os.path.exists(external):
os.remove(external)
h5 = h5py.File(external, "w")
h5["/data_at_top"] = h5py.ExternalLink(
self._h5File, "/entry/measurement/mca_soft/data")
h5["/entry/data"] = h5py.ExternalLink(
self._h5File, "/entry/measurement/mca_soft/data")
h5.flush()
h5.close()
h5 = None
fileList = [external]
for selection in [
{
"y": "/data_at_top"
}, # dataset at top level
{
"y": "/data"
}, # GOOD: selection inside /entry
{
"y": "/entry/data"
}
]: # WRONG: complete path
stack = HDF5Stack1D.HDF5Stack1D(fileList, selection)
info = stack.info
for key in ["McaCalib", "McaLiveTime"]:
self.assertTrue(
key in info,
"Key <%s> not present but it should be there")
readCalib = info["McaCalib"]
readLiveTime = info["McaLiveTime"]
self.assertTrue(abs(readCalib[0] - calibration[0]) < 1.0e-10,
"Calibration zero. Expected %f got %f" % \
(calibration[0], readCalib[0]))
self.assertTrue(abs(readCalib[1] - calibration[1]) < 1.0e-10,
"Calibration gain. Expected %f got %f" % \
(calibration[1], readCalib[0]))
self.assertTrue(abs(readCalib[2] - calibration[2]) < 1.0e-10,
"Calibration 2nd order. Expected %f got %f" % \
(calibration[2], readCalib[2]))
self.assertTrue(live_time.size == readLiveTime.size,
"Incorrect size of live time data")
self.assertTrue(numpy.allclose(live_time, readLiveTime),
"Incorrect live time read")
self.assertTrue(numpy.allclose(stack.x, channels),
"Incorrect channels read")
self.assertTrue(numpy.allclose(stack.data, data),
"Incorrect data read")
# check that the data can be read as a stack
fileList = [self._h5File]
for selection in [{
"y": "/measurement/mca_with_info/data"
}, {
"y": "/measurement/mca_soft/data"
}, {
"y": "/instrument/detector/data"
}]:
stack = HDF5Stack1D.HDF5Stack1D(fileList, selection)
info = stack.info
for key in ["McaCalib", "McaLiveTime"]:
self.assertTrue(
key in info,
"Key <%s> not present but it should be there")
readCalib = info["McaCalib"]
readLiveTime = info["McaLiveTime"]
self.assertTrue(abs(readCalib[0] - calibration[0]) < 1.0e-10,
"Calibration zero. Expected %f got %f" % \
(calibration[0], readCalib[0]))
self.assertTrue(abs(readCalib[1] - calibration[1]) < 1.0e-10,
"Calibration gain. Expected %f got %f" % \
(calibration[1], readCalib[0]))
self.assertTrue(abs(readCalib[2] - calibration[2]) < 1.0e-10,
"Calibration 2nd order. Expected %f got %f" % \
(calibration[2], readCalib[2]))
self.assertTrue(live_time.size == readLiveTime.size,
"Incorrect size of live time data")
self.assertTrue(numpy.allclose(live_time, readLiveTime),
"Incorrect live time read")
self.assertTrue(numpy.allclose(stack.x, channels),
"Incorrect channels read")
self.assertTrue(numpy.allclose(stack.data, data),
"Incorrect data read")
# TODO: this is done in PyMcaBatchTest on multiple input formats
# so not needed here
return
# perform the batch fit
self._outputDir = os.path.join(tempfile.gettempdir(), "SteelTestDir")
if not os.path.exists(self._outputDir):
os.mkdir(self._outputDir)
cfgFile = os.path.join(tempfile.gettempdir(), "SteelNew.cfg")
if os.path.exists(cfgFile):
try:
os.remove(cfgFile)
except:
print("Cannot remove file %s" % cfgFile)
# we need to make sure we use fundamental parameters and
# the time read from the file
configuration["concentrations"]["usematrix"] = 0
configuration["concentrations"]["useautotime"] = 1
if not os.path.exists(cfgFile):
configuration.write(cfgFile)
os.chmod(cfgFile, 0o777)
# Test batch fitting (legacy)
batch = LegacyMcaAdvancedFitBatch.McaAdvancedFitBatch(
cfgFile,
filelist=[self._h5File],
outputdir=self._outputDir,
concentrations=True,
selection=selection,
quiet=True)
batch.processList()
imageFile = os.path.join(self._outputDir, "IMAGES", "Steel.dat")
self._verifyBatchFitResult(imageFile,
nRows,
nColumns,
live_time,
nTimes,
legacy=True)
# Test batch fitting
batch = McaAdvancedFitBatch.McaAdvancedFitBatch(
cfgFile,
filelist=[self._h5File],
outputdir=self._outputDir,
concentrations=True,
selection=selection,
quiet=True)
batch.outbuffer.extensions = ['.dat']
batch.processList()
imageFile = batch.outbuffer.filename('.dat')
self._verifyBatchFitResult(imageFile, nRows, nColumns, live_time,
nTimes)
# Batch fitting went well
# Test the fast XRF
configuration["concentrations"]["usematrix"] = 0
configuration["concentrations"]["useautotime"] = 1
configuration['fit']['stripalgorithm'] = 1
self._verifyFastFit(stack, configuration, live_time, nTimes)