def main(argv=None):
if argv is None:
# Bevington data of Table 6-2
x = [0, 15, 30, 45, 60, 75, 90, 105, 120, 135]
y = [106, 80, 98, 75, 74, 73, 49, 38, 37, 22]
sigmay = numpy.sqrt(numpy.array(y))
slope, intercept, r, ddict = linregress(x, y, sigmay=sigmay, full_output=True)
print("WEIGHTED DATA")
print("LINREGRESS results")
print("SLOPE = ", ddict["slope"], " +/- ", ddict["sigma_slope"])
print("INTERCEPT = ", ddict["intercept"], " +/- ", ddict["sigma_intercept"])
from PyMca5.PyMcaMath.linalg import lstsq
derivatives = numpy.zeros((len(y), 2))
derivatives[:, 0] = numpy.array(x, dtype=numpy.float64)
derivatives[:, 1] = 1.0
print("LEAST SQUARES RESULT")
result = lstsq(derivatives, y, sigma_b=sigmay, weight=1, uncertainties=True)
print("SLOPE = ", result[0][0], " +/- ", result[1][0])
print("INTERCEPT = ", result[0][1], " +/- ", result[1][1])
print("\n\n")
# Bevington data of Table 6-1
x = [1, 2, 3, 4, 5, 6, 7, 8, 9]
y = [15.6, 17.5, 36.6, 43.8, 58.2, 61.6, 64.2, 70.4, 98.8]
print("UNWEIGHTED DATA")
slope, intercept, r, ddict = linregress(x, y, sigmay=None, full_output=True)
print("LINREGRESS results")
print("SLOPE = ", ddict["slope"], " +/- ", ddict["sigma_slope"])
print("INTERCEPT = ", ddict["intercept"], " +/- ", ddict["sigma_intercept"])
derivatives = numpy.zeros((len(y), 2))
derivatives[:, 0] = numpy.array(x, dtype=numpy.float64)
derivatives[:, 1] = 1.0
print("LEAST SQUARES RESULT")
result = lstsq(derivatives, y, sigma_b=None, weight=0, uncertainties=True)
print("SLOPE = ", result[0][0], " +/- ", result[1][0])
print("INTERCEPT = ", result[0][1], " +/- ", result[1][1])
print("\n\n")
elif len(argv) > 1:
# assume we have got a two (or three) column csv file
data = numpy.loadtxt(argv[1])
x = data[:, 0]
y = data[:, 1]
if data.shape[1] > 2:
sigmay = data[:, 2]
else:
sigmay = None
slope, intercept, r, ddict = linregress(x, y,
sigmay=sigmay,
full_output=True)
print("LINREGRESS results")
print("SLOPE = ", ddict["slope"], " +/- ", ddict["sigma_slope"])
print("INTERCEPT = ", ddict["intercept"], " +/- ", ddict["sigma_intercept"])
else:
print("RateLaw [csv_file_name]")
return
def main(argv=None):
if argv is None:
# Bevington data of Table 6-2
x = [0, 15, 30, 45, 60, 75, 90, 105, 120, 135]
y = [106, 80, 98, 75, 74, 73, 49, 38, 37, 22]
sigmay = numpy.sqrt(numpy.array(y))
slope, intercept, r, ddict = linregress(x, y, sigmay=sigmay, full_output=True)
print("WEIGHTED DATA")
print("LINREGRESS results")
print("SLOPE = ", ddict["slope"], " +/- ", ddict["sigma_slope"])
print("INTERCEPT = ", ddict["intercept"], " +/- ", ddict["sigma_intercept"])
from PyMca5.PyMcaMath.linalg import lstsq
derivatives = numpy.zeros((len(y), 2))
derivatives[:, 0] = numpy.array(x, dtype=numpy.float)
derivatives[:, 1] = 1.0
print("LEAST SQUARES RESULT")
result = lstsq(derivatives, y, sigma_b=sigmay, weight=1, uncertainties=True)
print("SLOPE = ", result[0][0], " +/- ", result[1][0])
print("INTERCEPT = ", result[0][1], " +/- ", result[1][1])
print("\n\n")
# Bevington data of Table 6-1
x = [1, 2, 3, 4, 5, 6, 7, 8, 9]
y = [15.6, 17.5, 36.6, 43.8, 58.2, 61.6, 64.2, 70.4, 98.8]
print("UNWEIGHTED DATA")
slope, intercept, r, ddict = linregress(x, y, sigmay=None, full_output=True)
print("LINREGRESS results")
print("SLOPE = ", ddict["slope"], " +/- ", ddict["sigma_slope"])
print("INTERCEPT = ", ddict["intercept"], " +/- ", ddict["sigma_intercept"])
derivatives = numpy.zeros((len(y), 2))
derivatives[:, 0] = numpy.array(x, dtype=numpy.float)
derivatives[:, 1] = 1.0
print("LEAST SQUARES RESULT")
result = lstsq(derivatives, y, sigma_b=None, weight=0, uncertainties=True)
print("SLOPE = ", result[0][0], " +/- ", result[1][0])
print("INTERCEPT = ", result[0][1], " +/- ", result[1][1])
print("\n\n")
elif len(argv) > 1:
# assume we have got a two (or three) column csv file
data = numpy.loadtxt(argv[1])
x = data[:, 0]
y = data[:, 1]
if data.shape[1] > 2:
sigmay = data[:, 2]
else:
sigmay = None
slope, intercept, r, ddict = linregress(x, y,
sigmay=sigmay,
full_output=True)
print("LINREGRESS results")
print("SLOPE = ", ddict["slope"], " +/- ", ddict["sigma_slope"])
print("INTERCEPT = ", ddict["intercept"], " +/- ", ddict["sigma_intercept"])
else:
print("RateLaw [csv_file_name]")
return
def _fitLstSqAll(self, data=None, sliceChan=None, mcaIndex=None,
derivatives=None, results=None, uncertainties=None,
fitmodel=None, config=None, anchorslist=None,
lstsq_kwargs=None):
"""
Fit all spectra
"""
nChan, nFree = derivatives.shape
bkgsub = bool(config['fit']['stripflag'])
nMca = self._numberOfSpectra(1, 'MiB', data=data, mcaIndex=mcaIndex,
sliceChan=sliceChan)
_logger.debug('Fit spectra in chunks of {}'.format(nMca))
chunkItems = self._dataChunkIter(McaStackView.FullView,
data=data,
fitmodel=fitmodel,
mcaSlice=sliceChan,
mcaAxis=mcaIndex,
nMca=nMca)
for chunk in chunkItems:
if fitmodel is None:
(idx, idxShape), chunk = chunk
chunkModel = None
else:
((idx, idxShape), chunk), (_, chunkModel) = chunk
chunkModel = chunkModel.T
chunk = chunk.T
# Subtract background
if bkgsub:
self._fitBkgSubtract(chunk, config=config,
anchorslist=anchorslist,
fitmodel=chunkModel)
# Solve linear system of equations
ddict = lstsq(derivatives, chunk, digested_output=True,
**lstsq_kwargs)
lstsq_kwargs['last_svd'] = ddict.get('svd', None)
# Save results
idx = (slice(None),) + idx
idxShape = (nFree,) + idxShape
results[idx] = ddict['parameters'].reshape(idxShape)
uncertainties[idx] = ddict['uncertainties'].reshape(idxShape)
if chunkModel is not None:
if bkgsub:
chunkModel += numpy.dot(derivatives, ddict['parameters'])
else:
chunkModel[()] = numpy.dot(derivatives, ddict['parameters'])
def fitMultipleSpectra(self, x=None, y=None, xmin=None, xmax=None,
configuration=None, concentrations=False,
ysum=None, weight=None, refit=True,
livetime=None):
"""
This method performs the actual fit. The y keyword is the only mandatory input argument.
:param x: 1D array containing the x axis (usually the channels) of the spectra.
:param y: 3D array containing the spectra as [nrows, ncolumns, nchannels]
:param xmin: lower limit of the fitting region
:param xmax: upper limit of the fitting region
:param weight: 0 Means no weight, 1 Use an average weight, 2 Individual weights (slow)
:param concentrations: 0 Means no calculation, 1 Calculate them
:param refit: if False, no check for negative results. Default is True.
:livetime: It will be used if not different from None and concentrations
are to be calculated by using fundamental parameters with
automatic time. The default is None.
:return: A dictionary with the parameters, uncertainties, concentrations and names as keys.
"""
if y is None:
raise RuntimeError("y keyword argument is mandatory!")
if hasattr(y, "info") and hasattr(y, "data"):
data = y.data
mcaIndex = y.info.get("McaIndex", -1)
else:
data = y
mcaIndex = -1
if x is None:
if hasattr(y, "info") and hasattr(y, "x"):
x = y.x[0]
if livetime is None:
if hasattr(y, "info"):
if "McaLiveTime" in y.info:
livetime = y.info["McaLiveTime"]
t0 = time.time()
if configuration is not None:
self._mcaTheory.setConfiguration(configuration)
elif self._config is None:
raise ValueError("Fit configuration missing")
else:
_logger.debug("Setting default configuration")
self._mcaTheory.setConfiguration(self._config)
# read the current configuration
# it is a copy, we can modify it at will
config = self._mcaTheory.getConfiguration()
if xmin is None:
xmin = config['fit']['xmin']
if xmax is None:
xmax = config['fit']['xmax']
toReconfigure = False
# if concentrations and use times, it needs to be reconfigured
# without using times and correct later on. If the concentrations
# are to be calculated from internal standard there is no need to
# raise an exception either.
autotime = 0
liveTimeFactor = 1.0
if not concentrations:
# ignore any time information to prevent unnecessary errors when
# setting the fitting data whithout the time information
if config['concentrations'].get("useautotime", 0):
config['concentrations']["useautotime"] = 0
toReconfigure = True
elif config["concentrations"]["usematrix"]:
if config['concentrations'].get("useautotime", 0):
config['concentrations']["useautotime"] = 0
toReconfigure = True
else:
# we are calculating concentrations from fundamental parameters
autotime = config['concentrations'].get("useautotime", 0)
nSpectra = data.size // data.shape[mcaIndex]
if autotime:
if livetime is None:
txt = "Automatic time requested but no time information provided"
raise RuntimeError(txt)
elif numpy.isscalar(livetime):
liveTimeFactor = \
float(config['concentrations']["time"]) / livetime
elif livetime.size == nSpectra:
liveTimeFactor = \
float(config['concentrations']["time"]) / livetime
else:
raise RuntimeError( \
"Number of live times not equal number of spectra")
config['concentrations']["useautotime"] = 0
toReconfigure = True
# use of strategies is not supported for the time being
strategy = config['fit'].get('strategyflag', 0)
if strategy:
raise RuntimeError("Strategies are incompatible with fast fit")
# background
if config['fit']['stripflag']:
if config['fit']['stripalgorithm'] == 1:
_logger.debug("SNIP")
else:
raise RuntimeError("Please use the faster SNIP background")
if weight is None:
# dictated by the file
weight = config['fit']['fitweight']
if weight:
# individual pixel weights (slow)
weightPolicy = 2
else:
# No weight
weightPolicy = 0
elif weight == 1:
# use average weight from the sum spectrum
weightPolicy = 1
if not config['fit']['fitweight']:
config['fit']['fitweight'] = 1
toReconfigure = True
elif weight == 2:
# individual pixel weights (slow)
weightPolicy = 2
if not config['fit']['fitweight']:
config['fit']['fitweight'] = 1
toReconfigure = True
weight = 1
else:
# No weight
weightPolicy = 0
if config['fit']['fitweight']:
config['fit']['fitweight'] = 0
toReconfigure = True
weight = 0
if not config['fit']['linearfitflag']:
#make sure we force a linear fit
config['fit']['linearfitflag'] = 1
toReconfigure = True
if toReconfigure:
# we must configure again the fit
self._mcaTheory.setConfiguration(config)
if len(data.shape) != 3:
txt = "For the time being only three dimensional arrays supported"
raise IndexError(txt)
elif mcaIndex not in [-1, 2]:
txt = "For the time being only mca arrays supported"
raise IndexError(txt)
else:
# if the cumulated spectrum is present it should be better
nRows = data.shape[0]
nColumns = data.shape[1]
nPixels = nRows * nColumns
if ysum is not None:
firstSpectrum = ysum
elif weightPolicy == 1:
# we need to calculate the sum spectrum to derive the uncertainties
totalSpectra = data.shape[0] * data.shape[1]
jStep = min(5000, data.shape[1])
ysum = numpy.zeros((data.shape[mcaIndex],), numpy.float)
for i in range(0, data.shape[0]):
if i == 0:
chunk = numpy.zeros((data.shape[0], jStep), numpy.float)
jStart = 0
while jStart < data.shape[1]:
jEnd = min(jStart + jStep, data.shape[1])
ysum += data[i, jStart:jEnd, :].sum(axis=0, dtype=numpy.float)
jStart = jEnd
firstSpectrum = ysum
elif not concentrations:
# just one spectrum is enough for the setup
firstSpectrum = data[0, 0, :]
else:
firstSpectrum = data[0, :, :].sum(axis=0, dtype=numpy.float)
# make sure we calculate the matrix of the contributions
self._mcaTheory.enableOptimizedLinearFit()
# initialize the fit
# print("xmin = ", xmin)
# print("xmax = ", xmax)
# print("firstShape = ", firstSpectrum.shape)
self._mcaTheory.setData(x=x, y=firstSpectrum, xmin=xmin, xmax=xmax)
# and initialize the derivatives
self._mcaTheory.estimate()
# now we can get the derivatives respect to the free parameters
# These are the "derivatives" respect to the peaks
# linearMatrix = self._mcaTheory.linearMatrix
# but we are still missing the derivatives from the background
nFree = 0
freeNames = []
nFreeBackgroundParameters = 0
for i, param in enumerate(self._mcaTheory.PARAMETERS):
if self._mcaTheory.codes[0][i] != ClassMcaTheory.Gefit.CFIXED:
nFree += 1
freeNames.append(param)
if i < self._mcaTheory.NGLOBAL:
nFreeBackgroundParameters += 1
if nFree == 0:
txt = "No free parameters to be fitted!\n"
txt += "No peaks inside fitting region?"
raise ValueError(txt)
#build the matrix of derivatives
derivatives = None
idx = 0
for i, param in enumerate(self._mcaTheory.PARAMETERS):
if self._mcaTheory.codes[0][i] == ClassMcaTheory.Gefit.CFIXED:
continue
deriv= self._mcaTheory.linearMcaTheoryDerivative(self._mcaTheory.parameters,
i,
self._mcaTheory.xdata)
deriv.shape = -1
if derivatives is None:
derivatives = numpy.zeros((deriv.shape[0], nFree), numpy.float)
derivatives[:, idx] = deriv
idx += 1
#loop for anchors
xdata = self._mcaTheory.xdata
if config['fit']['stripflag']:
anchorslist = []
if config['fit']['stripanchorsflag']:
if config['fit']['stripanchorslist'] is not None:
ravelled = numpy.ravel(xdata)
for channel in config['fit']['stripanchorslist']:
if channel <= ravelled[0]:continue
index = numpy.nonzero(ravelled >= channel)[0]
if len(index):
index = min(index)
if index > 0:
anchorslist.append(index)
if len(anchorslist) == 0:
anchorlist = [0, self._mcaTheory.ydata.size - 1]
anchorslist.sort()
# find the indices to be used for selecting the appropriate data
# if the original x data were not ordered we have a problem
# TODO: check for original ordering.
if x is None:
# we have an enumerated channels axis
iXMin = xdata[0]
iXMax = xdata[-1]
else:
iXMin = numpy.nonzero(x <= xdata[0])[0][-1]
iXMax = numpy.nonzero(x >= xdata[-1])[0][0]
# numpy 1.11.0 returns an array on previous expression
# and then complains about a future deprecation warning
# because of using an array and not an scalar in the selection
if hasattr(iXMin, "shape"):
if len(iXMin.shape):
iXMin = iXMin[0]
if hasattr(iXMax, "shape"):
if len(iXMax.shape):
iXMax = iXMax[0]
dummySpectrum = firstSpectrum[iXMin:iXMax+1].reshape(-1, 1)
# print("dummy = ", dummySpectrum.shape)
# allocate the output buffer
results = numpy.zeros((nFree, nRows, nColumns), numpy.float32)
uncertainties = numpy.zeros((nFree, nRows, nColumns), numpy.float32)
#perform the initial fit
_logger.debug("Configuration elapsed = %f", time.time() - t0)
t0 = time.time()
totalSpectra = data.shape[0] * data.shape[1]
jStep = min(100, data.shape[1])
if weightPolicy == 2:
SVD = False
sigma_b = None
elif weightPolicy == 1:
# the +1 is to prevent misbehavior due to weights less than 1.0
sigma_b = 1 + numpy.sqrt(dummySpectrum)/nPixels
SVD = True
else:
SVD = True
sigma_b = None
last_svd = None
for i in range(0, data.shape[0]):
#print(i)
#chunks of nColumns spectra
if i == 0:
chunk = numpy.zeros((dummySpectrum.shape[0],
jStep),
numpy.float)
jStart = 0
while jStart < data.shape[1]:
jEnd = min(jStart + jStep, data.shape[1])
chunk[:,:(jEnd - jStart)] = data[i, jStart:jEnd, iXMin:iXMax+1].T
if config['fit']['stripflag']:
for k in range(jStep):
# obtain the smoothed spectrum
background=SpecfitFuns.SavitskyGolay(chunk[:, k],
config['fit']['stripfilterwidth'])
lastAnchor = 0
for anchor in anchorslist:
if (anchor > lastAnchor) and (anchor < background.size):
background[lastAnchor:anchor] =\
SpecfitFuns.snip1d(background[lastAnchor:anchor],
config['fit']['snipwidth'],
0)
lastAnchor = anchor
if lastAnchor < background.size:
background[lastAnchor:] =\
SpecfitFuns.snip1d(background[lastAnchor:],
config['fit']['snipwidth'],
0)
chunk[:, k] -= background
# perform the multiple fit to all the spectra in the chunk
#print("SHAPES")
#print(derivatives.shape)
#print(chunk[:,:(jEnd - jStart)].shape)
ddict=lstsq(derivatives, chunk[:,:(jEnd - jStart)],
sigma_b=sigma_b,
weight=weight,
digested_output=True,
svd=SVD,
last_svd=last_svd)
last_svd = ddict.get('svd', None)
parameters = ddict['parameters']
results[:, i, jStart:jEnd] = parameters
uncertainties[:, i, jStart:jEnd] = ddict['uncertainties']
jStart = jEnd
t = time.time() - t0
_logger.debug("First fit elapsed = %f", t)
if t > 0.:
_logger.debug("Spectra per second = %f",
data.shape[0]*data.shape[1]/float(t))
t0 = time.time()
# cleanup zeros
# start with the parameter with the largest amount of negative values
if refit:
negativePresent = True
else:
negativePresent = False
nFits = 0
while negativePresent:
zeroList = []
#totalNegative = 0
for i in range(nFree):
#we have to skip the background parameters
if i >= nFreeBackgroundParameters:
t = results[i] < 0
tsum = t.sum()
if tsum > 0:
zeroList.append((tsum, i, t))
#totalNegative += tsum
#print("totalNegative = ", totalNegative)
if len(zeroList) == 0:
negativePresent = False
continue
if nFits > (2 * (nFree - nFreeBackgroundParameters)):
# we are probably in an endless loop
# force negative pixels
for item in zeroList:
i = item[1]
badMask = item[2]
results[i][badMask] = 0.0
_logger.warning("WARNING: %d pixels of parameter %s forced to zero",
item[0], freeNames[i])
continue
zeroList.sort()
zeroList.reverse()
badParameters = []
badParameters.append(zeroList[0][1])
badMask = zeroList[0][2]
if 1:
# prevent and endless loop if two or more parameters have common pixels where they are
# negative and one of them remains negative when forcing other one to zero
for i, item in enumerate(zeroList):
if item[1] not in badParameters:
if item[0] > 0:
#check if they have common negative pixels
t = badMask * item[-1]
if t.sum() > 0:
badParameters.append(item[1])
badMask = t
if badMask.sum() < (0.0025 * nPixels):
# fit not worth
for i in badParameters:
results[i][badMask] = 0.0
uncertainties[i][badMask] = 0.0
_logger.debug("WARNING: %d pixels of parameter %s set to zero",
badMask.sum(), freeNames[i])
else:
_logger.debug("Number of secondary fits = %d", nFits + 1)
nFits += 1
A = derivatives[:, [i for i in range(nFree) if i not in badParameters]]
#assume we'll not have too many spectra
if data.dtype not in [numpy.float32, numpy.float64]:
if data.itemsize < 5:
data_dtype = numpy.float32
else:
data_dtype = numpy.floa64
else:
data_dtype = data.dtype
try:
if data.dtype != data_dtype:
spectra = numpy.zeros((int(badMask.sum()), 1 + iXMax - iXMin),
data_dtype)
spectra[:] = data[badMask, iXMin:iXMax+1]
else:
spectra = data[badMask, iXMin:iXMax+1]
spectra.shape = badMask.sum(), -1
except TypeError:
# in case of dynamic arrays, two dimensional indices are not
# supported by h5py
spectra = numpy.zeros((int(badMask.sum()), 1 + iXMax - iXMin),
data_dtype)
selectedIndices = numpy.nonzero(badMask > 0)
tmpData = numpy.zeros((1, 1 + iXMax - iXMin), data_dtype)
oldDataRow = -1
j = 0
for i in range(len(selectedIndices[0])):
j = selectedIndices[0][i]
if j != oldDataRow:
tmpData = data[j]
olddataRow = j
spectra[i] = tmpData[selectedIndices[1][i], iXMin:iXMax+1]
spectra = spectra.T
#
if config['fit']['stripflag']:
for k in range(spectra.shape[1]):
# obtain the smoothed spectrum
background=SpecfitFuns.SavitskyGolay(spectra[:, k],
config['fit']['stripfilterwidth'])
lastAnchor = 0
for anchor in anchorslist:
if (anchor > lastAnchor) and (anchor < background.size):
background[lastAnchor:anchor] =\
SpecfitFuns.snip1d(background[lastAnchor:anchor],
config['fit']['snipwidth'],
0)
lastAnchor = anchor
if lastAnchor < background.size:
background[lastAnchor:] =\
SpecfitFuns.snip1d(background[lastAnchor:],
config['fit']['snipwidth'],
0)
spectra[:, k] -= background
ddict = lstsq(A, spectra,
sigma_b=sigma_b,
weight=weight,
digested_output=True,
svd=SVD)
idx = 0
for i in range(nFree):
if i in badParameters:
results[i][badMask] = 0.0
uncertainties[i][badMask] = 0.0
else:
results[i][badMask] = ddict['parameters'][idx]
uncertainties[i][badMask] = ddict['uncertainties'][idx]
idx += 1
if refit:
t = time.time() - t0
_logger.debug("Fit of negative peaks elapsed = %f", t)
t0 = time.time()
outputDict = {'parameters':results, 'uncertainties':uncertainties, 'names':freeNames}
if concentrations:
# check if an internal reference is used and if it is set to auto
####################################################
# CONCENTRATIONS
cTool = ConcentrationsTool.ConcentrationsTool()
cToolConf = cTool.configure()
cToolConf.update(config['concentrations'])
fitFirstSpectrum = False
if config['concentrations']['usematrix']:
_logger.debug("USING MATRIX")
if config['concentrations']['reference'].upper() == "AUTO":
fitFirstSpectrum = True
elif autotime:
# we have to calculate with the time in the configuration
# and correct later on
cToolConf["autotime"] = 0
fitresult = {}
if fitFirstSpectrum:
# we have to fit the "reference" spectrum just to get the reference element
mcafitresult = self._mcaTheory.startfit(digest=0, linear=True)
# if one of the elements has zero area this cannot be made directly
fitresult['result'] = self._mcaTheory.imagingDigestResult()
fitresult['result']['config'] = config
concentrationsResult, addInfo = cTool.processFitResult(config=cToolConf,
fitresult=fitresult,
elementsfrommatrix=False,
fluorates=self._mcaTheory._fluoRates,
addinfo=True)
# and we have to make sure that all the areas are positive
for group in fitresult['result']['groups']:
if fitresult['result'][group]['fitarea'] <= 0.0:
# give a tiny area
fitresult['result'][group]['fitarea'] = 1.0e-6
config['concentrations']['reference'] = addInfo['ReferenceElement']
else:
fitresult['result'] = {}
fitresult['result']['config'] = config
fitresult['result']['groups'] = []
idx = 0
for i, param in enumerate(self._mcaTheory.PARAMETERS):
if self._mcaTheory.codes[0][i] == Gefit.CFIXED:
continue
if i < self._mcaTheory.NGLOBAL:
# background
pass
else:
fitresult['result']['groups'].append(param)
fitresult['result'][param] = {}
# we are just interested on the factor to be applied to the area to get the
# concentrations
fitresult['result'][param]['fitarea'] = 1.0
fitresult['result'][param]['sigmaarea'] = 1.0
idx += 1
concentrationsResult, addInfo = cTool.processFitResult(config=cToolConf,
fitresult=fitresult,
elementsfrommatrix=False,
fluorates=self._mcaTheory._fluoRates,
addinfo=True)
nValues = 1
if len(concentrationsResult['layerlist']) > 1:
nValues += len(concentrationsResult['layerlist'])
nElements = len(list(concentrationsResult['mass fraction'].keys()))
massFractions = numpy.zeros((nValues * nElements, nRows, nColumns),
numpy.float32)
referenceElement = addInfo['ReferenceElement']
referenceTransitions = addInfo['ReferenceTransitions']
_logger.debug("Reference <%s> transition <%s>",
referenceElement, referenceTransitions)
if referenceElement in ["", None, "None"]:
_logger.debug("No reference")
counter = 0
for i, group in enumerate(fitresult['result']['groups']):
if group.lower().startswith("scatter"):
_logger.debug("skept %s", group)
continue
outputDict['names'].append("C(%s)" % group)
if counter == 0:
if hasattr(liveTimeFactor, "shape"):
liveTimeFactor.shape = results[nFreeBackgroundParameters+i].shape
massFractions[counter] = liveTimeFactor * \
results[nFreeBackgroundParameters+i] * \
(concentrationsResult['mass fraction'][group] / \
fitresult['result'][group]['fitarea'])
counter += 1
if len(concentrationsResult['layerlist']) > 1:
for layer in concentrationsResult['layerlist']:
outputDict['names'].append("C(%s)-%s" % (group, layer))
massFractions[counter] = liveTimeFactor * \
results[nFreeBackgroundParameters+i] * \
(concentrationsResult[layer]['mass fraction'][group] / \
fitresult['result'][group]['fitarea'])
counter += 1
else:
_logger.debug("With reference")
idx = None
testGroup = referenceElement+ " " + referenceTransitions.split()[0]
for i, group in enumerate(fitresult['result']['groups']):
if group == testGroup:
idx = i
if idx is None:
raise ValueError("Invalid reference: <%s> <%s>" %\
(referenceElement, referenceTransitions))
group = fitresult['result']['groups'][idx]
referenceArea = fitresult['result'][group]['fitarea']
referenceConcentrations = concentrationsResult['mass fraction'][group]
goodIdx = results[nFreeBackgroundParameters+idx] > 0
massFractions[idx] = referenceConcentrations
counter = 0
for i, group in enumerate(fitresult['result']['groups']):
if group.lower().startswith("scatter"):
_logger.debug("skept %s", group)
continue
outputDict['names'].append("C(%s)" % group)
goodI = results[nFreeBackgroundParameters+i] > 0
tmp = results[nFreeBackgroundParameters+idx][goodI]
massFractions[counter][goodI] = (results[nFreeBackgroundParameters+i][goodI]/(tmp + (tmp == 0))) *\
((referenceArea/fitresult['result'][group]['fitarea']) *\
(concentrationsResult['mass fraction'][group]))
counter += 1
if len(concentrationsResult['layerlist']) > 1:
for layer in concentrationsResult['layerlist']:
outputDict['names'].append("C(%s)-%s" % (group, layer))
massFractions[counter][goodI] = (results[nFreeBackgroundParameters+i][goodI]/(tmp + (tmp == 0))) *\
((referenceArea/fitresult['result'][group]['fitarea']) *\
(concentrationsResult[layer]['mass fraction'][group]))
counter += 1
outputDict['concentrations'] = massFractions
t = time.time() - t0
_logger.debug("Calculation of concentrations elapsed = %f", t)
####################################################
return outputDict
def fitMultipleSpectra(self, x=None, y=None, xmin=None, xmax=None,
configuration=None, concentrations=False,
ysum=None, weight=None, refit=True,
livetime=None):
"""
This method performs the actual fit. The y keyword is the only mandatory input argument.
:param x: 1D array containing the x axis (usually the channels) of the spectra.
:param y: 3D array containing the spectra as [nrows, ncolumns, nchannels]
:param xmin: lower limit of the fitting region
:param xmax: upper limit of the fitting region
:param weight: 0 Means no weight, 1 Use an average weight, 2 Individual weights (slow)
:param concentrations: 0 Means no calculation, 1 Calculate them
:param refit: if False, no check for negative results. Default is True.
:livetime: It will be used if not different from None and concentrations
are to be calculated by using fundamental parameters with
automatic time. The default is None.
:return: A dictionnary with the parameters, uncertainties, concentrations and names as keys.
"""
if y is None:
raise RuntimeError("y keyword argument is mandatory!")
if hasattr(y, "info") and hasattr(y, "data"):
data = y.data
mcaIndex = y.info.get("McaIndex", -1)
else:
data = y
mcaIndex = -1
if x is None:
if hasattr(y, "info") and hasattr(y, "x"):
x = y.x[0]
if livetime is None:
if hasattr(y, "info"):
if "McaLiveTime" in y.info:
livetime = y.info["McaLiveTime"]
t0 = time.time()
if configuration is not None:
self._mcaTheory.setConfiguration(configuration)
elif self._config is None:
raise ValueError("Fit configuration missing")
else:
_logger.debug("Setting default configuration")
self._mcaTheory.setConfiguration(self._config)
# read the current configuration
# it is a copy, we can modify it at will
config = self._mcaTheory.getConfiguration()
if xmin is None:
xmin = config['fit']['xmin']
if xmax is None:
xmax = config['fit']['xmax']
toReconfigure = False
# if concentrations and use times, it needs to be reconfigured
# without using times and correct later on. If the concentrations
# are to be calculated from internal standard there is no need to
# raise an exception either.
autotime = 0
liveTimeFactor = 1.0
if not concentrations:
# ignore any time information to prevent unnecessary errors when
# setting the fitting data whithout the time information
if config['concentrations'].get("useautotime", 0):
config['concentrations']["useautotime"] = 0
toReconfigure = True
elif config["concentrations"]["usematrix"]:
if config['concentrations'].get("useautotime", 0):
config['concentrations']["useautotime"] = 0
toReconfigure = True
else:
# we are calculating concentrations from fundamental parameters
autotime = config['concentrations'].get("useautotime", 0)
nSpectra = data.size // data.shape[mcaIndex]
if autotime:
if livetime is None:
txt = "Automatic time requested but no time information provided"
raise RuntimeError(txt)
elif numpy.isscalar(livetime):
liveTimeFactor = \
float(config['concentrations']["time"]) / livetime
elif livetime.size == nSpectra:
liveTimeFactor = \
float(config['concentrations']["time"]) / livetime
else:
raise RuntimeError( \
"Number of live times not equal number of spectra")
config['concentrations']["useautotime"] = 0
toReconfigure = True
# use of strategies is not supported for the time being
strategy = config['fit'].get('strategyflag', 0)
if strategy:
raise RuntimeError("Strategies are incompatible with fast fit")
# background
if config['fit']['stripflag']:
if config['fit']['stripalgorithm'] == 1:
_logger.debug("SNIP")
else:
raise RuntimeError("Please use the faster SNIP background")
if weight is None:
# dictated by the file
weight = config['fit']['fitweight']
if weight:
# individual pixel weights (slow)
weightPolicy = 2
else:
# No weight
weightPolicy = 0
elif weight == 1:
# use average weight from the sum spectrum
weightPolicy = 1
if not config['fit']['fitweight']:
config['fit']['fitweight'] = 1
toReconfigure = True
elif weight == 2:
# individual pixel weights (slow)
weightPolicy = 2
if not config['fit']['fitweight']:
config['fit']['fitweight'] = 1
toReconfigure = True
weight = 1
else:
# No weight
weightPolicy = 0
if config['fit']['fitweight']:
config['fit']['fitweight'] = 0
toReconfigure = True
weight = 0
if not config['fit']['linearfitflag']:
#make sure we force a linear fit
config['fit']['linearfitflag'] = 1
toReconfigure = True
if toReconfigure:
# we must configure again the fit
self._mcaTheory.setConfiguration(config)
if len(data.shape) != 3:
txt = "For the time being only three dimensional arrays supported"
raise IndexError(txt)
elif mcaIndex not in [-1, 2]:
txt = "For the time being only mca arrays supported"
raise IndexError(txt)
else:
# if the cumulated spectrum is present it should be better
nRows = data.shape[0]
nColumns = data.shape[1]
nPixels = nRows * nColumns
if ysum is not None:
firstSpectrum = ysum
elif weightPolicy == 1:
# we need to calculate the sum spectrum to derive the uncertainties
totalSpectra = data.shape[0] * data.shape[1]
jStep = min(5000, data.shape[1])
ysum = numpy.zeros((data.shape[mcaIndex],), numpy.float)
for i in range(0, data.shape[0]):
if i == 0:
chunk = numpy.zeros((data.shape[0], jStep), numpy.float)
jStart = 0
while jStart < data.shape[1]:
jEnd = min(jStart + jStep, data.shape[1])
ysum += data[i, jStart:jEnd, :].sum(axis=0, dtype=numpy.float)
jStart = jEnd
firstSpectrum = ysum
elif not concentrations:
# just one spectrum is enough for the setup
firstSpectrum = data[0, 0, :]
else:
firstSpectrum = data[0, :, :].sum(axis=0, dtype=numpy.float)
# make sure we calculate the matrix of the contributions
self._mcaTheory.enableOptimizedLinearFit()
# initialize the fit
# print("xmin = ", xmin)
# print("xmax = ", xmax)
# print("firstShape = ", firstSpectrum.shape)
self._mcaTheory.setData(x=x, y=firstSpectrum, xmin=xmin, xmax=xmax)
# and initialize the derivatives
self._mcaTheory.estimate()
# now we can get the derivatives respect to the free parameters
# These are the "derivatives" respect to the peaks
# linearMatrix = self._mcaTheory.linearMatrix
# but we are still missing the derivatives from the background
nFree = 0
freeNames = []
nFreeBackgroundParameters = 0
for i, param in enumerate(self._mcaTheory.PARAMETERS):
if self._mcaTheory.codes[0][i] != ClassMcaTheory.Gefit.CFIXED:
nFree += 1
freeNames.append(param)
if i < self._mcaTheory.NGLOBAL:
nFreeBackgroundParameters += 1
if nFree == 0:
txt = "No free parameters to be fitted!\n"
txt += "No peaks inside fitting region?"
raise ValueError(txt)
#build the matrix of derivatives
derivatives = None
idx = 0
for i, param in enumerate(self._mcaTheory.PARAMETERS):
if self._mcaTheory.codes[0][i] == ClassMcaTheory.Gefit.CFIXED:
continue
deriv= self._mcaTheory.linearMcaTheoryDerivative(self._mcaTheory.parameters,
i,
self._mcaTheory.xdata)
deriv.shape = -1
if derivatives is None:
derivatives = numpy.zeros((deriv.shape[0], nFree), numpy.float)
derivatives[:, idx] = deriv
idx += 1
#loop for anchors
xdata = self._mcaTheory.xdata
if config['fit']['stripflag']:
anchorslist = []
if config['fit']['stripanchorsflag']:
if config['fit']['stripanchorslist'] is not None:
ravelled = numpy.ravel(xdata)
for channel in config['fit']['stripanchorslist']:
if channel <= ravelled[0]:continue
index = numpy.nonzero(ravelled >= channel)[0]
if len(index):
index = min(index)
if index > 0:
anchorslist.append(index)
if len(anchorslist) == 0:
anchorlist = [0, self._mcaTheory.ydata.size - 1]
anchorslist.sort()
# find the indices to be used for selecting the appropriate data
# if the original x data were not ordered we have a problem
# TODO: check for original ordering.
if x is None:
# we have an enumerated channels axis
iXMin = xdata[0]
iXMax = xdata[-1]
else:
iXMin = numpy.nonzero(x <= xdata[0])[0][-1]
iXMax = numpy.nonzero(x >= xdata[-1])[0][0]
# numpy 1.11.0 returns an array on previous expression
# and then complains about a future deprecation warning
# because of using an array and not an scalar in the selection
if hasattr(iXMin, "shape"):
if len(iXMin.shape):
iXMin = iXMin[0]
if hasattr(iXMax, "shape"):
if len(iXMax.shape):
iXMax = iXMax[0]
dummySpectrum = firstSpectrum[iXMin:iXMax+1].reshape(-1, 1)
# print("dummy = ", dummySpectrum.shape)
# allocate the output buffer
results = numpy.zeros((nFree, nRows, nColumns), numpy.float32)
uncertainties = numpy.zeros((nFree, nRows, nColumns), numpy.float32)
#perform the initial fit
_logger.debug("Configuration elapsed = %f", time.time() - t0)
t0 = time.time()
totalSpectra = data.shape[0] * data.shape[1]
jStep = min(100, data.shape[1])
if weightPolicy == 2:
SVD = False
sigma_b = None
elif weightPolicy == 1:
# the +1 is to prevent misbehavior due to weights less than 1.0
sigma_b = 1 + numpy.sqrt(dummySpectrum)/nPixels
SVD = True
else:
SVD = True
sigma_b = None
last_svd = None
for i in range(0, data.shape[0]):
#print(i)
#chunks of nColumns spectra
if i == 0:
chunk = numpy.zeros((dummySpectrum.shape[0],
jStep),
numpy.float)
jStart = 0
while jStart < data.shape[1]:
jEnd = min(jStart + jStep, data.shape[1])
chunk[:,:(jEnd - jStart)] = data[i, jStart:jEnd, iXMin:iXMax+1].T
if config['fit']['stripflag']:
for k in range(jStep):
# obtain the smoothed spectrum
background=SpecfitFuns.SavitskyGolay(chunk[:, k],
config['fit']['stripfilterwidth'])
lastAnchor = 0
for anchor in anchorslist:
if (anchor > lastAnchor) and (anchor < background.size):
background[lastAnchor:anchor] =\
SpecfitFuns.snip1d(background[lastAnchor:anchor],
config['fit']['snipwidth'],
0)
lastAnchor = anchor
if lastAnchor < background.size:
background[lastAnchor:] =\
SpecfitFuns.snip1d(background[lastAnchor:],
config['fit']['snipwidth'],
0)
chunk[:, k] -= background
# perform the multiple fit to all the spectra in the chunk
#print("SHAPES")
#print(derivatives.shape)
#print(chunk[:,:(jEnd - jStart)].shape)
ddict=lstsq(derivatives, chunk[:,:(jEnd - jStart)],
sigma_b=sigma_b,
weight=weight,
digested_output=True,
svd=SVD,
last_svd=last_svd)
last_svd = ddict.get('svd', None)
parameters = ddict['parameters']
results[:, i, jStart:jEnd] = parameters
uncertainties[:, i, jStart:jEnd] = ddict['uncertainties']
jStart = jEnd
t = time.time() - t0
_logger.debug("First fit elapsed = %f", t)
if t > 0.:
_logger.debug("Spectra per second = %f",
data.shape[0]*data.shape[1]/float(t))
t0 = time.time()
# cleanup zeros
# start with the parameter with the largest amount of negative values
if refit:
negativePresent = True
else:
negativePresent = False
nFits = 0
while negativePresent:
zeroList = []
#totalNegative = 0
for i in range(nFree):
#we have to skip the background parameters
if i >= nFreeBackgroundParameters:
t = results[i] < 0
tsum = t.sum()
if tsum > 0:
zeroList.append((tsum, i, t))
#totalNegative += tsum
#print("totalNegative = ", totalNegative)
if len(zeroList) == 0:
negativePresent = False
continue
if nFits > (2 * (nFree - nFreeBackgroundParameters)):
# we are probably in an endless loop
# force negative pixels
for item in zeroList:
i = item[1]
badMask = item[2]
results[i][badMask] = 0.0
_logger.warning("WARNING: %d pixels of parameter %s forced to zero",
item[0], freeNames[i])
continue
zeroList.sort()
zeroList.reverse()
badParameters = []
badParameters.append(zeroList[0][1])
badMask = zeroList[0][2]
if 1:
# prevent and endless loop if two or more parameters have common pixels where they are
# negative and one of them remains negative when forcing other one to zero
for i, item in enumerate(zeroList):
if item[1] not in badParameters:
if item[0] > 0:
#check if they have common negative pixels
t = badMask * item[-1]
if t.sum() > 0:
badParameters.append(item[1])
badMask = t
if badMask.sum() < (0.0025 * nPixels):
# fit not worth
for i in badParameters:
results[i][badMask] = 0.0
uncertainties[i][badMask] = 0.0
_logger.debug("WARNING: %d pixels of parameter %s set to zero",
badMask.sum(), freeNames[i])
else:
_logger.debug("Number of secondary fits = %d", nFits + 1)
nFits += 1
A = derivatives[:, [i for i in range(nFree) if i not in badParameters]]
#assume we'll not have too many spectra
if data.dtype not in [numpy.float32, numpy.float64]:
if data.itemsize < 5:
data_dtype = numpy.float32
else:
data_dtype = numpy.floa64
else:
data_dtype = data.dtype
try:
if data.dtype != data_dtype:
spectra = numpy.zeros((int(badMask.sum()), 1 + iXMax - iXMin),
data_dtype)
spectra[:] = data[badMask, iXMin:iXMax+1]
else:
spectra = data[badMask, iXMin:iXMax+1]
spectra.shape = badMask.sum(), -1
except TypeError:
# in case of dynamic arrays, two dimensional indices are not
# supported by h5py
spectra = numpy.zeros((int(badMask.sum()), 1 + iXMax - iXMin),
data_dtype)
selectedIndices = numpy.nonzero(badMask > 0)
tmpData = numpy.zeros((1, 1 + iXMax - iXMin), data_dtype)
oldDataRow = -1
j = 0
for i in range(len(selectedIndices[0])):
j = selectedIndices[0][i]
if j != oldDataRow:
tmpData = data[j]
olddataRow = j
spectra[i] = tmpData[selectedIndices[1][i], iXMin:iXMax+1]
spectra = spectra.T
#
if config['fit']['stripflag']:
for k in range(spectra.shape[1]):
# obtain the smoothed spectrum
background=SpecfitFuns.SavitskyGolay(spectra[:, k],
config['fit']['stripfilterwidth'])
lastAnchor = 0
for anchor in anchorslist:
if (anchor > lastAnchor) and (anchor < background.size):
background[lastAnchor:anchor] =\
SpecfitFuns.snip1d(background[lastAnchor:anchor],
config['fit']['snipwidth'],
0)
lastAnchor = anchor
if lastAnchor < background.size:
background[lastAnchor:] =\
SpecfitFuns.snip1d(background[lastAnchor:],
config['fit']['snipwidth'],
0)
spectra[:, k] -= background
ddict = lstsq(A, spectra,
sigma_b=sigma_b,
weight=weight,
digested_output=True,
svd=SVD)
idx = 0
for i in range(nFree):
if i in badParameters:
results[i][badMask] = 0.0
uncertainties[i][badMask] = 0.0
else:
results[i][badMask] = ddict['parameters'][idx]
uncertainties[i][badMask] = ddict['uncertainties'][idx]
idx += 1
if refit:
t = time.time() - t0
_logger.debug("Fit of negative peaks elapsed = %f", t)
t0 = time.time()
outputDict = {'parameters':results, 'uncertainties':uncertainties, 'names':freeNames}
if concentrations:
# check if an internal reference is used and if it is set to auto
####################################################
# CONCENTRATIONS
cTool = ConcentrationsTool.ConcentrationsTool()
cToolConf = cTool.configure()
cToolConf.update(config['concentrations'])
fitFirstSpectrum = False
if config['concentrations']['usematrix']:
_logger.debug("USING MATRIX")
if config['concentrations']['reference'].upper() == "AUTO":
fitFirstSpectrum = True
elif autotime:
# we have to calculate with the time in the configuration
# and correct later on
cToolConf["autotime"] = 0
fitresult = {}
if fitFirstSpectrum:
# we have to fit the "reference" spectrum just to get the reference element
mcafitresult = self._mcaTheory.startfit(digest=0, linear=True)
# if one of the elements has zero area this cannot be made directly
fitresult['result'] = self._mcaTheory.imagingDigestResult()
fitresult['result']['config'] = config
concentrationsResult, addInfo = cTool.processFitResult(config=cToolConf,
fitresult=fitresult,
elementsfrommatrix=False,
fluorates=self._mcaTheory._fluoRates,
addinfo=True)
# and we have to make sure that all the areas are positive
for group in fitresult['result']['groups']:
if fitresult['result'][group]['fitarea'] <= 0.0:
# give a tiny area
fitresult['result'][group]['fitarea'] = 1.0e-6
config['concentrations']['reference'] = addInfo['ReferenceElement']
else:
fitresult['result'] = {}
fitresult['result']['config'] = config
fitresult['result']['groups'] = []
idx = 0
for i, param in enumerate(self._mcaTheory.PARAMETERS):
if self._mcaTheory.codes[0][i] == Gefit.CFIXED:
continue
if i < self._mcaTheory.NGLOBAL:
# background
pass
else:
fitresult['result']['groups'].append(param)
fitresult['result'][param] = {}
# we are just interested on the factor to be applied to the area to get the
# concentrations
fitresult['result'][param]['fitarea'] = 1.0
fitresult['result'][param]['sigmaarea'] = 1.0
idx += 1
concentrationsResult, addInfo = cTool.processFitResult(config=cToolConf,
fitresult=fitresult,
elementsfrommatrix=False,
fluorates=self._mcaTheory._fluoRates,
addinfo=True)
nValues = 1
if len(concentrationsResult['layerlist']) > 1:
nValues += len(concentrationsResult['layerlist'])
nElements = len(list(concentrationsResult['mass fraction'].keys()))
massFractions = numpy.zeros((nValues * nElements, nRows, nColumns),
numpy.float32)
referenceElement = addInfo['ReferenceElement']
referenceTransitions = addInfo['ReferenceTransitions']
_logger.debug("Reference <%s> transition <%s>",
referenceElement, referenceTransitions)
if referenceElement in ["", None, "None"]:
_logger.debug("No reference")
counter = 0
for i, group in enumerate(fitresult['result']['groups']):
if group.lower().startswith("scatter"):
_logger.debug("skept %s", group)
continue
outputDict['names'].append("C(%s)" % group)
if counter == 0:
if hasattr(liveTimeFactor, "shape"):
liveTimeFactor.shape = results[nFreeBackgroundParameters+i].shape
massFractions[counter] = liveTimeFactor * \
results[nFreeBackgroundParameters+i] * \
(concentrationsResult['mass fraction'][group] / \
fitresult['result'][group]['fitarea'])
counter += 1
if len(concentrationsResult['layerlist']) > 1:
for layer in concentrationsResult['layerlist']:
outputDict['names'].append("C(%s)-%s" % (group, layer))
massFractions[counter] = liveTimeFactor * \
results[nFreeBackgroundParameters+i] * \
(concentrationsResult[layer]['mass fraction'][group] / \
fitresult['result'][group]['fitarea'])
counter += 1
else:
_logger.debug("With reference")
idx = None
testGroup = referenceElement+ " " + referenceTransitions.split()[0]
for i, group in enumerate(fitresult['result']['groups']):
if group == testGroup:
idx = i
if idx is None:
raise ValueError("Invalid reference: <%s> <%s>" %\
(referenceElement, referenceTransitions))
group = fitresult['result']['groups'][idx]
referenceArea = fitresult['result'][group]['fitarea']
referenceConcentrations = concentrationsResult['mass fraction'][group]
goodIdx = results[nFreeBackgroundParameters+idx] > 0
massFractions[idx] = referenceConcentrations
counter = 0
for i, group in enumerate(fitresult['result']['groups']):
if group.lower().startswith("scatter"):
_logger.debug("skept %s", group)
continue
outputDict['names'].append("C(%s)" % group)
goodI = results[nFreeBackgroundParameters+i] > 0
tmp = results[nFreeBackgroundParameters+idx][goodI]
massFractions[counter][goodI] = (results[nFreeBackgroundParameters+i][goodI]/(tmp + (tmp == 0))) *\
((referenceArea/fitresult['result'][group]['fitarea']) *\
(concentrationsResult['mass fraction'][group]))
counter += 1
if len(concentrationsResult['layerlist']) > 1:
for layer in concentrationsResult['layerlist']:
outputDict['names'].append("C(%s)-%s" % (group, layer))
massFractions[counter][goodI] = (results[nFreeBackgroundParameters+i][goodI]/(tmp + (tmp == 0))) *\
((referenceArea/fitresult['result'][group]['fitarea']) *\
(concentrationsResult[layer]['mass fraction'][group]))
counter += 1
outputDict['concentrations'] = massFractions
t = time.time() - t0
_logger.debug("Calculation of concentrations elapsed = %f", t)
####################################################
return outputDict
def fitMultipleSpectra(self, x=None, y=None, xmin=None, xmax=None,
configuration=None, concentrations=False,
ysum=None, weight=None):
if y is None:
raise RuntimeError("y keyword argument is mandatory!")
#if concentrations:
# txt = "Fast concentration calculation not implemented yet"
# raise NotImplemented(txt)
if DEBUG:
t0 = time.time()
if configuration is not None:
self._mcaTheory.setConfiguration(configuration)
# read the current configuration
config = self._mcaTheory.getConfiguration()
# background
if config['fit']['stripflag']:
if config['fit']['stripalgorithm'] == 1:
if DEBUG:
print("SNIP")
else:
raise RuntimeError("Please use the faster SNIP background")
toReconfigure = False
if weight is None:
# dictated by the file
weight = config['fit']['fitweight']
if weight:
# individual pixel weights (slow)
weightPolicy = 2
else:
# No weight
weightPolicy = 0
elif weight == 1:
# use average weight from the sum spectrum
weightPolicy = 1
if not config['fit']['fitweight']:
config['fit']['fitweight'] = 1
toReconfigure = True
elif weight == 2:
# individual pixel weights (slow)
weightPolicy = 2
if not config['fit']['fitweight']:
config['fit']['fitweight'] = 1
toReconfigure = True
weight = 1
else:
# No weight
weightPolicy = 0
if config['fit']['fitweight']:
config['fit']['fitweight'] = 0
toReconfigure = True
weight = 0
if not config['fit']['linearfitflag']:
#make sure we force a linear fit
config['fit']['linearfitflag'] = 1
toReconfigure = True
if toReconfigure:
# we must configure again the fit
self._mcaTheory.setConfiguration(config)
if hasattr(y, "info") and hasattr(y, "data"):
data = y.data
mcaIndex = y.info.get("McaIndex", -1)
else:
data = y
mcaIndex = -1
if len(data.shape) != 3:
txt = "For the time being only three dimensional arrays supported"
raise IndexError(txt)
elif mcaIndex not in [-1, 2]:
txt = "For the time being only mca arrays supported"
raise IndexError(txt)
else:
# if the cumulated spectrum is present it should be better
nRows = data.shape[0]
nColumns = data.shape[1]
nPixels = nRows * nColumns
if ysum is not None:
firstSpectrum = ysum
elif weightPolicy == 1:
# we need to calculate the sum spectrum to derive the uncertainties
totalSpectra = data.shape[0] * data.shape[1]
jStep = min(5000, data.shape[1])
ysum = numpy.zeros((data.shape[mcaIndex],), numpy.float)
for i in range(0, data.shape[0]):
if i == 0:
chunk = numpy.zeros((data.shape[0], jStep), numpy.float)
jStart = 0
while jStart < data.shape[1]:
jEnd = min(jStart + jStep, data.shape[1])
ysum += data[i, jStart:jEnd, :].sum(axis=0, dtype=numpy.float)
jStart = jEnd
firstSpectrum = ysum
elif not concentrations:
# just one spectrum is enough for the setup
firstSpectrum = data[0, 0, :]
else:
firstSpectrum = data[0, :, :].sum(axis=0, dtype=numpy.float)
# make sure we calculate the matrix of the contributions
self._mcaTheory.enableOptimizedLinearFit()
# initialize the fit
# print("xmin = ", xmin)
# print("xmax = ", xmax)
# print("firstShape = ", firstSpectrum.shape)
self._mcaTheory.setData(x=x, y=firstSpectrum, xmin=xmin, xmax=xmax)
# and initialize the derivatives
self._mcaTheory.estimate()
# now we can get the derivatives respect to the free parameters
# These are the "derivatives" respect to the peaks
# linearMatrix = self._mcaTheory.linearMatrix
# but we are still missing the derivatives from the background
nFree = 0
freeNames = []
nFreeBackgroundParameters = 0
for i, param in enumerate(self._mcaTheory.PARAMETERS):
if self._mcaTheory.codes[0][i] != ClassMcaTheory.Gefit.CFIXED:
nFree += 1
freeNames.append(param)
if i < self._mcaTheory.NGLOBAL:
nFreeBackgroundParameters += 1
#build the matrix of derivatives
derivatives = None
idx = 0
for i, param in enumerate(self._mcaTheory.PARAMETERS):
if self._mcaTheory.codes[0][i] == ClassMcaTheory.Gefit.CFIXED:
continue
deriv= self._mcaTheory.linearMcaTheoryDerivative(self._mcaTheory.parameters,
i,
self._mcaTheory.xdata)
deriv.shape = -1
if derivatives is None:
derivatives = numpy.zeros((deriv.shape[0], nFree), numpy.float)
derivatives[:, idx] = deriv
idx += 1
#loop for anchors
xdata = self._mcaTheory.xdata
if config['fit']['stripflag']:
anchorslist = []
if config['fit']['stripanchorsflag']:
if config['fit']['stripanchorslist'] is not None:
ravelled = numpy.ravel(xdata)
for channel in config['fit']['stripanchorslist']:
if channel <= ravelled[0]:continue
index = numpy.nonzero(ravelled >= channel)[0]
if len(index):
index = min(index)
if index > 0:
anchorslist.append(index)
if len(anchorslist) == 0:
anchorlist = [0, self._mcaTheory.ydata.size - 1]
anchorslist.sort()
# find the indices to be used for selecting the appropriate data
# if the original x data were nor ordered we have a problem
# TODO: check for original ordering.
if x is None:
# we have an enumerated channels axis
iXMin = xdata[0]
iXMax = xdata[-1]
else:
iXMin = numpy.nonzero(x <= xdata[0])[0][-1]
iXMax = numpy.nonzero(x >= xdata[-1])[0][0]
dummySpectrum = firstSpectrum[iXMin:iXMax+1].reshape(-1, 1)
# print("dummy = ", dummySpectrum.shape)
# allocate the output buffer
results = numpy.zeros((nFree, nRows, nColumns), numpy.float32)
uncertainties = numpy.zeros((nFree, nRows, nColumns), numpy.float32)
#perform the initial fit
if DEBUG:
print("Configuration elapsed = %f" % (time.time() - t0))
t0 = time.time()
totalSpectra = data.shape[0] * data.shape[1]
jStep = min(100, data.shape[1])
if weightPolicy == 2:
SVD = False
sigma_b = None
elif weightPolicy == 1:
# the +1 is to prevent misbehavior due to weights less than 1.0
sigma_b = 1 + numpy.sqrt(dummySpectrum)/nPixels
SVD = True
else:
SVD = True
sigma_b = None
last_svd = None
for i in range(0, data.shape[0]):
#print(i)
#chunks of nColumns spectra
if i == 0:
chunk = numpy.zeros((dummySpectrum.shape[0],
jStep),
numpy.float)
jStart = 0
while jStart < data.shape[1]:
jEnd = min(jStart + jStep, data.shape[1])
chunk[:,:(jEnd - jStart)] = data[i, jStart:jEnd, iXMin:iXMax+1].T
if config['fit']['stripflag']:
for k in range(jStep):
# obtain the smoothed spectrum
background=SpecfitFuns.SavitskyGolay(chunk[:, k],
config['fit']['stripfilterwidth'])
lastAnchor = 0
for anchor in anchorslist:
if (anchor > lastAnchor) and (anchor < background.size):
background[lastAnchor:anchor] =\
SpecfitFuns.snip1d(background[lastAnchor:anchor],
config['fit']['snipwidth'],
0)
lastAnchor = anchor
if lastAnchor < background.size:
background[lastAnchor:] =\
SpecfitFuns.snip1d(background[lastAnchor:],
config['fit']['snipwidth'],
0)
chunk[:, k] -= background
# perform the multiple fit to all the spectra in the chunk
#print("SHAPES")
#print(derivatives.shape)
#print(chunk[:,:(jEnd - jStart)].shape)
ddict=lstsq(derivatives, chunk[:,:(jEnd - jStart)],
sigma_b=sigma_b,
weight=weight,
digested_output=True,
svd=SVD,
last_svd=last_svd)
last_svd = ddict.get('svd', None)
parameters = ddict['parameters']
results[:, i, jStart:jEnd] = parameters
uncertainties[:, i, jStart:jEnd] = ddict['uncertainties']
jStart = jEnd
if DEBUG:
t = time.time() - t0
print("First fit elapsed = %f" % t)
print("Spectra per second = %f" % (data.shape[0]*data.shape[1]/float(t)))
t0 = time.time()
# cleanup zeros
# start with the parameter with the largest amount of negative values
negativePresent = True
nFits = 0
while negativePresent:
zeroList = []
for i in range(nFree):
#we have to skip the background parameters
if i >= nFreeBackgroundParameters:
t = results[i] < 0
if t.sum() > 0:
zeroList.append((t.sum(), i, t))
if len(zeroList) == 0:
negativePresent = False
continue
if nFits > (2 * (nFree - nFreeBackgroundParameters)):
# we are probably in an endless loop
# force negative pixels
for item in zeroList:
i = item[1]
badMask = item[2]
results[i][badMask] = 0.0
print("WARNING: %d pixels of parameter %s set to zero" % (item[0], freeNames[i]))
continue
zeroList.sort()
zeroList.reverse()
badParameters = []
badParameters.append(zeroList[0][1])
badMask = zeroList[0][2]
if 1:
# prevent and endless loop if two or more parameters have common pixels where they are
# negative and one of them remains negative when forcing other one to zero
for i, item in enumerate(zeroList):
if item[1] not in badParameters:
if item[0] > 0:
#check if they have common negative pixels
t = badMask * item[-1]
if t.sum() > 0:
badParameters.append(item[1])
badMask = t
if badMask.sum() < (0.0025 * nPixels):
# fit not worth
for i in badParameters:
results[i][badMask] = 0.0
uncertainties[i][badMask] = 0.0
if DEBUG:
print("WARNING: %d pixels of parameter %s set to zero" % (badMask.sum(),
freeNames[i]))
else:
if DEBUG:
print("Number of secondary fits = %d" % (nFits + 1))
nFits += 1
A = derivatives[:, [i for i in range(nFree) if i not in badParameters]]
#assume we'll not have too many spectra
spectra = data[badMask, iXMin:iXMax+1]
spectra.shape = badMask.sum(), -1
spectra = spectra.T
#
if config['fit']['stripflag']:
for k in range(spectra.shape[1]):
# obtain the smoothed spectrum
background=SpecfitFuns.SavitskyGolay(spectra[:, k],
config['fit']['stripfilterwidth'])
lastAnchor = 0
for anchor in anchorslist:
if (anchor > lastAnchor) and (anchor < background.size):
background[lastAnchor:anchor] =\
SpecfitFuns.snip1d(background[lastAnchor:anchor],
config['fit']['snipwidth'],
0)
lastAnchor = anchor
if lastAnchor < background.size:
background[lastAnchor:] =\
SpecfitFuns.snip1d(background[lastAnchor:],
config['fit']['snipwidth'],
0)
spectra[:, k] -= background
ddict = lstsq(A, spectra,
sigma_b=sigma_b,
weight=weight,
digested_output=True,
svd=SVD)
idx = 0
for i in range(nFree):
if i in badParameters:
results[i][badMask] = 0.0
uncertainties[i][badMask] = 0.0
else:
results[i][badMask] = ddict['parameters'][idx]
uncertainties[i][badMask] = ddict['uncertainties'][idx]
idx += 1
if DEBUG:
t = time.time() - t0
print("Fit of negative peaks elapsed = %f" % t)
t0 = time.time()
outputDict = {'parameters':results, 'uncertainties':uncertainties, 'names':freeNames}
if concentrations:
# check if an internal reference is used and if it is set to auto
####################################################
# CONCENTRATIONS
cTool = ConcentrationsTool.ConcentrationsTool()
cToolConf = cTool.configure()
cToolConf.update(config['concentrations'])
fitFirstSpectrum = False
if config['concentrations']['usematrix']:
if DEBUG:
print("USING MATRIX")
if config['concentrations']['reference'].upper() == "AUTO":
fitFirstSpectrum = True
fitresult = {}
if fitFirstSpectrum:
# we have to fit the "reference" spectrum just to get the reference element
mcafitresult = self._mcaTheory.startfit(digest=0, linear=True)
# if one of the elements has zero area this cannot be made directly
fitresult['result'] = self._mcaTheory.imagingDigestResult()
fitresult['result']['config'] = config
concentrationsResult, addInfo = cTool.processFitResult(config=cToolConf,
fitresult=fitresult,
elementsfrommatrix=False,
fluorates=self._mcaTheory._fluoRates,
addinfo=True)
# and we have to make sure that all the areas are positive
for group in fitresult['result']['groups']:
if fitresult['result'][group]['fitarea'] <= 0.0:
# give a tiny area
fitresult['result'][group]['fitarea'] = 1.0e-6
config['concentrations']['reference'] = addInfo['ReferenceElement']
else:
fitresult['result'] = {}
fitresult['result']['config'] = config
fitresult['result']['groups'] = []
idx = 0
for i, param in enumerate(self._mcaTheory.PARAMETERS):
if self._mcaTheory.codes[0][i] == Gefit.CFIXED:
continue
if i < self._mcaTheory.NGLOBAL:
# background
pass
else:
fitresult['result']['groups'].append(param)
fitresult['result'][param] = {}
# we are just interested on the factor to be applied to the area to get the
# concentrations
fitresult['result'][param]['fitarea'] = 1.0
fitresult['result'][param]['sigmaarea'] = 1.0
idx += 1
concentrationsResult, addInfo = cTool.processFitResult(config=cToolConf,
fitresult=fitresult,
elementsfrommatrix=False,
fluorates=self._mcaTheory._fluoRates,
addinfo=True)
nValues = 1
if len(concentrationsResult['layerlist']) > 1:
nValues += len(concentrationsResult['layerlist'])
nElements = len(list(concentrationsResult['mass fraction'].keys()))
massFractions = numpy.zeros((nValues * nElements, nRows, nColumns),
numpy.float32)
referenceElement = addInfo['ReferenceElement']
referenceTransitions = addInfo['ReferenceTransitions']
if DEBUG:
print("Reference <%s> transition <%s>" % (referenceElement, referenceTransitions))
if referenceElement in ["", None, "None"]:
if DEBUG:
print("No reference")
counter = 0
for i, group in enumerate(fitresult['result']['groups']):
if group.lower().startswith("scatter"):
if DEBUG:
print("skept %s" % group)
continue
outputDict['names'].append("C(%s)" % group)
massFractions[counter] = results[nFreeBackgroundParameters+i] *\
(concentrationsResult['mass fraction'][group]/fitresult['result'][param]['fitarea'])
if len(concentrationsResult['layerlist']) > 1:
for layer in concentrationsResult['layerlist']:
outputDict['names'].append("C(%s)-%s" % (group, layer))
massFractions[counter] = results[nFreeBackgroundParameters+i] *\
(concentrationsResult[layer]['mass fraction'][group]/fitresult['result'][param]['fitarea'])
else:
if DEBUG:
print("With reference")
idx = None
testGroup = referenceElement+ " " + referenceTransitions.split()[0]
for i, group in enumerate(fitresult['result']['groups']):
if group == testGroup:
idx = i
if idx is None:
raise ValueError("Invalid reference: <%s> <%s>" %\
(referenceElement, referenceTransitions))
group = fitresult['result']['groups'][idx]
referenceArea = fitresult['result'][group]['fitarea']
referenceConcentrations = concentrationsResult['mass fraction'][group]
goodIdx = results[nFreeBackgroundParameters+idx] > 0
massFractions[idx] = referenceConcentrations
counter = 0
for i, group in enumerate(fitresult['result']['groups']):
if group.lower().startswith("scatter"):
if DEBUG:
print("skept %s" % group)
continue
outputDict['names'].append("C(%s)" % group)
if i == idx:
continue
goodI = results[nFreeBackgroundParameters+i] > 0
tmp = results[nFreeBackgroundParameters+idx][goodI]
massFractions[i][goodI] = (results[nFreeBackgroundParameters+i][goodI]/(tmp + (tmp == 0))) *\
((referenceArea/fitresult['result'][group]['fitarea']) *\
(concentrationsResult['mass fraction'][group]))
if len(concentrationsResult['layerlist']) > 1:
for layer in concentrationsResult['layerlist']:
outputDict['names'].append("C(%s)-%s" % (group, layer))
massFractions[i][goodI] = (results[nFreeBackgroundParameters+i][goodI]/(tmp + (tmp == 0))) *\
((referenceArea/fitresult['result'][group]['fitarea']) *\
(concentrationsResult[layer]['mass fraction'][group]))
outputDict['concentrations'] = massFractions
if DEBUG:
t = time.time() - t0
print("Calculation of concentrations elapsed = %f" % t)
t0 = time.time()
####################################################
return outputDict
def _fitLstSqReduced(self, data=None, sliceChan=None, mcaIndex=None,
derivatives=None, results=None, uncertainties=None,
fitmodel=None, config=None, anchorslist=None,
lstsq_kwargs=None, mask=None,
skipNames=None, skipParams=None,
nFreeParameters=None, nmin=None):
"""
Fit reduced number of spectra (mask) with a reduced model (skipped parameters will be set to zero)
"""
npixels = int(mask.sum())
nMca = self._numberOfSpectra(1, 'MiB', data=data, mcaIndex=mcaIndex,
sliceChan=sliceChan)
if npixels < nmin:
_logger.debug("Not worth refitting #%d pixels", npixels)
for iFree, name in zip(skipParams, skipNames):
results[iFree][mask] = 0.0
uncertainties[iFree][mask] = 0.0
_logger.debug("%d pixels of parameter %s set to zero",
npixels, name)
if nFreeParameters is not None:
nFreeParameters[mask] = 0
else:
_logger.debug("Refitting #{} spectra in chunks of {}".format(npixels, nMca))
nChan, nFreeOrg = derivatives.shape
idxFree = [i for i in range(nFreeOrg) if i not in skipParams]
nFree = len(idxFree)
A = derivatives[:, idxFree]
lstsq_kwargs['last_svd'] = None
# Fit all selected spectra in one chunk
bkgsub = bool(config['fit']['stripflag'])
chunkItems = self._dataChunkIter(McaStackView.MaskedView,
data=data,
fitmodel=fitmodel,
mask=mask,
mcaSlice=sliceChan,
mcaAxis=mcaIndex,
nMca=nMca)
for chunk in chunkItems:
if fitmodel is None:
(idx, idxShape), chunk = chunk
chunkModel = None
else:
((idx, idxShape), chunk), (_, chunkModel) = chunk
chunkModel = chunkModel.T
chunk = chunk.T
# Subtract background
if bkgsub:
self._fitBkgSubtract(chunk, config=config,
anchorslist=anchorslist,
fitmodel=chunkModel)
# Solve linear system of equations
ddict = lstsq(A, chunk, digested_output=True,
**lstsq_kwargs)
lstsq_kwargs['last_svd'] = ddict.get('svd', None)
# Save results
iParam = 0
for iFree in range(nFreeOrg):
if iFree in skipParams:
results[iFree][idx] = 0.0
uncertainties[iFree][idx] = 0.0
else:
results[iFree][idx] = ddict['parameters'][iParam]\
.reshape(idxShape)
uncertainties[iFree][idx] = ddict['uncertainties'][iParam]\
.reshape(idxShape)
iParam += 1
if chunkModel is not None:
if bkgsub:
chunkModel += numpy.dot(A, ddict['parameters'])
else:
chunkModel[()] = numpy.dot(A, ddict['parameters'])
if nFreeParameters is not None:
nFreeParameters[idx] = nFree
