def fitMultipleSpectra(self, x=None, y=None, xmin=None, xmax=None, configuration=None, concentrations=False, ysum=None, weight=None, refit=True, livetime=None): """ This method performs the actual fit. The y keyword is the only mandatory input argument. :param x: 1D array containing the x axis (usually the channels) of the spectra. :param y: 3D array containing the spectra as [nrows, ncolumns, nchannels] :param xmin: lower limit of the fitting region :param xmax: upper limit of the fitting region :param weight: 0 Means no weight, 1 Use an average weight, 2 Individual weights (slow) :param concentrations: 0 Means no calculation, 1 Calculate them :param refit: if False, no check for negative results. Default is True. :livetime: It will be used if not different from None and concentrations are to be calculated by using fundamental parameters with automatic time. The default is None. :return: A dictionary with the parameters, uncertainties, concentrations and names as keys. """ if y is None: raise RuntimeError("y keyword argument is mandatory!") if hasattr(y, "info") and hasattr(y, "data"): data = y.data mcaIndex = y.info.get("McaIndex", -1) else: data = y mcaIndex = -1 if x is None: if hasattr(y, "info") and hasattr(y, "x"): x = y.x[0] if livetime is None: if hasattr(y, "info"): if "McaLiveTime" in y.info: livetime = y.info["McaLiveTime"] t0 = time.time() if configuration is not None: self._mcaTheory.setConfiguration(configuration) elif self._config is None: raise ValueError("Fit configuration missing") else: _logger.debug("Setting default configuration") self._mcaTheory.setConfiguration(self._config) # read the current configuration # it is a copy, we can modify it at will config = self._mcaTheory.getConfiguration() if xmin is None: xmin = config['fit']['xmin'] if xmax is None: xmax = config['fit']['xmax'] toReconfigure = False # if concentrations and use times, it needs to be reconfigured # without using times and correct later on. If the concentrations # are to be calculated from internal standard there is no need to # raise an exception either. autotime = 0 liveTimeFactor = 1.0 if not concentrations: # ignore any time information to prevent unnecessary errors when # setting the fitting data whithout the time information if config['concentrations'].get("useautotime", 0): config['concentrations']["useautotime"] = 0 toReconfigure = True elif config["concentrations"]["usematrix"]: if config['concentrations'].get("useautotime", 0): config['concentrations']["useautotime"] = 0 toReconfigure = True else: # we are calculating concentrations from fundamental parameters autotime = config['concentrations'].get("useautotime", 0) nSpectra = data.size // data.shape[mcaIndex] if autotime: if livetime is None: txt = "Automatic time requested but no time information provided" raise RuntimeError(txt) elif numpy.isscalar(livetime): liveTimeFactor = \ float(config['concentrations']["time"]) / livetime elif livetime.size == nSpectra: liveTimeFactor = \ float(config['concentrations']["time"]) / livetime else: raise RuntimeError( \ "Number of live times not equal number of spectra") config['concentrations']["useautotime"] = 0 toReconfigure = True # use of strategies is not supported for the time being strategy = config['fit'].get('strategyflag', 0) if strategy: raise RuntimeError("Strategies are incompatible with fast fit") # background if config['fit']['stripflag']: if config['fit']['stripalgorithm'] == 1: _logger.debug("SNIP") else: raise RuntimeError("Please use the faster SNIP background") if weight is None: # dictated by the file weight = config['fit']['fitweight'] if weight: # individual pixel weights (slow) weightPolicy = 2 else: # No weight weightPolicy = 0 elif weight == 1: # use average weight from the sum spectrum weightPolicy = 1 if not config['fit']['fitweight']: config['fit']['fitweight'] = 1 toReconfigure = True elif weight == 2: # individual pixel weights (slow) weightPolicy = 2 if not config['fit']['fitweight']: config['fit']['fitweight'] = 1 toReconfigure = True weight = 1 else: # No weight weightPolicy = 0 if config['fit']['fitweight']: config['fit']['fitweight'] = 0 toReconfigure = True weight = 0 if not config['fit']['linearfitflag']: #make sure we force a linear fit config['fit']['linearfitflag'] = 1 toReconfigure = True if toReconfigure: # we must configure again the fit self._mcaTheory.setConfiguration(config) if len(data.shape) != 3: txt = "For the time being only three dimensional arrays supported" raise IndexError(txt) elif mcaIndex not in [-1, 2]: txt = "For the time being only mca arrays supported" raise IndexError(txt) else: # if the cumulated spectrum is present it should be better nRows = data.shape[0] nColumns = data.shape[1] nPixels = nRows * nColumns if ysum is not None: firstSpectrum = ysum elif weightPolicy == 1: # we need to calculate the sum spectrum to derive the uncertainties totalSpectra = data.shape[0] * data.shape[1] jStep = min(5000, data.shape[1]) ysum = numpy.zeros((data.shape[mcaIndex],), numpy.float) for i in range(0, data.shape[0]): if i == 0: chunk = numpy.zeros((data.shape[0], jStep), numpy.float) jStart = 0 while jStart < data.shape[1]: jEnd = min(jStart + jStep, data.shape[1]) ysum += data[i, jStart:jEnd, :].sum(axis=0, dtype=numpy.float) jStart = jEnd firstSpectrum = ysum elif not concentrations: # just one spectrum is enough for the setup firstSpectrum = data[0, 0, :] else: firstSpectrum = data[0, :, :].sum(axis=0, dtype=numpy.float) # make sure we calculate the matrix of the contributions self._mcaTheory.enableOptimizedLinearFit() # initialize the fit # print("xmin = ", xmin) # print("xmax = ", xmax) # print("firstShape = ", firstSpectrum.shape) self._mcaTheory.setData(x=x, y=firstSpectrum, xmin=xmin, xmax=xmax) # and initialize the derivatives self._mcaTheory.estimate() # now we can get the derivatives respect to the free parameters # These are the "derivatives" respect to the peaks # linearMatrix = self._mcaTheory.linearMatrix # but we are still missing the derivatives from the background nFree = 0 freeNames = [] nFreeBackgroundParameters = 0 for i, param in enumerate(self._mcaTheory.PARAMETERS): if self._mcaTheory.codes[0][i] != ClassMcaTheory.Gefit.CFIXED: nFree += 1 freeNames.append(param) if i < self._mcaTheory.NGLOBAL: nFreeBackgroundParameters += 1 if nFree == 0: txt = "No free parameters to be fitted!\n" txt += "No peaks inside fitting region?" raise ValueError(txt) #build the matrix of derivatives derivatives = None idx = 0 for i, param in enumerate(self._mcaTheory.PARAMETERS): if self._mcaTheory.codes[0][i] == ClassMcaTheory.Gefit.CFIXED: continue deriv= self._mcaTheory.linearMcaTheoryDerivative(self._mcaTheory.parameters, i, self._mcaTheory.xdata) deriv.shape = -1 if derivatives is None: derivatives = numpy.zeros((deriv.shape[0], nFree), numpy.float) derivatives[:, idx] = deriv idx += 1 #loop for anchors xdata = self._mcaTheory.xdata if config['fit']['stripflag']: anchorslist = [] if config['fit']['stripanchorsflag']: if config['fit']['stripanchorslist'] is not None: ravelled = numpy.ravel(xdata) for channel in config['fit']['stripanchorslist']: if channel <= ravelled[0]:continue index = numpy.nonzero(ravelled >= channel)[0] if len(index): index = min(index) if index > 0: anchorslist.append(index) if len(anchorslist) == 0: anchorlist = [0, self._mcaTheory.ydata.size - 1] anchorslist.sort() # find the indices to be used for selecting the appropriate data # if the original x data were not ordered we have a problem # TODO: check for original ordering. if x is None: # we have an enumerated channels axis iXMin = xdata[0] iXMax = xdata[-1] else: iXMin = numpy.nonzero(x <= xdata[0])[0][-1] iXMax = numpy.nonzero(x >= xdata[-1])[0][0] # numpy 1.11.0 returns an array on previous expression # and then complains about a future deprecation warning # because of using an array and not an scalar in the selection if hasattr(iXMin, "shape"): if len(iXMin.shape): iXMin = iXMin[0] if hasattr(iXMax, "shape"): if len(iXMax.shape): iXMax = iXMax[0] dummySpectrum = firstSpectrum[iXMin:iXMax+1].reshape(-1, 1) # print("dummy = ", dummySpectrum.shape) # allocate the output buffer results = numpy.zeros((nFree, nRows, nColumns), numpy.float32) uncertainties = numpy.zeros((nFree, nRows, nColumns), numpy.float32) #perform the initial fit _logger.debug("Configuration elapsed = %f", time.time() - t0) t0 = time.time() totalSpectra = data.shape[0] * data.shape[1] jStep = min(100, data.shape[1]) if weightPolicy == 2: SVD = False sigma_b = None elif weightPolicy == 1: # the +1 is to prevent misbehavior due to weights less than 1.0 sigma_b = 1 + numpy.sqrt(dummySpectrum)/nPixels SVD = True else: SVD = True sigma_b = None last_svd = None for i in range(0, data.shape[0]): #print(i) #chunks of nColumns spectra if i == 0: chunk = numpy.zeros((dummySpectrum.shape[0], jStep), numpy.float) jStart = 0 while jStart < data.shape[1]: jEnd = min(jStart + jStep, data.shape[1]) chunk[:,:(jEnd - jStart)] = data[i, jStart:jEnd, iXMin:iXMax+1].T if config['fit']['stripflag']: for k in range(jStep): # obtain the smoothed spectrum background=SpecfitFuns.SavitskyGolay(chunk[:, k], config['fit']['stripfilterwidth']) lastAnchor = 0 for anchor in anchorslist: if (anchor > lastAnchor) and (anchor < background.size): background[lastAnchor:anchor] =\ SpecfitFuns.snip1d(background[lastAnchor:anchor], config['fit']['snipwidth'], 0) lastAnchor = anchor if lastAnchor < background.size: background[lastAnchor:] =\ SpecfitFuns.snip1d(background[lastAnchor:], config['fit']['snipwidth'], 0) chunk[:, k] -= background # perform the multiple fit to all the spectra in the chunk #print("SHAPES") #print(derivatives.shape) #print(chunk[:,:(jEnd - jStart)].shape) ddict=lstsq(derivatives, chunk[:,:(jEnd - jStart)], sigma_b=sigma_b, weight=weight, digested_output=True, svd=SVD, last_svd=last_svd) last_svd = ddict.get('svd', None) parameters = ddict['parameters'] results[:, i, jStart:jEnd] = parameters uncertainties[:, i, jStart:jEnd] = ddict['uncertainties'] jStart = jEnd t = time.time() - t0 _logger.debug("First fit elapsed = %f", t) if t > 0.: _logger.debug("Spectra per second = %f", data.shape[0]*data.shape[1]/float(t)) t0 = time.time() # cleanup zeros # start with the parameter with the largest amount of negative values if refit: negativePresent = True else: negativePresent = False nFits = 0 while negativePresent: zeroList = [] #totalNegative = 0 for i in range(nFree): #we have to skip the background parameters if i >= nFreeBackgroundParameters: t = results[i] < 0 tsum = t.sum() if tsum > 0: zeroList.append((tsum, i, t)) #totalNegative += tsum #print("totalNegative = ", totalNegative) if len(zeroList) == 0: negativePresent = False continue if nFits > (2 * (nFree - nFreeBackgroundParameters)): # we are probably in an endless loop # force negative pixels for item in zeroList: i = item[1] badMask = item[2] results[i][badMask] = 0.0 _logger.warning("WARNING: %d pixels of parameter %s forced to zero", item[0], freeNames[i]) continue zeroList.sort() zeroList.reverse() badParameters = [] badParameters.append(zeroList[0][1]) badMask = zeroList[0][2] if 1: # prevent and endless loop if two or more parameters have common pixels where they are # negative and one of them remains negative when forcing other one to zero for i, item in enumerate(zeroList): if item[1] not in badParameters: if item[0] > 0: #check if they have common negative pixels t = badMask * item[-1] if t.sum() > 0: badParameters.append(item[1]) badMask = t if badMask.sum() < (0.0025 * nPixels): # fit not worth for i in badParameters: results[i][badMask] = 0.0 uncertainties[i][badMask] = 0.0 _logger.debug("WARNING: %d pixels of parameter %s set to zero", badMask.sum(), freeNames[i]) else: _logger.debug("Number of secondary fits = %d", nFits + 1) nFits += 1 A = derivatives[:, [i for i in range(nFree) if i not in badParameters]] #assume we'll not have too many spectra if data.dtype not in [numpy.float32, numpy.float64]: if data.itemsize < 5: data_dtype = numpy.float32 else: data_dtype = numpy.floa64 else: data_dtype = data.dtype try: if data.dtype != data_dtype: spectra = numpy.zeros((int(badMask.sum()), 1 + iXMax - iXMin), data_dtype) spectra[:] = data[badMask, iXMin:iXMax+1] else: spectra = data[badMask, iXMin:iXMax+1] spectra.shape = badMask.sum(), -1 except TypeError: # in case of dynamic arrays, two dimensional indices are not # supported by h5py spectra = numpy.zeros((int(badMask.sum()), 1 + iXMax - iXMin), data_dtype) selectedIndices = numpy.nonzero(badMask > 0) tmpData = numpy.zeros((1, 1 + iXMax - iXMin), data_dtype) oldDataRow = -1 j = 0 for i in range(len(selectedIndices[0])): j = selectedIndices[0][i] if j != oldDataRow: tmpData = data[j] olddataRow = j spectra[i] = tmpData[selectedIndices[1][i], iXMin:iXMax+1] spectra = spectra.T # if config['fit']['stripflag']: for k in range(spectra.shape[1]): # obtain the smoothed spectrum background=SpecfitFuns.SavitskyGolay(spectra[:, k], config['fit']['stripfilterwidth']) lastAnchor = 0 for anchor in anchorslist: if (anchor > lastAnchor) and (anchor < background.size): background[lastAnchor:anchor] =\ SpecfitFuns.snip1d(background[lastAnchor:anchor], config['fit']['snipwidth'], 0) lastAnchor = anchor if lastAnchor < background.size: background[lastAnchor:] =\ SpecfitFuns.snip1d(background[lastAnchor:], config['fit']['snipwidth'], 0) spectra[:, k] -= background ddict = lstsq(A, spectra, sigma_b=sigma_b, weight=weight, digested_output=True, svd=SVD) idx = 0 for i in range(nFree): if i in badParameters: results[i][badMask] = 0.0 uncertainties[i][badMask] = 0.0 else: results[i][badMask] = ddict['parameters'][idx] uncertainties[i][badMask] = ddict['uncertainties'][idx] idx += 1 if refit: t = time.time() - t0 _logger.debug("Fit of negative peaks elapsed = %f", t) t0 = time.time() outputDict = {'parameters':results, 'uncertainties':uncertainties, 'names':freeNames} if concentrations: # check if an internal reference is used and if it is set to auto #################################################### # CONCENTRATIONS cTool = ConcentrationsTool.ConcentrationsTool() cToolConf = cTool.configure() cToolConf.update(config['concentrations']) fitFirstSpectrum = False if config['concentrations']['usematrix']: _logger.debug("USING MATRIX") if config['concentrations']['reference'].upper() == "AUTO": fitFirstSpectrum = True elif autotime: # we have to calculate with the time in the configuration # and correct later on cToolConf["autotime"] = 0 fitresult = {} if fitFirstSpectrum: # we have to fit the "reference" spectrum just to get the reference element mcafitresult = self._mcaTheory.startfit(digest=0, linear=True) # if one of the elements has zero area this cannot be made directly fitresult['result'] = self._mcaTheory.imagingDigestResult() fitresult['result']['config'] = config concentrationsResult, addInfo = cTool.processFitResult(config=cToolConf, fitresult=fitresult, elementsfrommatrix=False, fluorates=self._mcaTheory._fluoRates, addinfo=True) # and we have to make sure that all the areas are positive for group in fitresult['result']['groups']: if fitresult['result'][group]['fitarea'] <= 0.0: # give a tiny area fitresult['result'][group]['fitarea'] = 1.0e-6 config['concentrations']['reference'] = addInfo['ReferenceElement'] else: fitresult['result'] = {} fitresult['result']['config'] = config fitresult['result']['groups'] = [] idx = 0 for i, param in enumerate(self._mcaTheory.PARAMETERS): if self._mcaTheory.codes[0][i] == Gefit.CFIXED: continue if i < self._mcaTheory.NGLOBAL: # background pass else: fitresult['result']['groups'].append(param) fitresult['result'][param] = {} # we are just interested on the factor to be applied to the area to get the # concentrations fitresult['result'][param]['fitarea'] = 1.0 fitresult['result'][param]['sigmaarea'] = 1.0 idx += 1 concentrationsResult, addInfo = cTool.processFitResult(config=cToolConf, fitresult=fitresult, elementsfrommatrix=False, fluorates=self._mcaTheory._fluoRates, addinfo=True) nValues = 1 if len(concentrationsResult['layerlist']) > 1: nValues += len(concentrationsResult['layerlist']) nElements = len(list(concentrationsResult['mass fraction'].keys())) massFractions = numpy.zeros((nValues * nElements, nRows, nColumns), numpy.float32) referenceElement = addInfo['ReferenceElement'] referenceTransitions = addInfo['ReferenceTransitions'] _logger.debug("Reference <%s> transition <%s>", referenceElement, referenceTransitions) if referenceElement in ["", None, "None"]: _logger.debug("No reference") counter = 0 for i, group in enumerate(fitresult['result']['groups']): if group.lower().startswith("scatter"): _logger.debug("skept %s", group) continue outputDict['names'].append("C(%s)" % group) if counter == 0: if hasattr(liveTimeFactor, "shape"): liveTimeFactor.shape = results[nFreeBackgroundParameters+i].shape massFractions[counter] = liveTimeFactor * \ results[nFreeBackgroundParameters+i] * \ (concentrationsResult['mass fraction'][group] / \ fitresult['result'][group]['fitarea']) counter += 1 if len(concentrationsResult['layerlist']) > 1: for layer in concentrationsResult['layerlist']: outputDict['names'].append("C(%s)-%s" % (group, layer)) massFractions[counter] = liveTimeFactor * \ results[nFreeBackgroundParameters+i] * \ (concentrationsResult[layer]['mass fraction'][group] / \ fitresult['result'][group]['fitarea']) counter += 1 else: _logger.debug("With reference") idx = None testGroup = referenceElement+ " " + referenceTransitions.split()[0] for i, group in enumerate(fitresult['result']['groups']): if group == testGroup: idx = i if idx is None: raise ValueError("Invalid reference: <%s> <%s>" %\ (referenceElement, referenceTransitions)) group = fitresult['result']['groups'][idx] referenceArea = fitresult['result'][group]['fitarea'] referenceConcentrations = concentrationsResult['mass fraction'][group] goodIdx = results[nFreeBackgroundParameters+idx] > 0 massFractions[idx] = referenceConcentrations counter = 0 for i, group in enumerate(fitresult['result']['groups']): if group.lower().startswith("scatter"): _logger.debug("skept %s", group) continue outputDict['names'].append("C(%s)" % group) goodI = results[nFreeBackgroundParameters+i] > 0 tmp = results[nFreeBackgroundParameters+idx][goodI] massFractions[counter][goodI] = (results[nFreeBackgroundParameters+i][goodI]/(tmp + (tmp == 0))) *\ ((referenceArea/fitresult['result'][group]['fitarea']) *\ (concentrationsResult['mass fraction'][group])) counter += 1 if len(concentrationsResult['layerlist']) > 1: for layer in concentrationsResult['layerlist']: outputDict['names'].append("C(%s)-%s" % (group, layer)) massFractions[counter][goodI] = (results[nFreeBackgroundParameters+i][goodI]/(tmp + (tmp == 0))) *\ ((referenceArea/fitresult['result'][group]['fitarea']) *\ (concentrationsResult[layer]['mass fraction'][group])) counter += 1 outputDict['concentrations'] = massFractions t = time.time() - t0 _logger.debug("Calculation of concentrations elapsed = %f", t) #################################################### return outputDict
def update(self): if self._y is None: return pars = self.getParameters() #smoothed data y = numpy.ravel(numpy.array(self._y)).astype(numpy.float64) ysmooth = SpecfitFuns.SavitskyGolay(y, pars['stripfilterwidth']) f = [0.25, 0.5, 0.25] ysmooth[1:-1] = numpy.convolve(ysmooth, f, mode=0) ysmooth[0] = 0.5 * (ysmooth[0] + ysmooth[1]) ysmooth[-1] = 0.5 * (ysmooth[-1] + ysmooth[-2]) #loop for anchors x = self._x niter = pars['stripiterations'] anchorslist = [] if pars['stripanchorsflag']: if pars['stripanchorslist'] is not None: ravelled = x for channel in pars['stripanchorslist']: if channel <= ravelled[0]: continue index = numpy.nonzero(ravelled >= channel)[0] if len(index): index = min(index) if index > 0: anchorslist.append(index) if niter > 1000: stripBackground = SpecfitFuns.subac(ysmooth, pars['stripconstant'], niter, pars['stripwidth'], anchorslist) #final smoothing stripBackground = SpecfitFuns.subac(stripBackground, pars['stripconstant'], 500, 1, anchorslist) elif niter > 0: stripBackground = SpecfitFuns.subac(ysmooth, pars['stripconstant'], niter, pars['stripwidth'], anchorslist) else: stripBackground = 0.0 * ysmooth + ysmooth.min() if len(anchorslist) == 0: anchorslist = [0, len(ysmooth) - 1] anchorslist.sort() snipBackground = 0.0 * ysmooth lastAnchor = 0 width = pars['snipwidth'] for anchor in anchorslist: if (anchor > lastAnchor) and (anchor < len(ysmooth)): snipBackground[lastAnchor:anchor] =\ SpecfitFuns.snip1d(ysmooth[lastAnchor:anchor], width, 0) lastAnchor = anchor if lastAnchor < len(ysmooth): snipBackground[lastAnchor:] =\ SpecfitFuns.snip1d(ysmooth[lastAnchor:], width, 0) self.graphWidget.addCurve(x, y, \ legend='Input Data',\ replace=True, replot=False) self.graphWidget.addCurve(x, stripBackground,\ legend='Strip Background',\ replot=False) self.graphWidget.addCurve(x, snipBackground,\ legend='SNIP Background', replot=True)