def polyfit(x,y,u=None): '''Determine the weighted least-squares fit for 2nd order polynomial''' x = np.asarray(x) y = np.asarray(y) if u is not None: u[u==0]=1 weight = 1./np.asarray(u) else: weight = np.ones_like(x) params = Parameters() params.add('a', value=0) params.add('b', value=(y.max()-y.min())/(x.max()-x.min())) params.add('c', value=0.0) def residual(pars, x, data=None,w=None): model = pars['a'].value + pars['b'].value*x + pars['c'].value*x**2 if data is None: return model return (model - data) #* w myfit = Minimizer(residual, params,fcn_args=(x,), fcn_kws={'data':y,'w':weight}) myfit.leastsq() return [params['c'].value,params['b'].value,params['a'].value]
def NIST_Test(DataSet, start='start2', plot=True): NISTdata = ReadNistData(DataSet) resid, npar, dimx = Models[DataSet] y = NISTdata['y'] x = NISTdata['x'] params = Parameters() for i in range(npar): pname = 'b%i' % (i+1) cval = NISTdata['cert_values'][i] cerr = NISTdata['cert_stderr'][i] pval1 = NISTdata[start][i] params.add(pname, value=pval1) myfit = Minimizer(resid, params, fcn_args=(x,), fcn_kws={'y':y}, scale_covar=True) myfit.prepare_fit() myfit.leastsq() digs = Compare_NIST_Results(DataSet, myfit, params, NISTdata) if plot and HASPYLAB: fit = -resid(params, x, ) pylab.plot(x, y, 'r+') pylab.plot(x, fit, 'ko--') pylab.show() return digs > 2
class LmModel(object): """ Base class for all models. Models take x and y and return """ def __init__(self,x,y): self.x, self.y=x,y self.parameters=Parameters() self.min=Minimizer(self.residual, self.parameters) def print_para(self): for i in self.parameters.values: print i def func(self,paras): raise NotImplementedError def est_startvals(self): raise NotImplementedError def residual(self,paras): return self.func(paras)-self.y def fit(self): self.min.leastsq() self.y_model=self.func(self.parameters)
def autobk(energy, mu, rbkg=1, nknots=None, group=None, e0=None, kmin=0, kmax=None, kw=1, dk=0, win=None, vary_e0=True, chi_std=None, nfft=2048, kstep=0.05, _larch=None): if _larch is None: raise Warning("cannot calculate autobk spline -- larch broken?") # get array indices for rkbg and e0: irbkg, ie0 rgrid = np.pi/(kstep*nfft) if rbkg < 2*rgrid: rbkg = 2*rgrid irbkg = int(1.01 + rbkg/rgrid) if e0 is None: e0 = find_e0(energy, mu, group=group, _larch=_larch) ie0 = _index_nearest(energy, e0) # save ungridded k (kraw) and grided k (kout) # and ftwin (*k-weighting) for FT in residual kraw = np.sqrt(ETOK*(energy[ie0:] - e0)) if kmax is None: kmax = max(kraw) kout = kstep * np.arange(int(1.01+kmax/kstep)) ftwin = kout**kw * ftwindow(kout, xmin=kmin, xmax=kmax, window=win, dx=dk) # calc k-value and initial guess for y-values of spline params nspline = max(4, min(60, 2*int(rbkg*(kmax-kmin)/np.pi) + 1)) spl_y = np.zeros(nspline) spl_k = np.zeros(nspline) for i in range(nspline): q = kmin + i*(kmax-kmin)/(nspline - 1) ik = _index_nearest(kraw, q) i1 = min(len(kraw)-1, ik + 5) i2 = max(0, ik - 5) spl_k[i] = kraw[ik] spl_y[i] = (2*mu[ik] + mu[i1] + mu[i2] ) / 4.0 # get spline represention: knots, coefs, order=3 # coefs will be varied in fit. knots, coefs, order = splrep(spl_k, spl_y) # set fit parameters from initial coefficients fparams = Parameters() for i, v in enumerate(coefs): fparams.add("c%i" % i, value=v, vary=i<len(spl_y)) fitkws = dict(knots=knots, order=order, kraw=kraw, mu=mu[ie0:], irbkg=irbkg, kout=kout, ftwin=ftwin, nfft=nfft) # do fit fit = Minimizer(__resid, fparams, fcn_kws=fitkws) fit.leastsq() # write final results coefs = [p.value for p in fparams.values()] bkg, chi = spline_eval(kraw, mu[ie0:], knots, coefs, order, kout) obkg = np.zeros(len(mu)) obkg[:ie0] = mu[:ie0] obkg[ie0:] = bkg if _larch.symtable.isgroup(group): setattr(group, 'bkg', obkg) setattr(group, 'chie', mu-obkg) setattr(group, 'k', kout) setattr(group, 'chi', chi)
def test_peakfit(): from lmfit.utilfuncs import gaussian def residual(pars, x, data=None): g1 = gaussian(x, pars['a1'].value, pars['c1'].value, pars['w1'].value) g2 = gaussian(x, pars['a2'].value, pars['c2'].value, pars['w2'].value) model = g1 + g2 if data is None: return model return (model - data) n = 601 xmin = 0. xmax = 15.0 noise = np.random.normal(scale=.65, size=n) x = np.linspace(xmin, xmax, n) org_params = Parameters() org_params.add_many(('a1', 12.0, True, None, None, None), ('c1', 5.3, True, None, None, None), ('w1', 1.0, True, None, None, None), ('a2', 9.1, True, None, None, None), ('c2', 8.1, True, None, None, None), ('w2', 2.5, True, None, None, None)) data = residual(org_params, x) + noise fit_params = Parameters() fit_params.add_many(('a1', 8.0, True, None, 14., None), ('c1', 5.0, True, None, None, None), ('w1', 0.7, True, None, None, None), ('a2', 3.1, True, None, None, None), ('c2', 8.8, True, None, None, None)) fit_params.add('w2', expr='2.5*w1') myfit = Minimizer(residual, fit_params, fcn_args=(x,), fcn_kws={'data':data}) myfit.prepare_fit() init = residual(fit_params, x) myfit.leastsq() print(' N fev = ', myfit.nfev) print(myfit.chisqr, myfit.redchi, myfit.nfree) report_fit(fit_params) fit = residual(fit_params, x) check_paras(fit_params, org_params)
def __FitEvent(self): try: dt = 1000./self.Fs # time-step in ms. # edat=np.asarray( np.abs(self.eventData), dtype='float64' ) edat=self.dataPolarity*np.asarray( self.eventData, dtype='float64' ) # control numpy error reporting np.seterr(invalid='ignore', over='ignore', under='ignore') ts = np.array([ t*dt for t in range(0,len(edat)) ], dtype='float64') # estimate initial guess for events initguess=self._characterizeevent(edat, np.abs(util.avg(edat[:10])), self.baseSD, self.InitThreshold, 6.) self.nStates=len(initguess)-1 # setup fit params params=Parameters() for i in range(1, len(initguess)): params.add('a'+str(i-1), value=initguess[i][0]-initguess[i-1][0]) params.add('mu'+str(i-1), value=initguess[i][1]*dt) params.add('tau'+str(i-1), value=dt*7.5) params.add('b', value=initguess[0][0]) optfit=Minimizer(self.__objfunc, params, fcn_args=(ts,edat,)) optfit.prepare_fit() optfit.leastsq(xtol=self.FitTol,ftol=self.FitTol,maxfev=self.FitIters) if optfit.success: self.__recordevent(optfit) else: #print optfit.message, optfit.lmdif_message self.rejectEvent('eFitConvergence') except KeyboardInterrupt: self.rejectEvent('eFitUserStop') raise except InvalidEvent: self.rejectEvent('eInvalidEvent') except: self.rejectEvent('eFitFailure') raise
def test_constraints1(): def residual(pars, x, sigma=None, data=None): yg = gaussian(x, pars['amp_g'], pars['cen_g'], pars['wid_g']) yl = lorentzian(x, pars['amp_l'], pars['cen_l'], pars['wid_l']) model = yg + yl + pars['line_off'] + x * pars['line_slope'] if data is None: return model if sigma is None: return (model - data) return (model - data)/sigma n = 601 xmin = 0. xmax = 20.0 x = linspace(xmin, xmax, n) data = (gaussian(x, 21, 8.1, 1.2) + lorentzian(x, 10, 9.6, 2.4) + random.normal(scale=0.23, size=n) + x*0.5) pfit = Parameters() pfit.add(name='amp_g', value=10) pfit.add(name='cen_g', value=9) pfit.add(name='wid_g', value=1) pfit.add(name='amp_tot', value=20) pfit.add(name='amp_l', expr='amp_tot - amp_g') pfit.add(name='cen_l', expr='1.5+cen_g') pfit.add(name='wid_l', expr='2*wid_g') pfit.add(name='line_slope', value=0.0) pfit.add(name='line_off', value=0.0) sigma = 0.021 # estimate of data error (for all data points) myfit = Minimizer(residual, pfit, fcn_args=(x,), fcn_kws={'sigma':sigma, 'data':data}, scale_covar=True) myfit.prepare_fit() init = residual(myfit.params, x) result = myfit.leastsq() print(' Nfev = ', result.nfev) print( result.chisqr, result.redchi, result.nfree) report_fit(result.params) pfit= result.params fit = residual(result.params, x) assert(pfit['cen_l'].value == 1.5 + pfit['cen_g'].value) assert(pfit['amp_l'].value == pfit['amp_tot'].value - pfit['amp_g'].value) assert(pfit['wid_l'].value == 2 * pfit['wid_g'].value)
def test_derive(): def func(pars, x, data=None): model = pars['a'] * np.exp(-pars['b'] * x) + pars['c'] if data is None: return model return model - data def dfunc(pars, x, data=None): v = np.exp(-pars['b']*x) return np.array([v, -pars['a']*x*v, np.ones(len(x))]) def f(var, x): return var[0] * np.exp(-var[1] * x) + var[2] params1 = Parameters() params1.add('a', value=10) params1.add('b', value=10) params1.add('c', value=10) params2 = Parameters() params2.add('a', value=10) params2.add('b', value=10) params2.add('c', value=10) a, b, c = 2.5, 1.3, 0.8 x = np.linspace(0, 4, 50) y = f([a, b, c], x) data = y + 0.15*np.random.normal(size=len(x)) # fit without analytic derivative min1 = Minimizer(func, params1, fcn_args=(x,), fcn_kws={'data': data}) out1 = min1.leastsq() # fit with analytic derivative min2 = Minimizer(func, params2, fcn_args=(x,), fcn_kws={'data': data}) out2 = min2.leastsq(Dfun=dfunc, col_deriv=1) check_wo_stderr(out1.params['a'], out2.params['a'].value, 0.00005) check_wo_stderr(out1.params['b'], out2.params['b'].value, 0.00005) check_wo_stderr(out1.params['c'], out2.params['c'].value, 0.00005)
def fitevent(self, edat, initguess): try: dt = 1000./self.Fs # time-step in ms. # control numpy error reporting np.seterr(invalid='ignore', over='ignore', under='ignore') ts = np.array([ t*dt for t in range(0,len(edat)) ], dtype='float64') self.nStates=len(initguess) initRCConst=dt*5. # setup fit params params=Parameters() for i in range(0, len(initguess)): params.add('a'+str(i), value=initguess[i][0]) params.add('mu'+str(i), value=initguess[i][1]) if self.LinkRCConst: if i==0: params.add('tau'+str(i), value=initRCConst) else: params.add('tau'+str(i), value=initRCConst, expr='tau0') else: params.add('tau'+str(i), value=initRCConst) params.add('b', value=self.baseMean ) igdict=params.valuesdict() optfit=Minimizer(self._objfunc, params, fcn_args=(ts,edat,)) optfit.prepare_fit() result=optfit.leastsq(xtol=self.FitTol,ftol=self.FitTol,maxfev=self.FitIters) if result.success: tt=[init[0] for init, final in zip(igdict.items(), (result.params.valuesdict()).items()) if init==final] if len(tt) > 0: self.flagEvent('wInitGuessUnchanged') self._recordevent(result) else: #print optfit.message, optfit.lmdif_message self.rejectEvent('eFitConvergence') except KeyboardInterrupt: self.rejectEvent('eFitUserStop') raise except InvalidEvent: self.rejectEvent('eInvalidEvent') except: self.rejectEvent('eFitFailure')
def time_confinterval(self): np.random.seed(0) x = np.linspace(0.3,10,100) y = 1/(0.1*x)+2+0.1*np.random.randn(x.size) p = Parameters() p.add_many(('a', 0.1), ('b', 1)) def residual(p): a = p['a'].value b = p['b'].value return 1/(a*x)+b-y minimizer = Minimizer(residual, p) out = minimizer.leastsq() return conf_interval(minimizer, out)
def test_peakfit(): def residual(pars, x, data=None): g1 = gaussian(x, pars['a1'], pars['c1'], pars['w1']) g2 = gaussian(x, pars['a2'], pars['c2'], pars['w2']) model = g1 + g2 if data is None: return model return (model - data) n = 601 xmin = 0. xmax = 15.0 noise = np.random.normal(scale=.65, size=n) x = np.linspace(xmin, xmax, n) org_params = Parameters() org_params.add_many(('a1', 12.0, True, None, None, None), ('c1', 5.3, True, None, None, None), ('w1', 1.0, True, None, None, None), ('a2', 9.1, True, None, None, None), ('c2', 8.1, True, None, None, None), ('w2', 2.5, True, None, None, None)) data = residual(org_params, x) + noise fit_params = Parameters() fit_params.add_many(('a1', 8.0, True, None, 14., None), ('c1', 5.0, True, None, None, None), ('w1', 0.7, True, None, None, None), ('a2', 3.1, True, None, None, None), ('c2', 8.8, True, None, None, None)) fit_params.add('w2', expr='2.5*w1') myfit = Minimizer(residual, fit_params, fcn_args=(x,), fcn_kws={'data': data}) myfit.prepare_fit() out = myfit.leastsq() check_paras(out.params, org_params)
def test_ci_report(): """test confidence interval report""" def residual(pars, x, data=None): argu = (x*pars['decay'])**2 shift = pars['shift'] if abs(shift) > np.pi/2: shift = shift - np.sign(shift)*np.pi model = pars['amp']*np.sin(shift + x/pars['period']) * np.exp(-argu) if data is None: return model return model - data p_true = Parameters() p_true.add('amp', value=14.0) p_true.add('period', value=5.33) p_true.add('shift', value=0.123) p_true.add('decay', value=0.010) n = 2500 xmin = 0. xmax = 250.0 x = np.linspace(xmin, xmax, n) data = residual(p_true, x) + np.random.normal(scale=0.7215, size=n) fit_params = Parameters() fit_params.add('amp', value=13.0) fit_params.add('period', value=2) fit_params.add('shift', value=0.0) fit_params.add('decay', value=0.02) mini = Minimizer(residual, fit_params, fcn_args=(x,), fcn_kws={'data': data}) out = mini.leastsq() report = fit_report(out) assert(len(report) > 500) ci, tr = conf_interval(mini, out, trace=True) report = ci_report(ci) assert(len(report) > 250)
def feffit(paramgroup, datasets, rmax_out=10, path_outputs=True, _larch=None, **kws): """execute a Feffit fit: a fit of feff paths to a list of datasets Parameters: ------------ paramgroup: group containing parameters for fit datasets: Feffit Dataset group or list of Feffit Dataset group. rmax_out: maximum R value to calculate output arrays. path_output: Flag to set whether all Path outputs should be written. Returns: --------- a fit results group. This will contain subgroups of: datasets: an array of FeffitDataSet groups used in the fit. params: This will be identical to the input parameter group. fit: an object which points to the low-level fit. Statistical parameters will be put into the params group. Each dataset will have a 'data' and 'model' subgroup, each with arrays: k wavenumber array of k chi chi(k). kwin window Omega(k) (length of input chi(k)). r uniform array of R, out to rmax_out. chir complex array of chi(R). chir_mag magnitude of chi(R). chir_pha phase of chi(R). chir_re real part of chi(R). chir_im imaginary part of chi(R). """ def _resid(params, datasets=None, paramgroup=None, **kwargs): """ this is the residual function""" params2group(params, paramgroup) return concatenate([d._residual(paramgroup) for d in datasets]) if isNamedClass(datasets, FeffitDataSet): datasets = [datasets] params = group2params(paramgroup, _larch=_larch) for ds in datasets: if not isNamedClass(ds, FeffitDataSet): print( "feffit needs a list of FeffitDataSets") return ds.prepare_fit() fit = Minimizer(_resid, params, fcn_kws=dict(datasets=datasets, paramgroup=paramgroup), scale_covar=True, **kws) result = fit.leastsq() params2group(result.params, paramgroup) dat = concatenate([d._residual(paramgroup, data_only=True) for d in datasets]) n_idp = 0 for ds in datasets: n_idp += ds.n_idp # here we rescale chi-square and reduced chi-square to n_idp npts = len(result.residual) chi_square = result.chisqr * n_idp*1.0 / npts chi_reduced = chi_square/(n_idp*1.0 - result.nvarys) rfactor = (result.residual**2).sum() / (dat**2).sum() # calculate 'aic', 'bic' rescaled to n_idp # note that neg2_loglikel is -2*log(likelihood) neg2_loglikel = n_idp * np.log(chi_square / n_idp) aic = neg2_loglikel + 2 * result.nvarys bic = neg2_loglikel + np.log(n_idp) * result.nvarys # With scale_covar = True, Minimizer() scales the uncertainties # by reduced chi-square assuming params.nfree is the correct value # for degrees-of-freedom. But n_idp-params.nvarys is a better measure, # so we rescale uncertainties here. covar = getattr(result, 'covar', None) # print("COVAR " , covar) if covar is not None: err_scale = (result.nfree / (n_idp - result.nvarys)) for name in result.var_names: p = result.params[name] if isParameter(p) and p.vary: p.stderr *= sqrt(err_scale) # next, propagate uncertainties to constraints and path parameters. result.covar *= err_scale vsave, vbest = {}, [] # 1. save current params for vname in result.var_names: par = result.params[vname] vsave[vname] = par vbest.append(par.value) # 2. get correlated uncertainties, set params accordingly uvars = correlated_values(vbest, result.covar) # 3. evaluate constrained params, save stderr for nam, obj in result.params.items(): eval_stderr(obj, uvars, result.var_names, result.params) # 3. evaluate path params, save stderr for ds in datasets: for p in ds.pathlist: p.store_feffdat() for pname in ('degen', 's02', 'e0', 'ei', 'deltar', 'sigma2', 'third', 'fourth'): obj = p.params[PATHPAR_FMT % (pname, p.label)] eval_stderr(obj, uvars, result.var_names, result.params) # restore saved parameters again for vname in result.var_names: # setattr(params, vname, vsave[vname]) params[vname] = vsave[vname] # clear any errors evaluting uncertainties if _larch is not None and (len(_larch.error) > 0): _larch.error = [] # reset the parameters group with the newly updated uncertainties params2group(result.params, paramgroup) # here we create outputs arrays for chi(k), chi(r): for ds in datasets: ds.save_ffts(rmax_out=rmax_out, path_outputs=path_outputs) out = Group(name='feffit results', datasets=datasets, fitter=fit, fit_details=result, chi_square=chi_square, n_independent=n_idp, chi_reduced=chi_reduced, rfactor=rfactor, aic=aic, bic=bic, covar=covar) for attr in ('params', 'nvarys', 'nfree', 'ndata', 'var_names', 'nfev', 'success', 'errorbars', 'message', 'lmdif_message'): setattr(out, attr, getattr(result, attr, None)) return out
def test_constraints2(): """add a user-defined function to symbol table""" def residual(pars, x, sigma=None, data=None): yg = gaussian(x, pars['amp_g'].value, pars['cen_g'].value, pars['wid_g'].value) yl = lorentzian(x, pars['amp_l'].value, pars['cen_l'].value, pars['wid_l'].value) slope = pars['line_slope'].value offset = pars['line_off'].value model = yg + yl + offset + x * slope if data is None: return model if sigma is None: return (model - data) return (model - data)/sigma n = 601 xmin = 0. xmax = 20.0 x = linspace(xmin, xmax, n) data = (gaussian(x, 21, 8.1, 1.2) + lorentzian(x, 10, 9.6, 2.4) + random.normal(scale=0.23, size=n) + x*0.5) pfit = Parameters() pfit.add(name='amp_g', value=10) pfit.add(name='cen_g', value=9) pfit.add(name='wid_g', value=1) pfit.add(name='amp_tot', value=20) pfit.add(name='amp_l', expr='amp_tot - amp_g') pfit.add(name='cen_l', expr='1.5+cen_g') pfit.add(name='line_slope', value=0.0) pfit.add(name='line_off', value=0.0) sigma = 0.021 # estimate of data error (for all data points) myfit = Minimizer(residual, pfit, fcn_args=(x,), fcn_kws={'sigma':sigma, 'data':data}, scale_covar=True) def width_func(wpar): """ """ return 2*wpar myfit.params._asteval.symtable['wfun'] = width_func try: myfit.params.add(name='wid_l', expr='wfun(wid_g)') except: assert(False) result = myfit.leastsq() print(' Nfev = ', result.nfev) print( result.chisqr, result.redchi, result.nfree) report_fit(result.params) pfit= result.params fit = residual(result.params, x) assert(pfit['cen_l'].value == 1.5 + pfit['cen_g'].value) assert(pfit['amp_l'].value == pfit['amp_tot'].value - pfit['amp_g'].value) assert(pfit['wid_l'].value == 2 * pfit['wid_g'].value)
def pre_edge(energy, mu=None, group=None, e0=None, step=None, nnorm=None, nvict=0, pre1=None, pre2=-50, norm1=100, norm2=None, make_flat=True, emin_area=None, _larch=None): """pre edge subtraction, normalization for XAFS This performs a number of steps: 1. determine E0 (if not supplied) from max of deriv(mu) 2. fit a line of polymonial to the region below the edge 3. fit a polymonial to the region above the edge 4. extrapolae the two curves to E0 to determine the edge jump 5. estimate area from emin_area to norm2, to get norm_area Arguments ---------- energy: array of x-ray energies, in eV, or group (see note) mu: array of mu(E) group: output group e0: edge energy, in eV. If None, it will be determined here. step: edge jump. If None, it will be determined here. pre1: low E range (relative to E0) for pre-edge fit pre2: high E range (relative to E0) for pre-edge fit nvict: energy exponent to use for pre-edg fit. See Note norm1: low E range (relative to E0) for post-edge fit norm2: high E range (relative to E0) for post-edge fit nnorm: degree of polynomial (ie, nnorm+1 coefficients will be found) for post-edge normalization curve. Default=None (see note) make_flat: boolean (Default True) to calculate flattened output. emin_area: energy threshold for area normalization (see note) Returns ------- None The following attributes will be written to the output group: e0 energy origin edge_step edge step norm normalized mu(E), using polynomial norm_area normalized mu(E), using integrated area flat flattened, normalized mu(E) pre_edge determined pre-edge curve post_edge determined post-edge, normalization curve dmude derivative of mu(E) (if the output group is None, _sys.xafsGroup will be written to) Notes ----- 1 If the first argument is a Group, it must contain 'energy' and 'mu'. If it exists, group.e0 will be used as e0. See First Argrument Group in Documentation 2 nvict gives an exponent to the energy term for the fits to the pre-edge and the post-edge region. For the pre-edge, a line (m * energy + b) is fit to mu(energy)*energy**nvict over the pre-edge region, energy=[e0+pre1, e0+pre2]. For the post-edge, a polynomial of order nnorm will be fit to mu(energy)*energy**nvict of the post-edge region energy=[e0+norm1, e0+norm2]. 3 nnorm will default to 2 in norm2-norm1>400, to 1 if 100>norm2-norm1>300, and to 0 in norm2-norm1<100. 4 norm_area will be estimated so that the area between emin_area and norm2 is equal to (norm2-emin_area). By default emin_area will be set to the *nominal* edge energy for the element and edge - 3*core_level_width """ energy, mu, group = parse_group_args(energy, members=('energy', 'mu'), defaults=(mu,), group=group, fcn_name='pre_edge') if len(energy.shape) > 1: energy = energy.squeeze() if len(mu.shape) > 1: mu = mu.squeeze() pre_dat = preedge(energy, mu, e0=e0, step=step, nnorm=nnorm, nvict=nvict, pre1=pre1, pre2=pre2, norm1=norm1, norm2=norm2) group = set_xafsGroup(group, _larch=_larch) e0 = pre_dat['e0'] norm = pre_dat['norm'] norm1 = pre_dat['norm1'] norm2 = pre_dat['norm2'] # generate flattened spectra, by fitting a quadratic to .norm # and removing that. flat = norm ie0 = index_nearest(energy, e0) p1 = index_of(energy, norm1+e0) p2 = index_nearest(energy, norm2+e0) if p2-p1 < 2: p2 = min(len(energy), p1 + 2) if make_flat and p2-p1 > 4: enx, mux = remove_nans2(energy[p1:p2], norm[p1:p2]) # enx, mux = (energy[p1:p2], norm[p1:p2]) fpars = Parameters() ncoefs = len(pre_dat['norm_coefs']) fpars.add('c0', value=0, vary=True) fpars.add('c1', value=0, vary=(ncoefs>1)) fpars.add('c2', value=0, vary=(ncoefs>2)) fit = Minimizer(flat_resid, fpars, fcn_args=(enx, mux)) result = fit.leastsq(xtol=1.e-6, ftol=1.e-6) fc0 = result.params['c0'].value fc1 = result.params['c1'].value fc2 = result.params['c2'].value flat_diff = fc0 + energy * (fc1 + energy * fc2) flat = norm - (flat_diff - flat_diff[ie0]) flat[:ie0] = norm[:ie0] group.e0 = e0 group.norm = norm group.norm_poly = 1.0*norm group.flat = flat group.dmude = np.gradient(mu)/np.gradient(energy) group.edge_step = pre_dat['edge_step'] group.edge_step_poly = pre_dat['edge_step'] group.pre_edge = pre_dat['pre_edge'] group.post_edge = pre_dat['post_edge'] group.pre_edge_details = Group() group.pre_edge_details.pre1 = pre_dat['pre1'] group.pre_edge_details.pre2 = pre_dat['pre2'] group.pre_edge_details.nnorm = pre_dat['nnorm'] group.pre_edge_details.norm1 = pre_dat['norm1'] group.pre_edge_details.norm2 = pre_dat['norm2'] group.pre_edge_details.nvict = pre_dat['nvict'] group.pre_edge_details.pre1_input = pre_dat['pre1_input'] group.pre_edge_details.norm2_input = pre_dat['norm2_input'] group.pre_edge_details.pre_slope = pre_dat['precoefs'][0] group.pre_edge_details.pre_offset = pre_dat['precoefs'][1] for i in range(MAX_NNORM): if hasattr(group, 'norm_c%i' % i): delattr(group, 'norm_c%i' % i) for i, c in enumerate(pre_dat['norm_coefs']): setattr(group.pre_edge_details, 'norm_c%i' % i, c) # guess element and edge group.atsym = getattr(group, 'atsym', None) group.edge = getattr(group, 'edge', None) if group.atsym is None or group.edge is None: _atsym, _edge = guess_edge(group.e0, _larch=_larch) if group.atsym is None: group.atsym = _atsym if group.edge is None: group.edge = _edge # calcuate area-normalization if emin_area is None: emin_area = (xray_edge(group.atsym, group.edge).edge - 2*core_width(group.atsym, group.edge)) i1 = index_of(energy, emin_area) i2 = index_of(energy, e0+norm2) en = energy[i1:i2] area_step = max(1.e-15, simps(norm[i1:i2], en) / en.ptp()) group.edge_step_area = group.edge_step_poly * area_step group.norm_area = norm/area_step group.pre_edge_details.emin_area = emin_area return
myfit.prepare_fit() # for scale_covar in (True, False): myfit.scale_covar = scale_covar print ' ==== scale_covar = ', myfit.scale_covar, ' ===' for sigma in (0.1, 0.2, 0.23, 0.5): myfit.userkws['sigma'] = sigma p_fit['amp_g'].value = 10 p_fit['cen_g'].value = 9 p_fit['wid_g'].value = 1 p_fit['line_slope'].value =0.0 p_fit['line_off'].value =0.0 myfit.leastsq() print ' sigma = ', sigma print ' chisqr = ', myfit.chisqr print ' reduced_chisqr = ', myfit.redchi report_errors(p_fit, modelpars=p_true, show_correl=False) print ' ==============================' # if HASPYLAB: # fit = residual(p_fit, x) # pylab.plot(x, fit, 'k-') # pylab.show() #
def test_derive(): def func(pars, x, data=None): a = pars['a'].value b = pars['b'].value c = pars['c'].value model=a * np.exp(-b * x)+c if data is None: return model return (model - data) def dfunc(pars, x, data=None): a = pars['a'].value b = pars['b'].value c = pars['c'].value v = np.exp(-b*x) return [v, -a*x*v, np.ones(len(x))] def f(var, x): return var[0]* np.exp(-var[1] * x)+var[2] params1 = Parameters() params1.add('a', value=10) params1.add('b', value=10) params1.add('c', value=10) params2 = Parameters() params2.add('a', value=10) params2.add('b', value=10) params2.add('c', value=10) a, b, c = 2.5, 1.3, 0.8 x = np.linspace(0,4,50) y = f([a, b, c], x) data = y + 0.15*np.random.normal(size=len(x)) # fit without analytic derivative min1 = Minimizer(func, params1, fcn_args=(x,), fcn_kws={'data':data}) min1.leastsq() fit1 = func(params1, x) # fit with analytic derivative min2 = Minimizer(func, params2, fcn_args=(x,), fcn_kws={'data':data}) min2.leastsq(Dfun=dfunc, col_deriv=1) fit2 = func(params2, x) print ('''Comparison of fit to exponential decay with and without analytic derivatives, to model = a*exp(-b*x) + c for a = %.2f, b = %.2f, c = %.2f ============================================== Statistic/Parameter| Without | With | ---------------------------------------------- N Function Calls | %3i | %3i | Chi-square | %.4f | %.4f | a | %.4f | %.4f | b | %.4f | %.4f | c | %.4f | %.4f | ---------------------------------------------- ''' % (a, b, c, min1.nfev, min2.nfev, min1.chisqr, min2.chisqr, params1['a'].value, params2['a'].value, params1['b'].value, params2['b'].value, params1['c'].value, params2['c'].value )) check_wo_stderr(params1['a'], params2['a'].value, 0.000001) check_wo_stderr(params1['b'], params2['b'].value, 0.000001) check_wo_stderr(params1['c'], params2['c'].value, 0.000001)
yBinSize=yBinSize, normFactor=normFactor) if not isinstance(points, np.ndarray): points = np.array(points) y, x = 0, 1 def residuals_func(model_params, times, flux, fluxerr): model = transit_line_model(model_params, times) return (model - flux) / fluxerr partial_residualspartial_ = partial(residuals_func, times=timeSliceKmod, flux=fluxSliceK / np.median(fluxSliceK), fluxerr=ferrSliceK / np.median(fluxSliceK)) mle0 = Minimizer(partial_residuals, initialParams) start = time() fitResult = mle0.leastsq() print("LMFIT operation took {} seconds".format(time() - start)) report_errors(fitResult.params) # # plt.scatter([p[0] for p in points], [p[1] for p in points], s=0.1, c=interpolFluxes) # plt.scatter(points.T[x], points.T[y], s=0.1, c=interpolFluxes) # plt.colorbar() # plt.show()
def __FitEvent(self): try: varyBlockedCurrent=True i0=np.abs(self.baseMean) i0sig=self.baseSD dt = 1000./self.Fs # time-step in ms. # edat=np.asarray( np.abs(self.eventData), dtype='float64' ) edat=self.dataPolarity*np.asarray( self.eventData, dtype='float64' ) blockedCurrent=min(edat) tauVal=dt estart = self.__eventStartIndex( self.__threadList( edat, range(0,len(edat)) ), i0, i0sig ) - 1 eend = self.__eventEndIndex( self.__threadList( edat, range(0,len(edat)) ), i0, i0sig ) - 2 # For long events, fix the blocked current to speed up the fit #if (eend-estart) > 1000: # blockedCurrent=np.mean(edat[estart+50:eend-50]) # control numpy error reporting np.seterr(invalid='ignore', over='ignore', under='ignore') ts = np.array([ t*dt for t in range(0,len(edat)) ], dtype='float64') #pl.plot(ts,edat) #pl.show() params=Parameters() # print self.absDataStartIndex params.add('mu1', value=estart * dt) params.add('mu2', value=eend * dt) params.add('a', value=(i0-blockedCurrent), vary=varyBlockedCurrent) params.add('b', value = i0) params.add('tau1', value = tauVal) if self.LinkRCConst: params.add('tau2', value = tauVal, expr='tau1') else: params.add('tau2', value = tauVal) optfit=Minimizer(self.__objfunc, params, fcn_args=(ts,edat,)) optfit.prepare_fit() result=optfit.leastsq(xtol=self.FitTol,ftol=self.FitTol,maxfev=self.FitIters) # print optfit.params['b'].value, optfit.params['b'].value - optfit.params['a'].value, optfit.params['mu1'].value, optfit.params['mu2'].value if result.success: if result.params['mu1'].value < 0.0 or result.params['mu2'].value < 0.0: # print 'eInvalidFitParams1', optfit.params['b'].value, optfit.params['b'].value - optfit.params['a'].value, optfit.params['mu1'].value, optfit.params['mu2'].value self.rejectEvent('eInvalidResTime') # The start of the event is set past the length of the data elif result.params['mu1'].value > ts[-1]: # print 'eInvalidFitParams2', optfit.params['b'].value, optfit.params['b'].value - optfit.params['a'].value, optfit.params['mu1'].value, optfit.params['mu2'].value self.rejectEvent('eInvalidEventStart') else: self.mdOpenChCurrent = result.params['b'].value self.mdBlockedCurrent = result.params['b'].value - result.params['a'].value self.mdEventStart = result.params['mu1'].value self.mdEventEnd = result.params['mu2'].value self.mdRCConst1 = result.params['tau1'].value self.mdRCConst2 = result.params['tau2'].value self.mdAbsEventStart = self.mdEventStart + self.absDataStartIndex * dt self.mdBlockDepth = self.mdBlockedCurrent/self.mdOpenChCurrent self.mdResTime = self.mdEventEnd - self.mdEventStart self.mdRedChiSq = result.chisqr/( np.var(result.residual) * (len(self.eventData) - result.nvarys -1) ) # if (eend-estart) > 1000: # print blockedCurrent, self.mdBlockedCurrent, self.mdOpenChCurrent, self.mdResTime, self.mdRiseTime, self.mdRedChiSq, optfit.chisqr # if self.mdBlockDepth > self.BlockRejectRatio: # # print 'eBlockDepthHigh', optfit.params['b'].value, optfit.params['b'].value - optfit.params['a'].value, optfit.params['mu1'].value, optfit.params['mu2'].value # self.rejectEvent('eBlockDepthHigh') if math.isnan(self.mdRedChiSq): self.rejectEvent('eInvalidChiSq') if self.mdBlockDepth < 0 or self.mdBlockDepth > 1: self.rejectEvent('eInvalidBlockDepth') if self.mdRCConst1 <= 0 or self.mdRCConst2 <= 0: self.rejectEvent('eInvalidRCConstant') #print i0, i0sig, [optfit.params['a'].value, optfit.params['b'].value, optfit.params['mu1'].value, optfit.params['mu2'].value, optfit.params['tau'].value] else: # print optfit.message, optfit.lmdif_message self.rejectEvent('eFitConvergence') except KeyboardInterrupt: self.rejectEvent('eFitUserStop') raise except: # print optfit.message, optfit.lmdif_message self.rejectEvent('eFitFailure')
params1.add('b', value=10) params1.add('c', value=10) params2 = Parameters() params2.add('a', value=10) params2.add('b', value=10) params2.add('c', value=10) a, b, c = 2.5, 1.3, 0.8 x = np.linspace(0,4,50) y = f([a, b, c], x) data = y + 0.15*np.random.normal(size=len(x)) # fit without analytic derivative min1 = Minimizer(func, params1, fcn_args=(x,), fcn_kws={'data':data}) min1.leastsq() fit1 = func(params1, x) # fit with analytic derivative min2 = Minimizer(func, params2, fcn_args=(x,), fcn_kws={'data':data}) min2.leastsq(Dfun=dfunc, col_deriv=1) fit2 = func(params2, x) print '''Comparison of fit to exponential decay with and without analytic derivatives, to model = a*exp(-b*x) + c for a = %.2f, b = %.2f, c = %.2f ============================================== Statistic/Parameter| Without | With | ---------------------------------------------- N Function Calls | %3i | %3i |
'data': data }) myfit.prepare_fit() # for scale_covar in (True, False): myfit.scale_covar = scale_covar print ' ==== scale_covar = ', myfit.scale_covar, ' ===' for sigma in (0.1, 0.2, 0.23, 0.5): myfit.userkws['sigma'] = sigma p_fit['amp_g'].value = 10 p_fit['cen_g'].value = 9 p_fit['wid_g'].value = 1 p_fit['line_slope'].value = 0.0 p_fit['line_off'].value = 0.0 out = myfit.leastsq() print ' sigma = ', sigma print ' chisqr = ', out.chisqr print ' reduced_chisqr = ', out.redchi report_fit(out.params, modelpars=p_true, show_correl=False) print ' ==============================' # if HASPYLAB: # fit = residual(p_fit, x) # pylab.plot(x, fit, 'k-') # pylab.show() #
def test_derive(): def func(pars, x, data=None): a = pars['a'].value b = pars['b'].value c = pars['c'].value model = a * np.exp(-b * x) + c if data is None: return model return (model - data) def dfunc(pars, x, data=None): a = pars['a'].value b = pars['b'].value c = pars['c'].value v = np.exp(-b * x) return [v, -a * x * v, np.ones(len(x))] def f(var, x): return var[0] * np.exp(-var[1] * x) + var[2] params1 = Parameters() params1.add('a', value=10) params1.add('b', value=10) params1.add('c', value=10) params2 = Parameters() params2.add('a', value=10) params2.add('b', value=10) params2.add('c', value=10) a, b, c = 2.5, 1.3, 0.8 x = np.linspace(0, 4, 50) y = f([a, b, c], x) data = y + 0.15 * np.random.normal(size=len(x)) # fit without analytic derivative min1 = Minimizer(func, params1, fcn_args=(x, ), fcn_kws={'data': data}) min1.leastsq() fit1 = func(params1, x) # fit with analytic derivative min2 = Minimizer(func, params2, fcn_args=(x, ), fcn_kws={'data': data}) min2.leastsq(Dfun=dfunc, col_deriv=1) fit2 = func(params2, x) print('''Comparison of fit to exponential decay with and without analytic derivatives, to model = a*exp(-b*x) + c for a = %.2f, b = %.2f, c = %.2f ============================================== Statistic/Parameter| Without | With | ---------------------------------------------- N Function Calls | %3i | %3i | Chi-square | %.4f | %.4f | a | %.4f | %.4f | b | %.4f | %.4f | c | %.4f | %.4f | ---------------------------------------------- ''' % (a, b, c, min1.nfev, min2.nfev, min1.chisqr, min2.chisqr, params1['a'].value, params2['a'].value, params1['b'].value, params2['b'].value, params1['c'].value, params2['c'].value)) check_wo_stderr(params1['a'], params2['a'].value, 0.000001) check_wo_stderr(params1['b'], params2['b'].value, 0.000001) check_wo_stderr(params1['c'], params2['c'].value, 0.000001)
return model - data n = 601 random.seed(0) x = linspace(0, 20.0, n) data = (gaussian(x, 21, 6.1, 1.2) + lorentzian(x, 10, 9.6, 1.3) + random.normal(scale=0.1, size=n)) pfit = Parameters() pfit.add(name='amp_g', value=10) pfit.add(name='amp_l', value=10) pfit.add(name='cen_g', value=5) pfit.add(name='peak_split', value=2.5, min=0, max=5, vary=True) pfit.add(name='cen_l', expr='peak_split+cen_g') pfit.add(name='wid_g', value=1) pfit.add(name='wid_l', expr='wid_g') mini = Minimizer(residual, pfit, fcn_args=(x, data)) out = mini.leastsq() report_fit(out.params) best_fit = data + out.residual if HASPYLAB: plt.plot(x, data, 'bo') plt.plot(x, best_fit, 'r--') plt.show()
params1.add('b', value=10) params1.add('c', value=10) params2 = Parameters() params2.add('a', value=10) params2.add('b', value=10) params2.add('c', value=10) a, b, c = 2.5, 1.3, 0.8 x = np.linspace(0, 4, 50) y = f([a, b, c], x) data = y + 0.15 * np.random.normal(size=len(x)) # fit without analytic derivative min1 = Minimizer(func, params1, fcn_args=(x, ), fcn_kws={'data': data}) out1 = min1.leastsq() fit1 = func(out1.params, x) # fit with analytic derivative min2 = Minimizer(func, params2, fcn_args=(x, ), fcn_kws={'data': data}) out2 = min2.leastsq(Dfun=dfunc, col_deriv=1) fit2 = func(out2.params, x) print '''Comparison of fit to exponential decay with and without analytic derivatives, to model = a*exp(-b*x) + c for a = %.2f, b = %.2f, c = %.2f ============================================== Statistic/Parameter| Without | With | ---------------------------------------------- N Function Calls | %3i | %3i |
def extractfeatures_inside(self, DICOMImages, image_pos_pat, image_ori_pat, series_path, phases_series, VOI_mesh): """ Start pixVals for collection pixel values at VOI """ pixVals = [] deltaS = {} # necessary to read point coords VOIPnt = [0,0,0] ijk = [0,0,0] pco = [0,0,0] for i in range(len(DICOMImages)): abspath_PhaseID = series_path+os.sep+str(phases_series[i]) print phases_series[i] # Get total number of files load = Inputs_init() [len_listSeries_files, FileNms_slices_sorted_stack] = load.ReadDicomfiles(abspath_PhaseID) mostleft_slice = FileNms_slices_sorted_stack.slices[0] # Get dicom header, retrieve dicomInfo_series = dicom.read_file(abspath_PhaseID+os.sep+str(mostleft_slice)) # (0008,0031) AT S Series Time # hh.mm.ss.frac seriesTime = str(dicomInfo_series[0x0008,0x0031].value) # (0008,0033) AT S Image Time # hh.mm.ss.frac imageTime = str(dicomInfo_series[0x0008,0x0033].value) # (0008,0032) AT S Acquisition Time # hh.mm.ss.frac ti = str(dicomInfo_series[0x0008,0x0032].value) acquisitionTimepoint = datetime.time(hour=int(ti[0:2]), minute=int(ti[2:4]), second=int(ti[4:6])) self.timepoints.append( datetime.datetime.combine(datetime.date.today(), acquisitionTimepoint) ) # find mapping to Dicom space [transformed_image, transform_cube] = Display().dicomTransform(DICOMImages[i], image_pos_pat, image_ori_pat) ### Get inside of VOI [VOI_scalars, VOIdims] = self.createMaskfromMesh(VOI_mesh, transformed_image) print "\n VOIdims" print VOIdims # get non zero elements image_scalars = transformed_image.GetPointData().GetScalars() numpy_VOI_imagedata = vtk_to_numpy(image_scalars) numpy_VOI_imagedata = numpy_VOI_imagedata.reshape(VOIdims[2], VOIdims[1], VOIdims[0]) numpy_VOI_imagedata = numpy_VOI_imagedata.transpose(2,1,0) print "Shape of VOI_imagedata: " print numpy_VOI_imagedata.shape #################### HERE GET IT AND MASK IT OUT self.nonzeroVOIextracted = nonzero(VOI_scalars) print self.nonzeroVOIextracted VOI_imagedata = numpy_VOI_imagedata[self.nonzeroVOIextracted] print "shape of VOI_imagedata Clipped:" print VOI_imagedata.shape for j in range( len(VOI_imagedata) ): pixValx = VOI_imagedata[j] pixVals.append(pixValx) # Now collect pixVals print "Saving %s" % 'delta'+str(i) deltaS['delta'+str(i)] = pixVals pixVals = [] print self.timepoints # Collecting timepoints in proper format t_delta = [] t_delta.append(0) total_time = 0 for i in range(len(DICOMImages)-1): current_time = self.timepoints[i+1] previous_time = self.timepoints[i] difference_time =current_time - previous_time timestop = divmod(difference_time.total_seconds(), 60) t_delta.append( t_delta[i] + timestop[0]+timestop[1]*(1./60)) total_time = total_time+timestop[0]+timestop[1]*(1./60) # finally print t_delta print t_delta t = array(t_delta) print "total_time" print total_time ############################################################## # Finished sampling deltaS # APply lmfit to deltaS # first sample the mean data_deltaS = []; t_deltaS = []; mean_deltaS = []; sd_deltaS = []; se_deltaS = []; n_deltaS = [] # append So and to data_deltaS.append( 0 ) t_deltaS.append(0) mean_deltaS.append( mean(deltaS['delta0']) ) sd_deltaS.append(0) se_deltaS.append(0) n_deltaS.append( len(deltaS['delta0']) ) for k in range(1,len(DICOMImages)): deltaS_i = ( mean(array(deltaS['delta'+str(k)]).astype(float)) - mean(deltaS['delta0']) )/ mean(deltaS['delta0']) data_deltaS.append( deltaS_i ) t_deltaS.append(k) print 'delta'+str(k) print data_deltaS[k] ############################################################## # Calculate data_error # estimate the population mean and SD from our samples to find SE # SE tells us the distribution of individual scores around the sampled mean. mean_deltaS_i = mean(array(deltaS['delta'+str(k)])) std_deltaS_i = std(array(deltaS['delta'+str(k)])) n_deltaS_i = len(array(deltaS['delta'+str(k)])) sd_deltaS.append( std_deltaS_i ) mean_deltaS.append( mean_deltaS_i ) # Standard Error of the mean SE # the smaller the variability in the data, the more confident we are that one value (the mean) accurately reflects them. se_deltaS.append(std_deltaS_i/sqrt(n_deltaS_i)) n_deltaS.append(n_deltaS_i) # make array for data_deltaS data = array(data_deltaS) print "\n================\nMean and SE (i.e VOI sample data)" print mean_deltaS print se_deltaS # create a set of Parameters params = Parameters() params.add('amp', value= 10, min=0) params.add('alpha', value= 1, min=0) params.add('beta', value= 0.05, min=0.0001, max=0.9) # do fit, here with leastsq model # define objective function: returns the array to be minimized def fcn2min(params, t, data): global model, model_res, x """ model EMM for Bilateral DCE-MRI, subtract data""" # unpack parameters: # extract .value attribute for each parameter amp = params['amp'].value # Upper limit of deltaS alpha = params['alpha'].value # rate of signal increase min-1 beta = params['beta'].value # rate of signal decrease min-1 model = amp * (1- exp(-alpha*t)) * exp(-beta*t) x = linspace(0, t[4], 101) model_res = amp * (1- exp(-alpha*x)) * exp(-beta*x) return model - data ##### myfit = Minimizer(fcn2min, params, fcn_args=(t,), fcn_kws={'data':data}) myfit.prepare_fit() myfit.leastsq() # On a successful fit using the leastsq method, several goodness-of-fit statistics # and values related to the uncertainty in the fitted variables will be calculated print "myfit.success" print myfit.success print "myfit.residual" print myfit.residual print "myfit.chisqr" print myfit.chisqr print "myfit.redchi" print myfit.redchi # calculate final result final = data + myfit.residual # write error report report_errors(params) # Calculate R-square # R_square = sum( y_fitted - y_mean)/ sum(y_data - y_mean) R_square = sum( (model - mean(data))**2 )/ sum( (data - mean(data))**2 ) print "R^2" print R_square self.amp = params['amp'].value self.alpha = params['alpha'].value self.beta = params['beta'].value ################################################## # Now Calculate Extract parameters from model self.iAUC1 = params['amp'].value *( ((1-exp(-params['beta'].value*t[1]))/params['beta'].value) + (exp((-params['alpha'].value+params['beta'].value)*t[1])-1)/(params['alpha'].value+params['beta'].value) ) print "iAUC1" print self.iAUC1 self.Slope_ini = params['amp'].value*params['alpha'].value print "Slope_ini" print self.Slope_ini self.Tpeak = (1/params['alpha'].value)*log(1+(params['alpha'].value/params['beta'].value)) print "Tpeak" print self.Tpeak self.Kpeak = -params['amp'].value * params['alpha'].value * params['beta'].value print "Kpeak" print self.Kpeak self.SER = exp( (t[4]-t[1])*params['beta'].value) * ( (1-exp(-params['alpha'].value*t[1]))/(1-exp(-params['alpha'].value*t[4])) ) print "SER" print self.SER ################################################## # Now Calculate enhancement Kinetic based features # Based on the course of signal intensity within the lesion print "\n Saving %s" % 'Crk' So = array(deltaS['delta0']).astype(float) Crk = {'Cr0': mean(So)} C = {} Carray = [] for k in range(1,len(DICOMImages)): Sk = array(deltaS['delta'+str(k)]).astype(float) Cr = 0 for j in range( len(So) ): # extract average enhancement over the lesion at each time point Cr = Cr + (Sk[j] - So[j])/So[j] Carray.append((Sk[j] - So[j])/So[j]) # compile C['C'+str(k)] = Carray Crk['Cr'+str(k)] = Cr/len(Sk) # Extract Fii_1 for k in range(1,5): currentCr = array(Crk['Cr'+str(k)]).astype(float) print currentCr if( self.maxCr < currentCr): self.maxCr = float(currentCr) self.peakCr = int(k) print "Maximum Upate (Fii_1) = %d " % self.maxCr print "Peak Cr (Fii_2) = %d " % self.peakCr # Uptake rate self.UptakeRate = float(self.maxCr/self.peakCr) print "Uptake rate (Fii_3) " print self.UptakeRate # WashOut Rate if( self.peakCr == 4): self.washoutRate = 0 else: self.washoutRate = float( (self.maxCr - array(Crk['Cr'+str(4)]).astype(float))/(4-self.peakCr) ) print "WashOut rate (Fii_4) " print self.washoutRate ################################################## # Now Calculate enhancement-variance Kinetic based features # Based on Crk['Cr'+str(k)] = Cr/len(Sk) print "\n Saving %s" % 'Vrk' Vrk = {} for k in range(1,5): Ci = array(C['C'+str(k)]).astype(float) Cri = array(Crk['Cr'+str(k)]).astype(float) Vr = 0 for j in range( len(Ci) ): # extract average enhancement over the lesion at each time point Vr = Vr + (Ci[j] - Cri)**2 # compile Vrk['Vr'+str(k)] = Vr/(len(Ci)-1) # Extract Fiii_1 for k in range(1,5): currentVr = array(Vrk['Vr'+str(k)]).astype(float) if( self.maxVr < currentVr): print currentVr self.maxVr = float(currentVr) self.peakVr = int(k) print "Maximum Variation of enhan (Fiii_1) = %d " % self.maxVr print "Peak Vr (Fii_2) = %d " % self.peakVr # Vr_increasingRate self.Vr_increasingRate = self.maxVr/self.peakVr print "Vr_increasingRate (Fiii_3)" print self.Vr_increasingRate # Vr_decreasingRate if( self.peakVr == 4): Vr_decreasingRate = 0 else: Vr_decreasingRate = float((self.maxVr - array(Vrk['Vr'+str(4)]).astype(float))/(4-self.peakVr)) print "Vr_decreasingRate (Fiii_4) " print Vr_decreasingRate # Vr_post_1 self.Vr_post_1 = float( array(Vrk['Vr'+str(1)]).astype(float)) print "Vr_post_1 (Fiii_5)" print self.Vr_post_1 ################################################## # orgamize into dataframe self.dynamicEMM_inside = DataFrame( data=array([[ self.amp, self.alpha, self.beta, self.iAUC1, self.Slope_ini, self.Tpeak, self.Kpeak, self.SER, self.maxCr, self.peakCr, self.UptakeRate, self.washoutRate, self.maxVr, self.peakVr, self.Vr_increasingRate, self.Vr_post_1]]), columns=['A.inside', 'alpha.inside', 'beta.inside', 'iAUC1.inside', 'Slope_ini.inside', 'Tpeak.inside', 'Kpeak.inside', 'SER.inside', 'maxCr.inside', 'peakCr.inside', 'UptakeRate.inside', 'washoutRate.inside', 'maxVr.inside', 'peakVr.inside','Vr_increasingRate.inside', 'Vr_post_1.inside']) ############################################################# # try to plot results pylab.figure() pylab.errorbar(t, data, yerr=se_deltaS, fmt='ro', label='data+SE') # data 'ro' red dots as markers pylab.plot(t, final, 'b+', label='data+residuals') # data+residuals 'b+' blue pluses pylab.plot(t, model, 'b', label='model') # model fit 'b' blue pylab.plot(x, model_res, 'k', label='model fit') # model fit 'k' blakc pylab.xlabel(" post-contrast time (min)") pylab.ylabel("delta S(t)") pylab.legend() return self.dynamicEMM_inside
def pre_edge(energy, mu=None, group=None, e0=None, step=None, nnorm=None, nvict=0, pre1=None, pre2=None, norm1=None, norm2=None, make_flat=True, _larch=None): """pre edge subtraction, normalization for XAFS This performs a number of steps: 1. determine E0 (if not supplied) from max of deriv(mu) 2. fit a line of polymonial to the region below the edge 3. fit a polymonial to the region above the edge 4. extrapolate the two curves to E0 and take their difference to determine the edge jump Arguments ---------- energy: array of x-ray energies, in eV, or group (see note 1) mu: array of mu(E) group: output group e0: edge energy, in eV. If None, it will be determined here. step: edge jump. If None, it will be determined here. pre1: low E range (relative to E0) for pre-edge fit pre2: high E range (relative to E0) for pre-edge fit nvict: energy exponent to use for pre-edg fit. See Notes. norm1: low E range (relative to E0) for post-edge fit norm2: high E range (relative to E0) for post-edge fit nnorm: degree of polynomial (ie, nnorm+1 coefficients will be found) for post-edge normalization curve. See Notes. make_flat: boolean (Default True) to calculate flattened output. Returns ------- None: The following attributes will be written to the output group: e0 energy origin edge_step edge step norm normalized mu(E), using polynomial norm_area normalized mu(E), using integrated area flat flattened, normalized mu(E) pre_edge determined pre-edge curve post_edge determined post-edge, normalization curve dmude derivative of mu(E) (if the output group is None, _sys.xafsGroup will be written to) Notes ----- 1. Supports `First Argument Group` convention, requiring group members `energy` and `mu`. 2. Support `Set XAFS Group` convention within Larch or if `_larch` is set. 3. pre_edge: a line is fit to mu(energy)*energy**nvict over the region, energy=[e0+pre1, e0+pre2]. pre1 and pre2 default to None, which will set pre1 = e0 - 2nd energy point, rounded to 5 eV pre2 = roughly pre1/3.0, rounded to 5 eV 4. post-edge: a polynomial of order nnorm is fit to mu(energy)*energy**nvict between energy=[e0+norm1, e0+norm2]. nnorm, norm1, norm2 default to None, which will set: norm2 = max energy - e0, rounded to 5 eV norm1 = roughly min(150, norm2/3.0), rounded to 5 eV nnorm = 2 in norm2-norm1>350, 1 if norm2-norm1>50, or 0 if less. 5. flattening fits a quadratic curve (no matter nnorm) to the post-edge normalized mu(E) and subtracts that curve from it. """ energy, mu, group = parse_group_args(energy, members=('energy', 'mu'), defaults=(mu, ), group=group, fcn_name='pre_edge') if len(energy.shape) > 1: energy = energy.squeeze() if len(mu.shape) > 1: mu = mu.squeeze() pre_dat = preedge(energy, mu, e0=e0, step=step, nnorm=nnorm, nvict=nvict, pre1=pre1, pre2=pre2, norm1=norm1, norm2=norm2) group = set_xafsGroup(group, _larch=_larch) e0 = pre_dat['e0'] norm = pre_dat['norm'] norm1 = pre_dat['norm1'] norm2 = pre_dat['norm2'] # generate flattened spectra, by fitting a quadratic to .norm # and removing that. flat = norm ie0 = index_nearest(energy, e0) p1 = index_of(energy, norm1 + e0) p2 = index_nearest(energy, norm2 + e0) if p2 - p1 < 2: p2 = min(len(energy), p1 + 2) if make_flat and p2 - p1 > 4: enx, mux = remove_nans2(energy[p1:p2], norm[p1:p2]) # enx, mux = (energy[p1:p2], norm[p1:p2]) fpars = Parameters() ncoefs = len(pre_dat['norm_coefs']) fpars.add('c0', value=0, vary=True) fpars.add('c1', value=0, vary=(ncoefs > 1)) fpars.add('c2', value=0, vary=(ncoefs > 2)) fit = Minimizer(flat_resid, fpars, fcn_args=(enx, mux)) result = fit.leastsq(xtol=1.e-6, ftol=1.e-6) fc0 = result.params['c0'].value fc1 = result.params['c1'].value fc2 = result.params['c2'].value flat_diff = fc0 + energy * (fc1 + energy * fc2) flat = norm - (flat_diff - flat_diff[ie0]) flat[:ie0] = norm[:ie0] group.e0 = e0 group.norm = norm group.norm_poly = 1.0 * norm group.flat = flat group.dmude = np.gradient(mu) / np.gradient(energy) group.edge_step = pre_dat['edge_step'] group.edge_step_poly = pre_dat['edge_step'] group.pre_edge = pre_dat['pre_edge'] group.post_edge = pre_dat['post_edge'] group.pre_edge_details = Group() for attr in ('pre1', 'pre2', 'norm1', 'norm2', 'nnorm', 'nvict'): setattr(group.pre_edge_details, attr, pre_dat.get(attr, None)) group.pre_edge_details.pre_slope = pre_dat['precoefs'][0] group.pre_edge_details.pre_offset = pre_dat['precoefs'][1] for i in range(MAX_NNORM): if hasattr(group, 'norm_c%i' % i): delattr(group, 'norm_c%i' % i) for i, c in enumerate(pre_dat['norm_coefs']): setattr(group.pre_edge_details, 'norm_c%i' % i, c) # guess element and edge group.atsym = getattr(group, 'atsym', None) group.edge = getattr(group, 'edge', None) if group.atsym is None or group.edge is None: _atsym, _edge = guess_edge(group.e0) if group.atsym is None: group.atsym = _atsym if group.edge is None: group.edge = _edge return
def featureMap(self, DICOMImages, img_features, time_points, featuresKeys, caseLabeloutput, path_outputFolder): """Extracts feature maps per pixel based on request from vector of keywords featuresKeys """ ## Retrive image data VOIshape = img_features['VOI0'].shape print VOIshape self.init_features(img_features, featuresKeys) data_deltaS=[] self.allvar_F_r_i=[] # append So and to data_deltaS.append( 0 ) # Based on the course of signal intensity within the lesion So = array(img_features['VOI0']).astype(float) Crk = {'Cr0': mean(So)} C = {} Carray = [] # iterate point-by-point to extract feature map for i in range(VOIshape[0]): for j in range(VOIshape[1]): for k in range(VOIshape[2]): for timep in range(1, len(DICOMImages)): pix_deltaS = (img_features['VOI'+str(timep)][i,j,k].astype(float) - img_features['VOI0'][i,j,k].astype(float))/img_features['VOI0'][i,j,k].astype(float) if pix_deltaS<0: pix_deltaS=0 data_deltaS.append( pix_deltaS ) F_r_i = array(img_features['VOI'+str(timep)]).astype(float) n_F_r_i, min_max_F_r_i, mean_F_r_i, var_F_r_i, skew_F_r_i, kurt_F_r_i = stats.describe(F_r_i) self.allvar_F_r_i.append(var_F_r_i) data = array(data_deltaS) #print data # create a set of Parameters params = Parameters() params.add('amp', value= 10, min=0) params.add('alpha', value= 1, min=0) params.add('beta', value= 0.05, min=0.0001, max=0.9) # do fit, here with leastsq self.model myfit = Minimizer(self.fcn2min, params, fcn_args=(time_points,), fcn_kws={'data':data}) myfit.prepare_fit() myfit.leastsq() #################################### # Calculate R-square: R_square = sum( y_fitted - y_mean)/ sum(y_data - y_mean) R_square = sum( (self.model - mean(data))**2 )/ sum( (data - mean(data))**2 ) #print "R^2:" #print R_square self.R_square_map[i,j,k] = R_square if 'amp' in featuresKeys: amp = params['amp'].value print "amp:" print amp self.amp_map[i,j,k] = amp if 'beta' in featuresKeys: beta = params['beta'].value self.beta_map[i,j,k] = beta if 'alpha' in featuresKeys: alpha = params['alpha'].value print "alpha:" print alpha self.alpha_map[i,j,k] = alpha if 'iAUC1' in featuresKeys: iAUC1 = params['amp'].value *( ((1-exp(-params['beta'].value*t[1]))/params['beta'].value) + (exp((-params['alpha'].value+params['beta'].value)*t[1])-1)/(params['alpha'].value+params['beta'].value) ) print "iAUC1" print iAUC1 self.iAUC1_map[i,j,k] = iAUC1 if 'Slope_ini' in featuresKeys: Slope_ini = params['amp'].value*params['alpha'].value print "Slope_ini" print Slope_ini self.Slope_ini_map[i,j,k] = Slope_ini if 'Tpeak' in featuresKeys: Tpeak = (1/params['alpha'].value)*log(1+(params['alpha'].value/params['beta'].value)) self.Tpeak_map[i,j,k] = Tpeak if 'Kpeak' in featuresKeys: Kpeak = -params['amp'].value * params['alpha'].value * params['beta'].value self.Kpeak_map[i,j,k] = Kpeak if 'SER' in featuresKeys: SER = exp( (t[4]-t[1])*params['beta'].value) * ( (1-exp(-params['alpha'].value*t[1]))/(1-exp(-params['alpha'].value*t[4])) ) print "SER" print SER self.SER_map[i,j,k] = SER if 'maxCr' in featuresKeys: print "Maximum Upate (Fii_1) = %d " % self.maxCr self.maxC_map[i,j,k] = self.maxCr if 'peakCr' in featuresKeys: print "Peak Cr (Fii_2) = %d " % self.peakCr self.peakCr_map[i,j,k] = self.peakCr if 'UptakeRate' in featuresKeys: self.UptakeRate = float(self.maxCr/self.peakCr) print "Uptake rate (Fii_3) " print self.UptakeRate self.UptakeRate_map[i,j,k] = self.UptakeRate if 'washoutRate' in featuresKeys: if( self.peakCr == 4): self.washoutRate = 0 else: self.washoutRate = float( (self.maxCr - array(Crk['Cr'+str(4)]).astype(float))/(4-self.peakCr) ) print "WashOut rate (Fii_4) " print self.washoutRate self.washoutRate_map[i,j,k] = self.washoutRate if 'var_F_r_i' in featuresKeys: print("Variance F_r_i: {0:8.6f}".format( mean(self.allvar_F_r_i) )) self.allvar_F_r_i_map[i,j,k] = mean(self.allvar_F_r_i) data_deltaS=[] data_deltaS.append( 0 ) # convert feature maps to image if 'beta' in featuresKeys: beta_map_stencil = self.convertfeatureMap2vtkImage(self.beta_map, self.imageStencil, 1000) print path_outputFolder print caseLabeloutput # ## save mask as metafile image os.chdir(path_outputFolder) vtkmask_w = vtk.vtkMetaImageWriter() vtkmask_w.SetInput(beta_map_stencil ) vtkmask_w.SetFileName( 'beta_'+os.sep+caseLabeloutput+'.mhd' ) vtkmask_w.Write() vtkmask_w.Update() self.xImagePlaneWidget.SetWindowLevel(640,75) self.yImagePlaneWidget.SetWindowLevel(640,75) self.zImagePlaneWidget.SetWindowLevel(640,75) self.renderer1.Render() self.visualize_map(beta_map_stencil) if 'Tpeak' in featuresKeys: Tpeak_map_stencil = self.convertfeatureMap2vtkImage(self.Tpeak_map, self.imageStencil, 1) print path_outputFolder print caseLabeloutput # ## save mask as metafile image os.chdir(path_outputFolder) vtkmask_w = vtk.vtkMetaImageWriter() vtkmask_w.SetInput( Tpeak_map_stencil ) vtkmask_w.SetFileName( 'Tpeak_'+os.sep+caseLabeloutput+'.mhd' ) vtkmask_w.Write() vtkmask_w.Update() self.xImagePlaneWidget.SetWindowLevel(240,35) self.yImagePlaneWidget.SetWindowLevel(240,35) self.zImagePlaneWidget.SetWindowLevel(240,35) self.renderer1.Render() self.visualize_map(Tpeak_map_stencil) if 'Kpeak' in featuresKeys: Kpeak_map_stencil = self.convertfeatureMap2vtkImage(self.Kpeak_map, self.imageStencil, 1) print path_outputFolder print caseLabeloutput # ## save mask as metafile image os.chdir(path_outputFolder) vtkmask_w = vtk.vtkMetaImageWriter() vtkmask_w.SetInput( Kpeak_map_stencil ) vtkmask_w.SetFileName( 'Kpeak_'+os.sep+caseLabeloutput+'.mhd' ) vtkmask_w.Write() vtkmask_w.Update() self.xImagePlaneWidget.SetWindowLevel(118,15) self.yImagePlaneWidget.SetWindowLevel(118,15) self.zImagePlaneWidget.SetWindowLevel(118,15) self.renderer1.Render() self.visualize_map(Kpeak_map_stencil) return
'data': data }) myfit.prepare_fit() # for scale_covar in (True, False): myfit.scale_covar = scale_covar print ' ==== scale_covar = ', myfit.scale_covar, ' ===' for sigma in (0.1, 0.2, 0.23, 0.5): myfit.userkws['sigma'] = sigma p_fit['amp_g'].value = 10 p_fit['cen_g'].value = 9 p_fit['wid_g'].value = 1 p_fit['line_slope'].value = 0.0 p_fit['line_off'].value = 0.0 myfit.leastsq() print ' sigma = ', sigma print ' chisqr = ', myfit.chisqr print ' reduced_chisqr = ', myfit.redchi report_errors(p_fit, modelpars=p_true, show_correl=False) print ' ==============================' # if HASPYLAB: # fit = residual(p_fit, x) # pylab.plot(x, fit, 'k-') # pylab.show() #
def feffit(paramgroup, datasets, rmax_out=10, path_outputs=True, _larch=None, **kws): """execute a Feffit fit: a fit of feff paths to a list of datasets Parameters: ------------ paramgroup: group containing parameters for fit datasets: Feffit Dataset group or list of Feffit Dataset group. rmax_out: maximum R value to calculate output arrays. path_output: Flag to set whether all Path outputs should be written. Returns: --------- a fit results group. This will contain subgroups of: datasets: an array of FeffitDataSet groups used in the fit. params: This will be identical to the input parameter group. fit: an object which points to the low-level fit. Statistical parameters will be put into the params group. Each dataset will have a 'data' and 'model' subgroup, each with arrays: k wavenumber array of k chi chi(k). kwin window Omega(k) (length of input chi(k)). r uniform array of R, out to rmax_out. chir complex array of chi(R). chir_mag magnitude of chi(R). chir_pha phase of chi(R). chir_re real part of chi(R). chir_im imaginary part of chi(R). """ def _resid(params, datasets=None, paramgroup=None, **kwargs): """ this is the residual function""" params2group(params, paramgroup) return concatenate([d._residual(paramgroup) for d in datasets]) if isNamedClass(datasets, FeffitDataSet): datasets = [datasets] params = group2params(paramgroup) for ds in datasets: if not isNamedClass(ds, FeffitDataSet): print("feffit needs a list of FeffitDataSets") return ds.prepare_fit(params=params) fit = Minimizer(_resid, params, fcn_kws=dict(datasets=datasets, paramgroup=paramgroup), scale_covar=True, **kws) result = fit.leastsq() params2group(result.params, paramgroup) dat = concatenate( [d._residual(paramgroup, data_only=True) for d in datasets]) n_idp = 0 for ds in datasets: n_idp += ds.n_idp # here we rescale chi-square and reduced chi-square to n_idp npts = len(result.residual) chi_square = result.chisqr * n_idp * 1.0 / npts chi_reduced = chi_square / (n_idp * 1.0 - result.nvarys) rfactor = (result.residual**2).sum() / (dat**2).sum() # calculate 'aic', 'bic' rescaled to n_idp # note that neg2_loglikel is -2*log(likelihood) neg2_loglikel = n_idp * np.log(chi_square / n_idp) aic = neg2_loglikel + 2 * result.nvarys bic = neg2_loglikel + np.log(n_idp) * result.nvarys # With scale_covar = True, Minimizer() scales the uncertainties # by reduced chi-square assuming params.nfree is the correct value # for degrees-of-freedom. But n_idp-params.nvarys is a better measure, # so we rescale uncertainties here. covar = getattr(result, 'covar', None) # print("COVAR " , covar) if covar is not None: err_scale = (result.nfree / (n_idp - result.nvarys)) for name in result.var_names: p = result.params[name] if isParameter(p) and p.vary: p.stderr *= sqrt(err_scale) # next, propagate uncertainties to constraints and path parameters. result.covar *= err_scale vsave, vbest = {}, [] # 1. save current params for vname in result.var_names: par = result.params[vname] vsave[vname] = par vbest.append(par.value) # 2. get correlated uncertainties, set params accordingly uvars = correlated_values(vbest, result.covar) # 3. evaluate constrained params, save stderr for nam, obj in result.params.items(): eval_stderr(obj, uvars, result.var_names, result.params) # 3. evaluate path params, save stderr for ds in datasets: for p in ds.pathlist: p.store_feffdat() for pname in ('degen', 's02', 'e0', 'ei', 'deltar', 'sigma2', 'third', 'fourth'): obj = p.params[PATHPAR_FMT % (pname, p.label)] eval_stderr(obj, uvars, result.var_names, result.params) # restore saved parameters again for vname in result.var_names: # setattr(params, vname, vsave[vname]) params[vname] = vsave[vname] # clear any errors evaluting uncertainties if _larch is not None and (len(_larch.error) > 0): _larch.error = [] # reset the parameters group with the newly updated uncertainties params2group(result.params, paramgroup) # here we create outputs arrays for chi(k), chi(r): for ds in datasets: ds.save_ffts(rmax_out=rmax_out, path_outputs=path_outputs) out = Group(name='feffit results', datasets=datasets, fitter=fit, fit_details=result, chi_square=chi_square, n_independent=n_idp, chi_reduced=chi_reduced, rfactor=rfactor, aic=aic, bic=bic, covar=covar) for attr in ('params', 'nvarys', 'nfree', 'ndata', 'var_names', 'nfev', 'success', 'errorbars', 'message', 'lmdif_message'): setattr(out, attr, getattr(result, attr, None)) return out
params.add('bigw2', value=bigw2, min=0, max=2 * np.pi) params.add('inc2', value=inc2, min=0, max=2 * np.pi) params.add('e2', value=0, vary=False) params.add('a2', value=a2, min=0) params.add('P2', value=period, vary=False) params.add('T2', value=T2, min=0) #params.add('pscale', value=1) #do fit, minimizer uses LM for least square fitting of model to data minner = Minimizer(triple_model, params, fcn_args=(xpos_all, ypos_all, t_all, error_maj_all, error_min_all, error_pa_all), nan_policy='omit') result = minner.leastsq(xtol=1e-5, ftol=1e-5) params_inner.append([ period, result.params['a2'], result.params['e2'], result.params['w2'], result.params['bigw2'], result.params['inc2'], result.params['T2'] ]) params_outer.append([ result.params['P'], result.params['a'], result.params['e'], result.params['w'], result.params['bigw'], result.params['inc'], result.params['T'] ]) chi2.append(result.redchi) params_inner = np.array(params_inner) params_outer = np.array(params_outer) chi2 = np.array(chi2) plt.plot(params_inner[:, 0], 1 / chi2, 'o-')
def test_constraints(with_plot=True): with_plot = with_plot and WITHPLOT def residual(pars, x, sigma=None, data=None): yg = gaussian(x, pars['amp_g'], pars['cen_g'], pars['wid_g']) yl = lorentzian(x, pars['amp_l'], pars['cen_l'], pars['wid_l']) model = yg + yl + pars['line_off'] + x * pars['line_slope'] if data is None: return model if sigma is None: return (model - data) return (model - data) / sigma n = 201 xmin = 0. xmax = 20.0 x = linspace(xmin, xmax, n) data = (gaussian(x, 21, 8.1, 1.2) + lorentzian(x, 10, 9.6, 2.4) + random.normal(scale=0.23, size=n) + x*0.5) if with_plot: pylab.plot(x, data, 'r+') pfit = Parameters() pfit.add(name='amp_g', value=10) pfit.add(name='cen_g', value=9) pfit.add(name='wid_g', value=1) pfit.add(name='amp_tot', value=20) pfit.add(name='amp_l', expr='amp_tot - amp_g') pfit.add(name='cen_l', expr='1.5+cen_g') pfit.add(name='wid_l', expr='2*wid_g') pfit.add(name='line_slope', value=0.0) pfit.add(name='line_off', value=0.0) sigma = 0.021 # estimate of data error (for all data points) myfit = Minimizer(residual, pfit, fcn_args=(x,), fcn_kws={'sigma':sigma, 'data':data}, scale_covar=True) myfit.prepare_fit() init = residual(myfit.params, x) result = myfit.leastsq() print(' Nfev = ', result.nfev) print( result.chisqr, result.redchi, result.nfree) report_fit(result.params, min_correl=0.3) fit = residual(result.params, x) if with_plot: pylab.plot(x, fit, 'b-') assert(result.params['cen_l'].value == 1.5 + result.params['cen_g'].value) assert(result.params['amp_l'].value == result.params['amp_tot'].value - result.params['amp_g'].value) assert(result.params['wid_l'].value == 2 * result.params['wid_g'].value) # now, change fit slightly and re-run myfit.params['wid_l'].expr = '1.25*wid_g' result = myfit.leastsq() report_fit(result.params, min_correl=0.4) fit2 = residual(result.params, x) if with_plot: pylab.plot(x, fit2, 'k') pylab.show() assert(result.params['cen_l'].value == 1.5 + result.params['cen_g'].value) assert(result.params['amp_l'].value == result.params['amp_tot'].value - result.params['amp_g'].value) assert(result.params['wid_l'].value == 1.25 * result.params['wid_g'].value)
def __FitEvent(self): try: varyBlockedCurrent=True i0=np.abs(self.baseMean) i0sig=self.baseSD dt = 1000./self.Fs # time-step in ms. # edat=np.asarray( np.abs(self.eventData), dtype='float64' ) edat=self.dataPolarity*np.asarray( self.eventData, dtype='float64' ) blockedCurrent=min(edat) tauVal=dt estart = self.__eventStartIndex( self.__threadList( edat, range(0,len(edat)) ), i0, i0sig ) - 1 eend = self.__eventEndIndex( self.__threadList( edat, range(0,len(edat)) ), i0, i0sig ) - 2 # For long events, fix the blocked current to speed up the fit if (eend-estart) > 1000: blockedCurrent=np.mean(edat[estart+50:eend-50]) # control numpy error reporting np.seterr(invalid='ignore', over='ignore', under='ignore') ts = np.array([ t*dt for t in range(0,len(edat)) ], dtype='float64') params=Parameters() # print self.absDataStartIndex params.add('mu1', value=estart * dt) params.add('mu2', value=eend * dt) params.add('a', value=(i0-blockedCurrent), vary=varyBlockedCurrent) params.add('b', value = i0) params.add('tau1', value = tauVal) if self.LinkRCConst: params.add('tau2', value = tauVal, expr='tau1') else: params.add('tau2', value = tauVal) optfit=Minimizer(self.__objfunc, params, fcn_args=(ts,edat,)) optfit.prepare_fit() result=optfit.leastsq(xtol=self.FitTol,ftol=self.FitTol,maxfev=self.FitIters) # print optfit.params['b'].value, optfit.params['b'].value - optfit.params['a'].value, optfit.params['mu1'].value, optfit.params['mu2'].value if result.success: if result.params['mu1'].value < 0.0 or result.params['mu2'].value < 0.0: # print 'eInvalidFitParams1', optfit.params['b'].value, optfit.params['b'].value - optfit.params['a'].value, optfit.params['mu1'].value, optfit.params['mu2'].value self.rejectEvent('eInvalidResTime') # The start of the event is set past the length of the data elif result.params['mu1'].value > ts[-1]: # print 'eInvalidFitParams2', optfit.params['b'].value, optfit.params['b'].value - optfit.params['a'].value, optfit.params['mu1'].value, optfit.params['mu2'].value self.rejectEvent('eInvalidEventStart') else: self.mdOpenChCurrent = result.params['b'].value self.mdBlockedCurrent = result.params['b'].value - result.params['a'].value self.mdEventStart = result.params['mu1'].value self.mdEventEnd = result.params['mu2'].value self.mdRCConst1 = result.params['tau1'].value self.mdRCConst2 = result.params['tau2'].value self.mdAbsEventStart = self.mdEventStart + self.absDataStartIndex * dt self.mdBlockDepth = self.mdBlockedCurrent/self.mdOpenChCurrent self.mdResTime = self.mdEventEnd - self.mdEventStart self.mdRedChiSq = result.chisqr/( np.var(result.residual) * (len(self.eventData) - result.nvarys -1) ) # if (eend-estart) > 1000: # print blockedCurrent, self.mdBlockedCurrent, self.mdOpenChCurrent, self.mdResTime, self.mdRiseTime, self.mdRedChiSq, optfit.chisqr # if self.mdBlockDepth > self.BlockRejectRatio: # # print 'eBlockDepthHigh', optfit.params['b'].value, optfit.params['b'].value - optfit.params['a'].value, optfit.params['mu1'].value, optfit.params['mu2'].value # self.rejectEvent('eBlockDepthHigh') if math.isnan(self.mdRedChiSq): self.rejectEvent('eInvalidChiSq') if self.mdBlockDepth < 0 or self.mdBlockDepth > 1: self.rejectEvent('eInvalidBlockDepth') if self.mdRCConst1 <= 0 or self.mdRCConst2 <= 0: self.rejectEvent('eInvalidRCConstant') #print i0, i0sig, [optfit.params['a'].value, optfit.params['b'].value, optfit.params['mu1'].value, optfit.params['mu2'].value, optfit.params['tau'].value] else: # print optfit.message, optfit.lmdif_message self.rejectEvent('eFitConvergence') except KeyboardInterrupt: self.rejectEvent('eFitUserStop') raise except: # print optfit.message, optfit.lmdif_message self.rejectEvent('eFitFailure')
pfit.add(name='cen_g', value=9) pfit.add(name='wid_g', value=1) pfit.add(name='amp_tot', value=20) pfit.add(name='amp_l', expr='amp_tot - amp_g') pfit.add(name='cen_l', expr='1.5+cen_g') pfit.add(name='wid_l', expr='2*wid_g') pfit.add(name='line_slope', value=0.0) pfit.add(name='line_off', value=0.0) sigma = 0.021 # estimate of data error (for all data points) myfit = Minimizer(residual, pfit, fcn_args=(x,), fcn_kws={'sigma': sigma, 'data': data}, scale_covar=True) myfit.prepare_fit() init = residual(myfit.params, x) if HASPYLAB: pylab.plot(x, init, 'b--') result = myfit.leastsq() report_fit(result) fit = residual(result.params, x) if HASPYLAB: pylab.plot(x, fit, 'k-') pylab.show()
# print config JSON info OutputFormatter.printConfig(configJson) config = Config(configJson) # retrieve params from config equationModel = config.getEquationModel() params = config.getParams() data = config.getData() exp = config.getExp() OutputFormatter.printExperimentalData(data, exp) minimizer = Minimizer(equationModel.residual, params, fcn_args=(data, exp, verbose)) out = minimizer.leastsq() # show output #lmfit.printfuncs.report_fit(out.params) print(lmfit.fit_report(out)) # confidence # ci = lmfit.conf_interval(minimizer, out) # show output # lmfit.printfuncs.report_ci(ci) calc = equationModel.model(params, data, False) # print results
myfit.prepare_fit() # for scale_covar in (True, False): myfit.scale_covar = scale_covar print ' ==== scale_covar = ', myfit.scale_covar, ' ===' for sigma in (0.1, 0.2, 0.23, 0.5): myfit.userkws['sigma'] = sigma p_fit['amp_g'].value = 10 p_fit['cen_g'].value = 9 p_fit['wid_g'].value = 1 p_fit['line_slope'].value =0.0 p_fit['line_off'].value =0.0 out = myfit.leastsq() print ' sigma = ', sigma print ' chisqr = ', out.chisqr print ' reduced_chisqr = ', out.redchi report_fit(out.params, modelpars=p_true, show_correl=False) print ' ==============================' # if HASPYLAB: # fit = residual(p_fit, x) # pylab.plot(x, fit, 'k-') # pylab.show() #
def pre_edge(energy, mu=None, group=None, e0=None, step=None, nnorm=2, nvict=0, pre1=None, pre2=-50, norm1=100, norm2=None, make_flat=True, _larch=None): """pre edge subtraction, normalization for XAFS This performs a number of steps: 1. determine E0 (if not supplied) from max of deriv(mu) 2. fit a line of polymonial to the region below the edge 3. fit a polymonial to the region above the edge 4. extrapolae the two curves to E0 to determine the edge jump Arguments ---------- energy: array of x-ray energies, in eV, or group (see note) mu: array of mu(E) group: output group e0: edge energy, in eV. If None, it will be determined here. step: edge jump. If None, it will be determined here. pre1: low E range (relative to E0) for pre-edge fit pre2: high E range (relative to E0) for pre-edge fit nvict: energy exponent to use for pre-edg fit. See Note norm1: low E range (relative to E0) for post-edge fit norm2: high E range (relative to E0) for post-edge fit nnorm: degree of polynomial (ie, nnorm+1 coefficients will be found) for post-edge normalization curve. Default=2 (quadratic), max=5 make_flat: boolean (Default True) to calculate flattened output. Returns ------- None The following attributes will be written to the output group: e0 energy origin edge_step edge step norm normalized mu(E) flat flattened, normalized mu(E) pre_edge determined pre-edge curve post_edge determined post-edge, normalization curve dmude derivative of mu(E) (if the output group is None, _sys.xafsGroup will be written to) Notes ----- 1 nvict gives an exponent to the energy term for the fits to the pre-edge and the post-edge region. For the pre-edge, a line (m * energy + b) is fit to mu(energy)*energy**nvict over the pre-edge region, energy=[e0+pre1, e0+pre2]. For the post-edge, a polynomial of order nnorm will be fit to mu(energy)*energy**nvict of the post-edge region energy=[e0+norm1, e0+norm2]. 2 If the first argument is a Group, it must contain 'energy' and 'mu'. If it exists, group.e0 will be used as e0. See First Argrument Group in Documentation """ energy, mu, group = parse_group_args(energy, members=('energy', 'mu'), defaults=(mu,), group=group, fcn_name='pre_edge') if len(energy.shape) > 1: energy = energy.squeeze() if len(mu.shape) > 1: mu = mu.squeeze() pre_dat = preedge(energy, mu, e0=e0, step=step, nnorm=nnorm, nvict=nvict, pre1=pre1, pre2=pre2, norm1=norm1, norm2=norm2) group = set_xafsGroup(group, _larch=_larch) e0 = pre_dat['e0'] norm = pre_dat['norm'] norm1 = pre_dat['norm1'] norm2 = pre_dat['norm2'] # generate flattened spectra, by fitting a quadratic to .norm # and removing that. flat = norm ie0 = index_nearest(energy, e0) p1 = index_of(energy, norm1+e0) p2 = index_nearest(energy, norm2+e0) if p2-p1 < 2: p2 = min(len(energy), p1 + 2) if make_flat and p2-p1 > 4: enx, mux = remove_nans2(energy[p1:p2], norm[p1:p2]) # enx, mux = (energy[p1:p2], norm[p1:p2]) fpars = Parameters() ncoefs = len(pre_dat['norm_coefs']) fpars.add('c0', value=0, vary=True) fpars.add('c1', value=0, vary=(ncoefs>1)) fpars.add('c2', value=0, vary=(ncoefs>2)) fit = Minimizer(flat_resid, fpars, fcn_args=(enx, mux)) result = fit.leastsq(xtol=1.e-6, ftol=1.e-6) fc0 = result.params['c0'].value fc1 = result.params['c1'].value fc2 = result.params['c2'].value flat_diff = fc0 + energy * (fc1 + energy * fc2) flat = norm - (flat_diff - flat_diff[ie0]) flat[:ie0] = norm[:ie0] group.e0 = e0 group.norm = norm group.flat = flat group.dmude = np.gradient(mu)/np.gradient(energy) group.edge_step = pre_dat['edge_step'] group.pre_edge = pre_dat['pre_edge'] group.post_edge = pre_dat['post_edge'] group.pre_edge_details = Group() group.pre_edge_details.pre1 = pre_dat['pre1'] group.pre_edge_details.pre2 = pre_dat['pre2'] group.pre_edge_details.nnorm = pre_dat['nnorm'] group.pre_edge_details.norm1 = pre_dat['norm1'] group.pre_edge_details.norm2 = pre_dat['norm2'] group.pre_edge_details.nvict = pre_dat['nvict'] group.pre_edge_details.pre1_input = pre_dat['pre1_input'] group.pre_edge_details.norm2_input = pre_dat['norm2_input'] group.pre_edge_details.pre_slope = pre_dat['precoefs'][0] group.pre_edge_details.pre_offset = pre_dat['precoefs'][1] for i in range(MAX_NNORM): if hasattr(group, 'norm_c%i' % i): delattr(group, 'norm_c%i' % i) for i, c in enumerate(pre_dat['norm_coefs']): setattr(group.pre_edge_details, 'norm_c%i' % i, c) return
params1.add('b', value=10) params1.add('c', value=10) params2 = Parameters() params2.add('a', value=10) params2.add('b', value=10) params2.add('c', value=10) a, b, c = 2.5, 1.3, 0.8 x = np.linspace(0, 4, 50) y = f([a, b, c], x) data = y + 0.15*np.random.normal(size=len(x)) # fit without analytic derivative min1 = Minimizer(func, params1, fcn_args=(x,), fcn_kws={'data': data}) out1 = min1.leastsq() fit1 = func(out1.params, x) # fit with analytic derivative min2 = Minimizer(func, params2, fcn_args=(x,), fcn_kws={'data': data}) out2 = min2.leastsq(Dfun=dfunc, col_deriv=1) fit2 = func(out2.params, x) print('''Comparison of fit to exponential decay with and without analytic derivatives, to model = a*exp(-b*x) + c for a = %.2f, b = %.2f, c = %.2f ============================================== Statistic/Parameter| Without | With | ---------------------------------------------- N Function Calls | %3i | %3i |
class FitModel(object): """base class for fitting models only supports polynomial background (offset, slop, quad) """ invalid_bkg_msg = """Warning: unrecoginzed background option '%s' expected one of the following: %s """ def __init__(self, background=None, **kws): self.params = Parameters() self.has_initial_guess = False self.bkg = None self.initialize_background(background=background, **kws) def initialize_background(self, background=None, offset=0, slope=0, quad=0): """initialize background parameters""" if background is None: return if background not in VALID_BKGS: print( self.invalid_bkg_msg % (repr(background), ', '.join(VALID_BKGS))) kwargs = {'offset':offset} if background.startswith('line'): kwargs['slope'] = slope if background.startswith('quad'): kwargs['quad'] = quad self.bkg = PolyBackground(**kwargs) for nam, par in self.bkg.params.items(): self.params[nam] = par def calc_background(self, x): if self.bkg is None: return 0 return self.bkg.calculate(x) def __objective(self, params, y=None, x=None, dy=None, **kws): """fit objective function""" bkg = 0 if x is not None: bkg = self.calc_background(x) if y is None: y = 0.0 if dy is None: dy = 1.0 model = self.model(self.params, x=x, dy=dy, **kws) return (model + bkg - y)/dy def model(self, params, x=None, **kws): raise NotImplementedError def guess_starting_values(self, params, y, x=None, **kws): raise NotImplementedError def fit_report(self, params=None, **kws): if params is None: params = self.params return lmfit.fit_report(params, **kws) def fit(self, y, x=None, dy=None, **kws): fcn_kws={'y':y, 'x':x, 'dy':dy} fcn_kws.update(kws) if not self.has_initial_guess: self.guess_starting_values(y, x=x, **kws) self.minimizer = Minimizer(self.__objective, self.params, fcn_kws=fcn_kws, scale_covar=True) self.minimizer.prepare_fit() self.init = self.model(self.params, x=x, **kws) self.minimizer.leastsq()
temp_night_fit, delta_T_fit, T_star, spider_params) print('Initializing Parameters') initialParams = Parameters() initialParams.add_many( ('nrg_ratio', np.max([nrg_ratio_fit, 0.0]), True, 0.0, 1.0), ('temp_night', temp_night_fit, True, 0.0, np.inf), ('delta_T', delta_T_fit, True, 0.0, np.inf)) from functools import partial partial_residuals = partial(chisq_lmfit, args=args) mle0 = Minimizer(partial_residuals, initialParams, nan_policy='omit') fitResult = mle0.leastsq(initialParams) def logprior_func(p): for key, val in p.items(): # Uniform Prior if val.min >= val.value >= val.max: return -np.inf # Establish that the limb darkening parameters # cannot sum to 1 or greater # Kipping et al 201? and Espinoza et al 201? if 'u1' in p.keys() and 'u2' in p.keys(): if p['u1'] + p['u2'] >= 1: return -np.inf return 0
def test_constraints(): def residual(pars, x, sigma=None, data=None): yg = gauss(x, pars['amp_g'].value, pars['cen_g'].value, pars['wid_g'].value) yl = loren(x, pars['amp_l'].value, pars['cen_l'].value, pars['wid_l'].value) slope = pars['line_slope'].value offset = pars['line_off'].value model = yg + yl + offset + x * slope if data is None: return model if sigma is None: return (model - data) return (model - data) / sigma n = 601 xmin = 0. xmax = 20.0 x = linspace(xmin, xmax, n) data = (gauss(x, 21, 8.1, 1.2) + loren(x, 10, 9.6, 2.4) + random.normal(scale=0.23, size=n) + x * 0.5) pfit = [ Parameter(name='amp_g', value=10), Parameter(name='cen_g', value=9), Parameter(name='wid_g', value=1), Parameter(name='amp_tot', value=20), Parameter(name='amp_l', expr='amp_tot - amp_g'), Parameter(name='cen_l', expr='1.5+cen_g'), Parameter(name='wid_l', expr='2*wid_g'), Parameter(name='line_slope', value=0.0), Parameter(name='line_off', value=0.0) ] sigma = 0.021 # estimate of data error (for all data points) myfit = Minimizer(residual, pfit, fcn_args=(x, ), fcn_kws={ 'sigma': sigma, 'data': data }, scale_covar=True) myfit.prepare_fit() init = residual(myfit.params, x) myfit.leastsq() print(' Nfev = ', myfit.nfev) print(myfit.chisqr, myfit.redchi, myfit.nfree) report_fit(myfit.params) pfit = myfit.params fit = residual(myfit.params, x) assert (pfit['cen_l'].value == 1.5 + pfit['cen_g'].value) assert (pfit['amp_l'].value == pfit['amp_tot'].value - pfit['amp_g'].value) assert (pfit['wid_l'].value == 2 * pfit['wid_g'].value)
pars['wid_l'].value) return (model - data) n = 601 random.seed(0) x = linspace(0, 20.0, n) data = (gaussian(x, 21, 6.1, 1.2) + lorentzian(x, 10, 9.6, 1.3) + random.normal(scale=0.1, size=n)) pfit = Parameters() pfit.add(name='amp_g', value=10) pfit.add(name='amp_l', value=10) pfit.add(name='cen_g', value=5) pfit.add(name='peak_split', value=2.5, min=0, max=5, vary=True) pfit.add(name='cen_l', expr='peak_split+cen_g') pfit.add(name='wid_g', value=1) pfit.add(name='wid_l', expr='wid_g') mini = Minimizer(residual, pfit, fcn_args=(x, data)) out = mini.leastsq() report_fit(out.params) best_fit = data + out.residual plt.plot(x, data, 'bo') plt.plot(x, best_fit, 'r--') plt.show()
def test_constraints(with_plot=True): with_plot = with_plot and WITHPLOT def residual(pars, x, sigma=None, data=None): yg = gaussian(x, pars['amp_g'], pars['cen_g'], pars['wid_g']) yl = lorentzian(x, pars['amp_l'], pars['cen_l'], pars['wid_l']) model = yg + yl + pars['line_off'] + x * pars['line_slope'] if data is None: return model if sigma is None: return (model - data) return (model - data) / sigma n = 201 xmin = 0. xmax = 20.0 x = linspace(xmin, xmax, n) data = (gaussian(x, 21, 8.1, 1.2) + lorentzian(x, 10, 9.6, 2.4) + random.normal(scale=0.23, size=n) + x * 0.5) if with_plot: pylab.plot(x, data, 'r+') pfit = Parameters() pfit.add(name='amp_g', value=10) pfit.add(name='cen_g', value=9) pfit.add(name='wid_g', value=1) pfit.add(name='amp_tot', value=20) pfit.add(name='amp_l', expr='amp_tot - amp_g') pfit.add(name='cen_l', expr='1.5+cen_g') pfit.add(name='wid_l', expr='2*wid_g') pfit.add(name='line_slope', value=0.0) pfit.add(name='line_off', value=0.0) sigma = 0.021 # estimate of data error (for all data points) myfit = Minimizer(residual, pfit, fcn_args=(x, ), fcn_kws={ 'sigma': sigma, 'data': data }, scale_covar=True) myfit.prepare_fit() init = residual(myfit.params, x) result = myfit.leastsq() print(' Nfev = ', result.nfev) print(result.chisqr, result.redchi, result.nfree) report_fit(result.params, min_correl=0.3) fit = residual(result.params, x) if with_plot: pylab.plot(x, fit, 'b-') assert (result.params['cen_l'].value == 1.5 + result.params['cen_g'].value) assert (result.params['amp_l'].value == result.params['amp_tot'].value - result.params['amp_g'].value) assert (result.params['wid_l'].value == 2 * result.params['wid_g'].value) # now, change fit slightly and re-run myfit.params['wid_l'].expr = '1.25*wid_g' result = myfit.leastsq() report_fit(result.params, min_correl=0.4) fit2 = residual(result.params, x) if with_plot: pylab.plot(x, fit2, 'k') pylab.show() assert (result.params['cen_l'].value == 1.5 + result.params['cen_g'].value) assert (result.params['amp_l'].value == result.params['amp_tot'].value - result.params['amp_g'].value) assert (result.params['wid_l'].value == 1.25 * result.params['wid_g'].value)
def extractfeatures_inside(self, DICOMImages, image_pos_pat, image_ori_pat, series_path, phases_series, VOI_mesh): """ Start pixVals for collection pixel values at VOI """ pixVals = [] deltaS = {} # necessary to read point coords VOIPnt = [0, 0, 0] ijk = [0, 0, 0] pco = [0, 0, 0] for i in range(len(DICOMImages)): abspath_PhaseID = series_path + os.sep + str(phases_series[i]) print phases_series[i] # Get total number of files load = Inputs_init() [len_listSeries_files, FileNms_slices_sorted_stack] = load.ReadDicomfiles(abspath_PhaseID) mostleft_slice = FileNms_slices_sorted_stack.slices[0] # Get dicom header, retrieve dicomInfo_series = dicom.read_file(abspath_PhaseID + os.sep + str(mostleft_slice)) # (0008,0031) AT S Series Time # hh.mm.ss.frac seriesTime = str(dicomInfo_series[0x0008, 0x0031].value) # (0008,0033) AT S Image Time # hh.mm.ss.frac imageTime = str(dicomInfo_series[0x0008, 0x0033].value) # (0008,0032) AT S Acquisition Time # hh.mm.ss.frac ti = str(dicomInfo_series[0x0008, 0x0032].value) acquisitionTimepoint = datetime.time(hour=int(ti[0:2]), minute=int(ti[2:4]), second=int(ti[4:6])) self.timepoints.append(datetime.datetime.combine(datetime.date.today(), acquisitionTimepoint)) # find mapping to Dicom space [transformed_image, transform_cube] = Display().dicomTransform(DICOMImages[i], image_pos_pat, image_ori_pat) ### Get inside of VOI [VOI_scalars, VOIdims] = self.createMaskfromMesh(VOI_mesh, transformed_image) print "\n VOIdims" print VOIdims # get non zero elements image_scalars = transformed_image.GetPointData().GetScalars() numpy_VOI_imagedata = vtk_to_numpy(image_scalars) numpy_VOI_imagedata = numpy_VOI_imagedata.reshape(VOIdims[2], VOIdims[1], VOIdims[0]) numpy_VOI_imagedata = numpy_VOI_imagedata.transpose(2, 1, 0) print "Shape of VOI_imagedata: " print numpy_VOI_imagedata.shape #################### HERE GET IT AND MASK IT OUT self.nonzeroVOIextracted = nonzero(VOI_scalars) print self.nonzeroVOIextracted VOI_imagedata = numpy_VOI_imagedata[self.nonzeroVOIextracted] print "shape of VOI_imagedata Clipped:" print VOI_imagedata.shape for j in range(len(VOI_imagedata)): pixValx = VOI_imagedata[j] pixVals.append(pixValx) # Now collect pixVals print "Saving %s" % "delta" + str(i) deltaS["delta" + str(i)] = pixVals pixVals = [] print self.timepoints # Collecting timepoints in proper format t_delta = [] t_delta.append(0) total_time = 0 for i in range(len(DICOMImages) - 1): current_time = self.timepoints[i + 1] previous_time = self.timepoints[i] difference_time = current_time - previous_time timestop = divmod(difference_time.total_seconds(), 60) t_delta.append(t_delta[i] + timestop[0] + timestop[1] * (1.0 / 60)) total_time = total_time + timestop[0] + timestop[1] * (1.0 / 60) # finally print t_delta print t_delta t = array(t_delta) print "total_time" print total_time ############################################################## # Finished sampling deltaS # APply lmfit to deltaS # first sample the mean data_deltaS = [] t_deltaS = [] mean_deltaS = [] sd_deltaS = [] se_deltaS = [] n_deltaS = [] # append So and to data_deltaS.append(0) t_deltaS.append(0) mean_deltaS.append(mean(deltaS["delta0"])) sd_deltaS.append(0) se_deltaS.append(0) n_deltaS.append(len(deltaS["delta0"])) for k in range(1, len(DICOMImages)): deltaS_i = (mean(array(deltaS["delta" + str(k)]).astype(float)) - mean(deltaS["delta0"])) / mean( deltaS["delta0"] ) data_deltaS.append(deltaS_i) t_deltaS.append(k) print "delta" + str(k) print data_deltaS[k] ############################################################## # Calculate data_error # estimate the population mean and SD from our samples to find SE # SE tells us the distribution of individual scores around the sampled mean. mean_deltaS_i = mean(array(deltaS["delta" + str(k)])) std_deltaS_i = std(array(deltaS["delta" + str(k)])) n_deltaS_i = len(array(deltaS["delta" + str(k)])) sd_deltaS.append(std_deltaS_i) mean_deltaS.append(mean_deltaS_i) # Standard Error of the mean SE # the smaller the variability in the data, the more confident we are that one value (the mean) accurately reflects them. se_deltaS.append(std_deltaS_i / sqrt(n_deltaS_i)) n_deltaS.append(n_deltaS_i) # make array for data_deltaS data = array(data_deltaS) print "\n================\nMean and SE (i.e VOI sample data)" print mean_deltaS print se_deltaS # create a set of Parameters params = Parameters() params.add("amp", value=10, min=0) params.add("alpha", value=1, min=0) params.add("beta", value=0.05, min=0.0001, max=0.9) # do fit, here with leastsq model # define objective function: returns the array to be minimized def fcn2min(params, t, data): global model, model_res, x """ model EMM for Bilateral DCE-MRI, subtract data""" # unpack parameters: # extract .value attribute for each parameter amp = params["amp"].value # Upper limit of deltaS alpha = params["alpha"].value # rate of signal increase min-1 beta = params["beta"].value # rate of signal decrease min-1 model = amp * (1 - exp(-alpha * t)) * exp(-beta * t) x = linspace(0, t[4], 101) model_res = amp * (1 - exp(-alpha * x)) * exp(-beta * x) return model - data ##### myfit = Minimizer(fcn2min, params, fcn_args=(t,), fcn_kws={"data": data}) myfit.prepare_fit() myfit.leastsq() # On a successful fit using the leastsq method, several goodness-of-fit statistics # and values related to the uncertainty in the fitted variables will be calculated print "myfit.success" print myfit.success print "myfit.residual" print myfit.residual print "myfit.chisqr" print myfit.chisqr print "myfit.redchi" print myfit.redchi # calculate final result final = data + myfit.residual # write error report report_errors(params) # Calculate R-square # R_square = sum( y_fitted - y_mean)/ sum(y_data - y_mean) R_square = sum((model - mean(data)) ** 2) / sum((data - mean(data)) ** 2) print "R^2" print R_square self.amp = params["amp"].value self.alpha = params["alpha"].value self.beta = params["beta"].value ################################################## # Now Calculate Extract parameters from model self.iAUC1 = params["amp"].value * ( ((1 - exp(-params["beta"].value * t[1])) / params["beta"].value) + (exp((-params["alpha"].value + params["beta"].value) * t[1]) - 1) / (params["alpha"].value + params["beta"].value) ) print "iAUC1" print self.iAUC1 self.Slope_ini = params["amp"].value * params["alpha"].value print "Slope_ini" print self.Slope_ini self.Tpeak = (1 / params["alpha"].value) * log(1 + (params["alpha"].value / params["beta"].value)) print "Tpeak" print self.Tpeak self.Kpeak = -params["amp"].value * params["alpha"].value * params["beta"].value print "Kpeak" print self.Kpeak self.SER = exp((t[4] - t[1]) * params["beta"].value) * ( (1 - exp(-params["alpha"].value * t[1])) / (1 - exp(-params["alpha"].value * t[4])) ) print "SER" print self.SER ################################################## # Now Calculate enhancement Kinetic based features # Based on the course of signal intensity within the lesion print "\n Saving %s" % "Crk" So = array(deltaS["delta0"]).astype(float) Crk = {"Cr0": mean(So)} C = {} Carray = [] for k in range(1, len(DICOMImages)): Sk = array(deltaS["delta" + str(k)]).astype(float) Cr = 0 for j in range(len(So)): # extract average enhancement over the lesion at each time point Cr = Cr + (Sk[j] - So[j]) / So[j] Carray.append((Sk[j] - So[j]) / So[j]) # compile C["C" + str(k)] = Carray Crk["Cr" + str(k)] = Cr / len(Sk) # Extract Fii_1 for k in range(1, 5): currentCr = array(Crk["Cr" + str(k)]).astype(float) print currentCr if self.maxCr < currentCr: self.maxCr = float(currentCr) self.peakCr = int(k) print "Maximum Upate (Fii_1) = %d " % self.maxCr print "Peak Cr (Fii_2) = %d " % self.peakCr # Uptake rate self.UptakeRate = float(self.maxCr / self.peakCr) print "Uptake rate (Fii_3) " print self.UptakeRate # WashOut Rate if self.peakCr == 4: self.washoutRate = 0 else: self.washoutRate = float((self.maxCr - array(Crk["Cr" + str(4)]).astype(float)) / (4 - self.peakCr)) print "WashOut rate (Fii_4) " print self.washoutRate ################################################## # Now Calculate enhancement-variance Kinetic based features # Based on Crk['Cr'+str(k)] = Cr/len(Sk) print "\n Saving %s" % "Vrk" Vrk = {} for k in range(1, 5): Ci = array(C["C" + str(k)]).astype(float) Cri = array(Crk["Cr" + str(k)]).astype(float) Vr = 0 for j in range(len(Ci)): # extract average enhancement over the lesion at each time point Vr = Vr + (Ci[j] - Cri) ** 2 # compile Vrk["Vr" + str(k)] = Vr / (len(Ci) - 1) # Extract Fiii_1 for k in range(1, 5): currentVr = array(Vrk["Vr" + str(k)]).astype(float) if self.maxVr < currentVr: print currentVr self.maxVr = float(currentVr) self.peakVr = int(k) print "Maximum Variation of enhan (Fiii_1) = %d " % self.maxVr print "Peak Vr (Fii_2) = %d " % self.peakVr # Vr_increasingRate self.Vr_increasingRate = self.maxVr / self.peakVr print "Vr_increasingRate (Fiii_3)" print self.Vr_increasingRate # Vr_decreasingRate if self.peakVr == 4: self.Vr_decreasingRate = 0 else: self.Vr_decreasingRate = float((self.maxVr - array(Vrk["Vr" + str(4)]).astype(float)) / (4 - self.peakVr)) print "Vr_decreasingRate (Fiii_4) " print self.Vr_decreasingRate # Vr_post_1 self.Vr_post_1 = float(array(Vrk["Vr" + str(1)]).astype(float)) print "Vr_post_1 (Fiii_5)" print self.Vr_post_1 ################################################## # orgamize into dataframe self.dynamicEMM_inside = DataFrame( data=array( [ [ self.amp, self.alpha, self.beta, self.iAUC1, self.Slope_ini, self.Tpeak, self.Kpeak, self.SER, self.maxCr, self.peakCr, self.UptakeRate, self.washoutRate, self.maxVr, self.peakVr, self.Vr_increasingRate, self.Vr_decreasingRate, self.Vr_post_1, ] ] ), columns=[ "A.inside", "alpha.inside", "beta.inside", "iAUC1.inside", "Slope_ini.inside", "Tpeak.inside", "Kpeak.inside", "SER.inside", "maxCr.inside", "peakCr.inside", "UptakeRate.inside", "washoutRate.inside", "maxVr.inside", "peakVr.inside", "Vr_increasingRate.inside", "Vr_decreasingRate.inside", "Vr_post_1.inside", ], ) ############################################################# # try to plot results pylab.figure() pylab.errorbar(t, data, yerr=se_deltaS, fmt="ro", label="data+SE") # data 'ro' red dots as markers pylab.plot(t, final, "b+", label="data+residuals") # data+residuals 'b+' blue pluses pylab.plot(t, model, "b", label="model") # model fit 'b' blue pylab.plot(x, model_res, "k", label="model fit") # model fit 'k' blakc pylab.xlabel(" post-contrast time (min)") pylab.ylabel("delta S(t)") pylab.legend() return self.dynamicEMM_inside
def extractfeatures_contour(self, DICOMImages, image_pos_pat, image_ori_pat, series_path, phases_series, VOI_mesh): """ Start pixVals for collection pixel values at VOI """ pixVals = [] deltaS = {} # necessary to read point coords VOIPnt = [0,0,0] ijk = [0,0,0] pco = [0,0,0] for i in range(len(DICOMImages)): abspath_PhaseID = series_path+os.sep+str(phases_series[i]) print phases_series[i] # Get total number of files [len_listSeries_files, FileNms_slices_sorted_stack] = processDicoms.ReadDicomfiles(abspath_PhaseID) mostleft_slice = FileNms_slices_sorted_stack.slices[0] # Get dicom header, retrieve dicomInfo_series = dicom.read_file(abspath_PhaseID+os.sep+str(mostleft_slice)) # (0008,0031) AT S Series Time # hh.mm.ss.frac seriesTime = str(dicomInfo_series[0x0008,0x0031].value) # (0008,0033) AT S Image Time # hh.mm.ss.frac imageTime = str(dicomInfo_series[0x0008,0x0033].value) # (0008,0032) AT S Acquisition Time # hh.mm.ss.frac ti = str(dicomInfo_series[0x0008,0x0032].value) acquisitionTimepoint = datetime.time(hour=int(ti[0:2]), minute=int(ti[2:4]), second=int(ti[4:6])) self.timepoints.append( datetime.datetime.combine(datetime.date.today(), acquisitionTimepoint) ) # find mapping to Dicom space [transformed_image, transform_cube] = Display().dicomTransform(DICOMImages[i], image_pos_pat, image_ori_pat) for j in range( VOI_mesh.GetNumberOfPoints() ): VOI_mesh.GetPoint(j, VOIPnt) # extract pixID at location VOIPnt pixId = transformed_image.FindPoint(VOIPnt[0], VOIPnt[1], VOIPnt[2]) im_pt = [0,0,0] transformed_image.GetPoint(pixId,im_pt) inorout = transformed_image.ComputeStructuredCoordinates( im_pt, ijk, pco) if(inorout == 0): pass else: pixValx = transformed_image.GetScalarComponentAsFloat( ijk[0], ijk[1], ijk[2], 0) pixVals.append(pixValx) # Now collect pixVals print "Saving %s" % 'delta'+str(i) deltaS['delta'+str(i)] = pixVals pixVals = [] print self.timepoints # Collecting timepoints in proper format t_delta = [] t_delta.append(0) total_time = 0 for i in range(len(DICOMImages)-1): current_time = self.timepoints[i+1] previous_time = self.timepoints[i] difference_time =current_time - previous_time timestop = divmod(difference_time.total_seconds(), 60) t_delta.append( t_delta[i] + timestop[0]+timestop[1]*(1./60)) total_time = total_time+timestop[0]+timestop[1]*(1./60) # finally print t_delta print t_delta t = array(t_delta) print "total_time" print total_time ############################################################## # Finished sampling deltaS # APply lmfit to deltaS # first sample the mean data_deltaS = []; t_deltaS = []; mean_deltaS = []; sd_deltaS = []; se_deltaS = []; n_deltaS = [] # append So and to data_deltaS.append( 0 ) t_deltaS.append(0) mean_deltaS.append( mean(deltaS['delta0']) ) sd_deltaS.append(0) se_deltaS.append(0) n_deltaS.append( len(deltaS['delta0']) ) for k in range(1,len(DICOMImages)): deltaS_i = ( mean(array(deltaS['delta'+str(k)]).astype(float)) - mean(deltaS['delta0']) )/ mean(deltaS['delta0']) data_deltaS.append( deltaS_i ) t_deltaS.append(k) print 'delta'+str(k) print data_deltaS[k] ############################################################## # Calculate data_error # estimate the population mean and SD from our samples to find SE # SE tells us the distribution of individual scores around the sampled mean. mean_deltaS_i = mean(array(deltaS['delta'+str(k)])) std_deltaS_i = std(array(deltaS['delta'+str(k)])) n_deltaS_i = len(array(deltaS['delta'+str(k)])) sd_deltaS.append( std_deltaS_i ) mean_deltaS.append( mean_deltaS_i ) # Standard Error of the mean SE # the smaller the variability in the data, the more confident we are that one value (the mean) accurately reflects them. se_deltaS.append(std_deltaS_i/sqrt(n_deltaS_i)) n_deltaS.append(n_deltaS_i) # make array for data_deltaS data = array(data_deltaS) print "\n================\nMean and SE (i.e VOI sample data)" print mean_deltaS print se_deltaS # create a set of Parameters params = Parameters() params.add('amp', value= 10, min=0) params.add('alpha', value= 1, min=0) params.add('beta', value= 0.05, min=0.0001, max=0.9) # do fit, here with leastsq model # define objective function: returns the array to be minimized def fcn2min(params, t, data): global model, model_res, x """ model EMM for Bilateral DCE-MRI, subtract data""" # unpack parameters: # extract .value attribute for each parameter amp = params['amp'].value # Upper limit of deltaS alpha = params['alpha'].value # rate of signal increase min-1 beta = params['beta'].value # rate of signal decrease min-1 model = amp * (1- exp(-alpha*t)) * exp(-beta*t) x = linspace(0, t[4], 101) model_res = amp * (1- exp(-alpha*x)) * exp(-beta*x) return model - data ##### myfit = Minimizer(fcn2min, params, fcn_args=(t,), fcn_kws={'data':data}) myfit.prepare_fit() myfit.leastsq() # On a successful fit using the leastsq method, several goodness-of-fit statistics # and values related to the uncertainty in the fitted variables will be calculated print "myfit.success" print myfit.success print "myfit.residual" print myfit.residual print "myfit.chisqr" print myfit.chisqr print "myfit.redchi" print myfit.redchi # calculate final result #final = data + myfit.residual # write error report report_errors(params) # Calculate R-square # R_square = sum( y_fitted - y_mean)/ sum(y_data - y_mean) R_square = sum( (model - mean(data))**2 )/ sum( (data - mean(data))**2 ) print "R^2" print R_square self.amp = params['amp'].value self.alpha = params['alpha'].value self.beta = params['beta'].value ################################################## # Now Calculate Extract parameters from model self.iAUC1 = params['amp'].value *( ((1-exp(-params['beta'].value*t[1]))/params['beta'].value) + (exp((-params['alpha'].value+params['beta'].value)*t[1])-1)/(params['alpha'].value+params['beta'].value) ) print "iAUC1" print self.iAUC1 self.Slope_ini = params['amp'].value*params['alpha'].value print "Slope_ini" print self.Slope_ini self.Tpeak = (1/params['alpha'].value)*log(1+(params['alpha'].value/params['beta'].value)) print "Tpeak" print self.Tpeak self.Kpeak = -params['amp'].value * params['alpha'].value * params['beta'].value print "Kpeak" print self.Kpeak self.SER = exp( (t[4]-t[1])*params['beta'].value) * ( (1-exp(-params['alpha'].value*t[1]))/(1-exp(-params['alpha'].value*t[4])) ) print "SER" print self.SER ################################################## # Now Calculate enhancement Kinetic based features # Based on the course of signal intensity within the lesion print "\n Saving %s" % 'Crk' So = array(deltaS['delta0']).astype(float) Crk = {'Cr0': mean(So)} C = {} for k in range(1,len(DICOMImages)): Sk = array(deltaS['delta'+str(k)]).astype(float) print Sk Cr = 0 Carray = [] for j in range( len(So) ): # extract average enhancement over the lesion at each time point Cr = Cr + (Sk[j] - So[j])/So[j] Carray.append((Sk[j] - So[j])/So[j]) # compile C['C'+str(k)] = Carray Crk['Cr'+str(k)] = float(Cr/len(Sk)) # Extract Fii_1 for k in range(1,5): currentCr = array(Crk['Cr'+str(k)]).astype(float) print currentCr if( self.maxCr < currentCr): self.maxCr = float(currentCr) self.peakCr = int(k) print "Maximum Upate (Fii_1) = %d " % self.maxCr print "Peak Cr (Fii_2) = %d " % self.peakCr # Uptake rate self.UptakeRate = float(self.maxCr/self.peakCr) print "Uptake rate (Fii_3) " print self.UptakeRate # WashOut Rate if( self.peakCr == 4): self.washoutRate = 0 else: self.washoutRate = float( (self.maxCr - array(Crk['Cr'+str(4)]).astype(float))/(4-self.peakCr) ) print "WashOut rate (Fii_4) " print self.washoutRate ################################################## # Now Calculate enhancement-variance Kinetic based features # Based on Crk['Cr'+str(k)] = Cr/len(Sk) print "\n Saving %s" % 'Vrk' Vrk = {} for k in range(1,5): Ci = array(C['C'+str(k)]).astype(float) Cri = array(Crk['Cr'+str(k)]).astype(float) Vr = 0 for j in range( len(Ci) ): # extract average enhancement over the lesion at each time point Vr = Vr + (Ci[j] - Cri)**2 # compile Vrk['Vr'+str(k)] = Vr/(len(Ci)-1) # Extract Fiii_1 for k in range(1,5): currentVr = array(Vrk['Vr'+str(k)]).astype(float) if( self.maxVr < currentVr): print currentVr self.maxVr = float(currentVr) self.peakVr = int(k) print "Maximum Variation of enhan (Fiii_1) = %d " % self.maxVr print "Peak Vr (Fii_2) = %d " % self.peakVr # Vr_increasingRate self.Vr_increasingRate = self.maxVr/self.peakVr print "Vr_increasingRate (Fiii_3)" print self.Vr_increasingRate # Vr_decreasingRate if( self.peakVr == 4): self.Vr_decreasingRate = 0 else: self.Vr_decreasingRate = float((self.maxVr - array(Vrk['Vr'+str(4)]).astype(float))/(4-self.peakVr)) print "Vr_decreasingRate (Fiii_4) " print self.Vr_decreasingRate # Vr_post_1 self.Vr_post_1 = float( array(Vrk['Vr'+str(1)]).astype(float)) print "Vr_post_1 (Fiii_5)" print self.Vr_post_1 ################################################## # orgamize into dataframe self.dynamicEMM_contour = DataFrame( data=array([[ self.amp, self.alpha, self.beta, self.iAUC1, self.Slope_ini, self.Tpeak, self.Kpeak, self.SER, self.maxCr, self.peakCr, self.UptakeRate, self.washoutRate, self.maxVr, self.peakVr, self.Vr_increasingRate, self.Vr_decreasingRate, self.Vr_post_1]]), columns=['A.contour', 'alpha.contour', 'beta.contour', 'iAUC1.contour', 'Slope_ini.contour', 'Tpeak.contour', 'Kpeak.contour', 'SER.contour', 'maxCr.contour', 'peakCr.contour', 'UptakeRate.contour', 'washoutRate.contour', 'maxVr.contour', 'peakVr.contour','Vr_increasingRate.contour', 'Vr_decreasingRate.contour', 'Vr_post_1.contour']) ############################################################# # try to plot results pylab.figure() pylab.errorbar(t, data, yerr=se_deltaS, fmt='ro', label='data+SE') # data 'ro' red dots as markers pylab.plot(t, model, 'b', label='model') # model fit 'b' blue pylab.plot(x, model_res, 'k', label='model fit') # model fit 'k' blakc pylab.xlabel(" post-contrast time (min)") pylab.ylabel("delta S(t)") pylab.legend() return self.dynamicEMM_contour