def func_wrapper(inst, *args, **kwargs): try: res, res_err, chi_dof, aicc, pcov = fit_func( inst, *args, **kwargs) if any(res_err == 0.0): raise ValueError("Fit error is zero!") except Exception as e: lg.info("\nWarning: Fit failed! Exception was:", e, "\n") if lg.isLevel("DEBUG"): traceback.print_exc() res = np.full(inst._nparams, np.nan) res_err = np.full(inst._nparams, np.nan) pcov = np.full((inst._nparams, inst._nparams), np.nan) chi_dof = np.inf aicc = np.inf if inst._btstr_fit: inst._samples.append([[(np.nan, ) * inst._nparams, (np.nan, ) * inst._nparams, np.nan, np.nan] for sitt in range(inst._nfit_samples) ]) return res, res_err, chi_dof, aicc, pcov
def _fit_loop(self): lg.info("\nStart fit loop from tmin = %d to tmax = %d" % (self._fit_interval[0], self._fit_interval[1] - 1)) lg.info( "\nYou will probably see a lot of errors\n" "This is normal, as we try a lot of different fitting methods\n" "and not all of them will work\n") for i in range(*self._fit_interval): xmin, xmax = i, self.xmax(i) self.perform_fit(xmin, xmax)
def _get_start_params_osc(self, xmin, xmax): self._osc_fit(xmin, xmax) self._no_fit(xmin, xmax) self._add_sep_data() if any(np.isnan(self._sep[-1])): start_params = None lg.info("Failed to estimate start parameters for oscillating fit") else: start_params = self._sep[-1] return start_params
def _write_data(self): os.makedirs(self._folder, exist_ok=True) if self._sym: method_string = "sym" else: method_string = "asym" if self._jack_data or self._std_err_data or self._ratio_data: method_string += "_jk" if self._btstr_data: method_string += "_btstr" if self._sample_data: method_string += "_fr_sample" if not self._direct_data: lg.info("Write effective mass...") if self._nstates_osc > 0: write_eff_mass(self._folder + "/effmass_no_" + method_string + self._file_string + ".txt", self._meff_no, self._meff_no_err, self._xdata_no, osc=True) write_eff_mass(self._folder + "/effmass_osc_" + method_string + self._file_string + ".txt", self._meff_osc, self._meff_osc_err, self._xdata_osc, osc=True) else: write_eff_mass( self._folder + "/effmass_" + method_string + self._file_string + ".txt", self._meff, self._meff_err, self._xdata) write_fit_mass(self._folder + "/fitmass_" + self._out_name + ".txt", self._ranges, self._res, self._res_err, self._chi_dof, self._aicc, self._nstates, self._nstates_osc) write_pcov(self._folder + "/pcov_" + self._out_name + ".txt", self._ranges, self._pcov, self._nstates, self._nstates_osc) write_corr(self._folder + "/corr_" + self._out_name + ".txt", self._xdata, self._ydata, self._edata) if self._btstr_fit: write_sample( self._folder + "/fitmass_sample_" + self._out_name + ".txt", self._ranges, self._samples, self._nstates, self._nstates_osc)
def calc_mult_const(self): # mult_const=1e-11 try: mult_const = np.abs(self._ydata[int(self._Nt / 2)]) except IndexError: try: mult_const = abs(self._ydata[-1]) #If data are empty except IndexError: mult_const = 1 lg.info("Normalize correlator with", mult_const) self._mult_const = abs(mult_const) #self._mult_const = 1 #print("WARNING MULT CONSTANT SET TO ONE") return self._mult_const
def _est_params_one_state_osc(self, xmin, xmax): even = self._xdata % 2 == 0 odd = self._xdata % 2 == 1 try: self.gen_fit_data(xmin, xmax, correlated=False, ind=even) #Prevent parent class from generating new fit data, as we set them manually here self._no_new_data = True lg.info("Fit even part...") start_params = self._est_params_one_state(xmin, xmax, ind=even) res_e = self.simple_corr_fit(xmin, xmax, start_params=start_params, correlated=False, nstates=1, nstates_osc=0)[0] print_res("Results from even fit", res_e, level="INFO") self.gen_fit_data(xmin, xmax, correlated=False, ind=odd) lg.info("Fit odd part...") start_params = self._est_params_one_state(xmin, xmax, ind=odd) res_o = self.simple_corr_fit(xmin, xmax, start_params=start_params, correlated=False, nstates=1, nstates_osc=0)[0] print_res("Results from odd fit", res_o, level="INFO") self._no_new_data = False #In principle res_e should be larger than res_o, but there are cases where this is #not the case. We have make sure that the amplitude is positive at his point A_osc = abs(res_e[0] - res_o[0]) / 2 A_no = (res_o[0] + res_e[0]) / 2 m = (res_e[1] + res_o[1]) / 2 start_params = np.array([A_no, m, A_osc, m]) finally: self._no_new_data = False return start_params
def _fit_corr_fr_data(self, data, xmin, xmax, start_params): lg.info() lg.info("Fitrange [ %d, %d ]" % (xmin, xmax)) if self._scnd_btstr: ydata, edata = bs.bootstr(self._calc_mean, data, self._numb_samples, err_by_dist=self._ng_btstr, same_rand_for_obs=False) corr_fitter = CorrFitter(self._xdata, ydata, edata, Nt=self._Nt) else: if self._correlated: ydata, edata, cov = self._calc_cov_and_mean(data) corr_fitter = CorrFitter(self._xdata, ydata, cov, Nt=self._Nt) else: ydata, edata = mean_and_err(data, axis=1) corr_fitter = CorrFitter(self._xdata, ydata, edata, Nt=self._Nt) res, res_err, chi_dof, aicc, pcov = corr_fitter.corr_fit( xmin, xmax, start_params, correlated=self._correlated, priorval=self._priorval, priorsigma=self._priorsigma, nstates=self._nstates, nstates_osc=self._nstates_osc) if self._nstates_osc > 0: if res[1] < res_err[1] or res[2 * self._nstates + 1] < res_err[2 * self._nstates + 1]: raise ValueError("Fit error larger than value") else: if res[1] < res_err[1]: raise ValueError("Fit error larger than value") return res, res_err, chi_dof, aicc, pcov
def _est_params_one_state(self, xmin, xmax, ind=None): # estimation of starting parameters via linear fit does not work for periodic # boundaries. Therefore we have to fit on the half interval. start_params = [1, 1] est_range = [int(xmin), int(min(xmax, self._Nt / 2))] #For estimating parameters of oscillating correlators, we use this function to estimate #the parameters for the even/odd part. We have to make sure that we use even/odd part #by this routine. lg.info("Estimate parameters: Linear fit for one state...") if ind is None: ind = (self._xdata > est_range[0]) & (self._xdata < est_range[1]) else: ind = (self._xdata > est_range[0]) & (self._xdata < est_range[1]) & ind try: (tmp_start_params, tmp_start_params_cov) = curve_fit( linear, self._xdata[ind], np.log(np.abs(self._ydata[ind])), sigma=1 / self._ydata[ind] * self._edata[ind]) if tmp_start_params[0] > 50 or tmp_start_params[1] > 50: start_params = [1, 1] else: start_params[0] = 1 / self._mult_const * np.exp(tmp_start_params[0])\ * np.exp(-tmp_start_params[1] * self._Nt / 2) * 2 start_params[1] = tmp_start_params[1] except Exception as e: lg.info("Linear fit failed. Error was", e) if lg.isLevel("DEBUG"): traceback.print_exc() start_params = [1, 1] print_res("Final start parameters for uncorrelated fit", start_params, level="INFO") return start_params
def _get_start_params(self, res, res_err, ratio=np.inf, ignore_ext=False): if not ignore_ext and self._start_params is not None: lg.info("Use start parameters from user") return self._start_params if len(res) < 1 or len(res_err) < 1: return None last_res = res[-1] last_err = res_err[-1] for i in range(len(last_res)): if np.isnan(last_res[i]): return None if last_err[i] / np.abs(last_res[i]) > ratio: return None return last_res
def run(self): if self._res_filename is not None: self._out_name += "_fr-file" self.read_in_results() if not self._plot_start: if self._res_filename is None: self._fit_loop() else: self._res.append(self._start_params) fit_range = (self._fit_interval[0], self.xmax(self._fit_interval[0])) self._ranges.append(fit_range) self._res_err.append(np.zeros_like(self._start_params)) self._out_name += "_start_params" # self._corr_fitter.set_mult_const(1) print_scl("Chi^2/d.o.f.", self._corr_fitter.calc_chisquare_dof( self._res[-1], *fit_range), level="INFO") lg.info("Plot data...") self.plot_corr() if self._correlated: lg.info("Plot covariance matrix...") self._plot_cov_mat() if not (self._plot_start or self._res_filename is not None): lg.info("Write out data...") self._write_data()
def _no_fit(self, xmin, xmax): # We do not use constraint fits here as this fit is for parameter estimation lg.info() start_params_no = self._get_start_params(self._res_no, self._res_no_err, ratio=1, ignore_ext=True) try: lg.info("Fit non-osc. correlator...") res_no, res_no_err, chi_dof_no, aicc_no, pcov_no = self._no_corr_fitter.corr_fit( xmin, xmax, start_params_no, correlated=False, nstates=self._nstates, nstates_osc=0) self._res_no.append(res_no) self._res_no_err.append(res_no_err) self._chi_dof_no.append(chi_dof_no) self._aicc_no.append(aicc_no) except Exception as e: lg.info("\nWarning. Fit failed:", e, "\n") if lg.isLevel("DEBUG"): traceback.print_exc() self._res_no.append(np.full(2 * self._nstates, np.nan)) self._res_no_err.append(np.full(2 * self._nstates, np.nan)) self._chi_dof_no.append(np.nan) self._aicc_no.append(np.nan)
def bs_fit(self, xmin, xmax, start_params): lg.info("\nFirst fit to estimate start parameters") start_params = self.direct_fit(xmin, xmax, start_params)[0] lg.info("\nStart bootstrap...") if self._correlated: (sample, (res, tmp_err, chi_dof, aicc, pcov), (res_err, tmp_err_err, chi_err, aicc_err, pcov_err)) = bs.bootstr(self._fit_corr_fr_data, self._data, self._nfit_samples, args=(xmin, xmax, start_params), same_rand_for_obs=True, err_by_dist=self._ng_btstr, return_sample=True, seed=self._seed, nmax_exceptions=0.5 * self._nfit_samples) else: (sample, (res, tmp_err, chi_dof, aicc, pcov), (res_err, tmp_err_err, chi_err, aicc_err, pcov_err)) = bs.bootstr(self._fit_corr_fr_data, self._data, self._nfit_samples, args=(xmin, xmax, start_params), return_sample=True, err_by_dist=self._ng_btstr, seed=self._seed, nmax_exceptions=0.5 * self._nfit_samples) self._samples.append(sample) all_res = [i[0] for i in sample] # Compute covariance matrix from sample pcov = calc_cov(np.array(all_res).transpose()) lg.info("\n\n\n") print_res("Final result for %d + %d bootstrap fit" % (self._nstates, self._nstates_osc), res, res_err, chi_dof, level="INFO") return res, res_err, chi_dof, aicc, pcov
def jk_fit(self, xmin, xmax, start_params): lg.info("\nFirst fit to estimate start parameters...") start_params = self.direct_fit(xmin, xmax, start_params)[0] lg.info("\nStart jackknife...") ((res, tmp_err, chi_dof, aicc, pcov), (res_err, tmp_err_err, chi_err, aicc_err, pcov_err)) = jk.jackknife(self._fit_corr_fr_data, self._data, self._numb_blocks, args=(xmin, xmax, start_params)) lg.info("\n\n\n") print_res("Final result for %d + %d jackknife fit" % (self._nstates, self._nstates_osc), res, res_err, chi_dof, level="INFO") return res, res_err, chi_dof, aicc, pcov
def _compute_samples(self, nknots): # For start parameter estimation perform a direct fit knots, res, res_err, chi_dof = multi_spline_fit( linear, 2, 1 / self._Nts**2, self._xdata, self._av_ydata, self._av_edata, randomization_factor=0, order=self._order, tol=self._tol, always_return=True, constraints=self._constraints, base_point=self._base_point, nknots=nknots, knots=self._knots, algorithms=["curve_fit"]) lg.info("Reference knots of full fit = ", knots) self._direct_knots.append(knots) lg.info("Chi^2/d.o.f. of first full fit:", chi_dof) if self._method == 'gauss_btstr': sample, tmp_res, tmp_res_err = bs.bootstr_from_gauss( self._fit_once, self._av_ydata, self._av_edata, self._nsamples, err_by_dist=True, return_sample=True, args={ 'edata': self._av_edata, 'start_params': res, 'nknots': nknots }, nmax_exceptions=0.25 * self._nsamples) elif self._method == 'from_sample': sample = [] nsamples = len(self._ydata[0][0]) nsteps = min(self._nsamples, nsamples) if nsteps < nsamples: lg.warn("WARNING not using all samples that are available") if nsteps < 100: step = 1 else: step = nsteps / 100 for i in range(nsteps): current_ydata = [] current_edata = [] for j in range(len(self._ydata)): current_ydata.append(self._ydata[j][:, i]) if self._no_edata: current_edata.append(self._av_edata[j]) else: current_edata.append(self._edata[j][:, i]) tmp_res = self._fit_once(current_ydata, current_edata, res, nknots) sample.append(tmp_res) if i % step == 0: lg.progress("%d%%" % ((i + 1) / nsteps * 100)) elif self._method == 'btstr': sample, tmp_res, tmp_res_err = bs.bootstr(self._fit_func, self._ydata, self._nsamples, conf_axis=2, same_rand_for_obs=True, err_by_dist=True, return_sample=True, args={ 'start_params': res, 'nknots': nknots }, nmax_exceptions=0.25 * self._nsamples) elif self._method == 'direct': sample = None else: raise ValueError("No such method: " + str(self._method)) return res, res_err, sample
def _est_params_osc(self, xmin, xmax, nstates, nstates_osc): lg.info( "Estimate parameters: Fit range for %d + %d state fit: [" % (nstates, nstates_osc), xmin, ",", xmax, "]") est_range = self._get_est_range( xmin, xmax, self.est_weights_osc[max(nstates, nstates_osc)]) nparams_no = 2 * nstates nparams = 2 * (nstates + nstates_osc) start_params = np.ones(nparams) if nstates + nstates_osc == 2: return self._est_params_one_state_osc(xmin, xmax) else: if nstates <= nstates_osc: tmp_params = self._est_params_osc(est_range[0], est_range[1], nstates, nstates_osc - 1) print_res("Start parameters for %d + %d fit" % (nstates, nstates_osc - 1), tmp_params, level="INFO") tmp_params = self.simple_corr_fit(*est_range, correlated=False, start_params=tmp_params, nstates=nstates, nstates_osc=nstates_osc - 1)[0] start_params[:nparams - 2] = tmp_params m = 5 / 4. * start_params[-3] A = self._ydata[int(xmin)] - self._func( xmin, tmp_params, nstates, nstates_osc - 1) A /= -np.cos(np.pi * xmin) * np.cosh( m * (xmin - self._Nt / 2)) * self._mult_const start_params[-2] = A start_params[-1] = m else: tmp_params = self._est_params_osc(est_range[0], est_range[1], nstates - 1, nstates_osc) print_res("Start parameters for %d + %d fit" % (nstates - 1, nstates_osc), tmp_params, level="INFO") tmp_params = self.simple_corr_fit(*est_range, correlated=False, start_params=tmp_params, nstates=nstates - 1, nstates_osc=nstates_osc)[0] start_params[:nparams_no - 2] = tmp_params[:nparams_no - 2] start_params[nparams_no:] = tmp_params[nparams_no - 2:] m = 5 / 4. * start_params[nparams_no - 3] A = self._ydata[int(xmin)] - self._func( xmin, tmp_params, nstates - 1, nstates_osc) A /= np.cosh(m * (xmin - self._Nt / 2)) * self._mult_const start_params[nparams_no - 2] = A start_params[nparams_no - 1] = m return start_params
def _compute_meff(self): lg.info("Cmpute effective mass...") # It is always better to use a jackknife for meff instead of a computation # based on error propagation. Therefore we use a Jacknife also if the data flag is set # to std_err_data. if self._jack_data or self._std_err_data or self._ratio_data: if self._nstates_osc > 0: rm_osc, rm_osc_err = jk.jackknife(remove_osc, self._data, self._numb_blocks, args=(self._xdata, )) else: eff_mass, eff_mass_err = jk.jackknife(calc_eff_mass, self._data, self._numb_blocks, args=(self._xdata, self._Nt)) if self._btstr_data: if self._nstates_osc > 0: rm_osc, rm_osc_err = bs.bootstr(remove_osc, self._data, self._numb_samples, err_by_dist=self._ng_btstr, args=(self._xdata, ), seed=self._seed) else: eff_mass, eff_mass_err = bs.bootstr(calc_eff_mass, self._data, self._numb_samples, err_by_dist=self._ng_btstr, args=(self._xdata, self._Nt), seed=self._seed) if self._sample_data: if self._nstates_osc > 0: results = [] for i in range(len(self._data[0])): results.append(remove_osc_av(self._data[:, i], self._xdata)) rm_osc = std_mean(results) rm_osc_err = std_dev(results) else: eff_masses = [] for i in range(len(self._data[0])): eff_masses.append( calc_eff_mass_direct(self._data[:, i], self._xdata, self._Nt)) eff_mass, eff_mass_err = mean_and_std_dev(eff_masses) if not self._direct_data: if self._nstates_osc > 0: # Nan destroys the whole fit, as the chi^2 becomes nan. Therefore we have to remove # those from the helper arrays for parameter estimation. rm_osc = remove_nan(*(rm_osc + rm_osc_err)) self._xdata_osc = rm_osc[0] self._xdata_no = rm_osc[0] self._ydata_no = rm_osc[1] self._ydata_osc = rm_osc[2] self._edata_no = rm_osc[8] self._edata_osc = rm_osc[9] self._meff_no = rm_osc[3] self._meff_osc = rm_osc[4] self._meff_no_err = rm_osc[10] self._meff_osc_err = rm_osc[11] else: self._meff = eff_mass self._meff_err = eff_mass_err
def _est_params_non_osc(self, xmin, xmax, nstates): lg.info("Estimate parameters: Fit range for", numToWords(nstates), "state fit: [", xmin, ",", xmax, "]") nparams = 2 * nstates start_params = np.ones(nparams) est_range = self._get_est_range(xmin, xmax, self.est_weights[nstates]) if nstates == 1: return self._est_params_one_state(xmin, xmax) else: start_params[0:nparams - 2] = self._est_params_non_osc( est_range[0], est_range[1], nstates - 1) print_res("Starting parameters for " + numToWords(nstates - 1) + " state fit", start_params[0:nparams - 2], level="INFO") start_params[0:nparams - 2] = self.simple_corr_fit( nstates=nstates - 1, correlated=False, xmin=est_range[0], xmax=est_range[1], start_params=start_params[0:nparams - 2])[0] #plot_func(self._func, xmin = 0, xmax = xmax, # args = (start_params[0:nparams - 2], nstates - 1, 0)) print_res("Results from " + numToWords(nstates - 1) + " state fit ", start_params[0:nparams - 2], level="INFO") #This is necessary to ensure that fit_ansatz_array gives the correct data. self.gen_fit_data(xmin, min(int(self._Nt / 2), xmax), correlated=False) higher_state_data = self._fit_ydata - self.fit_ansatz_array( start_params[0:nparams - 2]) sign_changed = False if higher_state_data[0] < 0: lg.info("Higher state seems to have negative amplitude!") higher_state_data *= -1 sign_changed = True lg.info("Estimate parameters: Linear fit for higher state...") tmp_start_params, tmp_start_params_cov = curve_fit( linear, self._fit_xdata, np.log(np.abs(higher_state_data)), sigma=1 / higher_state_data * self._fit_edata) start_params[nparams - 2] = np.exp(tmp_start_params[0]) * \ np.exp(-tmp_start_params[1] * self._Nt / 2) * 2 / self._mult_const #We already multiplied with -1, therefore > instead of < if sign_changed: start_params[nparams - 2] *= -1 start_params[nparams - 1] = tmp_start_params[1] lg.info("Estimate parameters: Start", numToWords(nstates), " state fit with fixed parameters...") def red_corr(x, params, fixed_params, nstates): return self._corr(x, fixed_params, nstates - 1, 0)\ + self._corr(x, params, 1, 0) self._func = red_corr self._args = (start_params[0:nparams - 2], nstates) self._grad_args = (1, 0) self._hess_args = (1, 0) print_res("Starting parameters for first " + numToWords(nstates) + " state fit", start_params, level="INFO") start_params[nparams - 2:], tmp_res_err, chi_dof = self.try_fit( self._std_algs, xmin=xmin, xmax=xmax, start_params=start_params[nparams - 2:], correlated=False) #Reset function self._func = self._corr self._args = (nstates, 0) self._grad_args = (nstates, 0) self._hess_args = (nstates, 0) #plot_dots(self._xdata, self._ydata, self._edata) #plt.yscale('log') return start_params
def perform_fit(self, xmin, xmax): lg.info( "==========================================================================" ) lg.info("Fitrange [ %d, %d ]" % (xmin, xmax)) self._ranges.append([xmin, xmax]) start_params_array = self._get_start_params_array(xmin, xmax) all_res = [] all_chi_dof = [] for (start_params, text) in start_params_array: lg.info("\n--------------------------------------") lg.info("Run with start parameters from", text, "\n") res = self.fit(xmin, xmax, start_params) all_chi_dof.append(res[2]) all_res.append(res) # Find smallest chi^2 min_ind = np.argmin(all_chi_dof) lg.info("\n--------------------------------------") lg.info("Choose fit with start parameters from", start_params_array[min_ind][1], "\n") res, res_err, chi_dof, aicc, pcov = all_res[min_ind] print_res("Final fit_res for xmin = %d, xmax = %d" % (xmin, xmax), res, res_err, chi_dof, level="INFO") print_scl("AICc", aicc, level="INFO") self._res.append(res) self._res_err.append(res_err) self._pcov.append(pcov) self._chi_dof.append(chi_dof) self._aicc.append(aicc) lg.info() return res, res_err, chi_dof, aicc
def corr_fit(self, xmin=-np.inf, xmax=np.inf, start_params=None, nstates=None, nstates_osc=None, correlated=None, priorsigma=None, priorval=None): if nstates == 0: raise ValueError("Require at least one non-oscillating state") if nstates is None: nstates = self._nstates if nstates_osc is None: nstates_osc = self._nstates_osc if correlated is None: correlated = self._cov_avail #To estimate parameters we usually perform a non-correlated fit before the actual #correlated fit. This is not neccessary if we already get start parameters. #However, for an oscillating fit is is reasonable to perform a non correlated fit #in any case. This is because start parameter estimation is usually performed outside #for oscillating fits. if correlated and start_params is not None: if nstates_osc > 0: skip_uncorr = False else: skip_uncorr = True else: skip_uncorr = False try: start_params, priorval, priorsigma = self.init_start_params( xmin, xmax, start_params, priorval, priorsigma, nstates, nstates_osc) except Exception as e: lg.info("Failed to estimate start parameters. Try direct fit") lg.details("Error was", e) if lg.isLevel("DEBUG"): traceback.print_exc() start_params = None #Save the states of the last fit. This must be ensured, even if the parameter #estimation fails. finally: self._nstates = nstates self._nstates_osc = nstates_osc print_res("Start parameters for %d + %d fit" % (nstates, nstates_osc), start_params, level="INFO") if not skip_uncorr: res, res_err, chi_dof, aicc, pcov = self.simple_corr_fit( xmin, xmax, start_params, correlated=False, priorval=priorval, priorsigma=priorsigma, nstates=nstates, nstates_osc=nstates_osc) start_params = np.copy(res) self._change_order(res, res_err, nstates, nstates_osc) res, res_err = self.remove_mult_const(res, res_err) pcov = self.remove_mult_const_pcov(pcov) lg.info() print_res("Fit result for uncorrelated %d + %d fit" % (nstates, nstates_osc), res, res_err, chi_dof, level="INFO") print_scl("AICc", aicc, level='INFO') if correlated: if not self._cov_avail: raise NotAvailableError("Covariance matrix is not available") res, res_err, chi_dof, aicc, pcov = self.simple_corr_fit( xmin, xmax, start_params=start_params, correlated=True, priorval=priorval, priorsigma=priorsigma, nstates=nstates, nstates_osc=nstates_osc) self._change_order(res, res_err, nstates, nstates_osc) res, res_err = self.remove_mult_const(res, res_err) pcov = self.remove_mult_const_pcov(pcov) lg.info() print_res("Fit result for correlated %d + %d fit" % (nstates, nstates_osc), res, res_err, chi_dof, level="INFO") print_scl("AICc", aicc) return res, res_err, chi_dof, aicc, pcov