def twoPeakLorentzianFit(self): try: nRow, nCol = self.dockedOpt.fileInfo() self.gausFit.binFitData = zeros((nRow, 0)) self.gausFit.TwoPkGausFitData = zeros((nCol, 12)) # Creates the empty 2D List for j in range(nCol): yy1 = [] yy2 = [] yy = self.dockedOpt.TT[:, j] i = 0 for y in yy: if i < len(yy)/2: yy1.append(y) else: yy2.append(y) i += 1 xx = arange(0, len(yy)) xx1 = arange(0, len(yy)/2) xx2 = arange(len(yy)/2, len(yy)) x1 = xx[0] x2 = xx[-1] y1 = yy[0] y2 = yy[-1] m = (y2 - y1) / (x2 - x1) b = y2 - m * x2 mod1 = LorentzianModel(prefix='p1_') mod2 = LorentzianModel(prefix='p2_') pars1 = mod1.guess(yy1, x=xx1) pars2 = mod2.guess(yy2, x=xx2) mod = mod1 + mod2 + LinearModel() pars = pars1 + pars2 pars.add('intercept', value=b, vary=True) pars.add('slope', value=m, vary=True) out = mod.fit(yy, pars, x=xx, slope=m) self.gausFit.TwoPkGausFitData[j, :] = (out.best_values['p1_amplitude'], 0, out.best_values['p1_center'], 0, out.best_values['p1_sigma'], 0, out.best_values['p2_amplitude'], 0, out.best_values['p2_center'], 0, out.best_values['p2_sigma'], 0) # Saves fitted data of each fit fitData = out.best_fit binFit = np.reshape(fitData, (len(fitData), 1)) self.gausFit.binFitData = np.concatenate((self.gausFit.binFitData, binFit), axis=1) if self.gausFit.continueGraphingEachFit == True: self.gausFit.graphEachFitRawData(xx, yy, out.best_fit, 'L') return False except Exception as e: QMessageBox.warning(self.myMainWindow, "Error", "There was an error \n\n Exception: " + str(e)) return True
def mult_params_peaks_Lorentzian(x, y): #http://cars9.uchicago.edu/software/python/lmfit/builtin_models.html loren_mod1 = LorentzianModel(prefix='l1_') pars = loren_mod1.guess(y, x) loren_mod2 = LorentzianModel(prefix='l2_') pars.update(loren_mod2.make_params()) loren_mod3 = LorentzianModel(prefix='l3_') pars.update(loren_mod3.make_params()) mod = loren_mod1 + loren_mod2 + loren_mod3 init = mod.eval(pars, x=x) out = mod.fit(y, pars, x=x) print(out.fit_report(min_correl=0.5)) plot_components = False plt.plot(x, y, 'b') plt.plot(x, init, 'k--') plt.plot(x, out.best_fit, 'r-') if plot_components: comps = out.eval_components(x=x) plt.plot(x, comps['l1_'], 'b--') plt.plot(x, comps['l2_'], 'b--') plt.plot(x, comps['l3_'], 'b--')
def fit_s21mag(x_val, y_val): peak = LorentzianModel() offset = ConstantModel() model = peak pars = peak.guess(y_val, x=x_val, amplitude=-0.05) result = model.fit(y_val, pars, x=x_val) return result
def check_energy(self, f, s, deg): p = int(np.argwhere(s == np.max(s))) freq = f[p] f_mask = (freq - 5e2 < f) & (f < freq + 5e2) x = f[f_mask] y = s[f_mask] mod = LorentzianModel() pars = mod.guess(y, x=x) out = mod.fit(y, pars, x=x) power = si.simps(out.best_fit, x) l = const.c / self.F0 # Calculate corresponding energy with formula: $ E = 0.5 m_{\mathrm{e}} [f_{\mathrm{r}} \lambda_\Re / (2 \cos\theta)]^2 $ E_plasma = (0.5 * const.m_e * (freq * l / (2 * np.cos(deg * np.pi / 180)))**2 / const.eV) res = 0 if self.vol == 1: if bool(15.58 < E_plasma < 18.42): res = 1 elif bool(22.47 < E_plasma < 23.75): res = 2 else: if bool(20.29 < E_plasma < 22.05): res = 1 elif bool(22.45 < E_plasma < 23.87): res = 2 elif bool(25.38 < E_plasma < 27.14): res = 3 return power, res, freq
def __LorentzianFit(self): """ Fitting by Lorentzian Lambda function will written in __LorentzianFunc Lorentzian parameter will be written in __fit_param amplitude: 'amplitude', center frequency: 'center', sigma: 'sigma', fwhm: 'fwhm', height: 'height' """ x, y = self.__freq, self.__intensity / self.__reference_intensity mod = LorentzianModel() pars = mod.guess(1 - y, x=x) out = mod.fit(1 - y, pars, x=x) def loren(a, x, x0, sigma): return a/np.pi * sigma / \ ((x-x0)**2+sigma**2) # Lorentzian template function self.__LorentzianFunc = lambda x: 1 - \ loren(out.values['amplitude'], x, out.values['center'], out.values['sigma']) self.__fit_param = out.values self.__CalcTemp(out.values['center'] * 1000)
def make_lor(df,num,center,length): """ This method do a single-peak Lorentzian deconvolution for a given dataframe Parameters ---------- df : pandas dataframe df is a column-wise dataframe that records the data you want to deconvolve. num : int a positional keyword, indicating which index of center array the Lorentzian in this method is building its center on. center : array the array recording the centers of all peaks. length : float the maximum allowed variance in the optimized position of peaks from their initila values indicated in the center array. Returns ------- dict model: the single-peak Lorentzian corresponded. paras: the parameters optimized through this function. This is useful in updating the parameters object paras """ pref='l'+str(num)+'_' model=LorentzianModel(prefix=pref) paras.update(model.guess(df,x=norm.Energy,center=center[num])) name=pref+'center' paras[name].set(value=center[num],min=center[num]-length,max=center[num]+length) paras[pref+'amplitude'].set(min=0.0) paras[pref+'sigma'].set(min=0.01) return {'model':model,'paras':paras}
def calculateQ_1peak(filename, time, taper, lambda0, slope, modulation_coefficient, rg): q = readlvm(filename) q = q.ravel() q = q / taper - 1 valley = q.min() valley_index = np.argmin(q) q_valley = q[valley_index - rg:valley_index + rg] l_valley = time[valley_index - rg:valley_index + rg] q_valley = q_valley * -1 peaks, peak_info = find_peaks(q_valley, height=-valley) #results_half = peak_widths(q_valley, peaks, rel_height=0.5) mod = LorentzianModel() x = np.asarray(list(range(0, 2 * rg))) pars = mod.guess(q_valley, x=x) out = mod.fit(q_valley, pars, x=x) res = out.fit_report() info = res.split("\n") variables = parse_info(info, 1) h_res, w_res, c_res = variables['height'], variables['fwhm'], variables[ 'center'] print(h_res, w_res, c_res) l = lambda0 + l_valley[int(float(c_res))] * slope * modulation_coefficient d_lambda = (l_valley[1] - l_valley[0]) * float(w_res) * slope * modulation_coefficient Q = l / d_lambda return Q, float(h_res) * 100, l
def lorentzian_model_w_lims(self, peak_pos, sigma, min_max_range): x, y = self.background_correction() lmodel = LorentzianModel(prefix='l1_') # calling lorentzian model pars = lmodel.guess(y, x=x) # parameters - center, width, height pars['l1_center'].set(peak_pos, min=min_max_range[0], max=min_max_range[1]) pars['l1_sigma'].set(sigma) result = lmodel.fit(y, pars, x=x, nan_policy='propagate') return result
def params_Lorentzian(x, y): mod = LorentzianModel() params = mod.guess(y, x) print(params) out = mod.fit(y, params, x=x) print(out.fit_report(min_correl=0.3)) init = mod.eval(params, x=x) plt.figure(2) plt.plot(x, y, 'b') plt.plot(x, init, 'k--') plt.plot(x, out.best_fit, 'r-')
def fit(): global fitx global fity fitx =fitx fity =fity a.clear mod = LorentzianModel() pars = mod.guess(fity,x=fitx) out = mod.fit(fity,pars, x=fitx) a.plot(fitx, fity) dataPlot.draw()
def lorentzian(x, y): # Lorentzian fit to a curve x_shifted = x - x.min() # Shifting to 0 y_shifted = y - y.min() # Shifting to 0 mod = LorentzianModel() # Setting model type pars = mod.guess(y_shifted, x=x_shifted) # Estimating fit out = mod.fit(y_shifted, pars, x=x_shifted) # Fitting fit # print(out.fit_report(min_correl=0.25)) # Outputting best fit results print("Lorentzian FWHM = ", out.params['fwhm'].value) # Outputting only FWHM out.plot() # Plotting fit
def fitcurve(self, w, f, xrange, nistlines, **kwargs): w = np.array(w)[xrange] f = np.array(f)[xrange] x = np.array(w) y = np.array(-f) + np.max( np.array(f)) #invert the spectrum upside down mod = LorentzianModel() pars = mod.guess(y, x=x) out = mod.fit(y, pars, x=x) report = out.fit_report(min_correl=0.25) """extract lorentzian parameters from the report output""" center = float(report.split('center:')[1].split( '+/-', 1)[0]) #x (wavelenth) value of the maximum amp = float(report.split('amplitude:')[1].split( '+/-', 1)[0]) #y (flux) value of the maximum fwhm = float(report.split('fwhm:')[1].split( '+/-', 1)[0]) #full width at half maximum #get chi-squared value of the fit chi_sq = float( report.split('reduced chi-square')[0].split( 'chi-square = ')[1]) iterations = int( report.split('# data points')[0].split('# function evals = ')[1]) #Plot inverted data points, Lorentzian curve, and absorption lines from NIST fig = plt.figure(figsize=(6, 4)) plt.plot(x, y, 'bo', label='Inverted') plt.plot(x, out.best_fit, 'r-', color='g', label='Best Lorentz fit') line_error = [] for i in range(len(nistlines)): line_error.append(center - nistlines[i]) plt.axvline(x=nistlines[i], ymin=0, ymax=1, linewidth=.5, color='r') plt.axvline(x=center - fwhm, ymin=0, ymax=1, linewidth=.5, color='k') plt.axvline(x=center + fwhm, ymin=0, ymax=1, linewidth=.5, color='k') #Print summary for each fitted curve if kwargs.get('showCurv', True) == True: plt.show() print 'Center of function: ', center print 'Fit iterations: ', iterations print 'Chi-squared: ', chi_sq print 'Wavelength range: ', w[0], '-', w[-1] else: plt.close() #don't show plot return center, amp, fwhm, chi_sq, line_error
def onePeakLorentzianFit(self): try: nRow, nCol = self.dockedOpt.fileInfo() self.gausFit.binFitData = plab.zeros((nRow, 0)) self.gausFit.OnePkFitData = plab.zeros( (nCol, 6)) # Creates the empty 2D List for j in range(nCol): yy = self.dockedOpt.TT[:, j] xx = plab.arange(0, len(yy)) x1 = xx[0] x2 = xx[-1] y1 = yy[0] y2 = yy[-1] m = (y2 - y1) / (x2 - x1) b = y2 - m * x2 mod = LorentzianModel() pars = mod.guess(yy, x=xx, slope=m) mod = mod + LinearModel() pars.add('intercept', value=b, vary=True) pars.add('slope', value=m, vary=True) out = mod.fit(yy, pars, x=xx, slope=m) self.gausFit.OnePkFitData[j, :] = ( out.best_values['amplitude'], 0, out.best_values['center'], 0, out.best_values['sigma'], 0) # Saves fitted data of each fit fitData = out.best_fit binFit = np.reshape(fitData, (len(fitData), 1)) self.gausFit.binFitData = np.concatenate( (self.gausFit.binFitData, binFit), axis=1) if self.gausFit.continueGraphingEachFit == True: self.gausFit.graphEachFitRawData(xx, yy, out.best_fit, 'L') return False except: return True
def onclick(event): global X plt.clf() x = event.xdata y = event.ydata print('waint...') L1 = LorentzianModel(prefix='L1_') pars = L1.guess(psd, x=freq) pars.update(L1.make_params()) pars['L1_center'].set(x, min=x - 10, max=x + 10) #pars['L1_sigma'].set(y, min=0) pars['L1_amplitude'].set(y, min=0) out = L1.fit(psd, pars, x=freq) X = out.best_fit plt.title(str(ordem) + " Lorenzian fit") plt.ylabel('PSD[ppm$^2$/$\mu$Hz]') plt.xlabel('Frequency [$\mu$Hz]') plt.minorticks_on() plt.tick_params(direction='in', which='major', top=True, right=True, left=True, length=8, width=1, labelsize=15) plt.tick_params(direction='in', which='minor', top=True, right=True, length=5, width=1, labelsize=15) plt.tight_layout() plt.loglog(freq, psd, 'k', lw=0.5, alpha=0.5) plt.plot(freq, out.best_fit, 'k-', lw=0.5, alpha=0.5) plt.xlim(min(freq), max(freq)) plt.ylim(min(psd), max(psd)) plt.draw() print('Done')
def guess_peak_lorentzian(data, x=None, **kwargs): y = np.abs(np.squeeze(np.real(data))) x = np.squeeze(x) idx = np.argsort(x) x = x[idx] y = y[idx] # prepare fitting a lorentzian m_lin = LinearModel() m_lorentzian = LorentzianModel() p_lin = m_lin.guess(y, x=x) p_lorentzian = m_lorentzian.guess(y - m_lin.eval(x=x, params=p_lin), x=x) m = m_lin + m_lorentzian p = p_lin + p_lorentzian r = m.fit(y, x=x, params=p) return (r.best_values["center"], r.best_values["sigma"], r.best_values["amplitude"] / (np.pi * r.best_values["sigma"]))
def test_convolution(): r"""Convolution of function with delta dirac should return the function""" # Reference Lorentzian parameter values amplitude = 42.0 sigma = 0.042 center = 0.0003 c1 = LorentzianModel(prefix='c1_') p = c1.make_params(amplitude=amplitude, center=center, sigma=sigma) c2 = DeltaDiracModel(prefix='c2_') p.update(c2.make_params(amplitude=1.0, center=0.0)) e = 0.0004 * np.arange(-250, 1500) # energies in meV # convolve Lorentzian with delta Dirac y1 = Convolve(c1, c2).eval(params=p, x=e) # should be the lorentzian # reverse order, convolve delta Dirac with Lorentzian y2 = Convolve(c2, c1).eval(params=p, x=e) # should be the lorentzian # We will fit a Lorentzian model against datasets y1 and y2 m = LorentzianModel() all_params = 'amplitude sigma center'.split() for y in (y1, y2): params = m.guess(y, x=e) # Set initial model Lorentzian parameters far from optimal solution params['amplitude'].set(value=amplitude * 10) params['sigma'].set(value=sigma * 4) params['center'].set(value=center * 7) # fit Lorentzian model against dataset y r = m.fit(y, params, x=e) # Compare the reference Lorentzian parameters against # parameters of the fitted model assert_allclose([amplitude, sigma, center], [r.params[p].value for p in all_params], rtol=0.01, atol=0.00001)
# Number of samplepoints N = projection.shape[0] # sample spacing T = nm_per_px y = projection yf = scipy.fftpack.fft(y) xf = np.linspace(0.0, 1.0/(2.0*T), N/2) fig1, ax = plt.subplots(1) thegraphy = 2.0/N * np.abs(yf[:N//2]) fromj = 9 toj = 17 ax.plot(xf[:50], thegraphy[:50], '-o') y = thegraphy[fromj:toj] x = xf[fromj:toj] pars = mod.guess(y, x=x) out = mod.fit(y, pars, x=x) x2 = np.linspace(np.min(x), np.max(x), 300) y2 = mod.eval(out.params, x=x2) plt.plot(x2, y2, '--', alpha=1) print(out.fit_report(min_correl=0.25)) fromj = 18 toj = 25 y = thegraphy[fromj:toj] x = xf[fromj:toj] pars = mod.guess(y, x=x) out = mod.fit(y, pars, x=x) x2 = np.linspace(np.min(x), np.max(x), 300) params2 = out.params # params2['center'] = 0.15390835*1.3
df_10 = summary_df.filter( col("datetaken").between(f'{start_date}', f'{end_date}')) print(df_10.toPandas().columns.tolist()) p_dfm = df_10.toPandas() # converting spark DF to Pandas DF # Non-Linear Least-Squares Minimization and Curve Fitting # Define model to be Lorentzian and deploy it model = LorentzianModel() n = len(p_dfm.columns) for i in range(n): if p_dfm.columns[i] != 'datetaken': # yyyyMM is x axis in integer # it goes through the loop and plots individual average curves one by one and then prints a report for each y value vcolumn = p_dfm.columns[i] print(vcolumn) params = model.guess(p_dfm[vcolumn], x=p_dfm['datetaken']) result = model.fit(p_dfm[vcolumn], params, x=p_dfm['datetaken']) result.plot_fit() plt.margins(0.15) plt.subplots_adjust(bottom=0.25) plt.xticks(rotation=90) plt.xlabel("year/month", fontdict=v.font) plt.text(0.35, 0.45, "Best-fit based on Non-Linear Lorentzian Model", transform=plt.gca().transAxes, color="grey", fontsize=9) plt.xlim(left=200900) plt.xlim(right=202100) if vcolumn == "flatprice": property = "Flat"
Matrix_results_name = root + CONDITION + '/' + SUBJECT_USE_ANALYSIS + '_' + brain_region + '_' + CONDITION + '_' + distance + '_' + Method_analysis + '.xlsx' Matrix_results_name= ub_wind_path(Matrix_results_name, system='wind') xls = pd.ExcelFile(Matrix_results_name) sheets = xls.sheet_names ## for sh in sheets: Matrix_results = pd.read_excel(Matrix_results_name, sheet_name=sh) df = Matrix_results.iloc[:180, :] ### just the quadrant df=pd.DataFrame(df) for TR in df.columns: data=df[TR] X=data.index.values Y=data.values mod = LorentzianModel() #mod =GaussianModel() pars = mod.guess(Y, x=X) out = mod.fit(Y, pars, x=X) Y_lorenz = mod.eval(pars, x=X) #print(out.fit_report(min_correl=0.25)) #plt.plot(X, Y_lorenz, 'k--', label='Lorentzian') #dec_angle_lorenz = (90-np.where(Y_lorenz==max(Y_lorenz))[0][0])/2 #out.params['center'].value / 2 #error = abs( (90-np.where(Y_lorenz==max(Y_lorenz))[0][0])/2 ) #round(ref_angle - dec_angle_lorenz, 3) error = (90-np.where(Y_lorenz==max(Y_lorenz))[0][0])/2 #round(ref_angle - dec_angle_lorenz, 3) results.append( [error, TR, CONDITION, SUBJECT_USE_ANALYSIS, sh[-1], brain_region]) df = pd.DataFrame(np.array(results)) df.columns = ['error', 'TR', 'CONDITION', 'subject', 'session', 'ROI']
def main(): regionname = sys.argv[1] ## parameter passed short = regionname.replace(" ", "").lower() print(f"""Getting plots for {regionname}""") appName = "ukhouseprices" spark = s.spark_session(appName) sc = s.sparkcontext() # # Get data from BigQuery table summaryTableName = v.fullyQualifiedoutputTableId start_date = "201001" end_date = "202001" lst = (spark.sql( "SELECT FROM_unixtime(unix_timestamp(), 'dd/MM/yyyy HH:mm:ss.ss') ") ).collect() print("\nStarted at") uf.println(lst) # Model predictions spark.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true") # read data from the Bigquery table summary print("\nreading data from " + v.fullyQualifiedoutputTableId) summary_df = spark.read. \ format("bigquery"). \ option("credentialsFile",v.jsonKeyFile). \ option("project", v.projectId). \ option("parentProject", v.projectId). \ option("dataset", v.targetDataset). \ option("table", v.targetTable). \ load() df_10 = summary_df.filter(F.col("Date").between(f'{start_date}', f'{end_date}')). \ select(F.date_format('Date',"yyyyMM").cast("Integer").alias("date"), 'flatprice', 'terracedprice', 'semidetachedprice', 'detachedprice') df_10.printSchema() print(df_10.toPandas().columns.tolist()) p_dfm = df_10.toPandas() # converting spark DF to Pandas DF # Non-Linear Least-Squares Minimization and Curve Fitting # Define model to be Lorentzian and deploy it model = LorentzianModel() n = len(p_dfm.columns) for i in range(n): if p_dfm.columns[i] != "date": # yyyyMM is x axis in integer # it goes through the loop and plots individual average curves one by one and then prints a report for each y value vcolumn = p_dfm.columns[i] print(vcolumn) params = model.guess(p_dfm[vcolumn], x=p_dfm['date']) result = model.fit(p_dfm[vcolumn], params, x=p_dfm['date']) result.plot_fit() plt.margins(0.15) plt.subplots_adjust(bottom=0.25) plt.xticks(rotation=90) plt.xlabel("year/month", fontdict=v.font) plt.text(0.35, 0.45, "Best-fit based on Non-Linear Lorentzian Model", transform=plt.gca().transAxes, color="grey", fontsize=9) plt.xlim(left=200900) plt.xlim(right=202100) if vcolumn == "flatprice": property = "Flat" if vcolumn == "terracedprice": property = "Terraced" if vcolumn == "semidetachedprice": property = "semi-detached" if vcolumn == "detachedprice": property = "detached" plt.ylabel(f"""{property} house prices in millions/GBP""", fontdict=v.font) plt.title( f"""Monthly {property} prices fluctuations in {regionname}""", fontdict=v.font) print(result.fit_report()) plt.show() plt.close() lst = (spark.sql( "SELECT FROM_unixtime(unix_timestamp(), 'dd/MM/yyyy HH:mm:ss.ss') ") ).collect() print("\nFinished at") uf.println(lst)
def fittingLoretzian(x, y): #fits a loretzian curve mod = LorentzianModel() pars = mod.guess(y, x=x) out = mod.fit(y, pars, x=x) print(out.fit_report(min_correl=0.25)) return out.best_fit
def plot_data(data): signal_format = 'hist' # 'line' or 'hist' or None Total_SM_label = False # for Total SM black line in plot and legend plot_label = r'$Z \rightarrow ll$' signal_label = plot_label signal = None for s in ZBosonSamples.samples.keys(): if s not in stack_order and s != 'data': signal = s for x_variable, hist in ZBosonHistograms.hist_dict.items(): h_bin_width = hist['bin_width'] h_num_bins = hist['num_bins'] h_xrange_min = hist['xrange_min'] h_xlabel = hist['xlabel'] h_log_y = hist['log_y'] h_y_label_x_position = hist['y_label_x_position'] h_legend_loc = hist['legend_loc'] h_log_top_margin = hist[ 'log_top_margin'] # to decrease the separation between data and the top of the figure, remove a 0 h_linear_top_margin = hist[ 'linear_top_margin'] # to decrease the separation between data and the top of the figure, pick a number closer to 1 bins = [h_xrange_min + x * h_bin_width for x in range(h_num_bins + 1)] bin_centres = [ h_xrange_min + h_bin_width / 2 + x * h_bin_width for x in range(h_num_bins) ] if store_histograms: stored_histos = {} if load_histograms: # not doing line for now npzfile = np.load(f'histograms/{x_variable}_hist_{fraction}.npz') # load bins loaded_bins = npzfile['bins'] if not np.array_equal(bins, loaded_bins): print('Bins mismatch. That\'s a problem') raise Exception # load data data_x = npzfile['data'] data_x_errors = np.sqrt(data_x) # load weighted signal signal_x_reshaped = npzfile[signal] signal_color = ZBosonSamples.samples[signal]['color'] # load backgrounds mc_x_heights_list = [] # mc_weights = [] mc_colors = [] mc_labels = [] mc_x_tot = np.zeros(len(bin_centres)) for s in stack_order: if not s in npzfile: continue mc_labels.append(s) # mc_x.append(data[s][x_variable].values) mc_colors.append(ZBosonSamples.samples[s]['color']) # mc_weights.append(data[s].totalWeight.values) mc_x_heights = npzfile[s] mc_x_heights_list.append(mc_x_heights) mc_x_tot = np.add(mc_x_tot, mc_x_heights) mc_x_err = np.sqrt(mc_x_tot) else: # ======== This creates histograms for the raw data events ======== # # no weights necessary (it's data) data_x, _ = np.histogram(data['data'][x_variable].values, bins=bins) data_x_errors = np.sqrt(data_x) if store_histograms: stored_histos[ 'data'] = data_x # saving histograms for later loading # ======== This creates histograms for signal simulation (Z->ll) ======== # # need to consider the event weights here signal_x = None if signal_format == 'line': signal_x, _ = np.histogram( data[signal][x_variable].values, bins=bins, weights=data[signal].totalWeight.values) elif signal_format == 'hist': signal_x = data[signal][x_variable].values signal_weights = data[signal].totalWeight.values signal_color = ZBosonSamples.samples[signal]['color'] signal_x_reshaped, _ = np.histogram( data[signal][x_variable].values, bins=bins, weights=data[signal].totalWeight.values) if store_histograms: stored_histos[ signal] = signal_x_reshaped # saving histograms for later loading # ======== This creates histograms for all of the background simulation ======== # # weights are also necessary here, since we produce an arbitrary number of MC events mc_x_heights_list = [] mc_weights = [] mc_colors = [] mc_labels = [] mc_x_tot = np.zeros(len(bin_centres)) for s in stack_order: if not s in data: continue if data[s].empty: continue mc_labels.append(s) # mc_x.append(data[s][x_variable].values) mc_colors.append(ZBosonSamples.samples[s]['color']) mc_weights.append(data[s].totalWeight.values) mc_x_heights, _ = np.histogram( data[s][x_variable].values, bins=bins, weights=data[s].totalWeight.values) #mc_heights? mc_x_heights_list.append(mc_x_heights) mc_x_tot = np.add(mc_x_tot, mc_x_heights) if store_histograms: stored_histos[ s] = mc_x_heights #saving histograms for later loading mc_x_err = np.sqrt(mc_x_tot) data_x_without_bkg = data_x - mc_x_tot # data fit # get rid of zero errors (maybe messy) : TODO a better way to do this? for i, e in enumerate(data_x_errors): if e == 0: data_x_errors[i] = np.inf if 0 in data_x_errors: print('please don\'t divide by zero') raise Exception bin_centres_array = np.asarray(bin_centres) # ************* # Models # ************* doniach_mod = DoniachModel() pars_doniach = doniach_mod.guess(data_x_without_bkg, x=bin_centres_array, amplitude=2100000 * fraction, center=90.5, sigma=2.3, height=10000 * fraction / 0.01, gamma=0) doniach = doniach_mod.fit(data_x_without_bkg, pars_doniach, x=bin_centres_array, weights=1 / data_x_errors) params_dict_doniach = doniach.params.valuesdict() gaussian_mod = GaussianModel() pars_gaussian = gaussian_mod.guess(data_x_without_bkg, x=bin_centres_array, amplitude=6000000 * fraction, center=90.5, sigma=3) gaussian = gaussian_mod.fit(data_x_without_bkg, pars_gaussian, x=bin_centres_array, weights=1 / data_x_errors) params_dict_gaussian = gaussian.params.valuesdict() lorentzian_mod = LorentzianModel() pars = lorentzian_mod.guess(data_x_without_bkg, x=bin_centres_array, amplitude=6000000 * fraction, center=90.5, sigma=2.9, gamma=1) lorentzian = lorentzian_mod.fit(data_x_without_bkg, pars, x=bin_centres_array, weights=1 / data_x_errors) params_dict_lorentzian = lorentzian.params.valuesdict() voigt_mod = VoigtModel() pars = voigt_mod.guess(data_x_without_bkg, x=bin_centres_array, amplitude=6800000 * fraction, center=90.5, sigma=1.7) voigt = voigt_mod.fit(data_x_without_bkg, pars, x=bin_centres_array, weights=1 / data_x_errors) params_dict_voigt = voigt.params.valuesdict() voigt_mod_2 = VoigtModel() polynomial = PolynomialModel(2) pars = voigt_mod_2.guess(data_x_without_bkg, x=bin_centres_array, amplitude=6800000 * fraction, center=90.5, sigma=1.7) pars += polynomial.guess(data_x_without_bkg, x=bin_centres_array, c0=data_x_without_bkg.max(), c1=0, c2=0) voigt_poly_mod = voigt_mod_2 + polynomial voigt_poly = voigt_poly_mod.fit(data_x_without_bkg, pars, x=bin_centres_array, weights=1 / data_x_errors) params_dict_voigt_poly = voigt_poly.params.valuesdict() if store_histograms: # save all histograms in npz format. different file for each variable. bins are common os.makedirs('histograms', exist_ok=True) np.savez(f'histograms/{x_variable}_hist.npz', bins=bins, **stored_histos) # ======== Now we start doing the fit ======== # # ************* # Main plot # ************* plt.clf() plt.axes([0.1, 0.3, 0.85, 0.65]) # (left, bottom, width, height) main_axes = plt.gca() main_axes.errorbar(x=bin_centres, y=data_x, yerr=data_x_errors, fmt='ko', label='Data') # this effectively makes a stacked histogram bottoms = np.zeros_like(bin_centres) for mc_x_height, mc_color, mc_label in zip(mc_x_heights_list, mc_colors, mc_labels): main_axes.bar(bin_centres, mc_x_height, bottom=bottoms, color=mc_color, label=mc_label, width=h_bin_width * 1.01) bottoms = np.add(bottoms, mc_x_height) main_axes.plot(bin_centres, doniach.best_fit, '-r', label='Doniach') main_axes.plot(bin_centres, gaussian.best_fit, '-g', label='Gaussian') main_axes.plot(bin_centres, lorentzian.best_fit, '-y', label='Lorentzian') main_axes.plot(bin_centres, voigt.best_fit, '--', label='Voigt') main_axes.plot(bin_centres, voigt_poly.best_fit, '-v', label='Voigt and Polynomial') if Total_SM_label: totalSM_handle, = main_axes.step(bins, np.insert(mc_x_tot, 0, mc_x_tot[0]), color='black') if signal_format == 'line': main_axes.step(bins, np.insert(signal_x, 0, signal_x[0]), color=ZBosonSamples.samples[signal]['color'], linestyle='--', label=signal) elif signal_format == 'hist': main_axes.bar(bin_centres, signal_x_reshaped, bottom=bottoms, color=signal_color, label=signal, width=h_bin_width * 1.01) bottoms = np.add(bottoms, signal_x_reshaped) main_axes.bar(bin_centres, 2 * mc_x_err, bottom=bottoms - mc_x_err, alpha=0.5, color='none', hatch="////", width=h_bin_width * 1.01, label='Stat. Unc.') mc_x_tot = bottoms main_axes.set_xlim(left=h_xrange_min, right=bins[-1]) main_axes.xaxis.set_minor_locator( AutoMinorLocator()) # separation of x axis minor ticks main_axes.tick_params(which='both', direction='in', top=True, labeltop=False, labelbottom=False, right=True, labelright=False) if h_log_y: main_axes.set_yscale('log') smallest_contribution = mc_x_heights_list[ 0] # TODO: mc_heights or mc_x_heights smallest_contribution.sort() bottom = smallest_contribution[-2] if bottom == 0: bottom = 0.001 # log doesn't like zero top = np.amax(data_x) * h_log_top_margin main_axes.set_ylim(bottom=bottom, top=top) main_axes.yaxis.set_major_formatter(CustomTicker()) locmin = LogLocator(base=10.0, subs=(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9), numticks=12) main_axes.yaxis.set_minor_locator(locmin) else: main_axes.set_ylim( bottom=0, top=(np.amax(data_x) + math.sqrt(np.amax(data_x))) * h_linear_top_margin) main_axes.yaxis.set_minor_locator(AutoMinorLocator()) main_axes.yaxis.get_major_ticks()[0].set_visible(False) plt.text(0.015, 0.97, 'ATLAS Open Data', ha="left", va="top", family='sans-serif', transform=main_axes.transAxes, fontsize=13) plt.text(0.015, 0.9, 'for education', ha="left", va="top", family='sans-serif', transform=main_axes.transAxes, style='italic', fontsize=8) plt.text(0.015, 0.86, r'$\sqrt{s}=13\,\mathrm{TeV},\;\int L\,dt=$' + str(lumi_used) + '$\,\mathrm{fb}^{-1}$', ha="left", va="top", family='sans-serif', transform=main_axes.transAxes) plt.text(0.015, 0.78, plot_label, ha="left", va="top", family='sans-serif', transform=main_axes.transAxes) plt.text(0.015, 0.72, r'$m_Z = $' + str(round(params_dict_doniach['center'], 4)) + ' GeV', ha="left", va="top", family='sans-serif', transform=main_axes.transAxes, fontsize=10) # Create new legend handles but use the colors from the existing ones handles, labels = main_axes.get_legend_handles_labels() if signal_format == 'line': handles[labels.index(signal)] = Line2D( [], [], c=ZBosonSamples.samples[signal]['color'], linestyle='dashed') uncertainty_handle = mpatches.Patch(facecolor='none', hatch='////') if Total_SM_label: handles.append((totalSM_handle, uncertainty_handle)) labels.append('Total SM') else: handles.append(uncertainty_handle) labels.append('Stat. Unc.') # specify order within legend new_handles = [ handles[labels.index('Data')], handles[labels.index('Doniach')], handles[labels.index('Gaussian')], handles[labels.index('Lorentzian')], handles[labels.index('Voigt')], handles[labels.index('Voigt and Polynomial')] ] new_labels = [ 'Data', 'Doniach', 'Gaussian', 'Lorentzian', 'Voigt', 'Voigt and Polynomial' ] for s in reversed(stack_order): if s not in labels: continue new_handles.append(handles[labels.index(s)]) new_labels.append(s) if signal is not None: new_handles.append(handles[labels.index(signal)]) new_labels.append(signal_label) if Total_SM_label: new_handles.append(handles[labels.index('Total SM')]) new_labels.append('Total SM') else: new_handles.append(handles[labels.index('Stat. Unc.')]) new_labels.append('Stat. Unc.') main_axes.legend(handles=new_handles, labels=new_labels, frameon=False, loc=h_legend_loc, fontsize='x-small') # ************* # Data / MC plot # ************* plt.axes([0.1, 0.1, 0.85, 0.2]) # (left, bottom, width, height) ratio_axes = plt.gca() ratio_axes.yaxis.set_major_locator( MaxNLocator(nbins='auto', symmetric=True)) ratio_axes.errorbar( x=bin_centres, y=data_x / signal_x_reshaped, fmt='ko' ) # TODO: yerr=data_x_errors produce error bars that are too big ratio_axes.set_xlim(left=h_xrange_min, right=bins[-1]) ratio_axes.plot(bins, np.ones(len(bins)), color='k') ratio_axes.xaxis.set_minor_locator( AutoMinorLocator()) # separation of x axis minor ticks ratio_axes.xaxis.set_label_coords( 0.9, -0.2) # (x,y) of x axis label # 0.2 down from x axis ratio_axes.set_xlabel(h_xlabel, fontname='sans-serif', fontsize=11) ratio_axes.set_ylim(bottom=0, top=2) ratio_axes.set_yticks([0, 1]) ratio_axes.tick_params(which='both', direction='in', top=True, labeltop=False, right=True, labelright=False) ratio_axes.yaxis.set_minor_locator(AutoMinorLocator()) ratio_axes.set_ylabel(r'Data / Pred', fontname='sans-serif', x=1, fontsize=11) # Generic features for both plots main_axes.yaxis.set_label_coords(h_y_label_x_position, 1) ratio_axes.yaxis.set_label_coords(h_y_label_x_position, 0.5) plt.savefig("ZBoson_" + x_variable + ".pdf", bbox_inches='tight') # ========== Statistics ========== # ========== Doniach ========== chisqr_doniach = mychisqr(doniach.residual, doniach.best_fit) redchisqr_doniach = chisqr_doniach / doniach.nfree center_doniach = params_dict_doniach['center'] sigma_doniach = params_dict_doniach['sigma'] rel_unc_center_doniach = doniach.params[ 'center'].stderr / doniach.params['center'].value rel_unc_sigma_doniach = doniach.params[ 'sigma'].stderr / doniach.params['sigma'].value # ========== Gaussian ========== chisqr_gaussian = mychisqr(gaussian.residual, gaussian.best_fit) redchisqr_gaussian = chisqr_gaussian / gaussian.nfree center_gaussian = params_dict_gaussian['center'] sigma_gaussian = params_dict_gaussian['sigma'] rel_unc_center_gaussian = gaussian.params[ 'center'].stderr / gaussian.params['center'].value rel_unc_sigma_gaussian = gaussian.params[ 'sigma'].stderr / gaussian.params['sigma'].value # ========== Lorentzian ========== chisqr_lorentzian = mychisqr(lorentzian.residual, lorentzian.best_fit) redchisqr_lorentzian = chisqr_lorentzian / lorentzian.nfree center_lorentzian = params_dict_lorentzian['center'] sigma_lorentzian = params_dict_lorentzian['sigma'] rel_unc_center_lorentzian = lorentzian.params[ 'center'].stderr / lorentzian.params['center'].value rel_unc_sigma_lorentzian = lorentzian.params[ 'sigma'].stderr / lorentzian.params['sigma'].value # ========== Voigt ========== chisqr_voigt = mychisqr(voigt.residual, voigt.best_fit) redchisqr_voigt = chisqr_voigt / voigt.nfree center_voigt = params_dict_voigt['center'] sigma_voigt = params_dict_voigt['sigma'] rel_unc_center_voigt = voigt.params['center'].stderr / voigt.params[ 'center'].value rel_unc_sigma_voigt = voigt.params['sigma'].stderr / voigt.params[ 'sigma'].value # ========== Voigt and Polynomial ========== chisqr_voigt_poly = mychisqr(voigt_poly.residual, voigt_poly.best_fit) redchisqr_voigt_poly = chisqr_voigt_poly / voigt_poly.nfree center_voigt_poly = params_dict_voigt_poly['center'] sigma_voigt_poly = params_dict_voigt_poly['sigma'] rel_unc_center_voigt_poly = voigt_poly.params[ 'center'].stderr / voigt_poly.params['center'].value rel_unc_sigma_voigt_poly = voigt_poly.params[ 'sigma'].stderr / voigt_poly.params['sigma'].value df_dict = { 'fraction': [fraction], 'luminosity': [lumi_used], 'doniach chisqr': [chisqr_doniach], 'doniach redchisqr': [redchisqr_doniach], 'doniach center': [rel_unc_center_doniach], 'doniach sigma': [rel_unc_sigma_doniach], 'gaussian chisqr': [chisqr_gaussian], 'gaussian redchisqr': [redchisqr_gaussian], 'gaussian center': [rel_unc_center_gaussian], 'gaussian sigma': [rel_unc_sigma_gaussian], 'lorentzian chisqr': [chisqr_lorentzian], 'lorentzian redchisqr': [redchisqr_lorentzian], 'lorentzian center': [rel_unc_center_lorentzian], 'lorentzian sigma': [rel_unc_sigma_lorentzian], 'voigt chisqr': [chisqr_voigt], 'voigt redchisqr': [redchisqr_voigt], 'voigt center': [rel_unc_center_voigt], 'voigt sigma': [rel_unc_sigma_voigt], 'voigt poly chisqr': [chisqr_voigt_poly], 'voigt poly redchisqr': [redchisqr_voigt_poly], 'voigt poly center': [rel_unc_center_voigt_poly], 'voigt poly sigma': [rel_unc_sigma_voigt_poly] } temp = pd.DataFrame(df_dict) fit_results = pd.read_csv('fit_results.csv') fit_results_concat = pd.concat([fit_results, temp]) fit_results_concat.to_csv('fit_results.csv', index=False) print("=====================================================") print("Statistics for the Doniach Model: ") print("\n") print("chi^2 = " + str(chisqr_doniach)) print("chi^2/dof = " + str(redchisqr_doniach)) print("center = " + str(center_doniach)) print("sigma = " + str(sigma_doniach)) print("Relative Uncertainty of Center = " + str(rel_unc_center_doniach)) print("Relative Uncertainty of Sigma = " + str(rel_unc_sigma_doniach)) print("\n") print("=====================================================") print("Statistics for the Gaussian Model: ") print("\n") print("chi^2 = " + str(chisqr_gaussian)) print("chi^2/dof = " + str(redchisqr_gaussian)) print("center = " + str(center_gaussian)) print("sigma = " + str(sigma_gaussian)) print("Relative Uncertainty of Center = " + str(rel_unc_center_gaussian)) print("Relative Uncertainty of Sigma = " + str(rel_unc_sigma_gaussian)) print("\n") print("=====================================================") print("Statistics for the Lorentzian Model: ") print("\n") print("chi^2 = " + str(chisqr_lorentzian)) print("chi^2/dof = " + str(redchisqr_lorentzian)) print("center = " + str(center_lorentzian)) print("sigma = " + str(sigma_lorentzian)) print("Relative Uncertainty of Center = " + str(rel_unc_center_lorentzian)) print("Relative Uncertainty of Sigma = " + str(rel_unc_sigma_lorentzian)) print("\n") print("=====================================================") print("Statistics for the Voigt Model: ") print("\n") print("chi^2 = " + str(chisqr_voigt)) print("chi^2/dof = " + str(redchisqr_voigt)) print("center = " + str(center_voigt)) print("sigma = " + str(sigma_voigt)) print("Relative Uncertainty of Center = " + str(rel_unc_center_voigt)) print("Relative Uncertainty of Sigma = " + str(rel_unc_sigma_voigt)) print("\n") print("=====================================================") print("Statistics for the Voigt and Polynomial Model: ") print("\n") print("chi^2 = " + str(chisqr_voigt_poly)) print("chi^2/dof = " + str(redchisqr_voigt_poly)) print("center = " + str(center_voigt_poly)) print("sigma = " + str(sigma_voigt_poly)) print("Relative Uncertainty of Center = " + str(rel_unc_center_voigt_poly)) print("Relative Uncertainty of Sigma = " + str(rel_unc_sigma_voigt_poly)) # ========= Plotting Residuals ========= # ========= Doniach Residuals ========= plt.clf() plt.axes([0.1, 0.3, 0.85, 0.65]) # (left, bottom, width, height) main_axes = plt.gca() main_axes.set_title("Doniach Model Residuals") main_axes.errorbar(x=bin_centres, y=doniach.residual, fmt='ko') main_axes.set_xlim(left=h_xrange_min, right=bins[-1]) main_axes.xaxis.set_minor_locator( AutoMinorLocator()) # separation of x axis minor ticks main_axes.tick_params(which='both', direction='in', top=True, labeltop=False, right=True, labelright=False) main_axes.set_xlabel(r'$M_Z$ GeV') main_axes.xaxis.get_major_ticks()[0].set_visible(False) main_axes.set_ylim(bottom=1.05 * doniach.residual.min(), top=1.05 * doniach.residual.max()) main_axes.yaxis.set_minor_locator(AutoMinorLocator()) main_axes.yaxis.get_major_ticks()[0].set_visible(False) main_axes.set_ylabel("Residual") plt.savefig("plots/doniach_residuals.pdf", bbox_inches='tight') # ========= Gaussian Residuals ========= plt.clf() plt.axes([0.1, 0.3, 0.85, 0.65]) # (left, bottom, width, height) main_axes = plt.gca() main_axes.set_title("Gaussian Model Residuals") main_axes.errorbar(x=bin_centres, y=gaussian.residual, fmt='ko') main_axes.set_xlim(left=h_xrange_min, right=bins[-1]) main_axes.xaxis.set_minor_locator( AutoMinorLocator()) # separation of x axis minor ticks main_axes.tick_params(which='both', direction='in', top=True, labeltop=False, right=True, labelright=False) main_axes.set_xlabel(r'$M_Z$ GeV') main_axes.xaxis.get_major_ticks()[0].set_visible(False) main_axes.set_ylim(bottom=1.05 * gaussian.residual.min(), top=1.05 * gaussian.residual.max()) main_axes.yaxis.set_minor_locator(AutoMinorLocator()) main_axes.yaxis.get_major_ticks()[0].set_visible(False) main_axes.set_ylabel("Residual") plt.savefig("plots/gaussian_residuals.pdf", bbox_inches='tight') # ========= Lorentzian Residuals ========= plt.clf() plt.axes([0.1, 0.3, 0.85, 0.65]) # (left, bottom, width, height) main_axes = plt.gca() main_axes.set_title("Lorentzian Model Residuals") main_axes.errorbar(x=bin_centres, y=lorentzian.residual, fmt='ko') main_axes.set_xlim(left=h_xrange_min, right=bins[-1]) main_axes.xaxis.set_minor_locator( AutoMinorLocator()) # separation of x axis minor ticks main_axes.tick_params(which='both', direction='in', top=True, labeltop=False, right=True, labelright=False) main_axes.set_xlabel(r'$M_Z$ GeV') main_axes.xaxis.get_major_ticks()[0].set_visible(False) main_axes.set_ylim(bottom=1.05 * lorentzian.residual.min(), top=1.05 * lorentzian.residual.max()) main_axes.yaxis.set_minor_locator(AutoMinorLocator()) main_axes.yaxis.get_major_ticks()[0].set_visible(False) main_axes.set_ylabel("Residual") plt.savefig("plots/lorentzian_residuals.pdf", bbox_inches='tight') # ========= Voigt Residuals ========= plt.clf() plt.axes([0.1, 0.3, 0.85, 0.65]) # (left, bottom, width, height) main_axes = plt.gca() main_axes.set_title("Voigt Model Residuals") main_axes.errorbar(x=bin_centres, y=voigt.residual, fmt='ko') main_axes.set_xlim(left=h_xrange_min, right=bins[-1]) main_axes.xaxis.set_minor_locator( AutoMinorLocator()) # separation of x axis minor ticks main_axes.tick_params(which='both', direction='in', top=True, labeltop=False, right=True, labelright=False) main_axes.set_xlabel(r'$M_Z$ GeV') main_axes.xaxis.get_major_ticks()[0].set_visible(False) main_axes.set_ylim(bottom=1.05 * voigt.residual.min(), top=1.05 * voigt.residual.max()) main_axes.yaxis.set_minor_locator(AutoMinorLocator()) main_axes.yaxis.get_major_ticks()[0].set_visible(False) main_axes.set_ylabel("Residual") plt.savefig("plots/voigt_residuals.pdf", bbox_inches='tight') # ========= Voigt and Polynomial Residuals ========= plt.clf() plt.axes([0.1, 0.3, 0.85, 0.65]) # (left, bottom, width, height) main_axes = plt.gca() main_axes.set_title("Voigt and Polynomial Model Residuals") main_axes.errorbar(x=bin_centres, y=voigt_poly.residual, fmt='ko') main_axes.set_xlim(left=h_xrange_min, right=bins[-1]) main_axes.xaxis.set_minor_locator( AutoMinorLocator()) # separation of x axis minor ticks main_axes.tick_params(which='both', direction='in', top=True, labeltop=False, right=True, labelright=False) main_axes.set_xlabel(r'$M_Z$ GeV') main_axes.xaxis.get_major_ticks()[0].set_visible(False) main_axes.set_ylim(bottom=1.05 * voigt_poly.residual.min(), top=1.05 * voigt_poly.residual.max()) main_axes.yaxis.set_minor_locator(AutoMinorLocator()) main_axes.yaxis.get_major_ticks()[0].set_visible(False) main_axes.set_ylabel("Residual") plt.savefig("plots/voigt_poly_residuals.pdf", bbox_inches='tight') if load_histograms: return None, None return signal_x, mc_x_tot
inicio = ind = list(abs(x - 805)).index(np.nanmin(abs(x - 805))) ind = list(abs(x - 815)).index(np.nanmin(abs(x - 815))) cavity_y = y[ind:] cavity_x = x[ind:] y = y[inicio:ind] x = x[inicio:ind] #Semillas que se dan iterativamente pars = L0_mod.make_params(center=center, amplitude=amplitude, sigma=sigma) pars += L1_mod.make_params(center=center1, amplitude=amplitude1, sigma=sigma1) pars2 = L2_mod.guess(cavity_y, x=cavity_x) pars += c_mod.make_params(c=c) pars2 += c2_mod.make_params(c=c) #bound_parameters_sigma(pars,2,5) #Defino funcion mod = L0_mod + c_mod + L1_mod #+ L2_mod mod2 = L2_mod + c2_mod #Fiteo out = mod.fit(y, pars, x=x) out_cavity = mod2.fit(cavity_y, pars2, x=cavity_x) #Imprimo el archivo fit log fit_log.write("\n\n\n\n\n\n Fitted from: %s%s \n" % (direction, filename)) now = datetime.datetime.now()
if len(x) == len(y) == 3: continue else: g_model = GaussianModel() g_pars = g_model.guess(y, x=x) g_out = g_model.fit(y, g_pars, x=x) g_qc = g_out.redchi # print(g_out.fit_report(min_correl=0.25)) l_model = LorentzianModel() l_pars = l_model.guess(y, x=x) l_out = l_model.fit(y, l_pars, x=x) l_qc = l_out.redchi # print(l_out.fit_report(min_correl=0.25)) v_model = VoigtModel() v_pars = v_model.guess(y, x=x) v_out = v_model.fit(y, v_pars, x=x) v_qc = v_out.redchi # print(v_out.fit_report(min_correl=0.25)) # other models to try
import matplotlib.pyplot as plt import pandas as pd from lmfit.models import LorentzianModel dframe = pd.read_csv('peak.csv') model = LorentzianModel() params = model.guess(dframe['y'], x=dframe['x']) result = model.fit(dframe['y'], params, x=dframe['x']) print(result.fit_report()) result.plot_fit() plt.show()
def residuals(force_pickrange=False): datafolder = filedialog.askopenfilenames( initialdir="C:\\Users\Josh\IdeaProjects\OpticalPumping", title="Select data for bulk plotting") for filename in datafolder: if 'data' in filename: global dat3 name_ext = filename.split('/')[-1] name_no_ending = name_ext.split('.csv')[0] parts = name_no_ending.split('_') print(parts) dat1 = read_csv( "C:\\Users\\Josh\\IdeaProjects\\OpticalPumping\\Sweep_dat\\{}". format(name_ext), names=['xdat', 'ydat']) dat1 = dat1[np.abs(dat1['xdat'] - dat1['xdat'].mean()) <= ( 3 * dat1['xdat'].std())] xdat = np.array(dat1['xdat']) ydat = np.array(dat1['ydat']) if force_pickrange: fig1, ax1 = plt.subplots() plt.title('Pick Ranges for Exponential decay fit') Figure1, = ax1.plot(xdat, ydat, '.') print(xdat[5], xdat[-5]) plt.xlim([xdat[5], xdat[-5]]) Sel3 = RangeTool(xdat, ydat, Figure1, ax1, name_no_ending) fullscreen() plt.show() dat3 = Sel3.return_range() if not force_pickrange: try: dat3 = read_csv( "C:\\Users\\Josh\\IdeaProjects\\OpticalPumping\\Sweep_ranges\\{}" .format(name_ext), names=[ 'Lower Bound', 'LowerIndex', 'Upper Bound', 'UpperIndex' ]) except FileNotFoundError or force_pickrange: fig1, ax1 = plt.subplots() plt.title('Pick Ranges for Exponential decay fit') Figure1, = ax1.plot(xdat, ydat, '.') print(xdat[5], xdat[-5]) plt.xlim([xdat[5], xdat[-5]]) Sel3 = RangeTool(xdat, ydat, Figure1, ax1, name_no_ending) fullscreen() plt.show() dat3 = Sel3.return_range() mdl = LorentzianModel() try: lowerindex = int(dat3.at[0, 'LowerIndex']) upperindex = int(dat3.at[0, 'UpperIndex']) except ValueError: pass try: params = mdl.guess(data=ydat[lowerindex:upperindex], x=xdat[lowerindex:upperindex]) result = mdl.fit(ydat[lowerindex:upperindex], params, x=xdat[lowerindex:upperindex]) resultdata = mdl.eval(x=xdat[lowerindex:upperindex], params=result.params) # print(result.fit_report()) MultiPlot(xdat[lowerindex:upperindex], ydat[lowerindex:upperindex], resultdata, xdat, ydat, name_no_ending, parts[1]) except UnboundLocalError: pass
def detecting_prominences_no_plot(self): # First we load the image test = cv2.imread(self.path) # Then we split the image to keep the inside using separate_images a, b = sun_img.separate_images(self) # Compute radius and center of the Sun r = sun_img.circle_properties(self)[0][2] x = sun_img.circle_properties(self)[0][0] y = sun_img.circle_properties(self)[0][1] center = (x,y) # Apply a polar coordinate transform to "flatten" the Sun circumpherence flat = cv2.linearPolar(a, center,r+50,cv2.WARP_FILL_OUTLIERS) # Make it a gray image and define its shape flat_g = cv2.cvtColor(flat, cv2.COLOR_BGR2GRAY) rows,cols = flat_g.shape # Apply a rotation matrix and rotate 90 degrees M = cv2.getRotationMatrix2D((cols/2,rows/2),90,1) dst = cv2.warpAffine(flat_g,M,(cols,rows)) # Zoom to the area where the information is contained zoom = dst[40:100,:] # Compute the mean and median intensity of the image in order to filter out the background mean_int=np.mean(zoom[np.nonzero(zoom)]) median_int=np.median(zoom[np.nonzero(zoom)]) # Filter out the background: very harshly, will only detect defined structures ret,thresh4 = cv2.threshold(zoom,1.35*mean_int,0,cv2.THRESH_TOZERO) ret5,thresh5 = cv2.threshold(dst,1.35*mean_int,0,cv2.THRESH_TOZERO) # Collapse the 2D image into a 1D signal by adding the y-axis values cropped_1D = np.sum(zoom, axis=0) #This is the unfiltered image filtered_1D = np.sum(thresh5, axis=0) # This is the filtered one # Smooth the signal using Scipy (https://scipy-cookbook.readthedocs.io/items/SignalSmooth.html) def smooth(x,window_len=11,window='hanning'): """smooth the data using a window with requested size. This method is based on the convolution of a scaled window with the signal. The signal is prepared by introducing reflected copies of the signal (with the window size) in both ends so that transient parts are minimized in the begining and end part of the output signal. input: x: the input signal window_len: the dimension of the smoothing window; should be an odd integer window: the type of window from 'flat', 'hanning', 'hamming', 'bartlett', 'blackman' flat window will produce a moving average smoothing. output: the smoothed signal example: t=linspace(-2,2,0.1) x=sin(t)+randn(len(t))*0.1 y=smooth(x) see also: numpy.hanning, numpy.hamming, numpy.bartlett, numpy.blackman, numpy.convolve scipy.signal.lfilter TODO: the window parameter could be the window itself if an array instead of a string NOTE: length(output) != length(input), to correct this: return y[(window_len/2-1):-(window_len/2)] instead of just y. """ if x.ndim != 1: raise ValueError("smooth only accepts 1 dimension arrays.") if x.size < window_len: raise ValueError("Input vector needs to be bigger than window size.") if window_len<3: return x if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']: raise ValueError("Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'") s=np.r_[x[window_len-1:0:-1],x,x[-2:-window_len-1:-1]] #print(len(s)) if window == 'flat': #moving average w=np.ones(window_len,'d') else: w=eval('np.'+window+'(window_len)') y=np.convolve(w/w.sum(),s,mode='valid') return y[int((window_len/2-1)):-int(window_len/2)] # Smooth the signal and obtain its length smoothed_1D = smooth(filtered_1D,window_len=11, window='hanning') sm_len = len(smoothed_1D) x = np.arange(sm_len) # Define initial maxima and minima in order to find the peak sm_max = np.max(smoothed_1D) sm_argmax = np.argmax(smoothed_1D) sm_std = np.std(smoothed_1D) sm_min =np.r_[True, smoothed_1D[1:] < smoothed_1D[:-1]] & np.r_[smoothed_1D[:-1] < smoothed_1D[1:], True] sm_argmin = np.where(sm_min==True) sm_argmin = np.asarray(sm_argmin[0]) sm_argmin= np.append(sm_argmin, x[-1]) # Define empty lists to store the results of the peak finding and fitting new_curve = smoothed_1D.copy() # parameters will contain the results of the fit as lmfit objects parameters = list() # ranges the x-range of each peak ranges = list() # peak_ind the x-value where the peak has the maxima peak_ind = list() # Find all peaks standing above 2*sigma over the background while sm_max > 2*sm_std: if (sm_argmax != sm_len -1) & (sm_argmax > sm_argmin[0]): min1 = sm_argmin[np.where((sm_argmin<sm_argmax))[0]][-1] min2 = sm_argmin[np.where((sm_argmin>sm_argmax))[0]][0] x_range =x[min1:min2] y_range = smoothed_1D[min1:min2] amp = sm_max mean = sm_argmax sigma = np.std(y_range) # Fit a Lorenztian + linear model using lmfit lore_mod = LorentzianModel(prefix='lore_') line_mod = LinearModel(prefix='line_') if len(x_range)>5: pars = line_mod.make_params(intercept=y_range.min(), slope=0) pars += lore_mod.guess(y_range, x=x_range) mod = lore_mod + line_mod out = mod.fit(y_range, pars, x=x_range) ranges.append(np.asarray((min1,min2))) parameters.append(out) peak = np.zeros(sm_len) peak[min1:min2]=out.best_fit peak_ind.append(sm_argmax) new_curve = new_curve - peak sm_max = np.max(new_curve) sm_argmax = np.argmax(new_curve) else: new_curve[sm_argmax]=0 sm_max = np.max(new_curve) sm_argmax = np.argmax(new_curve) else: new_curve[sm_argmax]=0 sm_max = np.max(new_curve) sm_argmax = np.argmax(new_curve) # Collect the results of the fit: centers, amplitudes and sigmas centers = np.asarray([i.best_values['lore_center'] for i in parameters]) amplitudes = np.asarray([i.best_values['lore_amplitude'] for i in parameters]) sigmas = np.asarray([i.best_values['lore_sigma'] for i in parameters]) # Define the amplitude of the peak over the background and the normalized amplitude h = np.asarray([np.max(parameters[i].best_fit)-np.min(parameters[i].best_fit) for i in range(len(parameters))]) norm_h = h/np.max(h) # Define a normalized smoothed curve to plot norm_smoothed = smoothed_1D/np.max(smoothed_1D) # Keep only the peaks that stand more than 10% over the background ind = np.where(norm_h>0.1) # Select the parameters consequently some_parameters = np.asarray(parameters)[ind[0]] some_ranges = np.asarray(ranges)[ind[0]] some_intensities = np.asarray(smoothed_1D[np.asarray(peak_ind)[ind]]) norm_fits = np.asarray([some_parameters[i].best_fit/np.max(smoothed_1D) for i in range(len(some_parameters))]) some_centers = np.asarray(centers[np.where(norm_h> 0.1)[0]]) some_centers = np.asarray([int(i) for i in some_centers]) some_centers[np.where(some_centers<0)]=0 some_centers[np.where(some_centers>len(x)-1)]=len(x)-1 some_amplitudes = np.asarray(amplitudes[np.where(norm_h>0.1)[0]]) some_sigmas = np.asarray(sigmas[np.where(norm_h>0.1)[0]]) some_normh = np.asarray(norm_h[np.where(norm_h>0.1)[0]]) # Remove double-fitting of peaks by forcing the peak maxima to be >1 x-step away from the next # First we sort the peak indices and single out the x-values that are problematic sort = np.sort(some_centers) sort_ind = np.argsort(some_centers) repeated_peaks = list() for i in np.arange(len(some_centers)-1): step = sort[i+1]-sort[i] if step <= 2: repeated_peaks.append(sort_ind[i]) # Select the parameters consequently some_parameters = np.delete(some_parameters, repeated_peaks) some_ranges = np.delete(some_ranges, repeated_peaks,0) some_intensities = np.delete(some_intensities, repeated_peaks) norm_fits = np.delete(norm_fits, repeated_peaks) some_centers = np.delete(some_centers, repeated_peaks) some_amplitudes = np.delete(some_amplitudes, repeated_peaks) some_sigmas = np.delete(some_sigmas, repeated_peaks) some_normh = np.delete(some_normh, repeated_peaks) # Define the shape of the image and undo the rotation rows1,cols1 = dst.shape M1 = cv2.getRotationMatrix2D((cols1/2,rows1/2),-90,1) rot = cv2.warpAffine(dst,M1,(cols1,rows1)) # Convert the position of the peaks in the flattened image to the circular one using # polar coordinates for a known r. Select r-10 as those coordinates will be where the # label for the prominence will be placed in the plot L = np.shape(rot)[0] cord = some_centers.copy()-1 cord_phi = [(2*np.pi*cord[i])/L for i in np.arange(len(cord))] cord_r = (r-10) cord_x = [(cord_r*np.cos(i))+L/2 for i in cord_phi] cord_y = [(cord_r*np.sin(i))+L/2 for i in cord_phi] # Return to the circular image recover1 = cv2.linearPolar(rot, center,r+50,cv2.WARP_FILL_OUTLIERS+cv2.WARP_INVERSE_MAP) # Store the results in a table results = np.empty((len(some_parameters)+1,6), dtype=object) for i in np.arange(len(some_parameters)): # Table header results[0,:]= ['Name', 'Flat Position.', 'Radial Position', 'Height of the peak over the noise', 'Width', 'Intensity of the peak'] # Name of the prominence: 1, 2, 3, 4... results[i+1,0]=str(i+1) # Coordinates of the prominence in flat projection results[i+1,1]= int(some_centers[i]) # Coordinates of the prominence in circular projection: results[i+1,2] = (cord_x[i], cord_y[i]) # Height of the peak over the noise results[i+1,3] = some_amplitudes[i] # Width of the peak results[i+1,4] = some_sigmas[i] # Intensity of the peak results[i+1,5] = some_intensities[i] return results
def FIT_PS(kic): """Essa rotina fita curvas Lorenzianas e Gaussianas no Power Spectrum somente com o kic da estrela""" path = 'TEMP/' + str(kic) + '/' file = np.loadtxt(str(path) + 'PS_' + str(kic) + '.txt') f = file[:, 0] p = file[:, 1] def fit_gauss(freq, psd): global X X = [], [] fig = plt.figure(figsize=(8, 5), dpi=130) plt.loglog(freq, psd, 'k', lw=0.5, alpha=0.5) plt.title("Select the gaussian fit region") plt.ylabel('PSD[ppm$^2$/$\mu$Hz]') plt.xlabel('Frequency [$\mu$Hz]') plt.xlim(min(freq), max(freq)) plt.ylim(min(psd), max(psd)) def onclick(event): global X x = event.xdata X = np.append(X, x) if len(X) == 1: plt.plot((x, x), (min(psd), max(psd)), 'r--', alpha=0.5, lw=0.8) plt.xlim(min(freq), max(freq)) plt.ylim(min(psd), max(psd)) plt.draw() if len(X) == 2: plt.fill_betweenx((min(psd), max(psd)), (X[-2], X[-2]), (X[-1], X[-1]), color='red', alpha=0.2) plt.plot((X[-1], X[-1]), (min(psd), max(psd)), 'r--', alpha=0.5, lw=0.8) plt.xlim(min(freq), max(freq)) plt.ylim(min(psd), max(psd)) plt.draw() if len(X) > 2: plt.clf() plt.title("Selecione a regiao do fit gaussiano") plt.ylabel('PSD[ppm$^2$/$\mu$Hz]') plt.xlabel('Frequency [$\mu$Hz]') plt.xlim(min(freq), max(freq)) plt.ylim(min(psd), max(psd)) plt.tight_layout() plt.loglog(freq, psd, 'k', lw=0.5, alpha=0.5) plt.plot((X[-2], X[-2]), (min(psd), max(psd)), 'r--', alpha=0.5, lw=0.8) plt.plot((X[-1], X[-1]), (min(psd), max(psd)), 'r--', alpha=0.5, lw=0.8) plt.fill_betweenx((min(psd), max(psd)), (X[-2], X[-2]), (X[-1], X[-1]), color='red', alpha=0.2) plt.draw() print('Ultimo click: x = ', x) fig.canvas.mpl_connect('button_press_event', onclick) plt.tight_layout() plt.show() plt.clf() dados = (X[-2], X[-1]) return min(dados), max(dados) def gauss(x, y): print( 'Selecione a regiao do fit gaussiano quantas vezes for necessario') l1, l2 = fit_gauss(x, y) index1 = np.where(abs(x - l1) == min(abs(x - l1)))[0] index1 = np.int(index1) index2 = np.where(abs(x - l2) == min(abs(x - l2)))[0] index2 = np.int(index2) print(index1, index2) x1 = x[index1:index2] y1 = y[index1:index2] mod = GaussianModel() pars = mod.guess(y1, x=x1) out = mod.fit(y1, pars, x=x1) xgauss = copy.copy(x1) ygauss = copy.copy(out.best_fit) #-####### x0 = x[0:index1] y0 = np.zeros(len(y[0:index1])) x2 = x[index2:] y2 = np.zeros(len(y[index2:])) x1 = np.append(x0, x1) x1 = np.append(x1, x2) y1 = np.append(y0, out.best_fit) y1 = np.append(y1, y2) print(out.fit_report(min_correl=0.25)) mod = GaussianModel() pars = mod.guess(y1, x=x1) out = mod.fit(y1, pars, x=x1) #-####### return x1, out.best_fit, xgauss, ygauss y = p * 1.0e12 x = f xx, yy, xgauss, ygauss = gauss(x, y) fiti = get_fit(x, y, 'Select the first') fiti2 = get_fit(x, y, 'Select the second') L1 = LorentzianModel(prefix='L1_') pars = L1.guess(y, x=x) out = L1.fit(y, pars, x=x) result = out.minimize('least_squares') plt.clf() plt.close('all') fig = plt.figure(figsize=(8, 5), dpi=130) plt.style.use('classic') plt.loglog(f, y, "#ff7f0e", lw=0.5) plt.plot(x, fiti, 'k--', lw=0.5) plt.plot(x, fiti2, 'k--', lw=0.5) plt.plot((x[0], x[-1]), (y[-1] / 2, y[-1] / 2), 'b--', lw=0.5) plt.plot(xgauss, ygauss, 'k--', lw=0.5) plt.plot(x, fiti + fiti2 - ((fiti + fiti2) / 2) + yy, 'r-', lw=0.8) plt.title('KIC ' + np.str(kic)) plt.ylabel('PSD[ppm$^2$/$\mu$Hz]') plt.xlabel('Frequency [$\mu$Hz]') plt.ylim(1.0e-4, max(y)) plt.xlim(min(f), max(f)) plt.tight_layout() plt.savefig(str(path) + 'PS_KIC' + str(kic) + '_fit.png', dpi=270) plt.show()
def main(): regionname = sys.argv[1] ## parameter passed short = regionname.replace(" ", "").lower() appName = config['common']['appName'] spark = s.spark_session(appName) spark = s.setSparkConfBQ(spark) # Get data from BigQuery table start_date = "201001" end_date = "202001" lst = (spark.sql( "SELECT FROM_unixtime(unix_timestamp(), 'dd/MM/yyyy HH:mm:ss.ss') ") ).collect() print("\nStarted at") uf.println(lst) # Model predictions read_df = s.loadTableFromBQ(spark, config['GCPVariables']['sourceDataset'], config['GCPVariables']['sourceTable']) df_10 = read_df.filter(F.date_format('Date',"yyyyMM").cast("Integer").between(f'{start_date}', f'{end_date}') & (lower(col("regionname"))== f'{regionname}'.lower())). \ select(F.date_format('Date',"yyyyMM").cast("Integer").alias("Date") \ , round(col("flatprice")).alias("flatprice") \ , round(col("terracedprice")).alias("terracedprice") , round(col("semidetachedprice")).alias("semidetachedprice") , round(col("detachedprice").alias("detachedprice"))) print(df_10.toPandas().columns.tolist()) p_dfm = df_10.toPandas() # converting spark DF to Pandas DF # Non-Linear Least-Squares Minimization and Curve Fitting # Define model to be Lorentzian and depoly it model = LorentzianModel() n = len(p_dfm.columns) for i in range(n): if (p_dfm.columns[i] != 'Date'): # yyyyMM is x axis in integer # it goes through the loop and plots individual average curves one by one and then prints a report for each y value vcolumn = p_dfm.columns[i] print(vcolumn) params = model.guess(p_dfm[vcolumn], x=p_dfm['Date']) result = model.fit(p_dfm[vcolumn], params, x=p_dfm['Date']) # plot the data points, initial fit and the best fit plt.plot(p_dfm['Date'], p_dfm[vcolumn], 'bo', label='data') plt.plot(p_dfm['Date'], result.init_fit, 'k--', label='initial fit') plt.plot(p_dfm['Date'], result.best_fit, 'r-', label='best fit') plt.legend(loc='upper left') plt.xlabel("Year/Month", fontdict=config['plot_fonts']['font']) plt.text(0.35, 0.55, "Fit Based on Non-Linear Lorentzian Model", transform=plt.gca().transAxes, color="grey", fontsize=9) if vcolumn == "flatprice": property = "Flat" if vcolumn == "terracedprice": property = "Terraced" if vcolumn == "semidetachedprice": property = "semi-detached" if vcolumn == "detachedprice": property = "detached" plt.ylabel(f"""{property} house prices in millions/GBP""", fontdict=config['plot_fonts']['font']) plt.title( f"""Monthly {property} price fluctuations in {regionname}""", fontdict=config['plot_fonts']['font']) plt.xlim(200901, 202101) print(result.fit_report()) plt.show() plt.close() lst = (spark.sql( "SELECT FROM_unixtime(unix_timestamp(), 'dd/MM/yyyy HH:mm:ss.ss') ") ).collect() print("\nFinished at") uf.println(lst)
import matplotlib.pyplot as plt from lmfit.models import LorentzianModel import pandas as pd dframe = pd.read_csv('peak.csv') model = LorentzianModel() params = model.guess(dframe['y'], x=dframe['x']) result = model.fit(dframe['y'], params, x=dframe['x']) print(result.fit_report()) result.plot_fit() plt.show()
data_energy = open("./fit_simple_lorentzian_energy.log", "a+") now = datetime.datetime.now() data_energy.write( "\n\nAt: %i/%i/%i, %i:%i:%i \n" % (now.day, now.month, now.year, now.hour, now.minute, now.second)) data_energy.write( "Archivo \t Curva \t \t Centro \t err \t \t \t\t Fwhm \t err \t\t\t\t Area \t Err\n" ) L_mod = LorentzianModel(prefix='L0') c_mod = ConstantModel() for filename in files: x, y = np.loadtxt(direction + filename, unpack=True) pars = L_mod.guess(y, x=x) pars += c_mod.guess(y, x=x) mod = L_mod + c_mod out = mod.fit(y, pars, x=x) # fit_log.write("\n\n\n\n\n\n Fitted from: %s%s \n" %(direction, filename)) # now= datetime.datetime.now() # fit_log.write("At: %i/%i/%i, %i:%i:%i \n" %(now.day,now.month,now.year,now.hour,now.minute,now.second)) # fit_log.write(out.fit_report(min_correl=0.2)) print_to_fit_log(filename, data_energy, out, 0) plt.plot(x, y, 'b') plt.plot(x, out.init_fit, 'g--') plt.plot(x, out.best_fit, 'r')