def score_models(self, y_test, Predict_matrix, verbose = 0): df_score = pd.DataFrame() for name_model in Predict_matrix: SS_residual = ((y_test - Predict_matrix[name_model])**2).sum() df_y = pd.DataFrame() df_y['y_test'] = y_test df_y['y_pred'] = Predict_matrix[name_model] df_y['diff'] = (y_test - Predict_matrix[name_model]) SS_Total = ((y_test - np.mean(y_test))**2).sum() r_square = 1 - (float(SS_residual))/SS_Total mae = mean_absolute_error(y_test, Predict_matrix[name_model]) mse = mean_squared_error(y_test, Predict_matrix[name_model]) r2 = r2_score(y_test, Predict_matrix[name_model]) df_score[name_model] = [round(mae,3), round(mse,3), round(r2,3), round(r_square,3), round(SS_residual,3)] df_score = df_score.T df_score.columns = ['MAE', 'MSE', 'R2', 'R2 a mano', 'residual'] if verbose == 1: fig, ax1 = ds().nuova_fig(123) ds().titoli(titolo='', xtag='model', ytag='MAE') ds().dati(x = df_score.index.tolist(), y = df_score['MAE'].to_numpy(), scat_plot = 'scat', larghezza_riga = 15) modelli = df_score.index.tolist() plt.xticks(rotation=90) fig.tight_layout() st.pyplot() return df_score
def disegna_marker(self, x, y, x_wide, y_wide, colore="#00A3E0"): x_wide = x_wide/2 y_wide = y_wide/2 x_tot = np.zeros(5) y_tot = np.zeros(5) x_tot[0] = x-x_wide x_tot[1] = x+x_wide x_tot[2] = x+x_wide x_tot[3] = x-x_wide x_tot[4] = x-x_wide y_tot[0] = y-y_wide y_tot[1] = y-y_wide y_tot[2] = y+y_wide y_tot[3] = y+y_wide y_tot[4] = y-y_wide ds().nuova_fig(1) ds().titoli(titolo="design preview", xtag="um", ytag="um") ds().dati(x_tot, y_tot, colore = colore, larghezza_riga = 0.5) x_tot[0] = x-y_wide x_tot[1] = x+y_wide x_tot[2] = x+y_wide x_tot[3] = x-y_wide x_tot[4] = x-y_wide y_tot[0] = y-x_wide y_tot[1] = y-x_wide y_tot[2] = y+x_wide y_tot[3] = y+x_wide y_tot[4] = y-x_wide ds().dati(x_tot, y_tot, colore = colore, larghezza_riga = 0.5)
def var_model_fit(self, x_train, x_test, df_time, lag, maxlag=None, verbose = 1): model = VAR(x_train) if maxlag != None:#studio hypersapce sul parametro lag vet_aic = [] vet_bic = [] vet_fpe = [] vet_hqic = [] for i in range(maxlag): result = model.fit(i) vet_aic.append(result.aic) vet_bic.append(result.bic) vet_fpe.append(result.fpe) vet_hqic.append(result.hqic) df_results = pd.DataFrame() df_results['AIC'] = vet_aic df_results['BIC'] = vet_bic df_results['FPE'] = vet_fpe df_results['HQIC'] = vet_hqic if verbose == 1: st.subheader('durbin_watson test') st.write('the closer the result is to 2 then there is no correlation, the closer to 0 or 4 then correlation implies') st.write(df_results) else:# fit diretto su un valore specifico di lag result = model.fit(lag) out = durbin_watson(result.resid) df_results = pd.DataFrame() for col, val in zip(x_train.columns, out): df_results[col] = [round(val, 2)] if verbose == 1: st.subheader('durbin_watson test') st.write('the closer the result is to 2 then there is no correlation, the closer to 0 or 4 then correlation implies') st.write(df_results.T) forecast_input = x_train.values[-lag:] fc = result.forecast(y=forecast_input, steps=x_test.shape[0])#y rappresenta i valori del training su cui verra prioritizzata la predizione per il fututo df_forecast = pd.DataFrame(fc, index=df_time['test'], columns=x_test.columns) x_test.index = df_time['test'] if verbose == 1: st.write(df_forecast) for i, col in enumerate(x_test): ds().nuova_fig(555+i) st.subheader(col) df_forecast[col].plot(label = 'Predicition') x_test[col].plot(label = 'True') ds().legenda() st.pyplot() return df_forecast
def summarize_performance(self, data, epoch, generator, discriminator, latent_dim, verbose=False): ''' funzione per fare lo scoring del training data: training set in formato pandas epoch: (int) epoca del training generator: modello del generatore discriminator: modello del discriminatore latent_dim: (int) dimensione del vettore dello spazio latente, deve essere lungo quanto il primo layer del modello verbose: boolean, if True print accuracy and plot return: epoch, acc_real (accuratezza nel identificare i dati reali), acc_fake (accuratezza nel identificare i dati fake) ''' # prepare real samples x_real, y_real = self.generate_real_samples(data, 100) # evaluate discriminator on real examples _, acc_real = discriminator.evaluate(x_real, y_real, verbose=0) # prepare fake examples x_fake, y_fake = self.generate_fake_samples(generator, latent_dim, 100) # evaluate discriminator on fake examples _, acc_fake = discriminator.evaluate(x_fake, y_fake, verbose=0) # summarize discriminator performance x_real, y_real = self.generate_real_samples(data, 1) x_fake, y_fake = self.generate_fake_samples(generator, latent_dim, 1) if verbose: print('epoch: ', epoch) print('how good the discriminator is to evaluate the real example', acc_real) print('how good the discriminator is to evaluate the fake example', acc_fake) # scatter plot real and fake data points ds().nuova_fig(1) X1 = np.arange(len(x_real.T[0])) ds().dati(x=X1, y=x_real.T[0], colore=palette[0], descrizione="real") ds().dati(x=X1, y=x_fake.T, colore=palette[1], descrizione="fake") ds().legenda() ds().porta_a_finestra() return epoch, acc_real, acc_fake
def varmax_model_fit(self, x_train, x_test, df_time, oreder = (1, 0), col_exog=[], verbose = 1): if col_exog: exo_train = pd.DataFrame() exo_test = pd.DataFrame() for col in col_exog: exo_train[col] = x_train[col] x_train.drop([col], axis=1, inplace = True) exo_test[col] = x_test[col] x_test.drop([col], axis=1, inplace = True) model = VARMAX(x_train, order=oreder, exog=exo_train) else: model = VARMAX(x_train, order=oreder) result = model.fit() out = durbin_watson(result.resid) df_results = pd.DataFrame() for col, val in zip(x_train.columns, out): df_results[col] = [round(val, 2)] if verbose == 1: st.subheader('durbin_watson test') st.write('the closer the result is to 2 then there is no correlation, the closer to 0 or 4 then correlation implies') st.write(df_results.T) if col_exog: df_forecast = result.forecast(steps=x_test.shape[0], exog = exo_test) else: df_forecast = result.forecast(steps=x_test.shape[0]) df_forecast.index = df_time['test'] df_forecast.columns = x_test.columns x_test.index = df_time['test'] if verbose == 1: st.write(df_forecast) for i, col in enumerate(x_test): fig = ds().nuova_fig(555+i) st.subheader(col) df_forecast[col].plot(label = 'Predicition') x_test[col].plot(label = 'True') ds().legenda() st.pyplot(fig) return df_forecast
def var_predict_sbs(self, x_train, x_test, df_time, lag = 5, oreder = (1,0), col_exog = [], model = 'VAR', verbose = 1): total_predict = [] total_true = [] df_train = pd.DataFrame(x_train['total_cases'].copy()) df_train.index = df_time['train'] for i in range(x_test.shape[0]): x_new = pd.DataFrame(x_test.iloc[i]).T df_time_new ={'test': [df_time['test'][i]]} if model == 'VAR': predicted = self.var_model_fit(x_train, x_new, df_time_new, lag = lag, verbose = 0) elif model == 'VARMAX': x_train = x_train[-lag:] predicted = self.varmax_model_fit(x_train, x_new, df_time_new, oreder = oreder, col_exog=[], verbose = 0) total_predict.append(predicted['total_cases'].values[0]) total_true.append(x_new['total_cases'].values[0]) x_new.drop(['total_cases'], axis=1, inplace=True) x_new['total_cases'] = predicted['total_cases'] x_train = pd.concat([x_train, x_new], ignore_index = True) if verbose == 1: st.subheader('VAR') df_results = pd.DataFrame() df_results['predicted'] = total_predict df_results['true'] = total_true df_results.index = df_time['test'] # st.write(df_results) ds().nuova_fig(444) df_train['total_cases'].plot(label = 'train') df_results['predicted'].plot(label = 'Predicition') df_results['true'].plot(label = 'True') ds().legenda() st.pyplot() predict_matrix = pd.DataFrame() predict_matrix['VAR'] = df_results['predicted'] predict_matrix = predict_matrix.reset_index(drop = True) return predict_matrix
def score_models(self, y_test, Predict_matrix, verbose=0): ''' generate the score with the actual test target :param y_test: array with the actual targets, pandas or numpy :param Predict_matrix: array with the predicted targets for each model, pandas :return: df_score.shape[0] = num of models + 1, df_score.shape[1] = 5 (MAE, MSE, R2, R2 a mano, residual) ''' df_score = pd.DataFrame() for name_model in Predict_matrix: SS_residual = ((y_test - Predict_matrix[name_model])**2).sum() df_y = pd.DataFrame() df_y['y_test'] = y_test df_y['y_pred'] = Predict_matrix[name_model] df_y['diff'] = (y_test - Predict_matrix[name_model]) SS_Total = ((y_test - np.mean(y_test))**2).sum() r_square = 1 - (float(SS_residual)) / SS_Total mae = mean_absolute_error(y_test, Predict_matrix[name_model]) mse = mean_squared_error(y_test, Predict_matrix[name_model]) r2 = r2_score(y_test, Predict_matrix[name_model]) df_score[name_model] = [ round(mae, 3), round(mse, 3), round(r2, 3), round(r_square, 3), round(SS_residual, 3) ] df_score = df_score.T df_score.columns = ['MAE', 'MSE', 'R2', 'R2 a mano', 'residual'] if verbose == 1: ds().nuova_fig(1) ds().titoli(titolo='target - Ensamble', xtag='real', ytag='predict') ds().dati(x=y_test, y=Predict_matrix['Ensamble'], scat_plot='scat', larghezza_riga=15) plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], linestyle='--') ds().porta_a_finestra() return df_score
def disegna_pattern(self, x, y, dimArr, colore="#00A3E0"): x_tot = np.zeros(5) y_tot = np.zeros(5) x_tot[0] = x x_tot[1] = x + dimArr x_tot[2] = x + dimArr x_tot[3] = x x_tot[4] = x y_tot[0] = y y_tot[1] = y y_tot[2] = y + dimArr y_tot[3] = y + dimArr y_tot[4] = y ds().nuova_fig(1) ds().titoli(titolo="design preview", xtag="um", ytag="um") ds().dati(x_tot, y_tot, colore = colore, larghezza_riga = 0.5)
def correlation_matrix(self, data, col_y, corr_value=0.95, corr_value_w_targhet=0.95, plot_matr='yes'): """ 'col_y' represent the target column """ corr = data.corr().abs() if plot_matr == 'yes': ds().nuova_fig(7, height=8, width=10) ds().titoli(titolo="Correlation matrix") sns.heatmap(corr[(corr >= 0.5)], cmap='viridis', vmax=1.0, vmin=-1.0, linewidths=0.1, annot=True, annot_kws={"size": 8}, square=True, linecolor="black") ds().aggiusta_la_finestra() st.pyplot() corr_with_target = corr[col_y] #correlation with the target relevant_feature_with_target = corr_with_target[ corr_with_target < corr_value_w_targhet].sort_values( ascending=False).reset_index() upper = corr.where(np.triu(np.ones(corr.shape), k=1).astype( np.bool)) #select upper triangle of correlation matrix correlation_between_parameters = [ column for column in upper.columns if any(upper[column] > corr_value) ] return relevant_feature_with_target, correlation_between_parameters
def plot_history(self, fitModel): if fitModel != 0: history_dict = fitModel.history history_dict.keys() loss = history_dict['loss'] val_loss = history_dict['val_loss'] epochs = range(1, len(loss) + 1) ds().nuova_fig(1) ds().titoli(titolo="Training loss", xtag='Epochs', ytag='Loss', griglia=0) ds().dati(epochs, loss, descrizione='Training loss', colore='red') ds().dati(epochs, val_loss, descrizione='Validation loss') ds().dati(epochs, loss, colore='red', scat_plot='scat', larghezza_riga=10) ds().dati(epochs, val_loss, scat_plot='scat', larghezza_riga=10) ds().range_plot(bottomY=np.array(val_loss).mean() - np.array(val_loss).std() * 6, topY=np.array(val_loss).mean() + np.array(val_loss).std() * 6) ds().legenda() st.pyplot()
name1 = 'sottomissioni/submission_var_new' name2 = 'sottomissioni/submission_var_old_dnn_average_on_20' df1 = pd.read_csv(name1 + '.csv') df2 = pd.read_csv(name2 + '.csv') st.write(df1) df1_sj = df1[df1['city'] == 'sj'] df1_iq = df1[df1['city'] == 'iq'] df2_sj = df2[df2['city'] == 'sj'] df2_iq = df2[df2['city'] == 'iq'] ds().nuova_fig(77) df1_sj['total_cases'].plot(label=name1) df2_sj['total_cases'].plot(label=name2) ds().legenda() st.pyplot() ds().nuova_fig(78) df1_iq['total_cases'].plot(label=name1) df2_iq['total_cases'].plot(label=name2) ds().legenda() st.pyplot() ########################## # df2_sj['total_cases'] = df2_sj['total_cases']-df2_sj['total_cases'].min() # df2_iq['total_cases'] = df2_iq['total_cases']-df2_iq['total_cases'].min()
tx_data['Date'] = pd.to_datetime(tx_data['Date']) #create a dataframe contaning CustomerID and first purchase date tx_min_purchase = tx_data.groupby('ID')['Date'].min().reset_index() tx_min_purchase.columns = ['ID','MinPurchaseDate'] #merge first purchase date column to our main dataframe (tx_uk) tx_data = pd.merge(tx_data, tx_min_purchase, on='ID') tx_data['UserType'] = 'Existing' loaction_first_order = tx_data['Date'].dt.to_period('M') == tx_data['MinPurchaseDate'].dt.to_period('M') tx_data.loc[loaction_first_order, 'UserType'] = 'New' st.write(tx_data.head()) # st.title('monthly revenue') # calcolare la revenue totale in ogni mese tx_revenue = tx_data.set_index('Date').groupby(pd.Grouper(freq='M'))['Revenue'].sum().reset_index() ds().nuova_fig(1, indice_subplot =121, width =12, height =5) ds().titoli(titolo="monthly revenue", ytag='revenue') ds().dati(x = tx_revenue['Date'], y = tx_revenue['Revenue']) # st.title('monthly active customers') #creating monthly active customers dataframe by counting unique Customer IDs tx_monthly_active = tx_data.set_index('Date').groupby(pd.Grouper(freq='M'))['ID'].nunique().reset_index() ds().nuova_fig(1, indice_subplot =122, width =12, height =5) ds().titoli(titolo="monthly active customers", ytag='Num Customers') ds().dati(x = tx_monthly_active['Date'], y = tx_monthly_active['ID']) if eda: st.pyplot() # st.title('Average Revenue per Order') # create a new dataframe for average revenue by taking the mean of it tx_monthly_order_avg = tx_data.set_index('Date').groupby(pd.Grouper(freq='M'))['Revenue'].mean().reset_index()
def plot1(T, colore): list_col = ['yy'+str(i) for i in range(numb_of_spect)] list_col_x = ['xx'+str(i) for i in range(numb_of_spect)] x_err = np.zeros((2, numb_of_spect)) x_err[0][:] = T['ex10'] x_err[1][:] = T['ex20'] x_tot = pd.DataFrame() y_tot = pd.DataFrame() for i in range(len(list_col)): x_tot = pd.concat([x_tot, T[list_col_x[i]]]) y_tot = pd.concat([y_tot, T[list_col[i]]]) x_tot = x_tot[0].to_numpy() y_tot = y_tot[0].to_numpy() y_fit2, m, m_err, q, q_err, y_fit2_up, y_fit2_down = self.interval_confidence(x_tot, y_tot) ds().nuova_fig(40) ds().titoli(xtag='P [uW]', ytag = 'T [k]', titolo='without quality filter') for i in range(numb_of_spect): ds().dati(x = T['xx0'], y = T['yy'+str(i)], colore=palette[i], scat_plot ='scat', larghezza_riga =15) ds().dati(x = T['xx0'], y = T[list_col].mean(axis = 1), colore=colore, scat_plot = 'scat', larghezza_riga =12) ds().dati(x = T['xx0'], y = T[list_col].mean(axis = 1), x_error = x_err, y_error= T[list_col].std(axis = 1), colore=colore, scat_plot = 'err') ds().dati(x_tot, y_fit2, colore=colore, descrizione='Y = '+str(round(m,2)) + '*X + ' + str(round(q,2))+'\n'+ 'm = '+str(round(m,2))+' +/- '+ str(round(m_err,2))+'\n q = '+str(round(q,2))+' +/- '+ str(round(q_err,2))) plt.fill_between(x_tot, y_fit2_down, y_fit2_up, color = 'black', alpha = 0.15) ds().legenda()
def plot2(T, T_quality_selected, average_T_quality_selected, colore): df_temp_tot = T_quality_selected.T x_tot = pd.DataFrame() y_tot = pd.DataFrame() df_tot = pd.DataFrame() for col in df_temp_tot.columns: y_tot = pd.concat([y_tot, df_temp_tot[col]]) x_tot = pd.concat([x_tot, T['xx0']]) df_tot['x'] = x_tot[0] df_tot.reset_index(inplace = True) df_tot.drop(['index'], axis=1, inplace=True) y_tot.reset_index(inplace = True) y_tot.drop(['index'], axis=1, inplace=True) df_tot['y'] = y_tot[0] df_tot.dropna(inplace = True) x_tot = df_tot['x'].to_numpy() y_tot = df_tot['y'].to_numpy() x_err = np.zeros((2, numb_of_spect)) x_err[0][:] = T['ex10'] x_err[1][:] = T['ex20'] y_fit2, m, m_err, q, q_err, y_fit2_up, y_fit2_down = self.interval_confidence(x_tot, y_tot) ds().nuova_fig(41) ds().titoli(xtag='P [uW]', ytag = 'T [k]', titolo='with quality filter') for i in range(T_quality_selected.shape[0]): ds().dati(x = T['xx0'], y = T_quality_selected.iloc[i], colore=palette[i], scat_plot ='scat', larghezza_riga =15) ds().dati(x = T['xx0'], y = average_T_quality_selected['Temp'], colore=colore, scat_plot = 'scat', larghezza_riga =12) ds().dati(x = T['xx0'], y = average_T_quality_selected['Temp'], x_error = x_err, y_error= average_T_quality_selected['Err Temp'], colore=colore, scat_plot = 'err') ds().dati(x_tot, y_fit2, colore=colore, descrizione='Y = '+str(round(m,2)) + '*X + ' + str(round(q,2))+'\n'+ 'm = '+str(round(m,2))+' +/- '+ str(round(m_err,2))+'\n q = '+str(round(q,2))+' +/- '+ str(round(q_err,2))) plt.fill_between(x_tot, y_fit2_down, y_fit2_up, color = 'black', alpha = 0.15) ds().legenda()
def summarize_performance(self, data, epoch, generator, discriminator, latent_dim, verbose=False, save_plot=False): ''' funzione per fare lo scoring del training data: training set in formato pandas epoch: (int) epoca del training generator: modello del generatore discriminator: modello del discriminatore latent_dim: (int) dimensione del vettore dello spazio latente, deve essere lungo quanto il primo layer del modello verbose: boolean, if True print accuracy and plot return: epoch, acc_real (accuratezza nel identificare i dati reali), acc_fake (accuratezza nel identificare i dati fake) ''' n_generate = 100 if n_generate > data.shape[0]: n_generate = data.shape[0] # prepare real samples x_real, y_real = self.generate_real_samples(data, n_generate) # evaluate discriminator on real examples _, acc_real = discriminator.evaluate(x_real, y_real, verbose=0) # prepare fake examples x_fake, y_fake = self.generate_fake_samples(generator, latent_dim, n_generate) # evaluate discriminator on fake examples _, acc_fake = discriminator.evaluate(x_fake, y_fake, verbose=0) # summarize discriminator performance x_real, y_real = self.generate_real_samples(data, 1) x_fake, y_fake = self.generate_fake_samples(generator, latent_dim, 1) if verbose: # scatter plot real and fake data points ds().nuova_fig(1) X1 = np.arange(len(x_real.T[0])) ds().dati(x=X1, y=x_real.T[0], colore=palette[0], descrizione="real") ds().dati(x=X1, y=x_fake.T, colore=palette[1], descrizione="fake") ds().legenda() if save_plot: filename = 'generated_plot_e%03d' % (epoch + 1) ds().salva_graf(filename) ds().porta_a_finestra(chiudi=save_plot) return epoch, acc_real, acc_fake
def eval_power(self, xy_mesh, intensity_sp, laser_power): if self.salva == 'yes': st.write('evaluating the real power on the particle') xx, yy = xy_mesh ds().nuova_fig(indice_fig=2, plot_dimension='3d', indice_subplot=322) ds().titoli(titolo='original with fit') ds().dati(xx, yy, scat_plot='3D', z=intensity_sp) ds().nuova_fig(indice_fig=9, plot_dimension='3d') ds().titoli(titolo='original with fit') ds().dati(xx, yy, scat_plot='3D', z=intensity_sp) st.pyplot() tot_intensity = sum(sum(intensity_sp)) intensity_sp_real = np.zeros( (int(len(intensity_sp[:, 0])), int(len(intensity_sp[0, :])))) intensity_sp_real = intensity_sp * ( laser_power / tot_intensity) * 1000 # the 1000 is in place to convert into uW return intensity_sp_real
def log_ratio(self, AS_min=588, AS_max=616, S_min=645, S_max=699, T_RT=295, laser_type=633): def funzione_raman2(x, p1, p0): kb = 8.617 * 1e-5 h_bar = 4.1356 * 1e-15 c = 299792458 return (((h_bar * c) / (kb * 1e-9)) * ((1 / T_RT) - (1 / p1))) / x + (((h_bar * c) / (kb * 1e-9 * laser_type)) * ((1 / p1) - (1 / T_RT))) + p0 data_cut_x, data_cut_y = self.cut_data_optimal_range( AS_min, AS_max, S_min, S_max) data_cut_x = np.delete(data_cut_x, -1) data_cut_y = np.delete(data_cut_y, -1, 0) data_cut_y = np.log(data_cut_y) T1 = np.zeros((self.number_of_scan)) P1 = np.zeros((self.number_of_scan)) r2 = np.zeros((self.number_of_scan)) ET1 = np.zeros((self.number_of_scan)) EP1 = np.zeros((self.number_of_scan)) resolution = data_cut_x[1] - data_cut_x[0] bandwidth = data_cut_x[-1] - data_cut_x[0] num_point = int(round(bandwidth / resolution)) x_fit = np.zeros(num_point) y_fit = np.zeros((num_point, self.number_of_scan)) for i in range(num_point): x_fit[i] = data_cut_x[0] + resolution * i data_cut_y = sm(data_cut_y).nansubstitute(data_cut_x) for imatr in range(self.number_of_scan): popt, pcov = curve_fit( funzione_raman2, data_cut_x, data_cut_y[:, imatr], p0=[300, 1]) #, bounds=([295, -np.inf],[self.temp_max, np.inf])) y_fit[:, imatr] = funzione_raman2(x_fit, popt[0], popt[1]) T1[imatr] = popt[0] P1[imatr] = popt[1] residual = data_cut_y[:, imatr] - funzione_raman2( data_cut_x, *popt) ss_res = np.sum(residual**2) ss_tot = np.sum( (data_cut_y[:, imatr] - np.mean(data_cut_y[:, imatr]))**2) if ss_tot == 0: ss_tot = 1 ss_res = 1 r2[imatr] = 1 - (ss_res / ss_tot) ############################################################ # p = len(popt) # n = len(data_cut_x) alpha = 0.05 #95% confidence interval dof = max(0, len(data_cut_x) - len(popt)) #degree of freedom tval = tstud.ppf( 1.0 - alpha / 2., dof) #t-student value for the dof and confidence level sigma = np.diag(pcov)**0.5 ET1[imatr] = sigma[0] * tval EP1[imatr] = sigma[1] * tval ######################################################## if self.salva == 'yes': ds().nuova_fig(indice_fig=3) ds().titoli(xtag='nm', ytag='ratio (log)', titolo='') for i in range(self.number_of_scan): ds().dati(data_cut_x, data_cut_y[:, i], colore=palette[i]) ds().dati(x_fit, y_fit[:, i], colore=palette[i]) st.pyplot() ds().nuova_fig(indice_fig=2, indice_subplot=326) ds().titoli(xtag='nm', ytag='ratio (log)', titolo='') for i in range(self.number_of_scan): ds().dati(data_cut_x, data_cut_y[:, i], colore=palette[i]) ds().dati(x_fit, y_fit[:, i], colore=palette[i]) return T1, r2, P1, ET1, EP1
def plot_history(self, fitModel, type='dnn'): if fitModel != 0 and type == 'dnn': history_dict = fitModel.history history_dict.keys() loss = history_dict['loss'] val_loss = history_dict['val_loss'] acc = history_dict['acc'] val_acc = history_dict['val_acc'] epochs = range(1, len(loss) + 1) ds().nuova_fig(1, indice_subplot=211) ds().titoli(titolo="Training loss", xtag='Epochs', ytag='Loss', griglia=0) ds().dati(epochs, loss, descrizione='Training loss', colore='red') ds().dati(epochs, val_loss, descrizione='Validation loss') ds().dati(epochs, loss, colore='red', scat_plot='scat', larghezza_riga=10) ds().dati(epochs, val_loss, scat_plot='scat', larghezza_riga=10) ds().range_plot(bottomY=np.array(val_loss).mean() - np.array(val_loss).std() * 6, topY=np.array(val_loss).mean() + np.array(val_loss).std() * 6) ds().legenda() ds().nuova_fig(1, indice_subplot=212) ds().titoli(titolo="Training loss", xtag='Epochs', ytag='Accuracy', griglia=0) ds().dati(epochs, acc, descrizione='Training loss', colore='red') ds().dati(epochs, val_acc, descrizione='Validation loss') ds().dati(epochs, acc, colore='red', scat_plot='scat', larghezza_riga=10) ds().dati(epochs, val_acc, scat_plot='scat', larghezza_riga=10) ds().range_plot( bottomY=np.array(val_acc).mean() - np.array(val_acc).std() * 6, topY=np.array(val_acc).mean() + np.array(val_acc).std() * 6) ds().legenda() plt.show() elif fitModel != 0 and type == 'xgb': epochs = len(fitModel['validation_0']['error']) x_axis = range(0, epochs) # plot log loss fig, ax = plt.subplots() ax.plot(x_axis, fitModel['validation_0']['logloss'], label='Train') ax.plot(x_axis, fitModel['validation_1']['logloss'], label='Test') ax.legend() plt.ylabel('Log Loss') plt.title('XGBoost Log Loss') plt.show() # plot classification error fig, ax = plt.subplots() ax.plot(x_axis, fitModel['validation_0']['error'], label='Train') ax.plot(x_axis, fitModel['validation_1']['error'], label='Test') ax.legend() plt.ylabel('Classification Error') plt.title('XGBoost Classification Error') plt.show()
def plot_fitness(self): ds().nuova_fig(1) ds().dati(x=np.arange(len(self.fit_min)), y=self.fit_min, colore=palette[0], descrizione='min') ds().dati(x=np.arange(len(self.fit_max)), y=self.fit_max, colore=palette[1], descrizione='max') ds().dati(x=np.arange(len(self.fit_mean)), y=self.fit_mean, colore=palette[2], descrizione='mean') ds().dati(x=np.arange(len(self.fit_top)), y=self.fit_top, colore=palette[3], descrizione='best_ever') ds().legenda() ds().porta_a_finestra()
def bar_plot(self, data_row): empty_plot1 = st.empty() empty_plot2 = st.empty() num_mat = len(data_row['material'].unique().tolist()) i = 0 xx = pd.DataFrame({'radius_nm': data_row['radius_nm'].unique()}) xx = xx['radius_nm'].to_numpy() yy = np.zeros(len(xx)) for material in data_row['material'].unique().tolist(): data = data_row[data_row['material'] == material] data_medie = pd.DataFrame({'radius_nm': data['radius_nm'].unique()}) raggi = data['radius_nm'].unique().tolist() media_quality = [] media_speed = [] media_error_q = [] media_error_s = [] media_material = [] for raggio in raggi: media_quality.append(data['normalized_signal_quality'][data['radius_nm'] == raggio].mean()) media_speed.append(data['signal_speed'][data['radius_nm'] == raggio].mean()) media_error_q.append(data['normalized_signal_quality'][data['radius_nm'] == raggio].std()) media_error_s.append(data['signal_speed'][data['radius_nm'] == raggio].std()) media_material.append(material) data_medie['normalized_signal_quality'] = media_quality data_medie['signal_speed'] = media_speed data_medie['normalized_signal_quality_err'] = media_error_q data_medie['signal_speed_err'] = media_error_s data_medie['material'] = media_material data_medie = data_medie.fillna(0) st.write(material) st.table(data) delta_delay = (12)/num_mat delay = -3 + delta_delay*i ds().nuova_fig(30) ds().titoli(titolo='Normalized Signal Intensity', xtag='radius[nm]', ytag='counts') ds().dati(x = data_medie['radius_nm'].to_numpy(), y = data_medie['normalized_signal_quality'].to_numpy(), scat_plot = 'bar', delay = delay, width = 3, descrizione=material) ds().dati(x = data_medie['radius_nm'].to_numpy()+delay/2, y = data_medie['normalized_signal_quality'].to_numpy(), y_error=data_medie['normalized_signal_quality_err'].to_numpy()/2, scat_plot = 'err', colore='black') ds().dati(x = data['radius_nm']+delay/2, y = data['normalized_signal_quality'], scat_plot ='scat', colore="blue", larghezza_riga =12, layer = 2) ds().dati(x = xx, y = yy, scat_plot ='bar', width = 3, delay = 0) ds().legenda() ds().nuova_fig(31) ds().titoli(titolo='Slope (C)', xtag='radius[nm]', ytag='T/I [k/uW]') ds().dati(x = data_medie['radius_nm'].to_numpy(), y = data_medie['signal_speed'].to_numpy(), scat_plot = 'bar', delay = delay, width = 3, descrizione=material) ds().dati(x = data_medie['radius_nm'].to_numpy()+delay/2, y = data_medie['signal_speed'].to_numpy(), y_error=data_medie['signal_speed_err'].to_numpy()/2, scat_plot = 'err', colore='black') ds().dati(x = data['radius_nm']+delay/2, y = data['signal_speed'], scat_plot ='scat', colore="blue", larghezza_riga =12, layer = 2) ds().dati(x = xx, y = yy, scat_plot ='bar', width = 3, delay = 0) ds().legenda() i = i+1 ds().nuova_fig(30) empty_plot1.pyplot() ds().nuova_fig(31) empty_plot2.pyplot()
def temperature_Raman(self, S_min=645, T_RT=295, laser_type=633): def funzione_raman(x, p1, p0): h_bar = 4.1356 * 1e-15 T_0 = T_RT kb = 8.617 * 1e-5 c = 299792458 l_laser = laser_type w_laser = c / (l_laser * 1e-9) # w_laser = 0 numeratore = np.exp( (h_bar * (c / (x * 1e-9) - w_laser)) / (kb * T_0)) - 1 denominatore = np.exp( (h_bar * (c / (x * 1e-9) - w_laser)) / (kb * p1)) - 1 return p0 * numeratore / denominatore data_cut_x, data_cut_y = self.data_x, self.data_y data_cut_x = np.asarray(data_cut_x) T1 = np.zeros((self.number_of_scan)) P1 = np.zeros((self.number_of_scan)) r2 = np.zeros((self.number_of_scan)) ET1 = np.zeros((self.number_of_scan)) EP1 = np.zeros((self.number_of_scan)) resolution = data_cut_x[1] - data_cut_x[0] bandwidth = data_cut_x[-1] - data_cut_x[0] num_point = int(round(bandwidth / resolution)) x_fit = np.zeros(num_point) y_fit = np.zeros((num_point, self.number_of_scan)) for i in range(num_point): x_fit[i] = data_cut_x[0] + resolution * i for k in range(len(data_cut_x)): if data_cut_x[k] <= S_min + 1: i_x_min_S = k #indice inferiore di dove parte lo Stokes per poi calcolare la potenza media da mettere come condizione nel fit data_cut_y = sm(data_cut_y).nansubstitute(data_cut_x) for imatr in range(self.number_of_scan): potenza_media = 0 for k in range(i_x_min_S, len(data_cut_x)): potenza_media = data_cut_y[k, imatr] + potenza_media potenza_media = potenza_media / (len(data_cut_x) - i_x_min_S) popt, pcov = curve_fit( funzione_raman, data_cut_x, data_cut_y[:, imatr], bounds=([295, potenza_media - (potenza_media / self.divP)], [ self.temp_max, potenza_media + (potenza_media / self.divP) ])) # popt, pcov = curve_fit(funzione_raman, data_cut_x, data_cut_y[:,imatr], bounds=([295, -np.inf],[self.temp_max, np.inf])) y_fit[:, imatr] = funzione_raman(x_fit, popt[0], popt[1]) T1[imatr] = popt[0] P1[imatr] = popt[1] residual = data_cut_y[:, imatr] - funzione_raman(data_cut_x, *popt) ss_res = np.sum(residual**2) ss_tot = np.sum( (data_cut_y[:, imatr] - np.mean(data_cut_y[:, imatr]))**2) if ss_tot == 0: ss_tot = 1 ss_res = 1 r2[imatr] = 1 - (ss_res / ss_tot) ############################################################ # p = len(popt) # n = len(data_cut_x) alpha = 0.05 #95% confidence interval dof = max(0, len(data_cut_x) - len(popt)) #degree of freedom tval = tstud.ppf( 1.0 - alpha / 2., dof) #t-student value for the dof and confidence level sigma = np.diag(pcov)**0.5 ET1[imatr] = sigma[0] * tval EP1[imatr] = sigma[1] * tval ######################################################## if self.salva == 'yes': ds().nuova_fig(indice_fig=7) ds().titoli(xtag='nm', ytag='ratio', titolo='') for i in range(self.number_of_scan): ds().dati(data_cut_x, data_cut_y[:, i], colore=palette[i]) ds().dati(x_fit, y_fit[:, i], colore=palette[i]) # st.pyplot() ds().nuova_fig(indice_fig=2, indice_subplot=324) ds().titoli(xtag='nm', ytag='ratio', titolo='') for i in range(self.number_of_scan): ds().dati(data_cut_x, data_cut_y[:, i], colore=palette[i]) ds().dati(x_fit, y_fit[:, i], colore=palette[i]) return T1, r2, P1, ET1, EP1
def plot_true_vs_predict(self, true, predict, naive, monica): ds().nuova_fig(10001) ds().titoli(titolo='X', xtag='real', ytag='predict') ds().dati(x=true[:, 0], y=predict[:, 0], scat_plot='scat', colore='green', larghezza_riga=15, descrizione='CNN') ds().dati(x=true[:, 0], y=naive[:, 0], scat_plot='scat', colore='blue', larghezza_riga=15, descrizione='naive') ds().dati(x=true[:, 0], y=monica[:, 0], scat_plot='scat', colore='red', larghezza_riga=15, descrizione='monica') plt.plot([true[:, 0].min(), true[:, 0].max()], [true[:, 0].min(), true[:, 0].max()], linestyle='--') ds().legenda() st.pyplot() ds().nuova_fig(10002) ds().titoli(titolo='Y', xtag='real', ytag='predict') ds().dati(x=true[:, 1], y=predict[:, 1], scat_plot='scat', colore='green', larghezza_riga=15, descrizione='CNN') ds().dati(x=true[:, 1], y=naive[:, 1], scat_plot='scat', colore='blue', larghezza_riga=15, descrizione='naive') ds().dati(x=true[:, 1], y=monica[:, 1], scat_plot='scat', colore='red', larghezza_riga=15, descrizione='monica') plt.plot([true[:, 1].min(), true[:, 1].max()], [true[:, 1].min(), true[:, 1].max()], linestyle='--') ds().legenda() st.pyplot()
def direct_standard(self, power, AS_min=588, AS_max=616, S_min=645, S_max=699, T_RT=295, laser_type=633, selected_scan=0): def funzione_direct(ratio, P1, P_ref, wave): h_bar = 4.1356 * 1e-15 kb = 8.617 * 1e-5 c = 299792458 l_laser = laser_type * 1e-9 wave = wave * 1e-9 numeratore = (h_bar * c / kb) * ((1 / wave) - (1 / l_laser)) denominatore = np.log((P1 / P_ref) * (1 / ratio) * (np.exp(((h_bar * c) / (kb * T_RT)) * (1 / (wave) - (1 / l_laser))) - 1) + 1) return numeratore / denominatore data_cut_x, data_cut_y = self.cut_data_optimal_range( AS_min, AS_max, S_min, S_max) media_x, media = self.average_over_wavelength(data_cut_x, data_cut_y, AS_min, AS_max, S_min, S_max) if self.salva == 'yes': ds().nuova_fig(indice_fig=7) ds().titoli(xtag='nm', ytag='ratio', titolo='') for i in range(self.number_of_scan): for j in range(len(media[:, 0])): ds().dati(media_x[j], media[j, i], colore=palette[j], scat_plot='scat', larghezza_riga=15) st.pyplot() range_of_wavelength = pd.DataFrame() for i, wave in enumerate(media_x): vet_temp = [] for j, potenza in enumerate(power): vet_temp.append( funzione_direct( media[i, j], potenza, power[self.number_of_scan - 1 - selected_scan], wave)) range_of_wavelength[wave] = vet_temp range_of_wavelength.index = power medie_temperature = range_of_wavelength.mean(axis=1) sigma_temperature = range_of_wavelength.std(axis=1) if self.salva == 'yes': st.write('Temperature density of states') ds().nuova_fig(20) ds().titoli(xtag='P [uW]', ytag='T [k]', titolo='') for j, col in enumerate(range_of_wavelength.columns): ds().dati(range_of_wavelength.index.tolist(), range_of_wavelength[col], colore=palette[j], scat_plot='scat', larghezza_riga=15, descrizione=str(round(col))) ds().legenda() st.pyplot() return medie_temperature, sigma_temperature
def plot_images(self, images, true=np.array([0]), predict=np.array([0]), naive=np.array([0]), monica=np.array([0]), image_selcted=None): x = [i for i in range(images.shape[1])] images = images.reshape(images.shape[0], images.shape[1], images.shape[2]).copy() if image_selcted is not None: ds().nuova_fig(image_selcted) ds().titoli(titolo=str(image_selcted)) ds().dati(x=x, y=x, z=images[image_selcted], scat_plot='cmap') if true.shape[0] > 2: ds().dati(x=true[image_selcted, 0], y=true[image_selcted, 1], scat_plot='scat', colore='black', larghezza_riga=15, layer=2, descrizione='true') if monica.shape[0] > 2: ds().dati(x=monica[image_selcted, 0], y=monica[image_selcted, 1], scat_plot='scat', colore='red', larghezza_riga=15, layer=2, descrizione='monica') st.pyplot() else: for image_num in range(images.shape[0]): ds().nuova_fig(image_num) ds().titoli(titolo=str(image_num)) ds().dati(x=x, y=x, z=images[image_num], scat_plot='cmap') if true.shape[0] > 2: ds().dati(x=true[image_num, 0], y=true[image_num, 1], scat_plot='scat', colore='black', larghezza_riga=15, layer=2, descrizione='true') if predict.shape[0] > 2: ds().dati(x=predict[image_num, 0], y=predict[image_num, 1], scat_plot='scat', colore='green', larghezza_riga=15, layer=2, descrizione='CNN') if naive.shape[0] > 2: ds().dati(x=naive[image_num, 0], y=naive[image_num, 1], scat_plot='scat', colore='blue', larghezza_riga=15, layer=2, descrizione='naive') if monica.shape[0] > 2: ds().dati(x=monica[image_num, 0], y=monica[image_num, 1], scat_plot='scat', colore='red', larghezza_riga=15, layer=2, descrizione='monica') ds().legenda() st.pyplot()
def fit_guass(self, x, y, xy_mesh, xc, yc, amp, intensity_sp, rad, laser_power, punti_cross_section_geometrica=10): def gaussian_2d(xy_mesh, amp, xc, yc, sigma_x, sigma_y, theta): (x, y) = xy_mesh a = np.cos(theta)**2 / (2 * sigma_x**2) + np.sin(theta)**2 / ( 2 * sigma_y**2) b = -np.sin(2 * theta) / (4 * sigma_x**2) + np.sin( 2 * theta) / (4 * sigma_y**2) c = np.sin(theta)**2 / (2 * sigma_x**2) + np.cos(theta)**2 / ( 2 * sigma_y**2) gauss = amp * np.exp(-(a * (x - xc)**2 + 2 * b * (x - xc) * (y - yc) + c * (y - yc)**2)) return np.ravel(gauss) # set initial parameters to build mock data sigma_x, sigma_y = 0.2, 0.2 theta = 0 guess_vals = [amp, xc, yc, sigma_x, sigma_y, theta] fit_params, cov_mat = curve_fit( gaussian_2d, xy_mesh, np.ravel(intensity_sp), p0=guess_vals ) #, bounds=((amp-20,xc-0.015,yc-0.015,0.1,0.1,-np.inf), (amp+20,xc+0.015,yc+0.015,0.2,0.2,np.inf))) intensity_gauss = np.zeros( (int(len(intensity_sp[:, 0])), int(len(intensity_sp[0, :])))) for i in range(len(intensity_sp[:, 0])): for j in range(len(intensity_sp[0, :])): intensity_gauss[i, j] = gaussian_2d( (x[j], y[i]), fit_params[0], fit_params[1], fit_params[2], fit_params[3], fit_params[4], fit_params[5]) xx, yy = xy_mesh if self.salva == 'yes': ds().nuova_fig(indice_fig=2, plot_dimension='3d', indice_subplot=322) ds().titoli(titolo='original with fit') ds().dati(xx, yy, scat_plot='3D', z=intensity_sp) ds().dati(xx, yy, scat_plot='3D_wire', z=intensity_gauss, colore='black') ds().nuova_fig(indice_fig=9, plot_dimension='3d') ds().titoli(titolo='original with fit') ds().dati(xx, yy, scat_plot='3D', z=intensity_sp) ds().dati(xx, yy, scat_plot='3D_wire', z=intensity_gauss, colore='black') st.pyplot() def area_particle(x0=0, y0=0): intensity_sp_sum = 0 x_rad = np.linspace(0, rad * 2, punti_cross_section_geometrica) y_rad = np.linspace(0, rad * 2, punti_cross_section_geometrica) for i in range(len(x_rad)): for j in range(len(y_rad)): x_pos = x_rad[i] - rad + x0 y_pos = y_rad[j] - rad + y0 dist = np.sqrt((x0 - x_pos)**2 + (y0 - y_pos)**2) if dist < rad: valore = gaussian_2d((x_pos, y_pos), fit_params[0], fit_params[1], fit_params[2], fit_params[3], fit_params[4], fit_params[5]) intensity_sp_sum = intensity_sp_sum + valore # intensity_sp_sum = intensity_sp_sum*(2*rad/len(x_rad))*(2*rad/len(y_rad)) intensity_sp_sum = intensity_sp_sum * (3.14 * rad * rad / (len(x_rad) * len(y_rad))) return intensity_sp_sum if self.salva == 'yes': st.text('calculating the geometrical cross-section') intensity_sp_real = np.zeros( (int(len(intensity_sp[:, 0])), int(len(intensity_sp[0, :])))) for i in range(len(intensity_sp[:, 0])): for j in range(len(intensity_sp[0, :])): intensity_sp_real[i, j] = area_particle(x[j], y[i]) tot_intensity = sum(sum(intensity_sp_real)) # intensity_sp = intensity_sp/10000 # intensity_sp_real[:,:] = intensity_sp_real[:,:]*(laser_power/tot_intensity) intensity_sp_real[:, :] = intensity_sp_real[:, :] * ( laser_power / tot_intensity) / (rad * rad * 3.14) # intensity_sp_real[:,:] = intensity_sp_real[:,:]*(laser_power/sum(sum(intensity_sp)))/(rad*rad*3.14) # intensity_sp_real[:,:] = intensity_sp_real[:,:]*(laser_power/sum(sum(intensity_sp))) return intensity_sp_real
def calc_deriv_thershold(self): raw_PL = self.raw_data y_deriv = np.zeros((len(raw_PL[0]), self.num_of_scan, self.num_of_scan)) dx = raw_PL[0][1] - raw_PL[0][0] for i in range(self.num_of_scan): for j in range(self.num_of_scan): y_deriv[:,i,j] = np.gradient(raw_PL[1][:,i,j], dx) S_min = 640 i_pl = 0 for i in range(len(raw_PL[0])): if raw_PL[0][i] < S_min+1: i_pl = i ds().nuova_fig(indice_fig=1, indice_subplot =211, width =12) ds().titoli(titolo='', xtag="",ytag="Derivate",griglia=2) max_S = 0 for i in range(self.num_of_scan): for j in range(self.num_of_scan): ds().dati(raw_PL[0], y_deriv[:,i,j], colore=palette[i]) if max_S < max(y_deriv[i_pl:,i,j]): max_S = max(y_deriv[i_pl:,i,j]) ds().range_plot(bottomY = 0, topY =100+max_S) ds().nuova_fig(indice_fig=1, indice_subplot =212, width =12) ds().titoli(titolo='', xtag="",ytag="PL",griglia=2) max_S = 0 for i in range(self.num_of_scan): for j in range(self.num_of_scan): ds().dati(raw_PL[0], raw_PL[1][:,i,j], colore=palette[i]) if max_S < max(raw_PL[1][i_pl:,i,j]): max_S = max(raw_PL[1][i_pl:,i,j]) ds().range_plot(bottomY = 0, topY =100+max_S) st.pyplot()
p = len(par1) n = len(x1) alpha = 0.05 #95% confidence interval dof = max(0, len(x1) - len(par1)) #degree of freedom tval = tstud.ppf( 1.0 - alpha / 2., dof) #t-student value for the dof and confidence level sigma = np.diag(par2)**0.5 m_err = sigma[0] * tval q_err = sigma[1] * tval y_fit2_up = retta(x1, m + (m_err / 2), q + (q_err / 2)) y_fit2_down = retta(x1, m - (m_err / 2), q - (q_err / 2)) ds().nuova_fig(indice_fig=15) ds().titoli(xtag='I [mW/um^2]', ytag='T [k]', titolo='') ds().dati(save_matr['xx0'], save_matr['yy0'], x_error=[save_matr['ex10'], save_matr['ex20']], y_error=save_matr['ey0'], scat_plot='err') ds().dati(x1, y_fit2, colore='black', descrizione=str(round(par1[0], 2)) + '*X + ' + str(round(par1[1], 2)) + '\n' + str(round(m, 2)) + ' +/- ' + str(round(m_err, 2)) + '\n' + str(round(q, 2)) + ' +/- ' + str(round(q_err, 2))) plt.fill_between(x1, y_fit2_down,
def good_to_go(self, AS_min=590, AS_max=616, S_min=645, S_max=699, wave_inf=591, wave_sup=616, num_bin=10, numb_of_spect=10, laser_power=1.75, raggio=0.07, lato_cella=0.05, riga_y_start=0, riga_y_stop=1, pixel_0=[3, 0], temp_max=550, T_RT=295, laser_type=633, salva='no', cut_min=0, cut_max=0, selected_scan=0, punti_cross_section_geometrica=10): def download_file(data, filename): testo = 'Download ' + filename + '.csv' csv = data.to_csv(index=False) b64 = base64.b64encode(csv.encode()).decode() href = f'<a href="data:file/csv;base64,{b64}" download="{filename}.csv">' + testo + '</a>' st.markdown(href, unsafe_allow_html=True) """ calculated the heat map wave_inf e wave_sup rappresentano l'intervallo in cui viene calcolato lo stokes per la heatmap """ # ██ ██████ █████ ██████ # ██ ██ ██ ██ ██ ██ ██ # ██ ██ ██ ███████ ██ ██ # ██ ██ ██ ██ ██ ██ ██ # ███████ ██████ ██ ██ ██████ if self.remove_cosmic_ray == 'yes': if salva == 'yes': st.text('cosmic ray cleaning') raw_PL = pp(self.raw_data, num_of_scan=self.num_of_scan).remove_cosmic_ray( self.soglia_derivata) else: raw_PL = self.raw_data # ██████ ███████ ███ ███ ██████ ██ ██ ███████ ██████ ██ ██ ██████ # ██ ██ ██ ████ ████ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ # ██████ █████ ██ ████ ██ ██ ██ ██ ██ █████ ██████ █████ ██ ███ # ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ # ██ ██ ███████ ██ ██ ██████ ████ ███████ ██████ ██ ██ ██████ if salva == 'yes': st.text('remove background') PL_x, PL_y = nr(file_data=raw_PL, map_conf='yes').bkg_map(laser_type=laser_type, bkg_position=self.bkg_position, x_bkg=self.x_bkg, y_bkg=self.y_bkg) # ██ ███ ██ ████████ ███████ ███ ██ ███████ ██ ████████ ██ ██ ███ ███ █████ ██████ # ██ ████ ██ ██ ██ ████ ██ ██ ██ ██ ██ ██ ████ ████ ██ ██ ██ ██ # ██ ██ ██ ██ ██ █████ ██ ██ ██ ███████ ██ ██ ████ ██ ████ ██ ███████ ██████ # ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ # ██ ██ ████ ██ ███████ ██ ████ ███████ ██ ██ ██ ██ ██ ██ ██ ██ if salva == 'yes': st.text('creating the intensity map') # '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' #sostituisce un certo numero di righe sulla intensity map con l'intensita del pixel_0 #serve per eliminare il rumore di fondo quando e' troppo alto in picocle porzioni del grafico for i in range(0, len(PL_y[0, :, 0])): # X for j in range(riga_y_start, riga_y_stop): # Y PL_y[:, j, i] = PL_y[:, pixel_0[0], pixel_0[1]] # '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' intensity_sp = np.zeros((len(PL_y[0, :, 0]), len(PL_y[0, 0, :]))) for k in range(len(PL_y[0, :, 0])): for i in range(len(PL_x)): if wave_inf > PL_x[i]: i591 = i for i in range(len(PL_x)): if wave_sup > PL_x[i]: i616 = i for j in range(self.num_of_scan): sum_temp = 0 for i in range(i591, i616): sum_temp = PL_y[i, k, j] + sum_temp sum_temp = sum_temp / (i616 - i591) intensity_sp[k, j] = sum_temp x = [i * lato_cella for i in range(len(intensity_sp[:, 0]))] y = [i * lato_cella for i in range(len(intensity_sp[0, :]))] xy = np.meshgrid(x, y) ##################################################################### if salva == 'yes': ds().nuova_fig(indice_fig=4) ds().titoli(titolo='', xtag='um', ytag='um') ds().dati(x, y, scat_plot='cmap', z=intensity_sp) ds().nuova_fig(indice_fig=2, indice_subplot=321, width=15, height=10) ds().titoli(titolo=self.nome_file, xtag='um', ytag='um') ds().dati(x, y, scat_plot='cmap', z=intensity_sp) ##################################################################### # ███████ ██████ ██████ ████████ ██ ███ ██ ████████ ███████ ███ ██ ███████ ██ ████████ ██ ██ # ██ ██ ██ ██ ██ ██ ██ ████ ██ ██ ██ ████ ██ ██ ██ ██ ██ ██ # ███████ ██ ██ ██████ ██ ██ ██ ██ ██ ██ █████ ██ ██ ██ ███████ ██ ██ ████ # ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ # ███████ ██████ ██ ██ ██ ██ ██ ████ ██ ███████ ██ ████ ███████ ██ ██ ██ if salva == 'yes': st.text('sorting the intensity') sorted_intensity0 = np.zeros(len(PL_y[0, :, 0]) * len(PL_y[0, 0, :])) k = 0 for i in range(len(intensity_sp[:, 0])): for j in range(len(intensity_sp[0, :])): sorted_intensity0[k] = intensity_sp[i, j] k = k + 1 sorted_intensity0 = sorted(sorted_intensity0, reverse=True) ##################################################################################### intens_coord = np.where(intensity_sp == sorted_intensity0[0]) if len(intens_coord) == 1: intens_coord_y = intens_coord[0] intens_coord_x = intens_coord[1] else: intens_coord_y = intens_coord[0][0] intens_coord_x = intens_coord[1][0] # xc = x[intens_coord_x] # yc = y[intens_coord_y] # intensity_sp2 = im(salva).fit_guass(x=x, y=y, xy_mesh= xy, amp=sorted_intensity0[0], intensity_sp=intensity_sp, xc=xc, yc=yc, rad = raggio, laser_power=laser_power, punti_cross_section_geometrica = punti_cross_section_geometrica) intensity_sp2 = im(salva).eval_power(xy_mesh=xy, intensity_sp=intensity_sp, laser_power=laser_power) ##################################################################################### sorted_intensity = np.zeros(len(PL_y[0, :, 0]) * len(PL_y[0, 0, :])) k = 0 for i in range(len(intensity_sp2[:, 0])): for j in range(len(intensity_sp2[0, :])): sorted_intensity[k] = intensity_sp2[i, j] k = k + 1 sorted_intensity = sorted(sorted_intensity, reverse=True) # tot_intensity = sum(sorted_intensity) # ██████ ██ ███ ██ ███ ██ ██ ███ ██ ██████ # ██ ██ ██ ████ ██ ████ ██ ██ ████ ██ ██ # ██████ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ███ # ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ # ██████ ██ ██ ████ ██ ████ ██ ██ ████ ██████ if salva == 'yes': st.text('binning') num_divisions = int(len(sorted_intensity) / num_bin) PL_bin = np.zeros((len(PL_y[:, 0, 0]), num_divisions)) average_intensity = np.zeros(num_divisions) average_intensity_vet = np.zeros(num_bin) average_intensity_err = np.zeros((2, num_divisions)) coordX_intesita = np.zeros((num_divisions, num_bin)) coordY_intesita = np.zeros((num_divisions, num_bin)) k = 0 for i in range(num_divisions): for j in range(num_bin): intens_coord = np.where(intensity_sp2 == sorted_intensity[k]) if len(intens_coord) == 1: intens_coord_y = intens_coord[0] intens_coord_x = intens_coord[1] else: intens_coord_y = intens_coord[0][0] intens_coord_x = intens_coord[1][0] coordX_intesita[i, j] = x[ intens_coord_x] + 0.025 #è normale che siano invertiti x e y coordY_intesita[i, j] = y[intens_coord_y] + 0.025 for l in range(len(PL_y[:, 0, 0])): PL_bin[l, i] = PL_y[l, intens_coord_y, intens_coord_x] + PL_bin[l, i] average_intensity_vet[j] = intensity_sp2[intens_coord_y, intens_coord_x] average_intensity[i] = intensity_sp2[ intens_coord_y, intens_coord_x] + average_intensity[i] k = k + 1 average_intensity[i] = average_intensity[i] / num_bin average_intensity_err[ 0, i] = average_intensity[i] - min(average_intensity_vet) average_intensity_err[ 1, i] = max(average_intensity_vet) - average_intensity[i] if salva == 'yes': ds().nuova_fig(indice_fig=4) for i in range(numb_of_spect): ds().dati(coordX_intesita[i, :], coordY_intesita[i, :], scat_plot='scat', colore=palette[i], larghezza_riga=30, layer=2) st.pyplot() ds().nuova_fig(indice_fig=2, indice_subplot=321) for i in range(numb_of_spect): ds().dati(coordX_intesita[i, :], coordY_intesita[i, :], scat_plot='scat', colore=palette[i], larghezza_riga=30, layer=2) # ██████ ███████ ███████ ██ ██ █████ ██████ ███████ # ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ # ██████ █████ ███████ ███████ ███████ ██████ █████ # ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ # ██ ██ ███████ ███████ ██ ██ ██ ██ ██ ███████ if salva == 'yes': st.text('select only the chosen number of spectrum') col_num = [i for i in range(numb_of_spect)] PL_bin = sm(PL_bin).rescape(col_num) #the code here add to all the selected spectrum the smallest value in the anistokes, so no spectrum goes below 0 #still experimenting with this # for i in range(len(PL_x)): # if AS_min>PL_x[i]: # iAS_min = i # if AS_max>PL_x[i]: # iAS_max = i # PL_bin = PL_bin + np.abs(PL_bin[iAS_min:iAS_max,:].min()) average_intensity_new = np.zeros(numb_of_spect) average_intensity_err_new = [[i] * numb_of_spect for i in range(2)] average_intensity_new = [ average_intensity[i] for i in range(numb_of_spect) ] for i in range(2): for j in range(numb_of_spect): average_intensity_err_new[i][j] = average_intensity_err[i, j] # ███████ ███ ███ ██████ ██████ ████████ ██ ██ # ██ ████ ████ ██ ██ ██ ██ ██ ██ ██ # ███████ ██ ████ ██ ██ ██ ██ ██ ██ ███████ # ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ # ███████ ██ ██ ██████ ██████ ██ ██ ██ if salva == 'yes': st.text('smoothing') ncycl = 4 y_smooth = pl(PL_x, PL_bin, len(PL_bin[0])).smoothing_fft(ncycl) y_smooth = pl(PL_x, y_smooth, len(PL_bin[0])).smoothing_fft(ncycl) y_smooth = pl(PL_x, y_smooth, len(PL_bin[0])).smoothing_fft(ncycl) y_smooth = pl(PL_x, y_smooth, len(PL_bin[0])).smoothing_fft(ncycl) ##################################################################### i_pl = 0 for i in range(len(PL_x)): if PL_x[i] < S_min + 1: i_pl = i if salva == 'yes': ds().nuova_fig(indice_fig=5) ds().titoli(titolo='', xtag="nm", ytag='count') for i in range(len(PL_bin[0, :])): ds().dati(PL_x, y_smooth[:, i], colore=palette[i], descrizione=str(i)) ds().range_plot(bottomX=AS_min, topX=S_max, bottomY=-10, topY=100 + max(y_smooth[i_pl:, 0])) # ds().range_plot(bottomX=AS_min, topX=AS_max, bottomY=-10, topY=1000) save_plot = pd.DataFrame() save_plot['x'] = PL_x for i in range(len(PL_bin[0, :])): save_plot[i] = y_smooth[:, i] download_file(save_plot, self.nome_file + '_PL') st.pyplot() ds().nuova_fig(indice_fig=2, indice_subplot=323) ds().titoli(titolo='', xtag="nm", ytag='count') for i in range(len(PL_bin[0, :])): ds().dati(PL_x, y_smooth[:, i], colore=palette[i], descrizione=str(i)) ds().range_plot(bottomX=AS_min, topX=S_max, bottomY=-10, topY=100 + max(y_smooth[i_pl:, 0])) signal_quality = max(y_smooth[i_pl:, 0]) ##################################################################### # ██████ █████ ████████ ██ ██████ # ██ ██ ██ ██ ██ ██ ██ ██ # ██████ ███████ ██ ██ ██ ██ # ██ ██ ██ ██ ██ ██ ██ ██ # ██ ██ ██ ██ ██ ██ ██████ # y_smooth = np.zeros((len(self.data_y), self.number_of_scan)) if salva == 'yes': st.text('creating the ratio') PL_ratio_raman = sa(PL_x, y_smooth, salva, len(y_smooth[0]), temp_max=temp_max).power_ratio_plain( AS_min=AS_min, AS_max=AS_max, S_min=S_min, S_max=S_max, cut_min=cut_min, cut_max=cut_max, selected_scan=selected_scan) if salva == 'yes': st.text('fit with the raman function') T_raman, R2_raman, _, ET_raman, _ = sa( PL_ratio_raman[0], PL_ratio_raman[1], salva, len(y_smooth[0]), temp_max=temp_max).temperature_Raman(S_min=S_min, T_RT=T_RT, laser_type=laser_type) if salva == 'yes': save_plot = pd.DataFrame() save_plot['x'] = PL_ratio_raman[0] for i in range(PL_ratio_raman[1].shape[1]): save_plot[i] = PL_ratio_raman[1][:, i] download_file(save_plot, self.nome_file + '_ratio') if self.log_scale == 3: T_direct_standard, ET_direct_standard = sa( PL_ratio_raman[0], PL_ratio_raman[1], salva, len(y_smooth[0]), temp_max=temp_max).direct_standard(average_intensity_new, S_min=S_min, S_max=S_max, AS_min=AS_min, AS_max=AS_max, T_RT=T_RT, laser_type=laser_type, selected_scan=selected_scan) else: T_dense, ET_dense = sa(PL_ratio_raman[0], PL_ratio_raman[1], salva, len(y_smooth[0]), temp_max=temp_max).ratio_vs_power( average_intensity_new, S_min=S_min, S_max=S_max, AS_min=AS_min, AS_max=AS_max, T_RT=T_RT, laser_type=laser_type, selected_scan=selected_scan) if salva == 'yes': st.text('Logaritmic scale') controllo_negativi = 0 for i in range(PL_ratio_raman[1].shape[0] * PL_ratio_raman[1].shape[1]): controllo_temp = PL_ratio_raman[1].flat[i] if controllo_temp <= 0: controllo_negativi = 1 if controllo_negativi == 0: T_log, R2_log, _, ET_log, _ = sa(PL_ratio_raman[0], PL_ratio_raman[1], salva, len(y_smooth[0]), temp_max=temp_max).log_ratio( S_min=S_max, S_max=S_max, AS_min=AS_min, AS_max=AS_max, T_RT=T_RT, laser_type=laser_type) else: if self.log_scale == 1: self.log_scale = 2 st.error( 'impossible calculate the log, a negative number is present, standard scale is used instead' ) #### y_nan = sm(PL_ratio_y).nansubstitute(PL_ratio_x) if self.log_scale == 1: T_raman = T_log R2_raman = R2_log ET_raman = ET_log elif self.log_scale == 2: T_raman = T_dense.tolist() R2_raman = [1 for i in range(len(T_dense))] ET_raman = ET_dense.tolist() elif self.log_scale == 3: T_raman = T_direct_standard.tolist() R2_raman = [1 for i in range(len(T_direct_standard))] ET_raman = ET_direct_standard.tolist() # ███████ ██ ████████ # ██ ██ ██ # █████ ██ ██ # ██ ██ ██ # ██ ██ ██ if salva == 'yes': st.text('fit the temperature with the power dependency') def retta(x, p0, p1): return p0 * x + p1 x1 = np.array(average_intensity_new, dtype="float") y2 = np.array(T_raman, dtype="float") par1, par2 = curve_fit(retta, x1, y2) y_fit2 = retta(x1, par1[0], par1[1]) m = par1[0] q = par1[1] # residual = y2 - y_fit2 # ss_res = np.sum(residual**2) ss_tot = np.sum((y2 - np.mean(y2))**2) if ss_tot == 0: ss_tot = 1 # ss_res = 1 # r2 = 1- (ss_res/ss_tot) # p = len(par1) # n = len(x1) alpha = 0.05 #95% confidence interval dof = max(0, len(x1) - len(par1)) #degree of freedom tval = tstud.ppf( 1.0 - alpha / 2., dof) #t-student value for the dof and confidence level sigma = np.diag(par2)**0.5 m_err = sigma[0] * tval q_err = sigma[1] * tval y_fit2_up = retta(x1, m + (m_err / 2), q + (q_err / 2)) y_fit2_down = retta(x1, m - (m_err / 2), q - (q_err / 2)) ##################################################################### if salva == 'yes': ds().nuova_fig(indice_fig=6) # ds().titoli(xtag='I [mW/um^2]', ytag = 'T [k]', titolo='') ds().titoli(xtag='P [uW]', ytag='T [k]', titolo='') ds().dati(average_intensity_new, T_raman, x_error=average_intensity_err_new, y_error=ET_raman, scat_plot='err') ds().dati(x1, y_fit2, colore='black', descrizione=str(round(par1[0], 2)) + '*X + ' + str(round(par1[1], 2)) + '\n' + str(round(m, 2)) + ' +/- ' + str(round(m_err, 2)) + '\n' + str(round(q, 2)) + ' +/- ' + str(round(q_err, 2))) plt.fill_between(x1, y_fit2_down, y_fit2_up, color='black', alpha=0.15) ds().legenda() st.pyplot() ds().nuova_fig(indice_fig=2, indice_subplot=325) # ds().titoli(xtag='I [mW/um^2]', ytag = 'T [k]', titolo='') ds().titoli(xtag='P [uW]', ytag='T [k]', titolo='') ds().dati(average_intensity_new, T_raman, x_error=average_intensity_err_new, y_error=ET_raman, scat_plot='err') ds().dati(x1, y_fit2, colore='black', descrizione=str(round(par1[0], 2)) + '*X + ' + str(round(par1[1], 2)) + '\n' + str(round(m, 2)) + ' +/- ' + str(round(m_err, 2)) + '\n' + str(round(q, 2)) + ' +/- ' + str(round(q_err, 2))) plt.fill_between(x1, y_fit2_down, y_fit2_up, color='black', alpha=0.15) ds().legenda() # if salva != 'yes': # ds().porta_a_finestra(chiudi = 1) ##################################################################### # ███████ █████ ██ ██ ███████ # ██ ██ ██ ██ ██ ██ # ███████ ███████ ██ ██ █████ # ██ ██ ██ ██ ██ ██ # ███████ ██ ██ ████ ███████ matrice_salve2 = pd.DataFrame() matrice_salve2['average_intensity_new'] = average_intensity_new matrice_salve2['Temperature'] = T_raman matrice_salve2[ 'average_intensity_err_low'] = average_intensity_err_new[0] matrice_salve2['average_intensity_err_up'] = average_intensity_err_new[ 1] matrice_salve2['Temperature_err'] = ET_raman matrice_salve2['Temperature_r2'] = R2_raman matrice_salve3 = pd.DataFrame() matrice_salve3['signal_quality'] = [signal_quality] matrice_salve3['signal_speed'] = [round(par1[0], 2)] #coefficiente angolare matrice_salve3['radius'] = [raggio] matrice_salve3['laser_type'] = [laser_type] matrice_salve3['laser_power'] = [laser_power] matrice_salve3['material'] = [self.material] if salva == 'yes': st.pyplot() return par1[1], matrice_salve2, matrice_salve3
def train(self, data, latent_dim, n_epochs=10000, n_batch=128, n_eval=2000, verbose=False, save=False): ''' training function. data: training set in formato pandas latent_dim: (int) dimensione del vettore dello spazio latente, deve essere lungo quanto il primo layer del modello n_epochs: (int) numero totale di epoche per il training n_batch: (int) numero di campioni da usare per il training, se usato un numero maggiore alla dimensione di data, il valore viene convertito alla lunghezza di data n_eval: (int) ogni quante epoche viene salvato il modello e valutata l'accuratezza verbose: (boolean) if True viene printata l'accuratezza ogni n_eval e generato il plot save: (boolean) if True viene salvato il modello ogni n_eval ''' size_single_sample = data.shape[1] # # create the discriminator discriminator = self.define_discriminator(n_inputs=size_single_sample) # # create the generator generator = self.define_generator(latent_dim, n_outputs=size_single_sample) # # create the gan gan_model = self.define_gan(generator, discriminator) # determine half the size of one batch, for updating the discriminator if n_batch > data.shape[1]: n_batch = data.shape[1] half_batch = int(n_batch / 2) sum_epoch, sum_acc_real, sum_acc_fake = [], [], [] sum_d_loss_real, sum_d_loss_fake, sum_g_loss = [], [], [] # manually enumerate epochs for i in range(n_epochs): # prepare real samples x_real, y_real = self.generate_real_samples(data, half_batch) # prepare fake examples x_fake, y_fake = self.generate_fake_samples( generator, latent_dim, half_batch) # update discriminator d_loss_real = discriminator.train_on_batch(x_real, y_real) d_loss_fake = discriminator.train_on_batch(x_fake, y_fake) # prepare points in latent space as input for the generator x_gan = self.generate_latent_points(latent_dim, n_batch) # create inverted labels for the fake samples y_gan = np.ones((n_batch, 1)) # update the generator via the discriminator✬s error g_loss = gan_model.train_on_batch(x_gan, y_gan) # evaluate the model every n_eval epochs if (i + 1) % n_eval == 0: temp_epoch, temp_real, temp_fake = self.summarize_performance( data, i, generator, discriminator, latent_dim, verbose) sum_epoch.append(temp_epoch) sum_acc_real.append(temp_real) sum_acc_fake.append(temp_fake) sum_d_loss_real.append(d_loss_real[0]) sum_d_loss_fake.append(d_loss_fake[0]) sum_g_loss.append(g_loss) if save: self.save_model(generator, filename='generator_model_%03d.h5' % (i + 1)) ds().nuova_fig(4) ds().dati(x=sum_epoch, y=sum_acc_real, colore=palette[0], descrizione='acc real') ds().dati(x=sum_epoch, y=sum_acc_fake, colore=palette[1], descrizione='acc fake') ds().legenda() ds().porta_a_finestra() ds().nuova_fig(5) ds().dati(x=sum_epoch, y=sum_d_loss_real, colore=palette[0], descrizione='loss dis real') ds().dati(x=sum_epoch, y=sum_d_loss_fake, colore=palette[1], descrizione='loss dis fake') ds().legenda() ds().porta_a_finestra() ds().nuova_fig(5) ds().dati(x=sum_epoch, y=sum_g_loss, colore=palette[2], descrizione='loss gan') ds().legenda() ds().porta_a_finestra()
def preview_generator(self, sample, num_elements_x, num_elements_y, raggio_iniziale, raggio_finale, step_size, distanza_tra_cerchi, dimArr, distanza_dosi_x, distanza_dosi_y, layer, step_dose, dose_base, width, vertici, rotazione, altezza, base, lato_cost, text_label, pitch_object_x, pitch_object_y, pitch_choice_text, dose_marker, starting_high, ending_high, high_step, angle, feature_check): if distanza_tra_cerchi < 30: distanza_tra_cerchi = 30 ############ for the markers ############ if dimArr > distanza_tra_cerchi: dimArr = distanza_tra_cerchi - 20 # size if the array which lay within the markers ############################################################ num_raggio = round((raggio_finale - raggio_iniziale) / step_size) if feature_check == 'bowtie_multi': num_raggio = round((ending_high - starting_high) / high_step) x = np.zeros(num_elements_x) y = np.zeros(num_elements_y) # distance between the elements with the same size (distance between two different doses) pitch_x = (distanza_tra_cerchi) * (num_raggio) + distanza_dosi_x pitch_y = distanza_dosi_y for i in range(num_elements_x): x[i] = i * pitch_x for j in range(num_elements_y): y[j] = j * pitch_y for k in range(num_raggio): dose_i = 0 for j in range(num_elements_y): for i in range(num_elements_x): # draw the preview of the pattern pattern().disegna_pattern(x[i] + k * distanza_tra_cerchi, y[j], dimArr) # draw the preview of the markers # pattern().disegna_pattern(x[i] + k*distanza_tra_cerchi - 10, y[j] - 10, dimArr+20, colore= 'black') pattern().disegna_marker(x[i] + k * distanza_tra_cerchi - 5, y[j] - 5, 1, 10, colore="#00A3E0") pattern().disegna_marker(x[i] + k * distanza_tra_cerchi - 5 + dimArr + 10, y[j] - 5, 1, 10, colore="#00A3E0") pattern().disegna_marker(x[i] + k * distanza_tra_cerchi - 5, y[j] - 5 + dimArr + 10, 1, 10, colore="#00A3E0") pattern().disegna_marker(x[i] + k * distanza_tra_cerchi - 5 + dimArr + 10, y[j] - 5 + dimArr + 10, 1, 10, colore="#00A3E0") # draw the preview of the labels for each array pattern().disegna_pattern(x[i] + (dimArr / 2) + k * distanza_tra_cerchi, y[j] + dimArr + 30, 1, colore='green') dose_i = dose_i + 1 dose_i = 0 for j in range(num_elements_y): for i in range(num_elements_x): # draw the preview of the dose label pattern().disegna_pattern( x[i] + (num_raggio) * distanza_tra_cerchi + 20, y[j] + dimArr / 2, 1) dose_i = dose_i + 1 for k in range(round(num_raggio / 2)): for j in range(num_elements_y + 3): for i in range(num_elements_x + 2): # draw the preview of the writing field pattern().disegna_pattern(x[0] - 10 + i * 100 + k * 100, y[0] - 10 + j * 100, 100, colore='#db3f3f') ds().aggiusta_la_finestra()