def on_epoch_end(self,epoch, batch, logs={}): X_val,X_val_gcc, y_val = self.validation_data[0], self.validation_data[1],self.validation_data[2] # Calculate the predictions on test data, in order to calculate ER and F scores pred = model.predict([X_val,X_val_gcc]) #È true o false pred_thresh = pred > 0.5 #0.5 threeshold vedi paper #print pred_thresh score_list = metrics.compute_scores(pred_thresh, y_val, frames_in_1_sec=frames_1_sec) #Y_TEST shape = (220,256,6) #score list è un dizionario self._f1 = score_list['f1_overall_1sec'] self._er = score_list['er_overall_1sec'] if self._er > self._er_prev: self._fail_count+=1 if self._fail_count >= 10: print('Epoch: ', epoch, ' Custom ER: ', self._er, ' Failcount: ', self._fail_count ) self.model.stop_training = True else: #resetto il patience count self._fail_count = 0 #aggiorno print(' Prev: ', self._er_prev ,' Custom ER: ', self._er,' Failcount: ', self._fail_count ) self._er_prev = self._er self._er_list.append(self._er) self._f1_list.append(self._f1) return
def test(model, data, args): print('Start testing ..<========') best_epoch, pat_cnt, best_er, f1_for_best_er, best_conf_mat = 0, 0, 99999, None, None tr_loss, val_loss, f1_overall_1sec_list, er_overall_1sec_list = [0] * nb_epoch, [0] * nb_epoch, [0] * nb_epoch, [0] * nb_epoch posterior_thresh = 0.8 x_test, y_test = data y_pred, x_recon = model.predict(x_test, batch_size=256) # print 'x_recon=', x_recon print('-'*30 + 'Begin: test' + '-'*30) print('Test acc:', np.sum(np.argmax(y_pred, 1) == np.argmax(y_test, 1))/y_test.shape[0]) for i in range(13000,13500): print(y_pred[i], '|', y_test[i]) # Calculate the predictions on test data, in order to calculate ER and F scores pred_thresh = y_pred > posterior_thresh # print('pred_thresh:', pred_thresh) score_list = metrics.compute_scores(pred_thresh, y_test, frames_in_1_sec=frames_1_sec) f1_overall_1sec_list = score_list['f1_overall_1sec'] er_overall_1sec_list = score_list['er_overall_1sec'] pat_cnt = pat_cnt + 1 # y_test = y_test > posterior_thresh y_test = y_test.astype(int) pred_thresh = pred_thresh.astype(int) print('Shape y_test:', y_test.shape) print('Shape y_test:', y_test) print('Shape pred_thresh:', pred_thresh.shape) print('Shape pred_thresh:', pred_thresh) # Results print('sklearn Macro-F1-Score:', f1_score(y_test, pred_thresh, average='macro')) print('Custom Macro-F1-Score:', K.eval(f1(y_test, pred_thresh))) # np.set_printoptions(threshold=np.inf) # print y_pred # Calculate confusion matrix # test_pred_cnt = np.sum(pred_thresh, 2) # y_test_cnt = np.sum(Y_test, 2) # conf_mat = confusion_matrix(y_test_cnt.reshape(-1), test_pred_cnt.reshape(-1)) # conf_mat = conf_mat / (utils.eps + np.sum(conf_mat, 1)[:, None].astype('float')) print('tr Er : {}, val Er : {}, F1_overall : {}, ER_overall : {} Best ER : {}, best_epoch: {}'.format( tr_loss, val_loss, f1_overall_1sec_list, er_overall_1sec_list, best_er, best_epoch)) # img = combine_images(np.concatenate([x_test[:50],x_recon[:50]])) # image = img * 255 # Image.fromarray(image.astype(np.uint8)).save(args.save_dir + "/real_and_recon.png") # print('Reconstructed images are saved to %s/real_and_recon.png' % args.save_dir) print('-' * 30 + 'End: test' + '-' * 30)
def on_epoch_end(self, epoch, batch, logs={}): X_val, X_val_gcc, y_val = self.validation_data[ 0], self.validation_data[1], self.validation_data[2] # Calculate the predictions on test data, in order to calculate ER and F scores pred = model.predict([X_val, X_val_gcc]) #È true o false pred_thresh = pred > 0.5 #0.5 threeshold vedi paper #print pred_thresh score_list = metrics.compute_scores( pred_thresh, y_val, frames_in_1_sec=frames_1_sec) #Y_TEST shape = (220,256,6) #score list è un dizionario self._f1 = score_list['f1_overall_1sec'] self._er = score_list['er_overall_1sec'] #error rate over epoch self.er_mean_batch += self._er self.er_mean = float(self.er_mean_batch) / (epoch + 1) # Calculate confusion matrix test_pred_cnt = np.sum(pred_thresh, 2) Y_test_cnt = np.sum(y_val, 2) conf_mat = confusion_matrix(Y_test_cnt.reshape(-1), test_pred_cnt.reshape(-1)) conf_mat = conf_mat / (utils.eps + np.sum(conf_mat, 1)[:, None].astype('float')) self._cf_list.append(conf_mat) #if self._er > self._er_prev: if self.er_mean >= self.er_mean_prev: self._fail_count += 1 if self._fail_count >= 100: print('Early stopping ', 'Custom ER: ', self._er, ' Failcount: ', self._fail_count) self.model.stop_training = True else: #resetto il patience count self._fail_count = 0 #aggiorno print(' Mean: ', self.er_mean, ' Custom ER: ', self._er, ' Failcount: ', self._fail_count, ' F1 :', self._f1) self._er_prev = self._er self.er_mean_prev = self.er_mean self._er_list.append(self._er) self._f1_list.append(self._f1) self.er_mean_list.append(self.er_mean) return
def main(bands, n_bin, model_file, output_file, metrics_code='jax_cosmo'): # Assume data in standard locations relative to current directory training_file = '../data/training.hdf5' validation_file = '../data/validation.hdf5' clf = load(model_file) print('clf parameters: ', clf.get_params()) #Apply model print('Apply model to validation set') tomo_bin = apply_model(clf, validation_file, bands) # Get a score z = load_redshift(validation_file) #JEC 20/7/20 z = z[:50000] if metrics_code == 'jax_cosmo': scores = jc_compute_scores(tomo_bin, z, metrics="SNR_3x2,FOM_3x2,FOM_DETF_3x2") else: scores = compute_scores(tomo_bin, z, metrics="SNR_3x2,FOM_3x2") # Get the galaxy distribution of redshifts into n_z bins of # equal number counts in each # p = np.linspace(0, 100, n_bin + 1) # z_edges = np.percentile(z, p) # result of above z_edges = np.array([0.02450252, 0.29473031, 0.44607811, 0.58123241,\ 0.70399809,0.8120476 , 0.91746771, 1.03793607,\ 1.20601892, 1.49989052, 3.03609133]) #DETF optim 25/7/2020 z_edges = np.array([0.02450252, 0.15961642, 0.37035, 0.475175,\ 0.58955, 0.700775 , 0.91746771, 1.03793607,\ 1.352945, 1.8, 3.03609133]) print("new set: ", z_edges) plot_distributions(z, tomo_bin, output_file, z_edges) # return return scores
validation_data=[X_test, Y_test], epochs=1, verbose=1 ) val_loss[i] = hist.history.get('val_loss')[-1] tr_loss[i] = hist.history.get('loss')[-1] # Calculate the predictions on test data, in order to calculate ER and F scores pred = model.predict(X_test) print(pred) print(pred.shape) os.system('pause') pred_thresh = pred > posterior_thresh score_list = metrics.compute_scores(pred_thresh, Y_test, frames_in_1_sec=frames_1_sec) f1_overall_1sec_list[i] = score_list['f1_overall_1sec'] er_overall_1sec_list[i] = score_list['er_overall_1sec'] pat_cnt = pat_cnt + 1 # Calculate confusion matrix test_pred_cnt = np.sum(pred_thresh, 2) Y_test_cnt = np.sum(Y_test, 2) conf_mat = confusion_matrix(Y_test_cnt.reshape(-1), test_pred_cnt.reshape(-1)) conf_mat = conf_mat / (utils.eps + np.sum(conf_mat, 1)[:, None].astype('float')) if er_overall_1sec_list[i] < best_er: best_conf_mat = conf_mat best_er = er_overall_1sec_list[i] f1_for_best_er = f1_overall_1sec_list[i]
print('Epoch : {} '.format(i), end='') hist = model.fit( [X_MBE,X_GCC], Y, batch_size=batch_size, validation_data=[[X_test_MBE,X_test_GCC], Y_test], epochs=1, verbose=1 ) val_loss[i] = hist.history.get('val_loss')[-1] tr_loss[i] = hist.history.get('loss')[-1] # Calculate the predictions on test data, in order to calculate ER and F scores pred = model.predict([X_test_MBE,X_test_GCC]) #È true o false pred_thresh = pred > posterior_thresh #0.5 threeshold vedi paper score_list = metrics.compute_scores(pred_thresh, Y_test, frames_in_1_sec=frames_1_sec) #Y_TEST shape = (220,256,6) #score list è un dizionario f1_overall_1sec_list[i] = score_list['f1_overall_1sec'] er_overall_1sec_list[i] = score_list['er_overall_1sec'] pat_cnt = pat_cnt + 1 # Calculate confusion matrix test_pred_cnt = np.sum(pred_thresh, 2) Y_test_cnt = np.sum(Y_test, 2) conf_mat = confusion_matrix(Y_test_cnt.reshape(-1), test_pred_cnt.reshape(-1)) conf_mat = conf_mat / (utils.eps + np.sum(conf_mat, 1)[:, None].astype('float')) if er_overall_1sec_list[i] < best_er: best_conf_mat = conf_mat best_er = er_overall_1sec_list[i] f1_for_best_er = f1_overall_1sec_list[i]