def on_epoch_end(self,epoch, batch, logs={}):
X_val,X_val_gcc, y_val = self.validation_data[0], self.validation_data[1],self.validation_data[2]
# Calculate the predictions on test data, in order to calculate ER and F scores
pred = model.predict([X_val,X_val_gcc])
#È true o false
pred_thresh = pred > 0.5 #0.5 threeshold vedi paper
#print pred_thresh
score_list = metrics.compute_scores(pred_thresh, y_val, frames_in_1_sec=frames_1_sec) #Y_TEST shape = (220,256,6)
#score list è un dizionario
self._f1 = score_list['f1_overall_1sec']
self._er = score_list['er_overall_1sec']
if self._er > self._er_prev:
self._fail_count+=1
if self._fail_count >= 10:
print('Epoch: ', epoch, ' Custom ER: ', self._er, ' Failcount: ', self._fail_count )
self.model.stop_training = True
else:
#resetto il patience count
self._fail_count = 0
#aggiorno
print(' Prev: ', self._er_prev ,' Custom ER: ', self._er,' Failcount: ', self._fail_count )
self._er_prev = self._er
self._er_list.append(self._er)
self._f1_list.append(self._f1)
return
def test(model, data, args):
print('Start testing ..<========')
best_epoch, pat_cnt, best_er, f1_for_best_er, best_conf_mat = 0, 0, 99999, None, None
tr_loss, val_loss, f1_overall_1sec_list, er_overall_1sec_list = [0] * nb_epoch, [0] * nb_epoch, [0] * nb_epoch, [0] * nb_epoch
posterior_thresh = 0.8
x_test, y_test = data
y_pred, x_recon = model.predict(x_test, batch_size=256)
# print 'x_recon=', x_recon
print('-'*30 + 'Begin: test' + '-'*30)
print('Test acc:', np.sum(np.argmax(y_pred, 1) == np.argmax(y_test, 1))/y_test.shape[0])
for i in range(13000,13500):
print(y_pred[i], '|', y_test[i])
# Calculate the predictions on test data, in order to calculate ER and F scores
pred_thresh = y_pred > posterior_thresh
# print('pred_thresh:', pred_thresh)
score_list = metrics.compute_scores(pred_thresh, y_test, frames_in_1_sec=frames_1_sec)
f1_overall_1sec_list = score_list['f1_overall_1sec']
er_overall_1sec_list = score_list['er_overall_1sec']
pat_cnt = pat_cnt + 1
# y_test = y_test > posterior_thresh
y_test = y_test.astype(int)
pred_thresh = pred_thresh.astype(int)
print('Shape y_test:', y_test.shape)
print('Shape y_test:', y_test)
print('Shape pred_thresh:', pred_thresh.shape)
print('Shape pred_thresh:', pred_thresh)
# Results
print('sklearn Macro-F1-Score:', f1_score(y_test, pred_thresh, average='macro'))
print('Custom Macro-F1-Score:', K.eval(f1(y_test, pred_thresh)))
# np.set_printoptions(threshold=np.inf)
# print y_pred
# Calculate confusion matrix
# test_pred_cnt = np.sum(pred_thresh, 2)
# y_test_cnt = np.sum(Y_test, 2)
# conf_mat = confusion_matrix(y_test_cnt.reshape(-1), test_pred_cnt.reshape(-1))
# conf_mat = conf_mat / (utils.eps + np.sum(conf_mat, 1)[:, None].astype('float'))
print('tr Er : {}, val Er : {}, F1_overall : {}, ER_overall : {} Best ER : {}, best_epoch: {}'.format(
tr_loss, val_loss, f1_overall_1sec_list, er_overall_1sec_list, best_er, best_epoch))
# img = combine_images(np.concatenate([x_test[:50],x_recon[:50]]))
# image = img * 255
# Image.fromarray(image.astype(np.uint8)).save(args.save_dir + "/real_and_recon.png")
# print('Reconstructed images are saved to %s/real_and_recon.png' % args.save_dir)
print('-' * 30 + 'End: test' + '-' * 30)
def on_epoch_end(self, epoch, batch, logs={}):
X_val, X_val_gcc, y_val = self.validation_data[
0], self.validation_data[1], self.validation_data[2]
# Calculate the predictions on test data, in order to calculate ER and F scores
pred = model.predict([X_val, X_val_gcc])
#È true o false
pred_thresh = pred > 0.5 #0.5 threeshold vedi paper
#print pred_thresh
score_list = metrics.compute_scores(
pred_thresh, y_val,
frames_in_1_sec=frames_1_sec) #Y_TEST shape = (220,256,6)
#score list è un dizionario
self._f1 = score_list['f1_overall_1sec']
self._er = score_list['er_overall_1sec']
#error rate over epoch
self.er_mean_batch += self._er
self.er_mean = float(self.er_mean_batch) / (epoch + 1)
# Calculate confusion matrix
test_pred_cnt = np.sum(pred_thresh, 2)
Y_test_cnt = np.sum(y_val, 2)
conf_mat = confusion_matrix(Y_test_cnt.reshape(-1),
test_pred_cnt.reshape(-1))
conf_mat = conf_mat / (utils.eps +
np.sum(conf_mat, 1)[:, None].astype('float'))
self._cf_list.append(conf_mat)
#if self._er > self._er_prev:
if self.er_mean >= self.er_mean_prev:
self._fail_count += 1
if self._fail_count >= 100:
print('Early stopping ', 'Custom ER: ', self._er,
' Failcount: ', self._fail_count)
self.model.stop_training = True
else:
#resetto il patience count
self._fail_count = 0
#aggiorno
print(' Mean: ', self.er_mean, ' Custom ER: ', self._er,
' Failcount: ', self._fail_count, ' F1 :', self._f1)
self._er_prev = self._er
self.er_mean_prev = self.er_mean
self._er_list.append(self._er)
self._f1_list.append(self._f1)
self.er_mean_list.append(self.er_mean)
return
def main(bands, n_bin, model_file, output_file, metrics_code='jax_cosmo'):
# Assume data in standard locations relative to current directory
training_file = '../data/training.hdf5'
validation_file = '../data/validation.hdf5'
clf = load(model_file)
print('clf parameters: ', clf.get_params())
#Apply model
print('Apply model to validation set')
tomo_bin = apply_model(clf, validation_file, bands)
# Get a score
z = load_redshift(validation_file)
#JEC 20/7/20
z = z[:50000]
if metrics_code == 'jax_cosmo':
scores = jc_compute_scores(tomo_bin,
z,
metrics="SNR_3x2,FOM_3x2,FOM_DETF_3x2")
else:
scores = compute_scores(tomo_bin, z, metrics="SNR_3x2,FOM_3x2")
# Get the galaxy distribution of redshifts into n_z bins of
# equal number counts in each
# p = np.linspace(0, 100, n_bin + 1)
# z_edges = np.percentile(z, p)
# result of above
z_edges = np.array([0.02450252, 0.29473031, 0.44607811, 0.58123241,\
0.70399809,0.8120476 , 0.91746771, 1.03793607,\
1.20601892, 1.49989052, 3.03609133])
#DETF optim 25/7/2020
z_edges = np.array([0.02450252, 0.15961642, 0.37035, 0.475175,\
0.58955, 0.700775 , 0.91746771, 1.03793607,\
1.352945, 1.8, 3.03609133])
print("new set: ", z_edges)
plot_distributions(z, tomo_bin, output_file, z_edges)
# return
return scores
validation_data=[X_test, Y_test],
epochs=1,
verbose=1
)
val_loss[i] = hist.history.get('val_loss')[-1]
tr_loss[i] = hist.history.get('loss')[-1]
# Calculate the predictions on test data, in order to calculate ER and F scores
pred = model.predict(X_test)
print(pred)
print(pred.shape)
os.system('pause')
pred_thresh = pred > posterior_thresh
score_list = metrics.compute_scores(pred_thresh, Y_test, frames_in_1_sec=frames_1_sec)
f1_overall_1sec_list[i] = score_list['f1_overall_1sec']
er_overall_1sec_list[i] = score_list['er_overall_1sec']
pat_cnt = pat_cnt + 1
# Calculate confusion matrix
test_pred_cnt = np.sum(pred_thresh, 2)
Y_test_cnt = np.sum(Y_test, 2)
conf_mat = confusion_matrix(Y_test_cnt.reshape(-1), test_pred_cnt.reshape(-1))
conf_mat = conf_mat / (utils.eps + np.sum(conf_mat, 1)[:, None].astype('float'))
if er_overall_1sec_list[i] < best_er:
best_conf_mat = conf_mat
best_er = er_overall_1sec_list[i]
f1_for_best_er = f1_overall_1sec_list[i]
print('Epoch : {} '.format(i), end='')
hist = model.fit(
[X_MBE,X_GCC], Y,
batch_size=batch_size,
validation_data=[[X_test_MBE,X_test_GCC], Y_test],
epochs=1,
verbose=1
)
val_loss[i] = hist.history.get('val_loss')[-1]
tr_loss[i] = hist.history.get('loss')[-1]
# Calculate the predictions on test data, in order to calculate ER and F scores
pred = model.predict([X_test_MBE,X_test_GCC])
#È true o false
pred_thresh = pred > posterior_thresh #0.5 threeshold vedi paper
score_list = metrics.compute_scores(pred_thresh, Y_test, frames_in_1_sec=frames_1_sec) #Y_TEST shape = (220,256,6)
#score list è un dizionario
f1_overall_1sec_list[i] = score_list['f1_overall_1sec']
er_overall_1sec_list[i] = score_list['er_overall_1sec']
pat_cnt = pat_cnt + 1
# Calculate confusion matrix
test_pred_cnt = np.sum(pred_thresh, 2)
Y_test_cnt = np.sum(Y_test, 2)
conf_mat = confusion_matrix(Y_test_cnt.reshape(-1), test_pred_cnt.reshape(-1))
conf_mat = conf_mat / (utils.eps + np.sum(conf_mat, 1)[:, None].astype('float'))
if er_overall_1sec_list[i] < best_er:
best_conf_mat = conf_mat
best_er = er_overall_1sec_list[i]
f1_for_best_er = f1_overall_1sec_list[i]
