def compare_convergences_classif(list_of_paths, list_of_labels=None):
# Parameters
# **********
steps_per_epoch = 0
smooth_n = 12
if list_of_labels is None:
list_of_labels = [str(i) for i in range(len(list_of_paths))]
# Read Logs
# *********
all_pred_epochs = []
all_val_OA = []
all_train_OA = []
all_vote_OA = []
all_vote_confs = []
for path in list_of_paths:
# Load parameters
config = Config()
config.load(list_of_paths[0])
# Get the number of classes
n_class = config.num_classes
# Load epochs
epochs, _, _, _, _, _ = load_training_results(path)
first_e = np.min(epochs)
# Get validation confusions
file = join(path, 'val_confs.txt')
val_C1 = load_confusions(file, n_class)
val_PRE, val_REC, val_F1, val_IoU, val_ACC = smooth_metrics(
val_C1, smooth_n=smooth_n)
# Get vote confusions
file = join(path, 'vote_confs.txt')
if exists(file):
vote_C2 = load_confusions(file, n_class)
vote_PRE, vote_REC, vote_F1, vote_IoU, vote_ACC = smooth_metrics(
vote_C2, smooth_n=2)
else:
vote_C2 = val_C1
vote_PRE, vote_REC, vote_F1, vote_IoU, vote_ACC = (val_PRE,
val_REC, val_F1,
val_IoU,
val_ACC)
# Aggregate results
all_pred_epochs += [
np.array([i + first_e for i in range(len(val_ACC))])
]
all_val_OA += [val_ACC]
all_vote_OA += [vote_ACC]
all_vote_confs += [vote_C2]
print()
# Best scores
# ***********
for i, label in enumerate(list_of_labels):
print('\n' + label + '\n' + '*' * len(label) + '\n')
print(list_of_paths[i])
best_epoch = np.argmax(all_vote_OA[i])
print('Best Accuracy : {:.1f} % (epoch {:d})'.format(
100 * all_vote_OA[i][best_epoch], best_epoch))
confs = all_vote_confs[i]
"""
s = ''
for cc in confs[best_epoch]:
for c in cc:
s += '{:.0f} '.format(c)
s += '\n'
print(s)
"""
TP_plus_FN = np.sum(confs, axis=-1, keepdims=True)
class_avg_confs = confs.astype(np.float32) / TP_plus_FN.astype(
np.float32)
diags = np.diagonal(class_avg_confs, axis1=-2, axis2=-1)
class_avg_ACC = np.sum(diags, axis=-1) / np.sum(class_avg_confs,
axis=(-1, -2))
print('Corresponding mAcc : {:.1f} %'.format(
100 * class_avg_ACC[best_epoch]))
# Plots
# *****
for fig_name, OA in zip(['Validation', 'Vote'], [all_val_OA, all_vote_OA]):
# Figure
fig = plt.figure(fig_name)
for i, label in enumerate(list_of_labels):
plt.plot(all_pred_epochs[i], OA[i], linewidth=1, label=label)
plt.xlabel('epochs')
plt.ylabel(fig_name + ' Accuracy')
# Set limits for y axis
#plt.ylim(0.55, 0.95)
# Display legends and title
plt.legend(loc=4)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
#ax.set_yticks(np.arange(0.8, 1.02, 0.02))
#for i, label in enumerate(list_of_labels):
# print(label, np.max(all_train_OA[i]), np.max(all_val_OA[i]))
# Show all
plt.show()
def compare_convergences_classif(dataset, list_of_paths, list_of_labels=None):
# Parameters
# **********
steps_per_epoch = 0
smooth_n = 2
if list_of_labels is None:
list_of_labels = [str(i) for i in range(len(list_of_paths))]
# Read Logs
# *********
all_pred_epochs = []
all_val_OA = []
all_train_OA = []
all_vote_OA = []
all_vote_confs = []
for path in list_of_paths:
# Load parameters
config = Config()
config.load(list_of_paths[0])
# Get the number of classes
n_class = config.num_classes
# Get validation confusions
file = join(path, 'val_confs.txt')
val_C1 = load_confusions(file, n_class)
val_PRE, val_REC, val_F1, val_IoU, val_ACC = smooth_metrics(val_C1, smooth_n=smooth_n)
# Get vote confusions
file = join(path, 'vote_confs.txt')
if exists(file):
vote_C2 = load_confusions(file, n_class)
vote_PRE, vote_REC, vote_F1, vote_IoU, vote_ACC = smooth_metrics(vote_C2, smooth_n=2)
else:
vote_C2 = val_C1
vote_PRE, vote_REC, vote_F1, vote_IoU, vote_ACC = (val_PRE, val_REC, val_F1, val_IoU, val_ACC)
# Get training confusions balanced
file = join(path, 'training_confs.txt')
train_C = load_confusions(file, n_class)
train_PRE, train_REC, train_F1, train_IoU, train_ACC = smooth_metrics(train_C, smooth_n=smooth_n)
# Aggregate results
all_pred_epochs += [np.array([i for i in range(len(val_ACC))])]
all_val_OA += [val_ACC]
all_vote_OA += [vote_ACC]
all_train_OA += [train_ACC]
all_vote_confs += [vote_C2]
#all_mean_IoU_scores += [running_mean(np.mean(val_IoU[:, 1:], axis=1), smooth_n)]
# Best scores
# ***********
for i, label in enumerate(list_of_labels):
print('\n' + label + '\n' + '*' * len(label) + '\n')
best_epoch = np.argmax(all_vote_OA[i])
print('Best Accuracy : {:.1f} % (epoch {:d})'.format(100 * all_vote_OA[i][best_epoch], best_epoch))
confs = all_vote_confs[i]
TP_plus_FN = np.sum(confs, axis=-1, keepdims=True)
class_avg_confs = confs.astype(np.float32) / TP_plus_FN.astype(np.float32)
diags = np.diagonal(class_avg_confs, axis1=-2, axis2=-1)
class_avg_ACC = np.sum(diags, axis=-1) / np.sum(class_avg_confs, axis=(-1, -2))
print('Corresponding mAcc : {:.1f} %'.format(100 * class_avg_ACC[best_epoch]))
# Plots
# *****
# Figure
fig = plt.figure('Validation')
for i, label in enumerate(list_of_labels):
plt.plot(all_pred_epochs[i], all_val_OA[i], linewidth=1, label=label)
plt.xlabel('epochs')
plt.ylabel('Validation Accuracy')
# Set limits for y axis
#plt.ylim(0.55, 0.95)
# Display legends and title
plt.legend(loc=4)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
#ax.set_yticks(np.arange(0.8, 1.02, 0.02))
# Figure
fig = plt.figure('Vote Validation')
for i, label in enumerate(list_of_labels):
plt.plot(all_pred_epochs[i], all_vote_OA[i], linewidth=1, label=label)
plt.xlabel('epochs')
plt.ylabel('Validation Accuracy')
# Set limits for y axis
#plt.ylim(0.55, 0.95)
# Display legends and title
plt.legend(loc=4)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
#ax.set_yticks(np.arange(0.8, 1.02, 0.02))
# Figure
fig = plt.figure('Training')
for i, label in enumerate(list_of_labels):
plt.plot(all_pred_epochs[i], all_train_OA[i], linewidth=1, label=label)
plt.xlabel('epochs')
plt.ylabel('Overall Accuracy')
# Set limits for y axis
#plt.ylim(0.8, 1)
# Display legends and title
plt.legend(loc=4)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
#ax.set_yticks(np.arange(0.8, 1.02, 0.02))
#for i, label in enumerate(list_of_labels):
# print(label, np.max(all_train_OA[i]), np.max(all_val_OA[i]))
# Show all
plt.show()