save_path = f'rapport/bayesian_results/histograms/'
save_path = pathlib.Path(save_path)
unc_names = ['sr', 'vr', 'pe', 'mi']#, 'au', 'eu']
start_tot = time()
start_exp = start_tot
for exp_idx, exp in enumerate(all_exps):
print(f'========================')
print(f'{exp_idx+1}/{len(all_exps)}')
res = {}
res_unseen_bay = {}
reload_modules()
verbose = True
bay_net_trained, arguments, _ = su.get_trained_model_and_args_and_groupnb(exp, path_to_res)
evalloader_seen = su.get_evalloader_seen(arguments, shuffle=False)
arguments['type_of_unseen'] = 'unseen_dataset'
evalloader_unseen = su.get_evalloader_unseen(arguments, shuffle=False)
dont_show = ['save_loss', 'save_observables', 'save_outputs', 'type_of_unseen', 'unseen_evalset',
'split_labels', 'number_of_tests', 'split_train', 'exp_nb']
is_determinist = arguments.get('determinist', False) or arguments.get('rho', 'determinist') == 'determinist'
assert not is_determinist, 'network determinist'
for number_of_tests in list_of_nb_of_tests:
bay_net_trained, arguments, _ = su.get_trained_model_and_args_and_groupnb(exp, path_to_res)
bay_net_trained.to(device)
if number_of_tests < 100 and os.path.exists(path_to_outputs / '100' / f'{exp}/true_labels_seen.pt'):
# title += f'{unc_name}: {round(unc[random_label_with_high_uncertainty].item(), 4)} || '
title += (
f'Index: {random_label_with_high_uncertainty.item()}'
f'\nTrue: {labels[evalloader.dataset.targets[random_label_with_high_uncertainty].item()]} || '
f'Pred: {labels[predicted_labels[random_label_with_high_uncertainty].item()]} || '
)
axs[k1, k2].set_title(title)
if show_title:
fig.suptitle(arguments, wrap=True)
fig.savefig(imgs_dir / f'{type_of_unc}_{number_of_row*number_of_col}_{group_nb}_{exp_nb}.png')
fig.tight_layout()
plt.close(fig)
arguments = get_args(exp_nb, exp_path)
evalloader = get_evalloader_seen(arguments, shuffle=False)
bay_net_trained, arguments, group_nb = get_trained_model_and_args_and_groupnb(exp_nb, exp_path=exp_path)
true_labels, all_outputs, _ = su.get_saved_outputs_labels_seen_unseen(
exp_nb=exp_nb,
number_of_tests=number_of_tests,
path_to_exps=exp_path,
shuffle=False,
)
print('Testing finished.')
uncs = get_uncs(all_outputs)
print('Uncertainty computed.')
if arguments['trainset'] == 'cifar10':
labels_name = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship','truck']
else:
def main(
exp_nbs=exp_nbs,
path_to_exps=path_to_exps,
path_to_results=save_path,
nb_of_runs=nb_of_runs,
nb_of_tests=nb_of_tests,
device='cpu',
**kwargs,
):
"""
Evaluates the accuracy for the experiments given. Writes it in a csv.
Args:
exp_nbs (list): list of int or str. Experiments to evaluate
path_to_exps (str): path to the experiments
path_to_results (str): path to the directory to save the results
nb_of_runs (int): number of times to run the same experiment for confidence interval
nb_of_tests (int): number of tests for inference for each prediction
device (torch.device): gpu or cpu, device to compute on
**kwargs: args to be able to put any arguments in our functions and not raise an error.
"""
save_path = pathlib.Path(path_to_results)
if save_output:
save_path.mkdir(exist_ok=True, parents=True)
if not os.path.exists(save_path / 'all_eval_accs.pkl'):
all_eval_accs = pd.DataFrame(columns=[
'exp_nb', 'group_nb', 'rho', 'std_prior', 'loss_type',
'number_of_tests'
])
else:
all_eval_accs = load_from_file(save_path / 'all_eval_accs.pkl', )
all_eval_accs.to_csv(save_path / 'all_eval_accs_backup.csv')
for _ in range(nb_of_runs):
for exp in exp_nbs:
print(f'Exp {exp}, computing accuracies...')
bay_net_trained, arguments, group_nb = get_trained_model_and_args_and_groupnb(
exp, path_to_exps)
evalloader_seen = get_evalloader_seen(arguments)
eval_acc, _ = eval_bayesian(
bay_net_trained,
evalloader_seen,
number_of_tests=nb_of_tests
if arguments.get('rho', 'determinist') != 'determinist' else 1,
return_accuracy=True,
device=device,
verbose=True,
)
all_eval_accs = all_eval_accs.append(
pd.DataFrame.from_dict({
'exp_nb': [exp],
'group_nb': [group_nb],
'split_labels': [arguments.get('split_labels', 10)],
'trainset': [arguments.get('trainset', 'mnist')],
'rho': [arguments.get('rho', 'determinist')],
'std_prior': [arguments.get('std_prior', -1)],
'epoch': [arguments['epoch']],
'loss_type': [arguments.get('loss_type', 'criterion')],
'number_of_tests': [nb_of_tests],
'eval_acc':
eval_acc,
}))
all_eval_accs.exp_nb = all_eval_accs.exp_nb.astype('int')
if save_output:
all_eval_accs.to_csv(save_path / 'all_eval_accs.csv')
all_eval_accs.to_pickle(save_path / 'all_eval_accs.pkl')
def compute_tps_fps(exp_nbs, get_uncs, unc_names, number_of_tests):
all_xs = dict({k: [] for k in unc_names})
all_ys = dict({k: [] for k in unc_names})
all_xs_pr = dict({k: [] for k in unc_names})
all_ys_pr = dict({k: [] for k in unc_names})
all_uncs = {}
for exp_nb in exp_nbs:
bay_net_trained, arguments, _ = get_trained_model_and_args_and_groupnb(
exp_nb, path_to_res)
if number_of_tests < 100 and os.path.exists(
path_to_outputs / '100' / f'{exp_nb}/true_labels_seen.pt'):
true_labels_seen = torch.load(path_to_outputs / f'100' /
f'{exp_nb}/true_labels_seen.pt')
all_outputs_seen = torch.load(path_to_outputs / f'100' /
f'{exp_nb}/all_outputs_seen.pt')
all_outputs_unseen = torch.load(
path_to_outputs / f'100' /
f'{exp_nb}/all_outputs_unseen.pt')
random_idx = np.arange(100)
np.random.shuffle(random_idx)
random_idx = random_idx[:number_of_tests]
all_outputs_seen = all_outputs_seen[random_idx]
all_outputs_unseen = all_outputs_unseen[random_idx]
elif os.path.exists(path_to_outputs / f'{number_of_tests}' /
f'{exp_nb}/true_labels_seen.pt'):
true_labels_seen = torch.load(path_to_outputs /
f'{number_of_tests}' /
f'{exp_nb}/true_labels_seen.pt')
all_outputs_seen = torch.load(path_to_outputs /
f'{number_of_tests}' /
f'{exp_nb}/all_outputs_seen.pt')
all_outputs_unseen = torch.load(
path_to_outputs / f'{number_of_tests}' /
f'{exp_nb}/all_outputs_unseen.pt')
else:
(path_to_outputs / f'{number_of_tests}' / f'{exp_nb}').mkdir(
exist_ok=True, parents=True)
evalloader_seen = get_evalloader_seen(arguments)
# BE CAREFUL: in the paper, the process is tested on the enterity of the unseen classes
evalloader_unseen = get_evalloader_unseen(arguments)
true_labels_seen, all_outputs_seen = eval_bayesian(
model=bay_net_trained,
evalloader=evalloader_seen,
number_of_tests=number_of_tests,
return_accuracy=False,
device=device,
verbose=True,
)
_, all_outputs_unseen = eval_bayesian(
model=bay_net_trained,
evalloader=evalloader_unseen,
number_of_tests=number_of_tests,
device=device,
verbose=True,
)
torch.save(
true_labels_seen, path_to_outputs / f'{number_of_tests}' /
f'{exp_nb}/true_labels_seen.pt')
torch.save(
all_outputs_seen, path_to_outputs / f'{number_of_tests}' /
f'{exp_nb}/all_outputs_seen.pt')
torch.save(
all_outputs_unseen, path_to_outputs /
f'{number_of_tests}' / f'{exp_nb}/all_outputs_unseen.pt')
preds = get_predictions_from_multiple_tests(
all_outputs_seen).float()
if typ == 'seen_unseen':
all_outputs_true = all_outputs_seen
all_outputs_false = all_outputs_unseen
elif typ == 'true_false':
all_outputs_true = all_outputs_seen[:, preds ==
true_labels_seen, :]
all_outputs_false = all_outputs_seen[:, preds !=
true_labels_seen, :]
else:
all_outputs_true = all_outputs_seen[:, preds ==
true_labels_seen, :]
all_outputs_false = torch.cat(
(all_outputs_seen[:, preds != true_labels_seen, :],
all_outputs_unseen), 1)
uncs_true = get_uncs(all_outputs_true)
uncs_false = get_uncs(all_outputs_false)
all_uncs[exp_nb] = pd.DataFrame()
for idx, unc_name in enumerate(unc_names):
all_uncs[exp_nb] = all_uncs[exp_nb].append(
pd.DataFrame().assign(unc=torch.cat((
uncs_true[idx],
uncs_false[idx]))).assign(is_ood=torch.cat((
torch.zeros_like(uncs_true[idx]),
torch.ones_like(uncs_false[idx])))).assign(
unc_name=unc_name).assign(
number_of_tests=number_of_tests))
this_df = all_uncs[exp_nb].loc[all_uncs[exp_nb].unc_name ==
unc_name]
positives = this_df.is_ood.sum()
negatives = (1 - this_df.is_ood).sum()
grouped_unc = this_df.groupby('unc')
to_plot = (grouped_unc.sum(
).assign(n=grouped_unc.apply(lambda df: len(df))).assign(
tp=lambda df: df.iloc[::-1].is_ood.cumsum()).assign(
fp=lambda df: df.iloc[::-1].n.cumsum() - df.tp).assign(
fn=lambda df: df.is_ood.cumsum()).assign(
precision=lambda df: df.tp / (df.tp + df.fp)).
assign(recall=lambda df: df.tp / (df.tp + df.fn))
).reset_index()
all_xs[unc_name].append((to_plot.fp / negatives))
all_ys[unc_name].append((to_plot.tp / positives))
all_xs_pr[unc_name].append((to_plot.recall))
all_ys_pr[unc_name].append(to_plot.precision)
# xs = to_plot.fp/negatives
# ys = to_plot.tp/positives
return all_xs, all_ys, all_xs_pr, all_ys_pr