def __init__(self, args):
# set up output directory
self.output_dir = os.path.join(args.experiment_dir, args.run_name)
if not os.path.exists(args.experiment_dir):
os.mkdir(args.experiment_dir)
if not os.path.exists(self.output_dir):
os.mkdir(self.output_dir)
if not os.path.exists(os.path.join(args.experiment_dir, "runs/")):
os.mkdir(os.path.join(args.experiment_dir, "runs/"))
# initialize model config
self.config = vars(args)
if args.real_run:
run_name = "{}-{}".format(args.experiment_dir, args.run_name)
else:
run_name = None
# initialize weights and biases
wandb.init(
name=run_name,
notes=args.a_nice_note,
project="coreference-detection",
config=self.config,
)
# check if there is a model to load
if args.old_model_dir is not None:
self.use_old_model = True
self.load_dir = args.old_model_dir
load_from_file(os.path.join(self.load_dir, "config.json"),
self.config)
# create vocab
self.vocab = Vocab()
self.vocab.load_from_dict(os.path.join(self.load_dir,
"vocab.json"))
self.update_vocab = False
self.config["min_count"] = 1
else:
self.use_old_model = False
self.vocab = None
self.update_vocab = True
# train
self.train_dataset = DialogueDataset(
os.path.join(self.config["dataset_filename"], "train_data.json"),
self.config["sentence_len"], self.vocab, self.update_vocab)
self.data_loader_train = torch.utils.data.DataLoader(
self.train_dataset, self.config["train_batch_size"], shuffle=True)
self.config["train_len"] = len(self.train_dataset)
self.vocab = self.train_dataset.vocab
# eval
self.val_dataset = DialogueDataset(
os.path.join(self.config["dataset_filename"], "val_data.json"),
self.config["sentence_len"], self.vocab, self.update_vocab)
self.data_loader_val = torch.utils.data.DataLoader(
self.val_dataset, self.config["val_batch_size"], shuffle=True)
self.config["val_len"] = len(self.val_dataset)
# update, and save vocab
self.vocab = self.val_dataset.vocab
self.train_dataset.vocab = self.vocab
if (self.config["min_count"] > 1):
self.config["old_vocab_size"] = len(self.vocab)
self.vocab.prune_vocab(self.config["min_count"])
self.vocab.save_to_dict(os.path.join(self.output_dir, "vocab.json"))
self.vocab_size = len(self.vocab)
self.config["vocab_size"] = self.vocab_size
# load embeddings
if self.config["pretrained_embeddings_dir"] is not None:
pretrained_embeddings = get_pretrained_embeddings(
self.config["pretrained_embeddings_dir"], self.vocab)
else:
pretrained_embeddings = None
# print and save the config file
print_config(self.config)
save_config(os.path.join(self.output_dir, "config.json"), self.config)
# set device
self.device = torch.device('cuda')
# create model
self.model = Transformer(
self.config["vocab_size"],
self.config["label_len"],
self.config["sentence_len"],
d_word_vec=self.config["embedding_dim"],
d_model=self.config["model_dim"],
d_inner=self.config["inner_dim"],
n_layers=self.config["num_layers"],
n_head=self.config["num_heads"],
d_k=self.config["dim_k"],
d_v=self.config["dim_v"],
dropout=self.config["dropout"],
pretrained_embeddings=pretrained_embeddings).to(self.device)
# create optimizer
self.optimizer = torch.optim.Adam(filter(lambda x: x.requires_grad,
self.model.parameters()),
betas=(0.9, 0.98),
eps=1e-09)
# load old model, optimizer if there is one
if self.use_old_model:
self.model, self.optimizer = load_checkpoint(
os.path.join(self.load_dir, "model.bin"), self.model,
self.optimizer, self.device)
# create a sceduled optimizer object
self.optimizer = ScheduledOptim(self.optimizer,
self.config["model_dim"],
self.config["warmup_steps"])
#self.optimizer.optimizer.to(torch.device('cpu'))
if self.config["weight"] is None:
self.weight = None
else:
self.weight = torch.Tensor(self.config["weight"]).to(self.device)
wandb.config.update(self.config)
wandb.watch(self.model)
def main(
exp_nbs=exp_nbs,
path_to_exps=path_to_exps,
path_to_results=save_path,
nb_of_runs=nb_of_runs,
nb_of_tests=nb_of_tests,
device='cpu',
**kwargs,
):
"""
Evaluates the accuracy for the experiments given. Writes it in a csv.
Args:
exp_nbs (list): list of int or str. Experiments to evaluate
path_to_exps (str): path to the experiments
path_to_results (str): path to the directory to save the results
nb_of_runs (int): number of times to run the same experiment for confidence interval
nb_of_tests (int): number of tests for inference for each prediction
device (torch.device): gpu or cpu, device to compute on
**kwargs: args to be able to put any arguments in our functions and not raise an error.
"""
save_path = pathlib.Path(path_to_results)
if save_output:
save_path.mkdir(exist_ok=True, parents=True)
if not os.path.exists(save_path / 'all_accs_uncs_mean.pkl'):
all_accs_uncs_mean = pd.DataFrame(columns=['exp_nb', 'group_nb', 'rho', 'std_prior', 'loss_type', 'number_of_tests'])
else:
all_accs_uncs_mean = load_from_file(save_path / 'all_accs_uncs_mean.pkl', )
all_accs_uncs_mean.to_csv(save_path / 'all_uncs_mean_backup.csv')
for _ in range(nb_of_runs):
for exp in exp_nbs:
print(f'Run {_+1}, Exp {exp}, computing accuracy and uncertainty...')
bay_net_trained, arguments, group_nb = get_trained_model_and_args_and_groupnb(exp, path_to_exps)
evalloader_seen = get_evalloader_seen(arguments)
evalloader_unseen = get_evalloader_unseen(arguments)
eval_acc, all_outputs_seen = eval_bayesian(
bay_net_trained,
evalloader_seen,
number_of_tests=nb_of_tests if arguments.get('rho', 'determinist') != 'determinist' else 1,
return_accuracy=True,
device=device,
verbose=True,
)
_, all_outputs_unseen = eval_bayesian(
bay_net_trained,
evalloader_unseen,
number_of_tests=nb_of_tests if arguments.get('rho', 'determinist') != 'determinist' else 1,
return_accuracy=True,
device=device,
verbose=True,
)
determinist = (
arguments.get('determinist', False) or
arguments.get('rho', 'determinist') == 'determinist'
)
if determinist:
get_unc_func = get_all_uncertainty_measures_not_bayesian
unc_names = ['us', 'pe']
else:
get_unc_func = get_all_uncertainty_measures_bayesian
unc_names = ['vr', 'pe', 'mi']
all_uncs_mean_seen = get_unc_func(all_outputs_seen)
all_uncs_mean_unseen = get_unc_func(all_outputs_unseen)
for unc_name, unc_seen, unc_unseen in zip(unc_names, all_uncs_mean_seen, all_uncs_mean_unseen):
all_accs_uncs_mean = all_accs_uncs_mean.append(pd.DataFrame.from_dict({
'exp_nb': [exp],
'group_nb': [group_nb],
'split_labels': [arguments.get('split_labels', 10)],
'trainset': [arguments.get('trainset', 'mnist')],
'rho': [arguments.get('rho', 'determinist')],
'std_prior': [arguments.get('std_prior', -1)],
'epoch': [arguments['epoch']],
'loss_type': [arguments.get('loss_type', 'criterion')],
'number_of_tests': [nb_of_tests],
'eval_acc': [eval_acc],
'unc_name': [unc_name],
'unc_seen': [unc_seen.mean().item()],
'unc_unseen': [unc_unseen.mean().item()],
'ratio': [unc_unseen.mean().item()/unc_seen.mean().item()],
}))
all_accs_uncs_mean.exp_nb = all_accs_uncs_mean.exp_nb.astype('int')
if save_output:
all_accs_uncs_mean.to_csv(save_path / 'all_accs_uncs_mean.csv')
all_accs_uncs_mean.to_pickle(save_path / 'all_accs_uncs_mean.pkl')
def main(
exp_nbs=exp_nbs,
path_to_results=save_path,
path_to_exps=path_to_exps,
n=n,
nb_of_runs=nb_of_runs,
number_of_tests=number_of_tests,
verbose=verbose,
nb_of_random=nb_of_random,
do_recompute_outputs=do_recompute_outputs,
save_csv=save_csv,
device='cpu',
):
save_path = pathlib.Path(path_to_results)
if do_recompute_outputs:
save_path = save_path / 'recomputed'
else:
save_path = save_path / 'saved_from_polyaxon'
save_path.mkdir(exist_ok=True, parents=True)
if not do_recompute_outputs:
nb_of_runs = 1
if not os.path.exists(save_path / 'deadzones.pkl'):
deadzones = pd.DataFrame(columns=['group_nb', 'exp_nb', 'unc_name'])
else:
deadzones = load_from_file(save_path / 'deadzones.pkl', )
deadzones.to_csv(save_path / 'deadzones.csv')
recomputed_exps = []
start_time = time()
for repeat_idx in range(nb_of_runs):
for exp_nb in exp_nbs:
print(
f'Repeat number {repeat_idx + 1} / {nb_of_runs}, Exp nb {exp_nb}'
)
arguments = get_args(exp_nb, path_to_exps)
determinist = arguments.get('rho', 'determinist') == 'determinist'
def recompute_outputs(deadzones):
bay_net_trained, arguments, group_nb = get_trained_model_and_args_and_groupnb(
exp_nb, exp_path=path_to_exps)
bay_net_trained.to(device)
arguments['number_of_tests'] = number_of_tests
all_eval_outputs, _ = get_seen_outputs_and_labels(
bay_net_trained,
arguments,
device=device,
verbose=verbose,
)
all_outputs_unseen = get_unseen_outputs(
bay_net_trained,
arguments,
nb_of_random,
device=device,
verbose=verbose,
)
if determinist:
dzs = get_deadzones(
all_eval_outputs, all_outputs_unseen,
get_all_uncertainty_measures_not_bayesian, n)
iterator = zip(['us', 'pe'], dzs)
else:
dzs = get_deadzones(all_eval_outputs, all_outputs_unseen,
get_all_uncertainty_measures_bayesian,
n)
iterator = zip(['vr', 'pe', 'mi'], dzs)
for unc_name, dz in iterator:
deadzones = deadzones.append(
pd.DataFrame.from_dict({
'group_nb': [group_nb],
'exp_nb': [exp_nb],
'trainset': [arguments.get('trainset', 'mnist')],
'type_of_unseen': [arguments['type_of_unseen']],
'epoch': [arguments['epoch']],
'number_of_tests': [arguments['number_of_tests']],
'unc_name': [unc_name],
f'dz_{n}': [dz],
}))
return deadzones
if do_recompute_outputs:
deadzones = recompute_outputs(deadzones)
else:
try:
results, arguments, group_nb = get_res_args_groupnb(
exp_nb, exp_path=path_to_exps)
except RuntimeError as e:
if e.__str__() == "Attempting to deserialize object on a CUDA device but torch.cuda.is_available() " \
"is False. If you are running on a CPU-only machine, please use torch.load with " \
"map_location='cpu' to map your storages to the CPU.":
recompute_outputs()
recomputed_exps.append(exp_nb)
continue
else:
raise e
def seen_and_unseen_and_n(results, unc, n):
return (results.get(
get_unc_key(results.columns, f'seen {unc}'),
[torch.tensor([-1], dtype=torch.float)])[0],
results.get(
get_unc_key(results.columns, f'unseen {unc}'),
[torch.tensor([-1], dtype=torch.float)])[0], n)
try:
dz_pe = get_deadzone_from_unc(
*seen_and_unseen_and_n(results, 'pe', n))
except:
dz_pe = -1
if determinist:
dz_us = get_deadzone_from_unc(
*seen_and_unseen_and_n(results, 'us', n))
iterator = zip(['us', 'pe'], [dz_us, dz_pe])
else:
dz_vr = get_deadzone_from_unc(
*seen_and_unseen_and_n(results, 'vr', n))
dz_mi = get_deadzone_from_unc(
*seen_and_unseen_and_n(results, 'mi', n))
iterator = zip(['vr', 'pe', 'mi'], [dz_vr, dz_pe, dz_mi])
for unc_name, dz in iterator:
deadzones = deadzones.append(
pd.DataFrame.from_dict({
'deadzone_number': [n],
'group_nb': [group_nb],
'trainset': [arguments.get('trainset', 'mnist')],
'exp_nb': [exp_nb],
'type_of_unseen': [arguments['type_of_unseen']],
'epoch': [arguments['epoch']],
'number_of_tests': [arguments['number_of_tests']],
'unc_name': [unc_name],
f'dz_{n}': [dz],
}))
print(f'Time Elapsed:{round(time() - start_time)} s.')
deadzones.exp_nb = deadzones.exp_nb.astype('int')
if save_csv:
save_to_file(arguments, save_path / 'arguments.pkl')
deadzones.sort_values('exp_nb')
deadzones.to_pickle(save_path / 'deadzones.pkl')
deadzones.to_csv(save_path / 'deadzones.csv')
print(deadzones)
import matplotlib.pyplot as plt
from src.utils import load_from_file
path_to_accs = 'polyaxon_results/groups/249/19888/accs_and_uncs.pkl'
path_to_pvalues = 'polyaxon_results/groups/249/19888/pvalues.pkl'
nb_bins = 10
df = load_from_file(path_to_accs)
figure = plt.figure()
ax1 = figure.add_subplot(111)
ax1.hist(df.accs1, label='dirac', bins=nb_bins, density=True)
ax1.hist(df.accs2, label='ce', bins=nb_bins, density=True)
ax1.legend()
ax1.set_xlabel('accuracy')
ax1.set_ylabel('density')
figure.show()
figure.savefig('results/dirac_ce.png')
from src.utils import get_file_and_dir_path_in_dir, load_from_file, save_to_file
path_to_exps = 'output/determinist_cifar10'
files, _ = get_file_and_dir_path_in_dir(path_to_exps, 'arguments.pkl')
for file in files:
args = load_from_file(file)
args['number_of_tests'] = 1
print(file, 'changed')
save_to_file(args, file)
def main(
path_to_acc=path_to_acc,
path_to_dz=path_to_dz,
path_to_auc=path_to_auc,
save_path=save_path,
):
save_path = pathlib.Path(save_path)
save_path.mkdir(exist_ok=True, parents=True)
df_dzs = load_from_file(path_to_dz)
df_aucs = load_from_file(path_to_auc)
df_accs_uncs = load_from_file(path_to_acc)
df_dzs_agg_show, df_dzs_agg_data = aggregate_df(df_dzs, indexs=[
'exp_nb',
'group_nb',
'trainset',
'epoch',
'number_of_tests',
'type_of_unseen',
'unc_name',
])
df_dzs_agg_show.to_csv(save_path / 'dzs_agg_show.csv')
df_dzs_agg_data.to_csv(save_path / 'dzs_agg_data.csv')
df_aucs_agg_show, df_aucs_agg_data = aggregate_df(df_aucs, indexs=[
'exp_nb',
'group_nb',
'trainset',
'rho',
'std_prior',
'loss_type',
'epoch',
'number_of_tests',
'unc_name',
])
df_aucs_agg_show.to_csv(save_path / 'aucs_agg_show.csv')
df_aucs_agg_data.to_csv(save_path / 'aucs_agg_data.csv')
df_accs_uncs_agg_show, df_accs_uncs_agg_data = aggregate_df(df_accs_uncs, indexs=[
'exp_nb',
'group_nb',
'trainset',
'rho',
'std_prior',
'loss_type',
'epoch',
'number_of_tests',
'split_labels',
])
df_accs_uncs_agg_show.to_csv(save_path / 'accs_uncs_agg_show.csv')
df_accs_uncs_agg_data.to_csv(save_path / 'accs_uncs_agg_data.csv')
first_join_col = ['exp_nb', 'group_nb', 'number_of_tests', 'rho', 'std_prior', 'epoch', 'loss_type', 'trainset',
'unc_name']
all_results_show = (
df_accs_uncs_agg_show
.merge(df_aucs_agg_show, on=first_join_col)
.merge(df_dzs_agg_show, on=['group_nb', 'exp_nb', 'number_of_tests', 'unc_name', 'epoch', 'trainset'])
)
all_results_data = (
df_accs_uncs_agg_data
.merge(df_aucs_agg_data, on=first_join_col)
.merge(df_dzs_agg_data, on=['group_nb', 'exp_nb', 'number_of_tests', 'unc_name', 'epoch', 'trainset'])
)
all_results_show = all_results_show.reindex([
'group_nb',
'exp_nb',
'trainset',
'type_of_unseen',
'split_labels',
'loss_type',
'std_prior',
'rho',
'epoch',
'number_of_tests',
'unc_name',
'eval_acc',
'auc',
'dz_100',
], axis=1)
all_results_data = all_results_data.reindex([
'group_nb',
'exp_nb',
'trainset',
'type_of_unseen',
'split_labels',
'loss_type',
'std_prior',
'rho',
'epoch',
'number_of_tests',
'unc_name',
'eval_acc',
'auc',
'dz_100',
], axis=1)
all_results_show.to_csv(save_path / 'all_results_show.csv')
all_results_show.to_pickle(save_path / 'all_results_show.pkl')
all_results_data.to_csv(save_path / 'all_results_data.csv')
all_results_data.to_pickle(save_path / 'all_results_data.pkl')
for group_nb in group_nbs:
all_path_exps += list(
set(get_file_and_dir_path_in_dir(join(result_dir, group_nb))[1]))
return all_path_exps
results = pd.DataFrame(columns=[
'exp_nb', 'exp_type', 'loss_type', 'nb_of_data', 'acc', 'vr', 'pe', 'mi'
])
result_dir = 'polyaxon_results/groups/'
all_exps = get_all_path_exps(group_nbs, result_dir)
for exp in all_exps:
try:
all_results = load_from_file(join(exp, 'results.pkl'))
arguments = load_from_file(join(exp, 'arguments.pkl'))
except FileNotFoundError:
print(exp, 'not found')
continue
exp_nb = exp.split('/')[-1]
if 'split_labels' in arguments.keys():
exp_type = 'unseen_classes'
elif 'dataset' in arguments.keys():
exp_type = 'unseen_dataset'
else:
exp_type = 'random'
results = results.append(
pd.DataFrame.from_dict({
'exp_nb': [exp_nb],
'exp_type': [exp_type],