def test_get_score_df(self):
# save a score_list
savedir_base = '.tmp'
exp_dict = {'model':{'name':'mlp', 'n_layers':30},
'dataset':'mnist', 'batch_size':1}
exp_dict2 = {'model':{'name':'mlp2', 'n_layers':30},
'dataset':'mnist', 'batch_size':1}
score_list = [{'epoch': 0, 'acc':0.5}, {'epoch': 0, 'acc':0.9}]
hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict),
'score_list.pkl'), score_list)
hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict),
'exp_dict.json'), exp_dict)
hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict2),
'exp_dict.json'), exp_dict)
# check if score_list can be loaded and viewed in pandas
exp_list = hu.get_exp_list(savedir_base=savedir_base)
score_df = hr.get_score_df(exp_list, savedir_base=savedir_base)
assert(np.array(score_df['dataset'])[0].strip("'") == 'mnist')
shutil.rmtree('.tmp')
def update_rm(self):
self.rm = hr.ResultManager(exp_list=self.rm_original.exp_list_all,
savedir_base=str(self.t_savedir_base.value),
filterby_list=get_dict_from_str(str(self.t_filterby_list.value)),
verbose=self.rm_original.verbose,
mode_key=self.rm_original.mode_key,
has_score_list=self.rm_original.has_score_list
)
if len(self.rm.exp_list) == 0:
if self.rm.n_exp_all > 0:
display('No experiments selected out of %d '
'for filtrby_list %s' % (self.rm.n_exp_all,
self.rm.filterby_list))
display('Table below shows all experiments.')
score_table = hr.get_score_df(exp_list=self.rm_original.exp_list_all,
savedir_base=self.rm_original.savedir_base)
display(score_table)
else:
display('No experiments exist...')
return
def test_get_score_df():
# save a score_list
savedir_base = ".tmp"
exp_dict = {
"model": {
"name": "mlp",
"n_layers": 30
},
"dataset": "mnist",
"batch_size": 1
}
exp_dict2 = {
"model": {
"name": "mlp2",
"n_layers": 30
},
"dataset": "mnist",
"batch_size": 1
}
score_list = [{"epoch": 0, "acc": 0.5}, {"epoch": 0, "acc": 0.9}]
hu.save_pkl(
os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"),
score_list)
hu.save_json(
os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"),
exp_dict)
hu.save_json(
os.path.join(savedir_base, hu.hash_dict(exp_dict2), "exp_dict.json"),
exp_dict)
# check if score_list can be loaded and viewed in pandas
exp_list = hu.get_exp_list(savedir_base=savedir_base)
score_df = hr.get_score_df(exp_list, savedir_base=savedir_base)
assert np.array(score_df["dataset"])[0].strip("'") == "mnist"
shutil.rmtree(".tmp")
"name": "mlp2",
"n_layers": 35
},
"dataset": "mnist",
"batch_size": 1
}
score_list = [{"epoch": 2, "acc": 0.1}, {"epoch": 6, "acc": 0.3}]
hu.save_pkl(
os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"),
score_list)
hu.save_json(
os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"),
exp_dict)
# check if score_list can be loaded and viewed in pandas
exp_list = hu.get_exp_list(savedir_base=savedir_base)
score_df = hr.get_score_df(exp_list, savedir_base=savedir_base)
print(
get_latex_table(
score_df,
columns=["acc", "epoch"],
rows=["model.name"],
filter_dict=dict(),
caption="Test",
float_format="%.2f",
label="ref:table1",
map_row_dict_dict=dict(),
map_col_dict=dict(),
))
print()
def test_avg_runs():
# save a score_list
savedir_base = ".tmp"
exp_dict = {
"run": 0,
"model": {
"name": "mlp",
"n_layers": 30
},
"dataset": "mnist",
"batch_size": 1
}
exp_dict2 = {
"run": 1,
"model": {
"name": "mlp",
"n_layers": 30
},
"dataset": "mnist",
"batch_size": 1
}
score_list = [
{
"epoch": 0,
"acc": 0.2,
},
{
"epoch": 1,
"acc": 1.0
},
{
"epoch": 2,
"acc": 0.9
},
]
hu.save_pkl(
os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"),
score_list)
hu.save_json(
os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"),
exp_dict)
score_list = [
{
"epoch": 0,
"acc": 0.4,
},
{
"epoch": 1,
"acc": 1.5
},
{
"epoch": 2,
"acc": 0.5
},
]
hu.save_pkl(
os.path.join(savedir_base, hu.hash_dict(exp_dict2), "score_list.pkl"),
score_list)
hu.save_json(
os.path.join(savedir_base, hu.hash_dict(exp_dict2), "exp_dict.json"),
exp_dict2)
# check if score_list can be loaded and viewed in pandas
exp_list = hu.get_exp_list(savedir_base=savedir_base)
score_df = hr.get_score_df(exp_list,
savedir_base=savedir_base,
avg_across="run")
assert score_df[("acc", "mean")].max() == 0.7
assert abs(score_df[("acc (min)", "mean")].max() - 0.3) < 1e-4
assert score_df[("acc (max)", "mean")].max() == 1.25
result_dict = hr.get_result_dict(exp_dict,
savedir_base,
exp_list=exp_list,
x_metric="epoch",
y_metric="acc",
avg_across="run")
np.testing.assert_array_almost_equal(result_dict["y_list"],
np.array([0.3, 1.25, 0.7]))
score_list = [{"epoch": 1, "acc": 1.5}, {"epoch": 2, "acc": 0.5}]
hu.save_pkl(
os.path.join(savedir_base, hu.hash_dict(exp_dict2), "score_list.pkl"),
score_list)
result_dict = hr.get_result_dict(exp_dict,
savedir_base,
exp_list=exp_list,
x_metric="epoch",
y_metric="acc",
avg_across="run")
np.testing.assert_array_almost_equal(result_dict["x_list"],
np.array([1, 2]))
np.testing.assert_array_almost_equal(result_dict["y_list"],
np.array([1.25, 0.7]))
shutil.rmtree(".tmp")
'test_spec': 'Spec.'
}
filter_dict = {
'Loss Function':
['Point Loss', 'CB Point Loss', 'W-CE (Full Sup.)']
}
caption_dict = {
'weakly_covid19_v1_c2': 'COVID19-A',
'weakly_covid19_v2_mixed_c2': 'COVID19-B-Mix',
'weakly_covid19_v3_mixed_c2': 'COVID19-C-Mix',
'weakly_covid19_v2_sep_c2': 'COVID19-B-Sep',
'weakly_covid19_v3_sep_c2': 'COVID19-C-Sep'
}
table = (hr.get_score_df(exp_list, savedir_base))
print(
create_latex_table(
table=table,
filter_dict=filter_dict,
map_row_dict_dict=map_row_dict_dict,
map_col_dict=map_col_dict,
float_format='%.2f',
caption=caption_dict[exp_group],
# label=caption_dict,
index=False))
for exp_dict in exp_list:
print(get_stats(exp_dict))
# save_images(exp_dict)
