def test_get_score_lists(self):
# save a score_list
savedir_base = '.tmp'
exp_dict = {
'model': {
'name': 'mlp',
'n_layers': 30
},
'dataset': 'mnist',
'batch_size': 1
}
score_list = [{'epoch': 0, 'acc': 0.5}, {'epoch': 0, 'acc': 0.9}]
hu.save_pkl(
os.path.join(savedir_base, hu.hash_dict(exp_dict),
'score_list.pkl'), score_list)
hu.save_json(
os.path.join(savedir_base, hu.hash_dict(exp_dict),
'exp_dict.json'), exp_dict)
# check if score_list can be loaded and viewed in pandas
exp_list = hr.get_exp_list(savedir_base=savedir_base)
score_lists = hr.get_score_lists(exp_list, savedir_base=savedir_base)
assert (score_lists[0][0]['acc'] == 0.5)
assert (score_lists[0][1]['acc'] == 0.9)
shutil.rmtree(savedir_base)
def test_zipdir(self):
# save a score_list
savedir_base = ".tmp"
exp_dict = {
"model": {
"name": "mlp",
"n_layers": 30
},
"dataset": "mnist",
"batch_size": 1
}
score_list = [{"epoch": 0, "acc": 0.5}, {"epoch": 0, "acc": 0.9}]
hu.save_pkl(
os.path.join(savedir_base, hu.hash_dict(exp_dict),
"score_list.pkl"), score_list)
hu.save_json(
os.path.join(savedir_base, hu.hash_dict(exp_dict),
"exp_dict.json"), exp_dict)
# check if score_list can be loaded and viewed in pandas
exp_list = hr.get_exp_list(savedir_base=savedir_base)
score_lists = hr.get_score_lists(exp_list, savedir_base=savedir_base)
assert score_lists[0][0]["acc"] == 0.5
assert score_lists[0][1]["acc"] == 0.9
from haven import haven_dropbox as hd
hd.zipdir([hu.hash_dict(exp_dict) for exp_dict in exp_list],
savedir_base,
src_fname=".tmp/results.zip")
shutil.rmtree(savedir_base)
def get_one_plot(exp_list, savedir_base, plot_names=None):
traceList = []
p_rules = np.unique([e['partition'] for e in exp_list])
s_rules = np.unique([e['selection'] for e in exp_list])
u_rules = np.unique([e['update'] for e in exp_list])
score_list_list = hr.get_score_lists(exp_list, savedir_base)
assert (len(exp_list) == len(score_list_list))
for exp_dict, score_list in zip(exp_list, score_list_list):
# single figure
score_df = pd.DataFrame(score_list)
if "converged" in score_df.columns:
ind = np.where(np.isnan(np.array(
score_df["converged"])))[0][-1] + 1
converged = {"Y": score_df["converged"][ind], "X": ind}
else:
converged = None
legend = ut.legendFunc(exp_dict['partition'],
exp_dict['selection'],
exp_dict['update'],
p_rules,
s_rules,
u_rules,
plot_names=plot_names)
trace = {
"Y": np.array(score_df["loss"]),
"X": np.array(score_df["iteration"]),
"legend": legend,
"converged": converged
}
traceList += [trace]
return traceList
def test_get_score_lists():
# save a score_list
savedir_base = ".tmp"
exp_dict = {
"model": {
"name": "mlp",
"n_layers": 30
},
"dataset": "mnist",
"batch_size": 1
}
score_list = [{"epoch": 0, "acc": 0.5}, {"epoch": 0, "acc": 0.9}]
hu.save_pkl(
os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"),
score_list)
hu.save_json(
os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"),
exp_dict)
# check if score_list can be loaded and viewed in pandas
exp_list = hu.get_exp_list(savedir_base=savedir_base)
score_lists = hr.get_score_lists(exp_list, savedir_base=savedir_base)
assert score_lists[0][0]["acc"] == 0.5
assert score_lists[0][1]["acc"] == 0.9
shutil.rmtree(savedir_base)