def test_get_score_df(self):
# save a score_list
savedir_base = '.tmp'
exp_dict = {'model':{'name':'mlp', 'n_layers':30},
'dataset':'mnist', 'batch_size':1}
exp_dict2 = {'model':{'name':'mlp2', 'n_layers':30},
'dataset':'mnist', 'batch_size':1}
score_list = [{'epoch': 0, 'acc':0.5}, {'epoch': 0, 'acc':0.9}]
hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict),
'score_list.pkl'), score_list)
hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict),
'exp_dict.json'), exp_dict)
hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict2),
'exp_dict.json'), exp_dict)
# check if score_list can be loaded and viewed in pandas
exp_list = hr.get_exp_list(savedir_base=savedir_base)
score_df = hr.get_score_df(exp_list, savedir_base=savedir_base)
assert(np.array(score_df['dataset'])[0].strip("'") == 'mnist')
shutil.rmtree('.tmp')
def test_get_score_lists(self):
# save a score_list
savedir_base = '.tmp'
exp_dict = {
'model': {
'name': 'mlp',
'n_layers': 30
},
'dataset': 'mnist',
'batch_size': 1
}
score_list = [{'epoch': 0, 'acc': 0.5}, {'epoch': 0, 'acc': 0.9}]
hu.save_pkl(
os.path.join(savedir_base, hu.hash_dict(exp_dict),
'score_list.pkl'), score_list)
hu.save_json(
os.path.join(savedir_base, hu.hash_dict(exp_dict),
'exp_dict.json'), exp_dict)
# check if score_list can be loaded and viewed in pandas
exp_list = hr.get_exp_list(savedir_base=savedir_base)
score_lists = hr.get_score_lists(exp_list, savedir_base=savedir_base)
assert (score_lists[0][0]['acc'] == 0.5)
assert (score_lists[0][1]['acc'] == 0.9)
shutil.rmtree(savedir_base)
def test_get_plot(self):
# save a score_list
savedir_base = '.tmp'
exp_dict = {'model':{'name':'mlp', 'n_layers':30},
'dataset':'mnist', 'batch_size':1}
score_list = [{'epoch': 0, 'acc':0.5}, {'epoch': 1, 'acc':0.9}]
hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict),
'score_list.pkl'), score_list)
hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict),
'exp_dict.json'), exp_dict)
# check if score_list can be loaded and viewed in pandas
exp_list = hr.get_exp_list(savedir_base=savedir_base)
fig, axis = hr.get_plot(exp_list,
savedir_base=savedir_base,
filterby_list=[({'model':{'name':'mlp'}},
{'style':{'color':'red'}})],
x_metric='epoch',
y_metric='acc')
# fig, axis = hr.get_plot(exp_list,
# savedir_base=savedir_base,
# x_metric='epoch',
# y_metric='acc',
# mode='pretty_plot')
fig, axis = hr.get_plot(exp_list,
savedir_base=savedir_base,
x_metric='epoch',
y_metric='acc',
mode='bar')
fig.savefig(os.path.join('.tmp',
'test.png'))
shutil.rmtree('.tmp')
def test_zipdir(self):
# save a score_list
savedir_base = ".tmp"
exp_dict = {
"model": {
"name": "mlp",
"n_layers": 30
},
"dataset": "mnist",
"batch_size": 1
}
score_list = [{"epoch": 0, "acc": 0.5}, {"epoch": 0, "acc": 0.9}]
hu.save_pkl(
os.path.join(savedir_base, hu.hash_dict(exp_dict),
"score_list.pkl"), score_list)
hu.save_json(
os.path.join(savedir_base, hu.hash_dict(exp_dict),
"exp_dict.json"), exp_dict)
# check if score_list can be loaded and viewed in pandas
exp_list = hr.get_exp_list(savedir_base=savedir_base)
score_lists = hr.get_score_lists(exp_list, savedir_base=savedir_base)
assert score_lists[0][0]["acc"] == 0.5
assert score_lists[0][1]["acc"] == 0.9
from haven import haven_dropbox as hd
hd.zipdir([hu.hash_dict(exp_dict) for exp_dict in exp_list],
savedir_base,
src_fname=".tmp/results.zip")
shutil.rmtree(savedir_base)