def test_get_score_lists(self): # save a score_list savedir_base = '.tmp' exp_dict = { 'model': { 'name': 'mlp', 'n_layers': 30 }, 'dataset': 'mnist', 'batch_size': 1 } score_list = [{'epoch': 0, 'acc': 0.5}, {'epoch': 0, 'acc': 0.9}] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict), 'score_list.pkl'), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict), 'exp_dict.json'), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hr.get_exp_list(savedir_base=savedir_base) score_lists = hr.get_score_lists(exp_list, savedir_base=savedir_base) assert (score_lists[0][0]['acc'] == 0.5) assert (score_lists[0][1]['acc'] == 0.9) shutil.rmtree(savedir_base)
def test_zipdir(self): # save a score_list savedir_base = ".tmp" exp_dict = { "model": { "name": "mlp", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } score_list = [{"epoch": 0, "acc": 0.5}, {"epoch": 0, "acc": 0.9}] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hr.get_exp_list(savedir_base=savedir_base) score_lists = hr.get_score_lists(exp_list, savedir_base=savedir_base) assert score_lists[0][0]["acc"] == 0.5 assert score_lists[0][1]["acc"] == 0.9 from haven import haven_dropbox as hd hd.zipdir([hu.hash_dict(exp_dict) for exp_dict in exp_list], savedir_base, src_fname=".tmp/results.zip") shutil.rmtree(savedir_base)
def get_one_plot(exp_list, savedir_base, plot_names=None): traceList = [] p_rules = np.unique([e['partition'] for e in exp_list]) s_rules = np.unique([e['selection'] for e in exp_list]) u_rules = np.unique([e['update'] for e in exp_list]) score_list_list = hr.get_score_lists(exp_list, savedir_base) assert (len(exp_list) == len(score_list_list)) for exp_dict, score_list in zip(exp_list, score_list_list): # single figure score_df = pd.DataFrame(score_list) if "converged" in score_df.columns: ind = np.where(np.isnan(np.array( score_df["converged"])))[0][-1] + 1 converged = {"Y": score_df["converged"][ind], "X": ind} else: converged = None legend = ut.legendFunc(exp_dict['partition'], exp_dict['selection'], exp_dict['update'], p_rules, s_rules, u_rules, plot_names=plot_names) trace = { "Y": np.array(score_df["loss"]), "X": np.array(score_df["iteration"]), "legend": legend, "converged": converged } traceList += [trace] return traceList
def test_get_score_lists(): # save a score_list savedir_base = ".tmp" exp_dict = { "model": { "name": "mlp", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } score_list = [{"epoch": 0, "acc": 0.5}, {"epoch": 0, "acc": 0.9}] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hu.get_exp_list(savedir_base=savedir_base) score_lists = hr.get_score_lists(exp_list, savedir_base=savedir_base) assert score_lists[0][0]["acc"] == 0.5 assert score_lists[0][1]["acc"] == 0.9 shutil.rmtree(savedir_base)