def test_get_score_df(self): # save a score_list savedir_base = '.tmp' exp_dict = {'model':{'name':'mlp', 'n_layers':30}, 'dataset':'mnist', 'batch_size':1} exp_dict2 = {'model':{'name':'mlp2', 'n_layers':30}, 'dataset':'mnist', 'batch_size':1} score_list = [{'epoch': 0, 'acc':0.5}, {'epoch': 0, 'acc':0.9}] hu.save_pkl(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'score_list.pkl'), score_list) hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict), 'exp_dict.json'), exp_dict) hu.save_json(os.path.join(savedir_base, hu.hash_dict(exp_dict2), 'exp_dict.json'), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hu.get_exp_list(savedir_base=savedir_base) score_df = hr.get_score_df(exp_list, savedir_base=savedir_base) assert(np.array(score_df['dataset'])[0].strip("'") == 'mnist') shutil.rmtree('.tmp')
def update_rm(self): self.rm = hr.ResultManager(exp_list=self.rm_original.exp_list_all, savedir_base=str(self.t_savedir_base.value), filterby_list=get_dict_from_str(str(self.t_filterby_list.value)), verbose=self.rm_original.verbose, mode_key=self.rm_original.mode_key, has_score_list=self.rm_original.has_score_list ) if len(self.rm.exp_list) == 0: if self.rm.n_exp_all > 0: display('No experiments selected out of %d ' 'for filtrby_list %s' % (self.rm.n_exp_all, self.rm.filterby_list)) display('Table below shows all experiments.') score_table = hr.get_score_df(exp_list=self.rm_original.exp_list_all, savedir_base=self.rm_original.savedir_base) display(score_table) else: display('No experiments exist...') return
def test_get_score_df(): # save a score_list savedir_base = ".tmp" exp_dict = { "model": { "name": "mlp", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } exp_dict2 = { "model": { "name": "mlp2", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } score_list = [{"epoch": 0, "acc": 0.5}, {"epoch": 0, "acc": 0.9}] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"), exp_dict) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict2), "exp_dict.json"), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hu.get_exp_list(savedir_base=savedir_base) score_df = hr.get_score_df(exp_list, savedir_base=savedir_base) assert np.array(score_df["dataset"])[0].strip("'") == "mnist" shutil.rmtree(".tmp")
"name": "mlp2", "n_layers": 35 }, "dataset": "mnist", "batch_size": 1 } score_list = [{"epoch": 2, "acc": 0.1}, {"epoch": 6, "acc": 0.3}] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"), exp_dict) # check if score_list can be loaded and viewed in pandas exp_list = hu.get_exp_list(savedir_base=savedir_base) score_df = hr.get_score_df(exp_list, savedir_base=savedir_base) print( get_latex_table( score_df, columns=["acc", "epoch"], rows=["model.name"], filter_dict=dict(), caption="Test", float_format="%.2f", label="ref:table1", map_row_dict_dict=dict(), map_col_dict=dict(), )) print()
def test_avg_runs(): # save a score_list savedir_base = ".tmp" exp_dict = { "run": 0, "model": { "name": "mlp", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } exp_dict2 = { "run": 1, "model": { "name": "mlp", "n_layers": 30 }, "dataset": "mnist", "batch_size": 1 } score_list = [ { "epoch": 0, "acc": 0.2, }, { "epoch": 1, "acc": 1.0 }, { "epoch": 2, "acc": 0.9 }, ] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict), "score_list.pkl"), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict), "exp_dict.json"), exp_dict) score_list = [ { "epoch": 0, "acc": 0.4, }, { "epoch": 1, "acc": 1.5 }, { "epoch": 2, "acc": 0.5 }, ] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict2), "score_list.pkl"), score_list) hu.save_json( os.path.join(savedir_base, hu.hash_dict(exp_dict2), "exp_dict.json"), exp_dict2) # check if score_list can be loaded and viewed in pandas exp_list = hu.get_exp_list(savedir_base=savedir_base) score_df = hr.get_score_df(exp_list, savedir_base=savedir_base, avg_across="run") assert score_df[("acc", "mean")].max() == 0.7 assert abs(score_df[("acc (min)", "mean")].max() - 0.3) < 1e-4 assert score_df[("acc (max)", "mean")].max() == 1.25 result_dict = hr.get_result_dict(exp_dict, savedir_base, exp_list=exp_list, x_metric="epoch", y_metric="acc", avg_across="run") np.testing.assert_array_almost_equal(result_dict["y_list"], np.array([0.3, 1.25, 0.7])) score_list = [{"epoch": 1, "acc": 1.5}, {"epoch": 2, "acc": 0.5}] hu.save_pkl( os.path.join(savedir_base, hu.hash_dict(exp_dict2), "score_list.pkl"), score_list) result_dict = hr.get_result_dict(exp_dict, savedir_base, exp_list=exp_list, x_metric="epoch", y_metric="acc", avg_across="run") np.testing.assert_array_almost_equal(result_dict["x_list"], np.array([1, 2])) np.testing.assert_array_almost_equal(result_dict["y_list"], np.array([1.25, 0.7])) shutil.rmtree(".tmp")
'test_spec': 'Spec.' } filter_dict = { 'Loss Function': ['Point Loss', 'CB Point Loss', 'W-CE (Full Sup.)'] } caption_dict = { 'weakly_covid19_v1_c2': 'COVID19-A', 'weakly_covid19_v2_mixed_c2': 'COVID19-B-Mix', 'weakly_covid19_v3_mixed_c2': 'COVID19-C-Mix', 'weakly_covid19_v2_sep_c2': 'COVID19-B-Sep', 'weakly_covid19_v3_sep_c2': 'COVID19-C-Sep' } table = (hr.get_score_df(exp_list, savedir_base)) print( create_latex_table( table=table, filter_dict=filter_dict, map_row_dict_dict=map_row_dict_dict, map_col_dict=map_col_dict, float_format='%.2f', caption=caption_dict[exp_group], # label=caption_dict, index=False)) for exp_dict in exp_list: print(get_stats(exp_dict)) # save_images(exp_dict)