def agg_batch(dir, metric_best='auto'): results = {'train': [], 'val': [], 'test': []} for run in os.listdir(dir): if run != 'agg': dict_name = name_to_dict(run) dir_run = os.path.join(dir, run, 'agg') if os.path.isdir(dir_run): for split in os.listdir(dir_run): dir_split = os.path.join(dir_run, split) fname_stats = os.path.join(dir_split, 'best.json') dict_stats = json_to_dict_list(fname_stats)[ -1] # get best val epoch rm_keys(dict_stats, ['lr', 'lr_std', 'eta', 'eta_std', 'params_std']) results[split].append({**dict_name, **dict_stats}) dir_out = os.path.join(dir, 'agg') makedirs_rm_exist(dir_out) for key in results: if len(results[key]) > 0: results[key] = pd.DataFrame(results[key]) results[key] = results[key].sort_values(list(dict_name.keys()), ascending=[True] * len(dict_name)) fname = os.path.join(dir_out, '{}_best.csv'.format(key)) results[key].to_csv(fname, index=False) results = {'train': [], 'val': [], 'test': []} for run in os.listdir(dir): if run != 'agg': dict_name = name_to_dict(run) dir_run = os.path.join(dir, run, 'agg') if os.path.isdir(dir_run): for split in os.listdir(dir_run): dir_split = os.path.join(dir_run, split) fname_stats = os.path.join(dir_split, 'stats.json') dict_stats = json_to_dict_list(fname_stats)[ -1] # get last epoch rm_keys(dict_stats, ['lr', 'lr_std', 'eta', 'eta_std', 'params_std']) results[split].append({**dict_name, **dict_stats}) dir_out = os.path.join(dir, 'agg') for key in results: if len(results[key]) > 0: results[key] = pd.DataFrame(results[key]) results[key] = results[key].sort_values(list(dict_name.keys()), ascending=[True] * len(dict_name)) fname = os.path.join(dir_out, '{}.csv'.format(key)) results[key].to_csv(fname, index=False) results = {'train': [], 'val': [], 'test': []} for run in os.listdir(dir): if run != 'agg': dict_name = name_to_dict(run) dir_run = os.path.join(dir, run, 'agg') if os.path.isdir(dir_run): for split in os.listdir(dir_run): dir_split = os.path.join(dir_run, split) fname_stats = os.path.join(dir_split, 'stats.json') dict_stats = json_to_dict_list( fname_stats) # get best epoch if metric_best == 'auto': metric = 'auc' if 'auc' in dict_stats[0] else 'accuracy' else: metric = metric_best performance_np = np.array( [stats[metric] for stats in dict_stats]) dict_stats = dict_stats[performance_np.argmax()] rm_keys(dict_stats, ['lr', 'lr_std', 'eta', 'eta_std', 'params_std']) results[split].append({**dict_name, **dict_stats}) dir_out = os.path.join(dir, 'agg') for key in results: if len(results[key]) > 0: results[key] = pd.DataFrame(results[key]) results[key] = results[key].sort_values(list(dict_name.keys()), ascending=[True] * len(dict_name)) fname = os.path.join(dir_out, '{}_bestepoch.csv'.format(key)) results[key].to_csv(fname, index=False) print('Results aggregated across models saved in {}'.format(dir_out))
def agg_runs(dir, metric_best='auto'): results = {'train': None, 'val': None, 'test': None} results_best = {'train': None, 'val': None, 'test': None} for seed in os.listdir(dir): if is_seed(seed): dir_seed = os.path.join(dir, seed) best_epoch = None # split = 'val' split = 'train' # hack: only consider train split for unsupervised case (the # only split there is) if split in os.listdir(dir_seed): dir_split = os.path.join(dir_seed, split) fname_stats = os.path.join(dir_split, 'stats.json') stats_list = json_to_dict_list(fname_stats) # read files if metric_best == 'auto': metric = 'auc' if 'auc' in stats_list[0] else 'accuracy' else: metric = metric_best performance_np = np.array( [stats[metric] for stats in stats_list]) best_epoch = stats_list[performance_np.argmax()]['epoch'] print('best epoch:', best_epoch) # hack to catch cases where best_epoch is not determined if best_epoch is None: best_epoch = 0 for split in os.listdir(dir_seed): if is_split(split): dir_split = os.path.join(dir_seed, split) fname_stats = os.path.join(dir_split, 'stats.json') stats_list = json_to_dict_list(fname_stats) # for each split, pick the stats of the epoch which gave the best # performance on the validation split (see above) stats_best = \ [stats for stats in stats_list if stats['epoch'] == best_epoch][0] # write to file dict_list_to_json([stats_best], os.path.join(dir_seed, "best.json")) print(stats_best) stats_list = [[stats] for stats in stats_list] if results[split] is None: results[split] = stats_list else: results[split] = join_list(results[split], stats_list) if results_best[split] is None: results_best[split] = [stats_best] else: results_best[split] += [stats_best] results = {k: v for k, v in results.items() if v is not None} # rm None results_best = {k: v for k, v in results_best.items() if v is not None} # rm None for key in results: for i in range(len(results[key])): results[key][i] = agg_dict_list(results[key][i]) for key in results_best: results_best[key] = agg_dict_list(results_best[key]) # save aggregated results for key, value in results.items(): dir_out = os.path.join(dir, 'agg', key) makedirs_rm_exist(dir_out) fname = os.path.join(dir_out, 'stats.json') dict_list_to_json(value, fname) if cfg.tensorboard_agg: writer = SummaryWriter(dir_out) dict_list_to_tb(value, writer) writer.close() for key, value in results_best.items(): dir_out = os.path.join(dir, 'agg', key) fname = os.path.join(dir_out, 'best.json') dict_to_json(value, fname) logging.info('Results aggregated across runs saved in {}'.format( os.path.join(dir, 'agg')))
def agg_runs(dir, metric_best='auto'): results = {'train': None, 'val': None, 'test': None} results_best = {'train': None, 'val': None, 'test': None} for seed in os.listdir(dir): if is_seed(seed): dir_seed = os.path.join(dir, seed) split = 'val' if split in os.listdir(dir_seed): dir_split = os.path.join(dir_seed, split) fname_stats = os.path.join(dir_split, 'stats.json') stats_list = json_to_dict_list(fname_stats) if metric_best == 'auto': metric = 'auc' if 'auc' in stats_list[0] else 'accuracy' else: metric = metric_best performance_np = np.array( [stats[metric] for stats in stats_list]) best_epoch = stats_list[performance_np.argmax()]['epoch'] print(best_epoch) for split in os.listdir(dir_seed): if is_split(split): dir_split = os.path.join(dir_seed, split) fname_stats = os.path.join(dir_split, 'stats.json') stats_list = json_to_dict_list(fname_stats) stats_best = [ stats for stats in stats_list if stats['epoch'] == best_epoch ][0] print(stats_best) stats_list = [[stats] for stats in stats_list] if results[split] is None: results[split] = stats_list else: results[split] = join_list(results[split], stats_list) if results_best[split] is None: results_best[split] = [stats_best] else: results_best[split] += [stats_best] results = {k: v for k, v in results.items() if v is not None} # rm None results_best = {k: v for k, v in results_best.items() if v is not None} # rm None for key in results: for i in range(len(results[key])): results[key][i] = agg_dict_list(results[key][i]) for key in results_best: results_best[key] = agg_dict_list(results_best[key]) # save aggregated results for key, value in results.items(): dir_out = os.path.join(dir, 'agg', key) makedirs_rm_exist(dir_out) fname = os.path.join(dir_out, 'stats.json') dict_list_to_json(value, fname) if cfg.tensorboard_agg: writer = SummaryWriter(dir_out) dict_list_to_tb(value, writer) writer.close() for key, value in results_best.items(): dir_out = os.path.join(dir, 'agg', key) fname = os.path.join(dir_out, 'best.json') dict_to_json(value, fname) logging.info('Results aggregated across runs saved in {}'.format( os.path.join(dir, 'agg')))