def agg_runs(dir, metric_best='auto'):
r'''
Aggregate over different random seeds of a single experiment
Args:
dir (str): Directory of the results, containing 1 experiment
metric_best (str, optional): The metric for selecting the best
validation performance. Options: auto, accuracy, auc.
'''
results = {'train': None, 'val': None}
results_best = {'train': None, 'val': None}
for seed in os.listdir(dir):
if is_seed(seed):
dir_seed = os.path.join(dir, seed)
split = 'val'
if split in os.listdir(dir_seed):
dir_split = os.path.join(dir_seed, split)
fname_stats = os.path.join(dir_split, 'stats.json')
stats_list = json_to_dict_list(fname_stats)
if metric_best == 'auto':
metric = 'auc' if 'auc' in stats_list[0] else 'accuracy'
else:
metric = metric_best
performance_np = np.array( # noqa
[stats[metric] for stats in stats_list])
best_epoch = \
stats_list[
eval("performance_np.{}()".format(cfg.metric_agg))][
'epoch']
print(best_epoch)
for split in os.listdir(dir_seed):
if is_split(split):
dir_split = os.path.join(dir_seed, split)
fname_stats = os.path.join(dir_split, 'stats.json')
stats_list = json_to_dict_list(fname_stats)
stats_best = [
stats for stats in stats_list
if stats['epoch'] == best_epoch
][0]
print(stats_best)
stats_list = [[stats] for stats in stats_list]
if results[split] is None:
results[split] = stats_list
else:
results[split] = join_list(results[split], stats_list)
if results_best[split] is None:
results_best[split] = [stats_best]
else:
results_best[split] += [stats_best]
results = {k: v for k, v in results.items() if v is not None} # rm None
results_best = {k: v
for k, v in results_best.items()
if v is not None} # rm None
for key in results:
for i in range(len(results[key])):
results[key][i] = agg_dict_list(results[key][i])
for key in results_best:
results_best[key] = agg_dict_list(results_best[key])
# save aggregated results
for key, value in results.items():
dir_out = os.path.join(dir, 'agg', key)
makedirs_rm_exist(dir_out)
fname = os.path.join(dir_out, 'stats.json')
dict_list_to_json(value, fname)
if cfg.tensorboard_agg:
if SummaryWriter is None:
raise ImportError(
'Tensorboard support requires `tensorboardX`.')
writer = SummaryWriter(dir_out)
dict_list_to_tb(value, writer)
writer.close()
for key, value in results_best.items():
dir_out = os.path.join(dir, 'agg', key)
fname = os.path.join(dir_out, 'best.json')
dict_to_json(value, fname)
logging.info('Results aggregated across runs saved in {}'.format(
os.path.join(dir, 'agg')))
def agg_batch(dir, metric_best='auto'):
r'''
Aggregate across results from multiple experiments via grid search
Args:
dir (str): Directory of the results, containing multiple experiments
metric_best (str, optional): The metric for selecting the best
validation performance. Options: auto, accuracy, auc.
'''
import pandas as pd
results = {'train': [], 'val': [], 'test': []}
for run in os.listdir(dir):
if run != 'agg':
dict_name = name_to_dict(run)
dir_run = os.path.join(dir, run, 'agg')
if os.path.isdir(dir_run):
for split in os.listdir(dir_run):
dir_split = os.path.join(dir_run, split)
fname_stats = os.path.join(dir_split, 'best.json')
dict_stats = json_to_dict_list(fname_stats)[
-1] # get best val epoch
rm_keys(dict_stats,
['lr', 'lr_std', 'eta', 'eta_std', 'params_std'])
results[split].append({**dict_name, **dict_stats})
dir_out = os.path.join(dir, 'agg')
makedirs_rm_exist(dir_out)
for key in results:
if len(results[key]) > 0:
results[key] = pd.DataFrame(results[key])
results[key] = results[key].sort_values(list(dict_name.keys()),
ascending=[True] *
len(dict_name))
fname = os.path.join(dir_out, '{}_best.csv'.format(key))
results[key].to_csv(fname, index=False)
results = {'train': [], 'val': [], 'test': []}
for run in os.listdir(dir):
if run != 'agg':
dict_name = name_to_dict(run)
dir_run = os.path.join(dir, run, 'agg')
if os.path.isdir(dir_run):
for split in os.listdir(dir_run):
dir_split = os.path.join(dir_run, split)
fname_stats = os.path.join(dir_split, 'stats.json')
dict_stats = json_to_dict_list(fname_stats)[
-1] # get last epoch
rm_keys(dict_stats,
['lr', 'lr_std', 'eta', 'eta_std', 'params_std'])
results[split].append({**dict_name, **dict_stats})
dir_out = os.path.join(dir, 'agg')
for key in results:
if len(results[key]) > 0:
results[key] = pd.DataFrame(results[key])
results[key] = results[key].sort_values(list(dict_name.keys()),
ascending=[True] *
len(dict_name))
fname = os.path.join(dir_out, '{}.csv'.format(key))
results[key].to_csv(fname, index=False)
results = {'train': [], 'val': [], 'test': []}
for run in os.listdir(dir):
if run != 'agg':
dict_name = name_to_dict(run)
dir_run = os.path.join(dir, run, 'agg')
if os.path.isdir(dir_run):
for split in os.listdir(dir_run):
dir_split = os.path.join(dir_run, split)
fname_stats = os.path.join(dir_split, 'stats.json')
dict_stats = json_to_dict_list(
fname_stats) # get best epoch
if metric_best == 'auto':
metric = 'auc' if 'auc' in dict_stats[0] \
else 'accuracy'
else:
metric = metric_best
performance_np = np.array( # noqa
[stats[metric] for stats in dict_stats])
dict_stats = dict_stats[eval("performance_np.{}()".format(
cfg.metric_agg))]
rm_keys(dict_stats,
['lr', 'lr_std', 'eta', 'eta_std', 'params_std'])
results[split].append({**dict_name, **dict_stats})
dir_out = os.path.join(dir, 'agg')
for key in results:
if len(results[key]) > 0:
results[key] = pd.DataFrame(results[key])
results[key] = results[key].sort_values(list(dict_name.keys()),
ascending=[True] *
len(dict_name))
fname = os.path.join(dir_out, '{}_bestepoch.csv'.format(key))
results[key].to_csv(fname, index=False)
print('Results aggregated across models saved in {}'.format(dir_out))
