def write_epoch(self, cur_epoch):
basic_stats = self.basic()
# Try to load customized metrics
task_stats = {}
for custom_metric in cfg.custom_metrics:
func = register.metric_dict.get(custom_metric)
if not func:
raise ValueError(
f'Unknown custom metric function name: {custom_metric}')
custom_metric_score = func(self._true, self._pred, self.task_type)
task_stats[custom_metric] = custom_metric_score
if not task_stats: # use default metrics if no matching custom metric
if self.task_type == 'regression':
task_stats = self.regression()
elif self.task_type == 'classification_binary':
task_stats = self.classification_binary()
elif self.task_type == 'classification_multi':
task_stats = self.classification_multi()
else:
raise ValueError('Task has to be regression or classification')
epoch_stats = {'epoch': cur_epoch}
eta_stats = {'eta': round(self.eta(cur_epoch), cfg.round)}
custom_stats = self.custom()
if self.name == 'train':
stats = {
**epoch_stats,
**eta_stats,
**basic_stats,
**task_stats,
**custom_stats
}
else:
stats = {
**epoch_stats,
**basic_stats,
**task_stats,
**custom_stats
}
# print
logging.info('{}: {}'.format(self.name, stats))
# json
dict_to_json(stats, '{}/stats.json'.format(self.out_dir))
# tensorboard
if cfg.tensorboard_each_run:
dict_to_tb(stats, self.tb_writer, cur_epoch)
self.reset()
def write_epoch(self, cur_epoch):
basic_stats = self.basic()
if self.task_type == 'regression':
task_stats = self.regression()
elif self.task_type == 'classification_binary':
task_stats = self.classification_binary()
elif self.task_type == 'classification_multi':
task_stats = self.classification_multi()
else:
raise ValueError('Task has to be regression or classification')
epoch_stats = {'epoch': cur_epoch}
eta_stats = {'eta': round(self.eta(cur_epoch), cfg.round)}
custom_stats = self.custom()
if self.name == 'train':
stats = {
**epoch_stats,
**eta_stats,
**basic_stats,
**task_stats,
**custom_stats
}
else:
stats = {
**epoch_stats,
**basic_stats,
**task_stats,
**custom_stats
}
# print
logging.info('{}: {}'.format(self.name, stats))
# json
dict_to_json(stats, '{}/stats.json'.format(self.out_dir))
# tensorboard
if cfg.tensorboard_each_run:
dict_to_tb(stats, self.tb_writer, cur_epoch)
self.reset()
def agg_runs(dir, metric_best='auto'):
r'''
Aggregate over different random seeds of a single experiment
Args:
dir (str): Directory of the results, containing 1 experiment
metric_best (str, optional): The metric for selecting the best
validation performance. Options: auto, accuracy, auc.
'''
results = {'train': None, 'val': None}
results_best = {'train': None, 'val': None}
for seed in os.listdir(dir):
if is_seed(seed):
dir_seed = os.path.join(dir, seed)
split = 'val'
if split in os.listdir(dir_seed):
dir_split = os.path.join(dir_seed, split)
fname_stats = os.path.join(dir_split, 'stats.json')
stats_list = json_to_dict_list(fname_stats)
if metric_best == 'auto':
metric = 'auc' if 'auc' in stats_list[0] else 'accuracy'
else:
metric = metric_best
performance_np = np.array( # noqa
[stats[metric] for stats in stats_list])
best_epoch = \
stats_list[
eval("performance_np.{}()".format(cfg.metric_agg))][
'epoch']
print(best_epoch)
for split in os.listdir(dir_seed):
if is_split(split):
dir_split = os.path.join(dir_seed, split)
fname_stats = os.path.join(dir_split, 'stats.json')
stats_list = json_to_dict_list(fname_stats)
stats_best = [
stats for stats in stats_list
if stats['epoch'] == best_epoch
][0]
print(stats_best)
stats_list = [[stats] for stats in stats_list]
if results[split] is None:
results[split] = stats_list
else:
results[split] = join_list(results[split], stats_list)
if results_best[split] is None:
results_best[split] = [stats_best]
else:
results_best[split] += [stats_best]
results = {k: v for k, v in results.items() if v is not None} # rm None
results_best = {k: v
for k, v in results_best.items()
if v is not None} # rm None
for key in results:
for i in range(len(results[key])):
results[key][i] = agg_dict_list(results[key][i])
for key in results_best:
results_best[key] = agg_dict_list(results_best[key])
# save aggregated results
for key, value in results.items():
dir_out = os.path.join(dir, 'agg', key)
makedirs_rm_exist(dir_out)
fname = os.path.join(dir_out, 'stats.json')
dict_list_to_json(value, fname)
if cfg.tensorboard_agg:
if SummaryWriter is None:
raise ImportError(
'Tensorboard support requires `tensorboardX`.')
writer = SummaryWriter(dir_out)
dict_list_to_tb(value, writer)
writer.close()
for key, value in results_best.items():
dir_out = os.path.join(dir, 'agg', key)
fname = os.path.join(dir_out, 'best.json')
dict_to_json(value, fname)
logging.info('Results aggregated across runs saved in {}'.format(
os.path.join(dir, 'agg')))