def write_epoch(self, cur_epoch):
basic_stats = self.basic()
if self.task_type == 'regression':
task_stats = self.regression()
elif self.task_type == 'classification_binary':
task_stats = self.classification_binary()
elif self.task_type == 'classification_multi':
task_stats = self.classification_multi()
else:
raise ValueError('Task has to be regression or classification')
epoch_stats = {'epoch': cur_epoch}
eta_stats = {'eta': round(self.eta(cur_epoch), cfg.round)}
custom_stats = self.custom()
if self.name == 'train':
stats = {**epoch_stats, **eta_stats, **basic_stats, **task_stats,
**custom_stats}
else:
stats = {**epoch_stats, **basic_stats, **task_stats, **custom_stats}
# print
logging.info('{}: {}'.format(self.name, stats))
# json
dict_to_json(stats, '{}/stats.json'.format(self.out_dir))
# tensorboard
if cfg.tensorboard_each_run:
dict_to_tb(stats, self.tb_writer, cur_epoch)
self.reset()
def write_epoch(self, cur_epoch, loader=None):
# additionally pass in loader to access graph structure for community metrics
basic_stats = self.basic()
if self.task_type == 'regression':
task_stats = self.regression()
elif self.task_type == 'classification_binary':
task_stats = self.classification_binary()
elif self.task_type == 'classification_multi':
task_stats = self.classification_multi()
elif self.task_type == 'community':
# TODO code smell, these should be registered like loaders
task_stats = self.community_metrics(loader)
else:
raise ValueError('unknown task type')
epoch_stats = {'epoch': cur_epoch}
eta_stats = {'eta': round(self.eta(cur_epoch), cfg.round)}
if self.name == 'train':
stats = {**epoch_stats, **eta_stats, **basic_stats, **task_stats}
else:
stats = {**epoch_stats, **basic_stats, **task_stats}
# print
# logging.info('{}: {}'.format(self.name, stats))
# json
dict_to_json(stats, '{}/stats.json'.format(self.out_dir))
# tensorboard
if cfg.tensorboard_each_run:
dict_to_tb(stats, self.tb_writer, cur_epoch)
self.reset()
def agg_runs(dir, metric_best='auto'):
results = {'train': None, 'val': None, 'test': None}
results_best = {'train': None, 'val': None, 'test': None}
for seed in os.listdir(dir):
if is_seed(seed):
dir_seed = os.path.join(dir, seed)
best_epoch = None
# split = 'val'
split = 'train' # hack: only consider train split for unsupervised case (the
# only split there is)
if split in os.listdir(dir_seed):
dir_split = os.path.join(dir_seed, split)
fname_stats = os.path.join(dir_split, 'stats.json')
stats_list = json_to_dict_list(fname_stats) # read files
if metric_best == 'auto':
metric = 'auc' if 'auc' in stats_list[0] else 'accuracy'
else:
metric = metric_best
performance_np = np.array(
[stats[metric] for stats in stats_list])
best_epoch = stats_list[performance_np.argmax()]['epoch']
print('best epoch:', best_epoch)
# hack to catch cases where best_epoch is not determined
if best_epoch is None:
best_epoch = 0
for split in os.listdir(dir_seed):
if is_split(split):
dir_split = os.path.join(dir_seed, split)
fname_stats = os.path.join(dir_split, 'stats.json')
stats_list = json_to_dict_list(fname_stats)
# for each split, pick the stats of the epoch which gave the best
# performance on the validation split (see above)
stats_best = \
[stats for stats in stats_list if stats['epoch'] == best_epoch][0]
# write to file
dict_list_to_json([stats_best],
os.path.join(dir_seed, "best.json"))
print(stats_best)
stats_list = [[stats] for stats in stats_list]
if results[split] is None:
results[split] = stats_list
else:
results[split] = join_list(results[split], stats_list)
if results_best[split] is None:
results_best[split] = [stats_best]
else:
results_best[split] += [stats_best]
results = {k: v for k, v in results.items() if v is not None} # rm None
results_best = {k: v
for k, v in results_best.items()
if v is not None} # rm None
for key in results:
for i in range(len(results[key])):
results[key][i] = agg_dict_list(results[key][i])
for key in results_best:
results_best[key] = agg_dict_list(results_best[key])
# save aggregated results
for key, value in results.items():
dir_out = os.path.join(dir, 'agg', key)
makedirs_rm_exist(dir_out)
fname = os.path.join(dir_out, 'stats.json')
dict_list_to_json(value, fname)
if cfg.tensorboard_agg:
writer = SummaryWriter(dir_out)
dict_list_to_tb(value, writer)
writer.close()
for key, value in results_best.items():
dir_out = os.path.join(dir, 'agg', key)
fname = os.path.join(dir_out, 'best.json')
dict_to_json(value, fname)
logging.info('Results aggregated across runs saved in {}'.format(
os.path.join(dir, 'agg')))
def agg_runs(dir, metric_best='auto'):
results = {'train': None, 'val': None, 'test': None}
results_best = {'train': None, 'val': None, 'test': None}
for seed in os.listdir(dir):
if is_seed(seed):
dir_seed = os.path.join(dir, seed)
split = 'val'
if split in os.listdir(dir_seed):
dir_split = os.path.join(dir_seed, split)
fname_stats = os.path.join(dir_split, 'stats.json')
stats_list = json_to_dict_list(fname_stats)
if metric_best == 'auto':
metric = 'auc' if 'auc' in stats_list[0] else 'accuracy'
else:
metric = metric_best
performance_np = np.array(
[stats[metric] for stats in stats_list])
best_epoch = stats_list[performance_np.argmax()]['epoch']
print(best_epoch)
for split in os.listdir(dir_seed):
if is_split(split):
dir_split = os.path.join(dir_seed, split)
fname_stats = os.path.join(dir_split, 'stats.json')
stats_list = json_to_dict_list(fname_stats)
stats_best = [
stats for stats in stats_list
if stats['epoch'] == best_epoch
][0]
print(stats_best)
stats_list = [[stats] for stats in stats_list]
if results[split] is None:
results[split] = stats_list
else:
results[split] = join_list(results[split], stats_list)
if results_best[split] is None:
results_best[split] = [stats_best]
else:
results_best[split] += [stats_best]
results = {k: v for k, v in results.items() if v is not None} # rm None
results_best = {k: v
for k, v in results_best.items()
if v is not None} # rm None
for key in results:
for i in range(len(results[key])):
results[key][i] = agg_dict_list(results[key][i])
for key in results_best:
results_best[key] = agg_dict_list(results_best[key])
# save aggregated results
for key, value in results.items():
dir_out = os.path.join(dir, 'agg', key)
makedirs_rm_exist(dir_out)
fname = os.path.join(dir_out, 'stats.json')
dict_list_to_json(value, fname)
if cfg.tensorboard_agg:
writer = SummaryWriter(dir_out)
dict_list_to_tb(value, writer)
writer.close()
for key, value in results_best.items():
dir_out = os.path.join(dir, 'agg', key)
fname = os.path.join(dir_out, 'best.json')
dict_to_json(value, fname)
logging.info('Results aggregated across runs saved in {}'.format(
os.path.join(dir, 'agg')))