def compare_results(self):
model_results = {}
dataset_results = {}
for eval_space in self.errors.keys():
model_results[eval_space] = Evaluator.means_per_metric(
self.errors[eval_space])
dataset_results[eval_space] = Evaluator.means_per_metric(
self.dataset_errors[eval_space])
Evaluator.print_comparison(model_results, dataset_results)
def _full_evaluation(self, model, eval_space):
default_results = Evaluator.means_per_metric(
Evaluator.to_model(self.val_loader, model, 'default'))
eval_results = {'default': default_results}
if eval_space == 'original':
original_results = Evaluator.means_per_metric(
Evaluator.to_model(self.val_loader, model, 'original'))
eval_results['original'] = original_results
for eval_space_results in eval_results.values():
Evaluator.results_to_cpu(eval_space_results)
return eval_results
def _intermediate_eval(self, session, batch):
if session.params['eval_space'] == 'original':
batch.original_poses = self.normalizer.denormalize(
batch.poses, batch.normalization_params)
train_results = {
'DEFAULT': Evaluator.to_batch(batch, session.params['eval_space'])
}
train_mean_results = Evaluator.means_per_metric(train_results)
Evaluator.results_to_cpu(train_mean_results)
val_mean_results = session.test_model(self.val_loader)
Evaluator.results_to_cpu(val_mean_results)
return train_mean_results, val_mean_results
def test_means_per_metric():
true_mean_results = {
'sub1': {
'coord_diff': torch.tensor(1.0),
'distance': torch.tensor(1.0),
'bone_length': torch.tensor(1.0),
'proportion': torch.tensor(1.0),
},
'sub2': {
'coord_diff': torch.tensor(2.0),
'distance': torch.tensor(2.0),
'bone_length': torch.tensor(3.0),
'proportion': torch.tensor(3.0),
}
}
mean_results = Evaluator.means_per_metric(example_results)
for subset_name in mean_results.keys():
for metric_name in Evaluator.metric_names:
assert torch.allclose(mean_results[subset_name][metric_name],
true_mean_results[subset_name][metric_name])
"""
Compute the mean evaluation metrics on a dataset (before applying any corrections).
"""
import os
import torch
from data_utils import datasets
from evaluation.evaluator import Evaluator
# Config
dataset_name = 'HANDS17_DPREN_SubjClust_val'
########################################################################
dataset = datasets.PairedPoseDataset(dataset_name, use_preset=True)
data_loader = datasets.DataLoader(dataset, 100000)
results = Evaluator.means_per_metric(Evaluator.to_dataset(data_loader, 'default'))
torch.save(results, os.path.join('results', 'datasets', dataset_name + '.pt'))