def compare_results(self): model_results = {} dataset_results = {} for eval_space in self.errors.keys(): model_results[eval_space] = Evaluator.means_per_metric( self.errors[eval_space]) dataset_results[eval_space] = Evaluator.means_per_metric( self.dataset_errors[eval_space]) Evaluator.print_comparison(model_results, dataset_results)
def _full_evaluation(self, model, eval_space): default_results = Evaluator.means_per_metric( Evaluator.to_model(self.val_loader, model, 'default')) eval_results = {'default': default_results} if eval_space == 'original': original_results = Evaluator.means_per_metric( Evaluator.to_model(self.val_loader, model, 'original')) eval_results['original'] = original_results for eval_space_results in eval_results.values(): Evaluator.results_to_cpu(eval_space_results) return eval_results
def _intermediate_eval(self, session, batch): if session.params['eval_space'] == 'original': batch.original_poses = self.normalizer.denormalize( batch.poses, batch.normalization_params) train_results = { 'DEFAULT': Evaluator.to_batch(batch, session.params['eval_space']) } train_mean_results = Evaluator.means_per_metric(train_results) Evaluator.results_to_cpu(train_mean_results) val_mean_results = session.test_model(self.val_loader) Evaluator.results_to_cpu(val_mean_results) return train_mean_results, val_mean_results
def test_means_per_metric(): true_mean_results = { 'sub1': { 'coord_diff': torch.tensor(1.0), 'distance': torch.tensor(1.0), 'bone_length': torch.tensor(1.0), 'proportion': torch.tensor(1.0), }, 'sub2': { 'coord_diff': torch.tensor(2.0), 'distance': torch.tensor(2.0), 'bone_length': torch.tensor(3.0), 'proportion': torch.tensor(3.0), } } mean_results = Evaluator.means_per_metric(example_results) for subset_name in mean_results.keys(): for metric_name in Evaluator.metric_names: assert torch.allclose(mean_results[subset_name][metric_name], true_mean_results[subset_name][metric_name])
""" Compute the mean evaluation metrics on a dataset (before applying any corrections). """ import os import torch from data_utils import datasets from evaluation.evaluator import Evaluator # Config dataset_name = 'HANDS17_DPREN_SubjClust_val' ######################################################################## dataset = datasets.PairedPoseDataset(dataset_name, use_preset=True) data_loader = datasets.DataLoader(dataset, 100000) results = Evaluator.means_per_metric(Evaluator.to_dataset(data_loader, 'default')) torch.save(results, os.path.join('results', 'datasets', dataset_name + '.pt'))