def test_data_loader(): distorter = distorters.NoDistorter() dataset_plain = datasets.NormalizedPairedPoseDataset( 'unit_test/dummy42', distorter, norm.NoNorm, True, dataset_size, 'cuda:0') dataset_dict = datasets.NormalizedPairedPoseDataset( 'unit_test/dummy42_dict', distorter, norm.NoNorm, True, dataset_size, 'cuda:0') true_sum_sub1 = dataset_dict[:].poses.sum() dataset_dict.select_subset('sub2') true_sum_sub2 = dataset_dict[:].poses.sum() data_loader_plain = datasets.DataLoader(dataset_plain, 6) data_loader_dict = datasets.DataLoader(dataset_dict, 6) plain_batch = next(iter(data_loader_plain)) subset_names_plain = data_loader_plain.get_subset_names() data_loader_plain.select_subset(subset_names_plain[0]) all_batches = {} sum_of_subsets = {} for subset_name in data_loader_dict.get_subset_names(): data_loader_dict.select_subset(subset_name) all_batches[subset_name] = list(data_loader_dict) sum_of_subsets[subset_name] = sum( batch.poses.sum() for batch in all_batches[subset_name]) assert type(plain_batch) is datasets.PoseCorrectionBatch assert subset_names_plain == ['DEFAULT'] assert list(all_batches.keys()) == ['sub1', 'sub2'] assert len(all_batches['sub1']) == 7 assert type(all_batches['sub1'][0]) == datasets.PoseCorrectionBatch assert all_batches['sub1'][0].labels.shape == (6, 21, 3) assert torch.allclose(sum_of_subsets['sub1'], true_sum_sub1) assert torch.allclose(sum_of_subsets['sub2'], true_sum_sub2)
def test_training_session(): training_set = datasets.NormalizedPairedPoseDataset('unit_test/dummy42', distorter, norm.NoNorm, False, None, device='cuda:0') validation_set = datasets.NormalizedPairedPoseDataset( 'unit_test/dummy42_dict', distorters.NoDistorter(), norm.NoNorm, True, None, device='cuda:0') training_batch = training_set[:] val_loader = datasets.DataLoader(validation_set, batch_size=6) training_session = TrainingSession(model, hyperparams, norm.NoNorm) training_session.schedule_learning_rate() loss, result = training_session.train_batch(training_batch) test_results = training_session.test_model(val_loader) assert loss.numel() == 1 assert loss.device == torch.device('cpu') assert training_batch.poses.is_same_size(result.poses) assert list(test_results.keys()) == ['sub1', 'sub2'] assert list(test_results['sub1'].keys()) == Evaluator.metric_names assert test_results['sub1']['distance'].numel() == 1
def test_solver(): training_set = datasets.NormalizedPairedPoseDataset('unit_test/dummy42', distorter, norm.NoNorm, False, None, device='cuda:0') validation_set = datasets.NormalizedPairedPoseDataset( 'unit_test/dummy42_dict', distorters.NoDistorter(), norm.NoNorm, True, None, device='cuda:0') solver_params = { 'log_frequency': 2, 'log_loss': True, 'log_grad': True, 'verbose': False, 'show_plots': False, 'num_epochs': 5, 'batch_size': 6, 'interest_keys': [(None, 'loss_function'), ('optimizer_args', 'weight_decay'), ('optimizer_args', 'lr')], 'val_example_indices': [0, 1], 'val_example_subset': 'sub1' } solver = Solver(solver_params, training_set, validation_set) log, eval_results, weights, example_predictions = solver.train( model, hyperparams) assert len(log['train']['loss']) == 35 assert len(log['val']['sub1'][Evaluator.metric_names[0]]) == 10 assert example_predictions.shape == (10, 2, 21, 3) assert type(eval_results) is dict assert list(eval_results.keys()) == ['default', 'original'] assert log['train']['loss'][0].device == torch.device('cpu') assert log['train']['distance'][0].device == torch.device('cpu') assert log['val']['sub1']['distance'][0].device == torch.device('cpu')
def test_to_model(): distorter = distorters.NoDistorter() model = helpers.DummyModel() dataset_no_subs = datasets.NormalizedPairedPoseDataset('unit_test/dummy42', distorter, norm.NoNorm, False, device='cuda:0') dataset_subs = datasets.NormalizedPairedPoseDataset('unit_test/ident42', distorter, norm.NoNorm, True, device='cuda:0') data_loader_no_subs = datasets.DataLoader(dataset_no_subs, 6) data_loader_subs = datasets.DataLoader(dataset_subs, 6) batch_size = 42 true_results = { 'coord_diff': torch.zeros(batch_size, device='cuda:0'), 'distance': torch.zeros(batch_size, device='cuda:0'), 'bone_length': torch.zeros(batch_size, device='cuda:0'), 'proportion': torch.zeros(batch_size, device='cuda:0'), } results_norm_no_subs = Evaluator.to_model(data_loader_no_subs, model) results_orig_no_subs = Evaluator.to_model(data_loader_no_subs, model, space='original') results_norm_subs = Evaluator.to_model(data_loader_subs, model) results_orig_subs = Evaluator.to_model(data_loader_subs, model, space='original') for metric_name in Evaluator.metric_names: for subset_name in ['sub1', 'sub2']: assert torch.allclose(results_norm_subs[subset_name][metric_name], true_results[metric_name], atol=1e-5) assert torch.allclose(results_orig_subs[subset_name][metric_name], true_results[metric_name], atol=1e-5) assert torch.allclose(results_norm_no_subs['DEFAULT'][metric_name], true_results[metric_name], atol=1e-5) assert torch.allclose(results_orig_no_subs['DEFAULT'][metric_name], true_results[metric_name], atol=1e-5)
def test_normalized_paired_pose_dataset(): plain_size = 20 some_size = 10 distorter = distorters.SyntheticDistorter(distorter_config) dataset_plain = datasets.NormalizedPairedPoseDataset( 'unit_test/dummy42', distorter, norm.NoNorm, False, plain_size, 'cpu') dataset_dict = datasets.NormalizedPairedPoseDataset( 'unit_test/dummy42_dict', distorter, norm.NoNorm, True, device='cuda:0') some = dataset_plain[:some_size] single_sub1 = dataset_dict[0] dataset_dict.select_subset('sub2') single_sub2 = dataset_dict[1] assert type(some) is datasets.PoseCorrectionBatch assert len(dataset_plain) == plain_size assert len(dataset_dict) == dataset_size assert some.poses.shape == (some_size, 21, 3) assert some.poses.dtype == torch.float32 assert some.poses.is_same_size(some.labels) assert some.poses.is_same_size(some.original_poses) assert some.poses.is_same_size(some.original_labels) assert some.normalization_params is not None assert some.device == torch.device('cpu') assert dataset_dict.device == torch.device('cuda:0') assert single_sub1.device == torch.device('cuda:0') assert single_sub1.poses.shape == (1, 21, 3) assert not torch.allclose(single_sub1.poses, single_sub2.poses) assert not torch.allclose(single_sub1.poses, single_sub1.labels) assert dataset_dict.get_subset_names() == ['sub1', 'sub2'] assert dataset_dict.has_subsets
def run(self): distorter = self.base_hyperparams['distorter'](self.base_hyperparams['distorter_args']) if self.config['experiment']['normalizer'] is not None: train_data = datasets.NormalizedPairedPoseDataset( self.config['experiment']['train_set'], distorter, self.config['experiment']['normalizer'], self.config['experiment']['use_preset'], self.config['experiment']['train_set_size'], self.config['experiment']['target_device']) val_data = datasets.NormalizedPairedPoseDataset( self.config['experiment']['val_set'], distorters.NoDistorter(), self.config['experiment']['normalizer'], True, self.config['experiment']['val_set_size'], self.config['experiment']['target_device']) else: train_data = datasets.PairedPoseDataset(self.config['experiment']['train_set'], distorter, self.config['experiment']['use_preset'], self.config['experiment']['train_set_size'], self.config['experiment']['target_device']) val_data = datasets.PairedPoseDataset(self.config['experiment']['val_set'], distorters.NoDistorter(), True, self.config['experiment']['val_set_size'], self.config['experiment']['target_device']) self.config['experiment']['train_set_size'] = len(train_data) self.config['experiment']['val_set_size'] = len(val_data) experiment_dir = os.path.join('results', self.config['experiment']['name']) os.mkdir(experiment_dir) torch.save(self.config, os.path.join(experiment_dir, 'config.pt')) solver = Solver(self.config['solver'], train_data, val_data) combinations_of_configs = self._generate_combinations() for i, hyperparams in enumerate(combinations_of_configs): print('\n\n' + '#' * 100) print('START OF SESSION {}/{}'.format(i + 1, len(combinations_of_configs))) results_dir = os.path.join(experiment_dir, str(i)) os.mkdir(results_dir) torch.save(hyperparams, os.path.join(results_dir, 'params.pt')) distorter = hyperparams['distorter'](hyperparams['distorter_args']) train_data.distorter = distorter for j in range(self.config['experiment']['n_repetitions']): print('\nRepetition {}/{} ({}):'.format(j + 1, self.config['experiment']['n_repetitions'], self.config['experiment']['name'])) print('*' * 50) if self.config['experiment']['deterministic_mode']: torch.manual_seed(0) model = self._create_model_and_normalizer(hyperparams) log, eval_results, weights, example_predictions = solver.train(model, hyperparams) repetition_dir = os.path.join(results_dir, str(j)) os.mkdir(repetition_dir) torch.save(log, os.path.join(repetition_dir, 'log.pt')) torch.save(eval_results, os.path.join(repetition_dir, 'eval.pt')) torch.save(weights, os.path.join(repetition_dir, 'weights.pt')) torch.save(example_predictions, os.path.join(repetition_dir, 'examples.pt')) print('\nExperiment >> {} << finished.\n'.format(self.config['experiment']['name']))
cont = False max_iter = 2500 cheat_metric = False print( 'Starting TSNE analysis with: \n\t{}\n\t{}\n\t{}\n\t{}\n\t{}\n\t{}\n\t{}\n\t{}' .format(dataset_names, eval_spaces, subset_name, error_name, perplexities, cont, max_iter, cheat_metric)) ######################################################################## for dataset_name in dataset_names: for eval_space in eval_spaces: for perplexity in perplexities: if eval_space == 'normalized': dataset = datasets.NormalizedPairedPoseDataset( dataset_name, distorters.NoDistorter(), True) else: dataset = datasets.PairedPoseDataset(dataset_name, distorters.NoDistorter(), True) dataset.select_subset(subset_name) all_data = dataset[:] if error_name == 'distance': data = errors.distance_error(all_data.poses, all_data.labels) elif error_name == 'bone_length': data = errors.bone_length_error(all_data.poses, all_data.labels) elif error_name == 'dist_bone_cat': distance_errors = errors.distance_error(