def test_TrainerGrad_MutualInfo(self): set_seed(2) mi_history = [] class MutualInfoNPEETDebug(MutualInfoNPEET): def plot(self_mi, viz): mi_history.append(self_mi.information['mlp.2']) super().plot(viz) data_loader = DataLoader(MNIST, eval_size=100, transform=TransformDefault.mnist()) trainer = TrainerGrad(self.model, criterion=nn.CrossEntropyLoss(), data_loader=data_loader, optimizer=self.optimizer, mutual_info=MutualInfoNPEETDebug( data_loader=data_loader, pca_size=None), scheduler=self.scheduler) trainer.monitor.advanced_monitoring(level=MonitorLevel.FULL) trainer.train(n_epochs=1, mutual_info_layers=1) mi_history = np.vstack(mi_history) assert_array_less(0, mi_history) x_last, y_last = mi_history[-1] # the model is well trained self.assertGreater(y_last, 2.5) self.assertGreater(x_last, 2.2) self.assertLessEqual(x_last, y_last)
def test_reshape(self): set_seed(2) model = Reshape(height=10, width=12) tensor = torch.rand(5, 3 * model.height * model.width) output = model(tensor) assert_array_equal(output, tensor.view(5, 3, model.height, model.width))
def test_rand_pairs(self): set_seed(1) outputs = torch.randn(self.labels.shape[0], 30) for loss_model in self.loss_models: with self.subTest(loss_model=loss_model): loss = loss_model(outputs, self.labels) self.assertGreater(loss, 0.)
def test_mlp(self): set_seed(3) in_features, hidden_features, output_features = 32, 12, 17 model = MLP(in_features, hidden_features, output_features) tensor = torch.rand(5, in_features) output = model(tensor) self.assertEqual(output.shape, (tensor.shape[0], output_features))
def test_exponential_moving_average_psnr(self): set_seed(1) noise = torch.rand(100) smoothed = exponential_moving_average(noise, window=3) psnr = peak_to_signal_noise_ratio(noise, smoothed) self.assertGreaterEqual(psnr, 15.7) self.assertEqual(smoothed.shape, noise.shape) self.assertLess(smoothed.std(), noise.std())
def test_softshink(self): set_seed(16) lambd = 0.1 softshrink = Softshrink(n_features=1) softshrink.lambd.data[:] = lambd softshrink_gt = nn.Softshrink(lambd=lambd) tensor = torch.randn(10, 20) assert_array_almost_equal( softshrink(tensor).detach(), softshrink_gt(tensor))
def test_NormalizeInverse(self): set_seed(0) mean = torch.rand(3) std = torch.rand_like(mean) normalize = Normalize(mean=mean, std=std, inplace=False) normalize_inverse = NormalizeInverse(mean=mean, std=std) tensor = torch.rand(5, 3, 12, 12) tensor_normalized = normalize(tensor) tensor_restored = normalize_inverse(tensor_normalized) assert_array_almost_equal(tensor_restored, tensor)
def test_TrainerAutoencoder(self): set_seed(4) model = AutoencoderLinear(784, 64) trainer = TrainerAutoencoder(model, criterion=nn.BCEWithLogitsLoss(), data_loader=self.data_loader, optimizer=self.optimizer, scheduler=self.scheduler) loss_epochs = trainer.train(n_epochs=1, mutual_info_layers=0) assert_array_almost_equal(loss_epochs, [0.69737625122])
def test_get_normalize_inverse(self): set_seed(1) mean = torch.rand(3) std = torch.rand_like(mean) normalize = Normalize(mean=mean, std=std) normalize_inverse = NormalizeInverse(mean=mean, std=std) normalize_inverse2 = get_normalize_inverse(normalize) assert_array_almost_equal(normalize_inverse2.mean, normalize_inverse.mean) assert_array_almost_equal(normalize_inverse2.std, normalize_inverse.std)
def test_TrainerEmbedding(self): set_seed(3) criterion = TripletLossSampler(nn.TripletMarginLoss()) trainer = TrainerEmbedding(self.model, criterion=criterion, data_loader=self.data_loader, optimizer=self.optimizer, scheduler=self.scheduler) loss_epochs = trainer.train(n_epochs=1, mutual_info_layers=0) # CircleCI outputs 0.103 assert_array_almost_equal(loss_epochs, [0.09936], decimal=2)
def setUp(self): set_seed(0) n_classes = 3 labels = torch.arange(n_classes) self.labels = torch.cat([labels, labels]) outputs_same = torch.randn(n_classes, 30) self.outputs_same = torch.cat([outputs_same, outputs_same]) self.loss_models = (ContrastiveLossSampler( nn.CosineEmbeddingLoss(margin=0.5)), TripletLossSampler(nn.TripletMarginLoss()), TripletLossSampler(TripletCosineLoss()))
def setUp(self): set_seed(1) self.model = MLP(784, 64, 10) self.data_loader = DataLoader(MNIST, eval_size=10000, transform=TransformDefault.mnist()) self.optimizer = torch.optim.Adam(filter( lambda param: param.requires_grad, self.model.parameters()), lr=1e-3, weight_decay=1e-5) self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( self.optimizer)
def test_AutoEncoderLinear(self): set_seed(0) in_features, hidden_features = 16, 64 batch_size = 5 model = AutoencoderLinear(in_features, hidden_features) self.assertEqual(model.encoding_dim, hidden_features) self.assertIsInstance(model.encoder, nn.Module) self.assertIsInstance(model.decoder, nn.Module) tensor = torch.rand(batch_size, in_features) output = model(tensor) self.assertIsInstance(output, AutoencoderOutput) self.assertEqual(output.latent.shape, (batch_size, hidden_features)) self.assertEqual(output.reconstructed.shape, (batch_size, in_features))
def test_TrainerGrad(self): set_seed(2) trainer = TrainerGrad(self.model, criterion=nn.CrossEntropyLoss(), data_loader=self.data_loader, optimizer=self.optimizer, scheduler=self.scheduler) loss_epochs = trainer.train(n_epochs=1, mutual_info_layers=0, mask_explain_params=dict()) assert_array_almost_equal(loss_epochs, [0.219992]) trainer.save() trainer.restore()
def test_basis_pursuit_admm_convergence(self): set_seed(12) n_features, n_atoms = 10, 50 dictionary = torch.randn(n_features, n_atoms) tensor_x = torch.randn(3, n_features) solver = BasisPursuitADMM(lambd=0.1, max_iters=1000) solver.save_stats = True tensor_z = solver.solve(A=dictionary, b=tensor_x) iterations = solver.online['iterations'].get_mean().item() self.assertLessEqual(iterations, solver.max_iters) dv_norm = solver.online['dv_norm'].get_mean().item() self.assertLessEqual(dv_norm, solver.tol) x_restored = tensor_z.matmul(dictionary.t()) assert_array_almost_equal(tensor_x, x_restored, decimal=1) psnr = solver.online['psnr'].get_mean().item() self.assertGreater(psnr, 41.) # 41 is taken from the output
def test_force_update(self): set_seed(1) mi_instance_pca = self._init_mutual_info(pca_size=20) mi_instance_pca.prepare(self.model) self.mi_instance.force_update(self.model) mi_instance_pca.force_update(self.model) mi_no_pca = self.mi_instance.information mi_pca = mi_instance_pca.information self.assertEqual(mi_pca.keys(), mi_no_pca.keys()) for mi_layers_dict in (mi_pca, mi_no_pca): for mi_x, mi_y in mi_layers_dict.values(): # MI must be non-negative self.assertGreater(mi_x, 0) self.assertGreater(mi_y, 0) for layer_name in mi_pca.keys(): self._test_estimated_values(mi_pca[layer_name], mi_no_pca[layer_name]) mi_instance_pca.plot(self.viz) mi_instance_pca.plot_activations_hist(self.viz)
def test_lista_forward_best(self): set_seed(16) solver = BasisPursuitADMM() in_features = 10 out_features = 40 lista = LISTA(in_features=in_features, out_features=out_features, solver=solver) mp = MatchingPursuit(in_features=in_features, out_features=out_features, solver=solver) tensor = torch.randn(5, in_features) with torch.no_grad(): mp.normalize_weight() lista.weight_input.data = mp.weight.data.clone() mp_output = mp(tensor) lista_output_best = lista.forward_best(tensor) assert_array_almost_equal(lista_output_best.latent, mp_output.latent) assert_array_almost_equal(lista_output_best.reconstructed, mp_output.reconstructed)
def test_TrainerAutoencoder_cached(self): set_seed(3) model = AutoencoderLinear(784, 64) trainer = TrainerAutoencoder( model, criterion=nn.BCEWithLogitsLoss(), data_loader=self.data_loader, optimizer=self.optimizer, scheduler=self.scheduler, accuracy_measure=AccuracyEmbedding(cache=True)) trainer.open_monitor(offline=True) # TripletLoss is not deterministic; fix the seed set_seed(4) loss_cached = trainer.full_forward_pass(train=True) accuracy_cached = trainer.update_accuracy(train=True) trainer.accuracy_measure.cache = False trainer.accuracy_measure.reset() set_seed(4) loss = trainer.full_forward_pass(train=True) accuracy = trainer.update_accuracy(train=True) self.assertAlmostEqual(loss_cached.item(), loss.item()) self.assertAlmostEqual(accuracy_cached, accuracy)
def test_TrainerEmbedding_cached(self): set_seed(3) criterion = TripletLossSampler(nn.TripletMarginLoss()) trainer = TrainerEmbedding( self.model, criterion=criterion, data_loader=self.data_loader, optimizer=self.optimizer, scheduler=self.scheduler, accuracy_measure=AccuracyEmbedding(cache=True)) trainer.open_monitor(offline=True) # TripletLoss is not deterministic; fix the seed set_seed(4) loss_cached = trainer.full_forward_pass(train=True) accuracy_cached = trainer.update_accuracy(train=True) trainer.accuracy_measure.cache = False trainer.accuracy_measure.reset() set_seed(4) loss = trainer.full_forward_pass(train=True) accuracy = trainer.update_accuracy(train=True) self.assertAlmostEqual(loss_cached.item(), loss.item()) self.assertAlmostEqual(accuracy_cached, accuracy)
def test_basis_pursuit_admm_as_numpy(self): set_seed(12) n_features, n_atoms = 10, 50 dictionary = torch.randn(n_features, n_atoms) tensor_x = torch.randn(3, n_features) lambd = 0.1 tol_decimal = 4 tol = 10**(-tol_decimal) max_iters = 100 z_pytorch = bp_pytorch(A=dictionary, b=tensor_x, lambd=lambd, tol=tol, max_iters=max_iters).numpy() z_numpy = [ bp_numpy(A=dictionary.numpy(), b=b, lambd=lambd, tol=tol, max_iters=max_iters) for b in tensor_x ] assert_array_almost_equal(z_pytorch, z_numpy, decimal=tol_decimal)
def test_flatten(self): set_seed(1) model = Flatten() tensor = torch.rand(5, 1, 28, 28) output = model(tensor) assert_array_equal(output, tensor.flatten(start_dim=1))
def setUp(self): set_seed(10) self.image = torch.rand(1, 10, 10) self.model = MLP(self.image.nelement(), 10) self.label = 3
def test_cifar10(self): set_seed(0) transform = TransformDefault.cifar10() loader = DataLoader(dataset_cls=CIFAR10, transform=transform) self._test_dataset(loader)
def test_kmeans(): outputs, labels = synthesize_log_softmax_data(n_samples=20000, n_classes=10, p_argmax=0.99) estimator = MutualInfoKMeans(n_bins=20, debug=False) quantized = estimator.quantize(outputs) estimated = MutualInfoKMeans.compute_mutual_info(quantized, labels) print(f"KMeans Mutual Information estimate: {estimated:.3f}") def test_gcmi(): """ Test Gaussian-Copula Mutual Information estimator """ outputs, labels = synthesize_log_softmax_data(n_samples=20000, n_classes=10, p_argmax=0.99) print(type(labels)) estimated = micd(outputs.numpy().T, labels.numpy()) print(f"Gaussian-Copula Mutual Information estimate: {estimated:.3f}") if __name__ == '__main__': set_seed(26) # expected estimated value: log2(10) ~ 3.322 test_kmeans() test_mine() test_gcmi()
def setUp(self): set_seed(0) self.mi_instance = self._init_mutual_info(pca_size=None) self.mi_instance.prepare(self.model)
n_epochs = max(n_epochs - trainer.epoch, 0) trainer.train(n_epochs=n_epochs, mutual_info_layers=0) return trainer.model def train_lista(dataset_cls=MNIST): model_ref = train_matching_pursuit(dataset_cls=dataset_cls) model = LISTA(784, out_features=model_ref.out_features) data_loader = DataLoader(dataset_cls, transform=TransformDefault.mnist( normalize=None )) criterion = nn.MSELoss() optimizer, scheduler = get_optimizer_scheduler(model) trainer = TrainLISTA(model, model_reference=model_ref, criterion=criterion, data_loader=data_loader, optimizer=optimizer, scheduler=scheduler, accuracy_measure=AccuracyEmbedding()) # trainer.monitor.advanced_monitoring(level=MonitorLevel.FULL) trainer.train(n_epochs=10, mutual_info_layers=0) if __name__ == '__main__': set_seed(28) # test_matching_pursuit() # train_matching_pursuit() train_lista()
def test_mnist(self): set_seed(0) transform = TransformDefault.mnist() loader = DataLoader(dataset_cls=MNIST, transform=transform) self._test_dataset(loader)