def test_linear_transformation(self): c, h, w = 3, 24, 32 tensor, _ = self._create_data(h, w, channels=c, device=self.device) matrix = torch.rand(c * h * w, c * h * w, device=self.device) mean_vector = torch.rand(c * h * w, device=self.device) fn = T.LinearTransformation(matrix, mean_vector) scripted_fn = torch.jit.script(fn) self._test_transform_vs_scripted(fn, scripted_fn, tensor) batch_tensors = torch.rand(4, c, h, w, device=self.device) # We skip some tests from _test_transform_vs_scripted_on_batch as # results for scripted and non-scripted transformations are not exactly the same torch.manual_seed(12) torch.set_deterministic(True) transformed_batch = fn(batch_tensors) torch.manual_seed(12) s_transformed_batch = scripted_fn(batch_tensors) self.assertTrue(transformed_batch.equal(s_transformed_batch)) with get_tmp_dir() as tmp_dir: scripted_fn.save(os.path.join(tmp_dir, "t_norm.pt"))
def __init__(self, config): # Constants for data loading self.VAL_SET_SPLIT = config.VAL_SET_SPLIT self.BATCH_SIZE = config.BATCH_SIZE # Compute dataset statistics mean, std, zca = get_dataset_stats(self.VAL_SET_SPLIT) # Define transformations to be applied on the data # Basic transformations T = transforms.Compose([ # The first transform is ToTensor, which transforms the raw CIFAR10 data to a tensor in the form # [depth, width, height]. Additionally, pixel values are mapped from the range [0, 255] to the range [0, 1] transforms.ToTensor(), # The Normalize transform subtracts mean values from each channel (passed in the first tuple) and divides each # channel by std dev values (passed in the second tuple). In this case we bring each channel to zero mean and # unitary std dev, i.e. from range [0, 1] to [-1, 1] transforms.Normalize(mean, std) ]) # Add whitening transformation, if needed if config.WHITEN_DATA: T = transforms.Compose([ T, transforms.LinearTransformation(zca, torch.zeros(zca.size(1))) ]) self.T_train = T self.T_test = T # Extra transformations for data augmentation if config.AUGMENT_DATA: T_augm = transforms.Compose([ transforms.RandomApply([ transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=20 / 360) ], p=0.5), transforms.RandomApply([transforms.ColorJitter(saturation=1)], p=0.5), transforms.RandomHorizontalFlip(), transforms.Pad(8), transforms.RandomApply([ transforms.Lambda( lambda x: TF.resize(x, (48 + random.randint(-6, 6), 48 + random.randint(-6, 6)))) ], p=0.3), transforms.RandomApply( [transforms.RandomAffine(degrees=10, shear=10)], p=0.3), transforms.CenterCrop(40), transforms.RandomApply([transforms.RandomCrop(32)], p=0.5), transforms.CenterCrop(32), ]) self.T_train = transforms.Compose([T_augm, self.T_train])
def test_linear_transformation(self): x = torch.randn(250, 10, 10, 3) flat_x = x.view(x.size(0), x.size(1) * x.size(2) * x.size(3)) # compute principal components sigma = torch.mm(flat_x.t(), flat_x) / flat_x.size(0) u, s, _ = np.linalg.svd(sigma.numpy()) zca_epsilon = 1e-10 # avoid division by 0 d = torch.Tensor(np.diag(1. / np.sqrt(s + zca_epsilon))) u = torch.Tensor(u) principal_components = torch.mm(torch.mm(u, d), u.t()) # initialize whitening matrix whitening = transforms.LinearTransformation(principal_components) # pass first vector xwhite = whitening(x[0].view(10, 10, 3)) # estimate covariance xwhite = xwhite.view(1, 300).numpy() cov = np.dot(xwhite, xwhite.T) / x.size(0) assert np.allclose(cov, np.identity(1), rtol=1e-3)
def load_subset_mnist(max_label=2, max_size=1000, reshape=True): root = '.' input_size = 28 channels = 1 num_classes = 10 k = max_size input_transform = transforms.Compose([ transforms.ToTensor(), transforms.LinearTransformation(torch.eye(input_size**2), torch.zeros(input_size**2)) ]) mnist = datasets.MNIST(root + "data/MNIST", train=True, transform=input_transform, target_transform=None, download=True) x = mnist.data.reshape(-1, input_size**2) y = mnist.targets.numpy().reshape(-1) test = datasets.MNIST(root + "data/MNIST", train=False, transform=input_transform, target_transform=None, download=True) xtest = test.data.reshape(-1, input_size**2) ytest = test.targets.numpy().reshape(-1) x = x.numpy() # Subset generation x = x[np.where(y < max_label)] #[:k] y = y[np.where(y < max_label)] #[:k] x = x[:k] y = y[:k] xtest = xtest.numpy() xtest = xtest[np.where(ytest < max_label)] ytest = ytest[np.where(ytest < max_label)] return xtest, ytest, x, y
def test_linear_transformation(device, tmpdir): c, h, w = 3, 24, 32 tensor, _ = _create_data(h, w, channels=c, device=device) matrix = torch.rand(c * h * w, c * h * w, device=device) mean_vector = torch.rand(c * h * w, device=device) fn = T.LinearTransformation(matrix, mean_vector) scripted_fn = torch.jit.script(fn) _test_transform_vs_scripted(fn, scripted_fn, tensor) batch_tensors = torch.rand(4, c, h, w, device=device) # We skip some tests from _test_transform_vs_scripted_on_batch as # results for scripted and non-scripted transformations are not exactly the same torch.manual_seed(12) transformed_batch = fn(batch_tensors) torch.manual_seed(12) s_transformed_batch = scripted_fn(batch_tensors) assert_equal(transformed_batch, s_transformed_batch) scripted_fn.save(os.path.join(tmpdir, "t_norm.pt"))
def __init__(self, learning_rate=0.1, momentum=0.0, weight_decay=0.0, data_dir="data", savepoint_dir="savepoints", no_cuda=False, prep_dir="cache", batch_size=20, max_savepoints=20, num_workers=2, sp_serial=-1, no_save_savepoints=False, no_save_prep=False, save_after_batches=1000): self.prep_dir = prep_dir self.savepoint_dir = savepoint_dir self.save_after_batches = save_after_batches self.no_save_prep = no_save_prep self.no_save_savepoints = no_save_savepoints self.sp_serial = sp_serial self.learning_rate = learning_rate self.momentum = momentum self.weight_decay = weight_decay self.batch_size = batch_size self.max_savepoints = max_savepoints self.num_workers = num_workers self.data_dir = data_dir self.dataset = ImageFolder(root=self.data_dir) # sample_idx = random.randint(0, len(self.dataset)) # plt.imshow(self.dataset[sample_idx][0]) # plt.show() mean, std, whiten_matrix, mean_vec = self._loadOrCalcTransformValues() self.transforms = transforms.Compose([ transforms.Grayscale(1), transforms.RandomAffine(0, translate=(.1, .1)), transforms.ToTensor(), transforms.Normalize((mean, ), (std, )), transforms.LinearTransformation(whiten_matrix, mean_vec), Rescale(0.0, 1.0) ]) self.dataset.transform = self.transforms # (img, label) = self.dataset[sample_idx] # label = self.dataset.classes[label] # plt.title(label) # plt.imshow(transforms.functional.to_pil_image(img), cmap="gray") # plt.show() # exit() self.net = Net(num_classes=len(self.dataset.classes)) if (not no_cuda) and torch.cuda.is_available(): self.net.cuda() self.device = "cuda" print(f"Device :: CUDA {torch.cuda.get_device_name()}") else: self.device = "cpu" print(f"Device :: CPU") # TODO: dynamic learning rate # Define optimizer AFTER device is set self.optimizer = optim.RMSprop(self.net.parameters(), lr=self.learning_rate, momentum=self.momentum, weight_decay=self.weight_decay) self.criterion = torch.nn.CrossEntropyLoss() # load savepoints if available savepoints = os.listdir(self.savepoint_dir) if os.path.isdir( self.savepoint_dir) else [] if not savepoints == []: self._loadSavepoint(savepoints) else: self.epoch = 0 self.current_loss = None print("No savepoints found!") self.trainloader = torch.utils.data.DataLoader( self.dataset, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers) self.testloader = torch.utils.data.DataLoader( self.dataset, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers)