def test_linear_transformation(self):
c, h, w = 3, 24, 32
tensor, _ = self._create_data(h, w, channels=c, device=self.device)
matrix = torch.rand(c * h * w, c * h * w, device=self.device)
mean_vector = torch.rand(c * h * w, device=self.device)
fn = T.LinearTransformation(matrix, mean_vector)
scripted_fn = torch.jit.script(fn)
self._test_transform_vs_scripted(fn, scripted_fn, tensor)
batch_tensors = torch.rand(4, c, h, w, device=self.device)
# We skip some tests from _test_transform_vs_scripted_on_batch as
# results for scripted and non-scripted transformations are not exactly the same
torch.manual_seed(12)
torch.set_deterministic(True)
transformed_batch = fn(batch_tensors)
torch.manual_seed(12)
s_transformed_batch = scripted_fn(batch_tensors)
self.assertTrue(transformed_batch.equal(s_transformed_batch))
with get_tmp_dir() as tmp_dir:
scripted_fn.save(os.path.join(tmp_dir, "t_norm.pt"))
def __init__(self, config):
# Constants for data loading
self.VAL_SET_SPLIT = config.VAL_SET_SPLIT
self.BATCH_SIZE = config.BATCH_SIZE
# Compute dataset statistics
mean, std, zca = get_dataset_stats(self.VAL_SET_SPLIT)
# Define transformations to be applied on the data
# Basic transformations
T = transforms.Compose([
# The first transform is ToTensor, which transforms the raw CIFAR10 data to a tensor in the form
# [depth, width, height]. Additionally, pixel values are mapped from the range [0, 255] to the range [0, 1]
transforms.ToTensor(),
# The Normalize transform subtracts mean values from each channel (passed in the first tuple) and divides each
# channel by std dev values (passed in the second tuple). In this case we bring each channel to zero mean and
# unitary std dev, i.e. from range [0, 1] to [-1, 1]
transforms.Normalize(mean, std)
])
# Add whitening transformation, if needed
if config.WHITEN_DATA:
T = transforms.Compose([
T,
transforms.LinearTransformation(zca, torch.zeros(zca.size(1)))
])
self.T_train = T
self.T_test = T
# Extra transformations for data augmentation
if config.AUGMENT_DATA:
T_augm = transforms.Compose([
transforms.RandomApply([
transforms.ColorJitter(brightness=0.1,
contrast=0.1,
saturation=0.1,
hue=20 / 360)
],
p=0.5),
transforms.RandomApply([transforms.ColorJitter(saturation=1)],
p=0.5),
transforms.RandomHorizontalFlip(),
transforms.Pad(8),
transforms.RandomApply([
transforms.Lambda(
lambda x: TF.resize(x, (48 + random.randint(-6, 6), 48
+ random.randint(-6, 6))))
],
p=0.3),
transforms.RandomApply(
[transforms.RandomAffine(degrees=10, shear=10)], p=0.3),
transforms.CenterCrop(40),
transforms.RandomApply([transforms.RandomCrop(32)], p=0.5),
transforms.CenterCrop(32),
])
self.T_train = transforms.Compose([T_augm, self.T_train])
def test_linear_transformation(self):
x = torch.randn(250, 10, 10, 3)
flat_x = x.view(x.size(0), x.size(1) * x.size(2) * x.size(3))
# compute principal components
sigma = torch.mm(flat_x.t(), flat_x) / flat_x.size(0)
u, s, _ = np.linalg.svd(sigma.numpy())
zca_epsilon = 1e-10 # avoid division by 0
d = torch.Tensor(np.diag(1. / np.sqrt(s + zca_epsilon)))
u = torch.Tensor(u)
principal_components = torch.mm(torch.mm(u, d), u.t())
# initialize whitening matrix
whitening = transforms.LinearTransformation(principal_components)
# pass first vector
xwhite = whitening(x[0].view(10, 10, 3))
# estimate covariance
xwhite = xwhite.view(1, 300).numpy()
cov = np.dot(xwhite, xwhite.T) / x.size(0)
assert np.allclose(cov, np.identity(1), rtol=1e-3)
def load_subset_mnist(max_label=2, max_size=1000, reshape=True):
root = '.'
input_size = 28
channels = 1
num_classes = 10
k = max_size
input_transform = transforms.Compose([
transforms.ToTensor(),
transforms.LinearTransformation(torch.eye(input_size**2),
torch.zeros(input_size**2))
])
mnist = datasets.MNIST(root + "data/MNIST",
train=True,
transform=input_transform,
target_transform=None,
download=True)
x = mnist.data.reshape(-1, input_size**2)
y = mnist.targets.numpy().reshape(-1)
test = datasets.MNIST(root + "data/MNIST",
train=False,
transform=input_transform,
target_transform=None,
download=True)
xtest = test.data.reshape(-1, input_size**2)
ytest = test.targets.numpy().reshape(-1)
x = x.numpy()
# Subset generation
x = x[np.where(y < max_label)] #[:k]
y = y[np.where(y < max_label)] #[:k]
x = x[:k]
y = y[:k]
xtest = xtest.numpy()
xtest = xtest[np.where(ytest < max_label)]
ytest = ytest[np.where(ytest < max_label)]
return xtest, ytest, x, y
def test_linear_transformation(device, tmpdir):
c, h, w = 3, 24, 32
tensor, _ = _create_data(h, w, channels=c, device=device)
matrix = torch.rand(c * h * w, c * h * w, device=device)
mean_vector = torch.rand(c * h * w, device=device)
fn = T.LinearTransformation(matrix, mean_vector)
scripted_fn = torch.jit.script(fn)
_test_transform_vs_scripted(fn, scripted_fn, tensor)
batch_tensors = torch.rand(4, c, h, w, device=device)
# We skip some tests from _test_transform_vs_scripted_on_batch as
# results for scripted and non-scripted transformations are not exactly the same
torch.manual_seed(12)
transformed_batch = fn(batch_tensors)
torch.manual_seed(12)
s_transformed_batch = scripted_fn(batch_tensors)
assert_equal(transformed_batch, s_transformed_batch)
scripted_fn.save(os.path.join(tmpdir, "t_norm.pt"))
def __init__(self,
learning_rate=0.1,
momentum=0.0,
weight_decay=0.0,
data_dir="data",
savepoint_dir="savepoints",
no_cuda=False,
prep_dir="cache",
batch_size=20,
max_savepoints=20,
num_workers=2,
sp_serial=-1,
no_save_savepoints=False,
no_save_prep=False,
save_after_batches=1000):
self.prep_dir = prep_dir
self.savepoint_dir = savepoint_dir
self.save_after_batches = save_after_batches
self.no_save_prep = no_save_prep
self.no_save_savepoints = no_save_savepoints
self.sp_serial = sp_serial
self.learning_rate = learning_rate
self.momentum = momentum
self.weight_decay = weight_decay
self.batch_size = batch_size
self.max_savepoints = max_savepoints
self.num_workers = num_workers
self.data_dir = data_dir
self.dataset = ImageFolder(root=self.data_dir)
# sample_idx = random.randint(0, len(self.dataset))
# plt.imshow(self.dataset[sample_idx][0])
# plt.show()
mean, std, whiten_matrix, mean_vec = self._loadOrCalcTransformValues()
self.transforms = transforms.Compose([
transforms.Grayscale(1),
transforms.RandomAffine(0, translate=(.1, .1)),
transforms.ToTensor(),
transforms.Normalize((mean, ), (std, )),
transforms.LinearTransformation(whiten_matrix, mean_vec),
Rescale(0.0, 1.0)
])
self.dataset.transform = self.transforms
# (img, label) = self.dataset[sample_idx]
# label = self.dataset.classes[label]
# plt.title(label)
# plt.imshow(transforms.functional.to_pil_image(img), cmap="gray")
# plt.show()
# exit()
self.net = Net(num_classes=len(self.dataset.classes))
if (not no_cuda) and torch.cuda.is_available():
self.net.cuda()
self.device = "cuda"
print(f"Device :: CUDA {torch.cuda.get_device_name()}")
else:
self.device = "cpu"
print(f"Device :: CPU")
# TODO: dynamic learning rate
# Define optimizer AFTER device is set
self.optimizer = optim.RMSprop(self.net.parameters(),
lr=self.learning_rate,
momentum=self.momentum,
weight_decay=self.weight_decay)
self.criterion = torch.nn.CrossEntropyLoss()
# load savepoints if available
savepoints = os.listdir(self.savepoint_dir) if os.path.isdir(
self.savepoint_dir) else []
if not savepoints == []:
self._loadSavepoint(savepoints)
else:
self.epoch = 0
self.current_loss = None
print("No savepoints found!")
self.trainloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=self.batch_size,
shuffle=True,
num_workers=self.num_workers)
self.testloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=self.batch_size,
shuffle=True,
num_workers=self.num_workers)
