def get_CIFAR10_data(validation_ratio = 0.02, flatten=False):
"""
Load the CIFAR-10 dataset from disk and perform preprocessing to prepare
it for the linear classifier. These are the same steps as we used for the
SVM, but condensed to a single function.
"""
X_train, y_train, X_test, y_test = eecs598.data.cifar10()
# load every data on cuda
X_train = X_train.cuda()
y_train = y_train.cuda()
X_test = X_test.cuda()
y_test = y_test.cuda()
# 0. Visualize some examples from the dataset.
classes = [
'plane', 'car', 'bird', 'cat', 'deer',
'dog', 'frog', 'horse', 'ship', 'truck'
]
samples_per_class = 12
samples = []
eecs598.reset_seed(0)
for y, cls in enumerate(classes):
plt.text(-4, 34 * y + 18, cls, ha='right')
idxs, = (y_train == y).nonzero(as_tuple=True)
for i in range(samples_per_class):
idx = idxs[random.randrange(idxs.shape[0])].item()
samples.append(X_train[idx])
img = torchvision.utils.make_grid(samples, nrow=samples_per_class)
plt.imshow(eecs598.tensor_to_image(img))
plt.axis('off')
plt.show()
# 1. Normalize the data: subtract the mean RGB (zero mean)
mean_image = X_train.mean(dim=0, keepdim=True).mean(dim=2, keepdim=True).mean(dim=3, keepdim=True)
X_train -= mean_image
X_test -= mean_image
# 2. Reshape the image data into rows
if flatten:
X_train = X_train.reshape(X_train.shape[0], -1)
X_test = X_test.reshape(X_test.shape[0], -1)
# 3. take the validation set from the training set
# Note: It should not be taken from the test set
# For random permumation, you can use torch.randperm or torch.randint
# But, for this homework, we use slicing instead.
num_training = int( X_train.shape[0] * (1.0 - validation_ratio) )
num_validation = X_train.shape[0] - num_training
# return the dataset
data_dict = {}
data_dict['X_val'] = X_train[num_training:num_training + num_validation]
data_dict['y_val'] = y_train[num_training:num_training + num_validation]
data_dict['X_train'] = X_train[0:num_training]
data_dict['y_train'] = y_train[0:num_training]
data_dict['X_test'] = X_test
data_dict['y_test'] = y_test
return data_dict
def get_toy_data(num_inputs=5,
input_size=4,
hidden_size=10,
num_classes=3,
dtype=torch.float32,
device='cuda'):
"""
Get toy data for use when developing a two-layer-net.
Inputs:
- num_inputs: Integer N giving the data set size
- input_size: Integer D giving the dimension of input data
- hidden_size: Integer H giving the number of hidden units in the model
- num_classes: Integer C giving the number of categories
- dtype: torch datatype for all returned data
- device: device on which the output tensors will reside
Returns a tuple of:
- toy_X: `dtype` tensor of shape (N, D) giving data points
- toy_y: int64 tensor of shape (N,) giving labels, where each element is an
integer in the range [0, C)
- params: A dictionary of toy model parameters, with keys:
- 'W1': `dtype` tensor of shape (D, H) giving first-layer weights
- 'b1': `dtype` tensor of shape (H,) giving first-layer biases
- 'W2': `dtype` tensor of shape (H, C) giving second-layer weights
- 'b2': `dtype` tensor of shape (C,) giving second-layer biases
"""
N = num_inputs
D = input_size
H = hidden_size
C = num_classes
# We set the random seed for repeatable experiments.
eecs598.reset_seed(0)
# Generate some random parameters, storing them in a dict
params = {}
params['W1'] = 1e-4 * torch.randn(D, H, device=device, dtype=dtype)
params['b1'] = torch.zeros(H, device=device, dtype=dtype)
params['W2'] = 1e-4 * torch.randn(H, C, device=device, dtype=dtype)
params['b2'] = torch.zeros(C, device=device, dtype=dtype)
# Generate some random inputs and labels
toy_X = 10.0 * torch.randn(N, D, device=device, dtype=dtype)
toy_y = torch.tensor([0, 1, 2, 2, 1], device=device, dtype=torch.int64)
return toy_X, toy_y, params
def grad_check_sparse(f, x, analytic_grad, num_checks=10, h=1e-7):
"""
Utility function to perform numeric gradient checking. We use the centered
difference formula to compute a numeric derivative:
f'(x) =~ (f(x + h) - f(x - h)) / (2h)
Rather than computing a full numeric gradient, we sparsely sample a few
dimensions along which to compute numeric derivatives.
Inputs:
- f: A function that inputs a torch tensor and returns a torch scalar
- x: A torch tensor giving the point at which to evaluate the numeric gradient
- analytic_grad: A torch tensor giving the analytic gradient of f at x
- num_checks: The number of dimensions along which to check
- h: Step size for computing numeric derivatives
"""
# fix random seed to 0
eecs598.reset_seed(0)
for i in range(num_checks):
ix = tuple([random.randrange(m) for m in x.shape])
oldval = x[ix].item()
x[ix] = oldval + h # increment by h
fxph = f(x).item() # evaluate f(x + h)
x[ix] = oldval - h # increment by h
fxmh = f(x).item() # evaluate f(x - h)
x[ix] = oldval # reset
grad_numerical = (fxph - fxmh) / (2 * h)
grad_analytic = analytic_grad[ix]
rel_error_top = abs(grad_numerical - grad_analytic)
rel_error_bot = (abs(grad_numerical) + abs(grad_analytic) + 1e-12)
rel_error = rel_error_top / rel_error_bot
msg = 'numerical: %f analytic: %f, relative error: %e'
print(msg % (grad_numerical, grad_analytic, rel_error))
def preprocess_cifar10(cuda=False,
show_examples=True,
bias_trick=False,
validation_ratio=0.2,
dtype=torch.float32):
"""
Returns a preprocessed version of the CIFAR10 dataset, automatically
downloading if necessary. We perform the following steps:
(0) [Optional] Visualize some images from the dataset
(1) Normalize the data by subtracting the mean
(2) Reshape each image of shape (3, 32, 32) into a vector of shape (3072,)
(3) [Optional] Bias trick: add an extra dimension of ones to the data
(4) Carve out a validation set from the training set
Inputs:
- cuda: If true, move the entire dataset to the GPU
- validation_ratio: Float in the range (0, 1) giving the fraction of the train
set to reserve for validation
- bias_trick: Boolean telling whether or not to apply the bias trick
- show_examples: Boolean telling whether or not to visualize data samples
- dtype: Optional, data type of the input image X
Returns a dictionary with the following keys:
- 'X_train': `dtype` tensor of shape (N_train, D) giving training images
- 'X_val': `dtype` tensor of shape (N_val, D) giving val images
- 'X_test': `dtype` tensor of shape (N_test, D) giving test images
- 'y_train': int64 tensor of shape (N_train,) giving training labels
- 'y_val': int64 tensor of shape (N_val,) giving val labels
- 'y_test': int64 tensor of shape (N_test,) giving test labels
N_train, N_val, and N_test are the number of examples in the train, val, and
test sets respectively. The precise values of N_train and N_val are determined
by the input parameter validation_ratio. D is the dimension of the image data;
if bias_trick is False, then D = 32 * 32 * 3 = 3072;
if bias_trick is True then D = 1 + 32 * 32 * 3 = 3073.
"""
X_train, y_train, X_test, y_test = cifar10(x_dtype=dtype)
# Move data to the GPU
if cuda:
X_train = X_train.cuda()
y_train = y_train.cuda()
X_test = X_test.cuda()
y_test = y_test.cuda()
# 0. Visualize some examples from the dataset.
if show_examples:
classes = [
'plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck'
]
samples_per_class = 12
samples = []
eecs598.reset_seed(0)
for y, cls in enumerate(classes):
plt.text(-4, 34 * y + 18, cls, ha='right')
idxs, = (y_train == y).nonzero(as_tuple=True)
for i in range(samples_per_class):
idx = idxs[random.randrange(idxs.shape[0])].item()
samples.append(X_train[idx])
img = torchvision.utils.make_grid(samples, nrow=samples_per_class)
plt.imshow(eecs598.tensor_to_image(img))
plt.axis('off')
plt.show()
# 1. Normalize the data: subtract the mean RGB (zero mean)
mean_image = X_train.mean(dim=(0, 2, 3), keepdim=True)
X_train -= mean_image
X_test -= mean_image
# 2. Reshape the image data into rows
X_train = X_train.reshape(X_train.shape[0], -1)
X_test = X_test.reshape(X_test.shape[0], -1)
# 3. Add bias dimension and transform into columns
if bias_trick:
ones_train = torch.ones(X_train.shape[0], 1, device=X_train.device)
X_train = torch.cat([X_train, ones_train], dim=1)
ones_test = torch.ones(X_test.shape[0], 1, device=X_test.device)
X_test = torch.cat([X_test, ones_test], dim=1)
# 4. take the validation set from the training set
# Note: It should not be taken from the test set
# For random permumation, you can use torch.randperm or torch.randint
# But, for this homework, we use slicing instead.
num_training = int(X_train.shape[0] * (1.0 - validation_ratio))
num_validation = X_train.shape[0] - num_training
# return the dataset
data_dict = {}
data_dict['X_val'] = X_train[num_training:num_training + num_validation]
data_dict['y_val'] = y_train[num_training:num_training + num_validation]
data_dict['X_train'] = X_train[0:num_training]
data_dict['y_train'] = y_train[0:num_training]
data_dict['X_test'] = X_test
data_dict['y_test'] = y_test
return data_dict