def forward_(mytorch_model, mytorch_criterion, pytorch_model,
pytorch_criterion, x, y):
"""
Calls forward on both mytorch and pytorch models.
x: ndrray (batch_size, in_features)
y: ndrray (batch_size,)
Returns (passed, (mytorch x, mytorch y, pytorch x, pytorch y)),
where passed is whether the test passed
"""
# forward
pytorch_x = Variable(torch.tensor(x).double(), requires_grad=True)
pytorch_y = pytorch_model(pytorch_x)
if not pytorch_criterion is None:
pytorch_y = pytorch_criterion(pytorch_y, torch.LongTensor(y))
mytorch_x = Tensor(x, requires_grad=True)
mytorch_y = mytorch_model(mytorch_x)
if not mytorch_criterion is None:
mytorch_y = mytorch_criterion(mytorch_y, Tensor(y))
# check that model is correctly configured
check_model_param_settings(mytorch_model)
# forward check
if not assertions_all(mytorch_y.data, pytorch_y.detach().numpy(), 'y'):
return False, (mytorch_x, mytorch_y, pytorch_x, pytorch_y)
return True, (mytorch_x, mytorch_y, pytorch_x, pytorch_y)
def test5():
a = Tensor(1, requires_grad=True)
a_torch = get_same_torch_tensor(a)
b = Tensor(2, requires_grad=True)
b_torch = get_same_torch_tensor(b)
c = Tensor(3, requires_grad=True)
c_torch = get_same_torch_tensor(c)
# d = a + a * b
z1 = a * b
z1_torch = a_torch * b_torch
d = a + z1
d_torch = a_torch + z1_torch
# e = (d + c) + 3
z2 = d + c
z2_torch = d_torch + c_torch
e = z2 + Tensor(3)
e_torch = z2_torch + 3
e.backward()
e_torch.sum().backward()
assert check_val_and_grad(a, a_torch)
assert check_val_and_grad(b, b_torch)
assert check_val_and_grad(c, c_torch)
assert check_val_and_grad(z1, z1_torch)
assert check_val_and_grad(d, d_torch)
assert check_val_and_grad(z2, z2_torch)
assert check_val_and_grad(e, e_torch)
def test_dropout_forward_backward():
np.random.seed(11785)
# run on small model, forward backward (no step)
model = Sequential(Linear(10, 20), ReLU(), Dropout(p=0.6))
x, y = generate_dataset_for_mytorch_model(model, 5)
x, y = Tensor(x), Tensor(y)
criterion = CrossEntropyLoss()
out = model(x)
test_out = load_numpy_array(
'autograder/hw1_bonus_autograder/outputs/backward_output.npy')
if not assertions_all(out.data, test_out,
"test_dropout_forward_backward_output", 1e-5, 1e-6):
return False
loss = criterion(out, y)
loss.backward()
assert model[0].weight.grad is not None, "Linear layer must have gradient."
assert model[
0].weight.grad.grad is None, "Final gradient tensor must not have its own gradient"
assert model[
0].weight.grad.grad_fn is None, "Final gradient tensor must not have its own grad function"
assert model[
0].weight.requires_grad, "Weight tensor must have requires_grad==True"
assert model[
0].weight.is_parameter, "Weight tensor must be marked as a parameter tensor"
test_grad = load_numpy_array(
'autograder/hw1_bonus_autograder/outputs/backward_grad.npy')
return assertions_all(model[0].weight.grad.data, test_grad,
"test_dropout_forward_backward_grad", 1e-5, 1e-6)
def testbroadcast():
"""Tests addition WITH broadcasting matches torch's"""
# shape of tensor to test
# get mytorch and torch tensor: 'a'
a = Tensor.randn(3, 4)
a.requires_grad = True
a_torch = get_same_torch_tensor(a)
# get mytorch and torch tensor: 'b'
b = Tensor.randn(4)
b.requires_grad = True
b_torch = get_same_torch_tensor(b)
# run mytorch and torch forward: 'c = a + b'
c = a + b
c_torch = a_torch + b_torch
# run mytorch and torch addition backward
c.backward()
c_torch.sum().backward()
#print(b.grad); print(b_torch.grad)
# check that c matches
assert check_val_and_grad(c, c_torch)
# check that dc/da and dc/db respectively match
assert check_val_and_grad(a, a_torch)
assert check_val_and_grad(b, b_torch)
def test6():
a = Tensor.randn(2, 3)
a.requires_grad = True
a_torch = get_same_torch_tensor(a)
b = Tensor.randn(2, 3)
b.requires_grad = True
b_torch = get_same_torch_tensor(b)
c = a / b
c_torch = a_torch / b_torch
d = a - b
d_torch = a_torch - b_torch
e = c + d
e_torch = c_torch + d_torch
e.backward()
e_torch.sum().backward()
assert check_val_and_grad(a, a_torch)
assert check_val_and_grad(b, b_torch)
assert check_val_and_grad(c, c_torch)
assert check_val_and_grad(d, d_torch)
assert check_val_and_grad(e, e_torch)
def train(model,
optimizer,
criterion,
train_x,
train_y,
val_x,
val_y,
num_epochs=3):
"""Problem 3.2: Training routine that runs for `num_epochs` epochs.
Returns:
val_accuracies (list): (num_epochs,)
"""
val_accuracies = []
# TODO: Implement me! (Pseudocode on writeup)
model.train()
for epoch in range(num_epochs):
shuffler = np.random.permutation(len(train_y))
train_x = train_x[shuffler]
train_y = train_y[shuffler]
batches = split_data_into_batches(train_x, train_y, 100)
for i, (batch_data, batch_labels) in enumerate(batches):
optimizer.zero_grad() # clear any previous gradients
out = model(Tensor(batch_data))
loss = criterion(out, Tensor(batch_labels))
loss.backward()
optimizer.step() # update weights with new gradients
if i % 100 == 0:
accuracy = validate(model, val_x, val_y)
val_accuracies.append(accuracy)
model.train()
return val_accuracies
def __init__(self, input_size, hidden_size, nonlinearity='tanh'):
super(RNNUnit, self).__init__()
# Initializing parameters
self.weight_ih = Tensor(np.random.randn(hidden_size, input_size),
requires_grad=True,
is_parameter=True)
self.bias_ih = Tensor(np.zeros(hidden_size),
requires_grad=True,
is_parameter=True)
self.weight_hh = Tensor(np.random.randn(hidden_size, hidden_size),
requires_grad=True,
is_parameter=True)
self.bias_hh = Tensor(np.zeros(hidden_size),
requires_grad=True,
is_parameter=True)
self.hidden_size = hidden_size
# Setting the Activation Unit
if nonlinearity == 'tanh':
self.act = Tanh()
elif nonlinearity == 'relu':
self.act = ReLU()
def test7():
# a = 3
a = Tensor(3., requires_grad=False)
a_torch = get_same_torch_tensor(a)
# b = 4
b = Tensor(4., requires_grad=False)
b_torch = get_same_torch_tensor(b)
# c = 5
c = Tensor(5., requires_grad=True)
c_torch = get_same_torch_tensor(c)
# out = a * b + 3 * c
z1 = a * b
z1_torch = a_torch * b_torch
z2 = Tensor(3) * c
z2_torch = 3 * c_torch
out = z1 + z2
out_torch = z1_torch + z2_torch
out_torch.sum().backward()
out.backward()
assert check_val_and_grad(a, a_torch)
assert check_val_and_grad(b, b_torch)
assert check_val_and_grad(c, c_torch)
assert check_val_and_grad(z1, z1_torch)
assert check_val_and_grad(z2, z2_torch)
assert check_val_and_grad(out, out_torch)
def train(model, optimizer, criterion, train_x, train_y, val_x, val_y, num_epochs=3):
"""Problem 3.2: Training routine that runs for `num_epochs` epochs.
Returns:
val_accuracies (list): (num_epochs,)
"""
model.train()
store_validation_accuracy = []
store_loss = []
Avg_loss = []
for j in range(num_epochs):
p = np.random.permutation(len(train_x))
shuffled_x = train_x[p]
shuffled_y = train_y[p]
xx = np.split(shuffled_x,len(shuffled_x)/BATCH_SIZE)
yy = np.split(shuffled_y,len(shuffled_y)/BATCH_SIZE)
for i, (batch_data,batch_labels) in enumerate(zip(xx,yy)):
optimizer.zero_grad() # clear any previous gradients
out = model(Tensor(batch_data))
loss = criterion(out,Tensor(batch_labels))
store_loss.append(loss.data)
loss.backward()
optimizer.step()
if i % 100 == 0:
Avg_loss.append(np.mean(np.array(store_loss)))
store_loss = []
accuracy = validate(model,val_x,val_y)
store_validation_accuracy.append(accuracy)
model.train()
print(Avg_loss)
return store_validation_accuracy
def test_mul():
"""Tests that mytorch's elementwise multiplication matches torch's"""
# shape of tensor to test
shape = (1, 2, 3)
# get mytorch and torch tensor: 'a'
a = Tensor.randn(*shape)
a.requires_grad = True
a_torch = get_same_torch_tensor(a)
# get mytorch and torch tensor: 'b'
b = Tensor.randn(*shape)
b.requires_grad = True
b_torch = get_same_torch_tensor(b)
# run mytorch and torch forward: 'c = a * b'
ctx = ContextManager()
c = Mul.forward(ctx, a, b)
c_torch = a_torch * b_torch
# run mytorch and torch multiplication backward
back = Mul.backward(ctx, Tensor.ones(*shape))
c_torch.sum().backward()
# check that c matches
assert check_val_and_grad(c, c_torch)
# check that dc/da and dc/db respectively match
assert check_val(back[0], a_torch.grad)
assert check_val(back[1], b_torch.grad)
# ensure * is overridden
c_using_override = a * b
assert check_val(c_using_override, c_torch)
return True
def forward_(mytorch_model, mytorch_criterion, pytorch_model,
pytorch_criterion, x, y):
"""
Calls forward on both mytorch and pytorch models.
x: ndrray (batch_size, in_features)
y: ndrray (batch_size,)
Returns (passed, (mytorch x, mytorch y, pytorch x, pytorch y)),
where passed is whether the test passed
"""
# forward
pytorch_x = Variable(torch.tensor(x).double(), requires_grad=True)
pytorch_y = pytorch_model(pytorch_x)
if not pytorch_criterion is None:
pytorch_y = pytorch_criterion(pytorch_y, torch.LongTensor(y))
mytorch_x = Tensor(x, requires_grad=True)
mytorch_y = mytorch_model(mytorch_x)
if not mytorch_criterion is None:
mytorch_y = mytorch_criterion(mytorch_y, Tensor(y))
# forward check
assert assertions_all(mytorch_y.data,
pytorch_y.detach().numpy(), 'y'), "Forward Failed"
return (mytorch_x, mytorch_y, pytorch_x, pytorch_y)
def init_weights(self, weights):
"""Use the 3 weight matrices of linear MLP to init the weights of the CNN.
Args:
weights (tuple(np.array)): shapes ((8, 192), (16, 8), (4, 16))
Think of each as a Linear.weight.data, shaped (out_features, in_features)
"""
w1, w2, w3 = weights
# TODO: Convert the linear weights into Conv1d weights
# Make sure to not add nodes to the comp graph!
w1 = w1[0:2:, 0:48]
w2 = w2[0:8:, 0:4]
conv1_weights = np.reshape(
w1, (self.conv1.out_channel, self.conv1.kernel_size,
self.conv1.in_channel))
conv1_weights = np.transpose(conv1_weights, axes=(0, 2, 1))
self.conv1.weight = Tensor(conv1_weights,
is_parameter=True,
requires_grad=True)
conv2_weights = np.reshape(
w2, (self.conv2.out_channel, self.conv2.kernel_size,
self.conv2.in_channel))
conv2_weights = np.transpose(conv2_weights, axes=(0, 2, 1))
self.conv2.weight = Tensor(conv2_weights,
is_parameter=True,
requires_grad=True)
conv3_weights = np.reshape(
w3, (self.conv3.out_channel, self.conv3.kernel_size,
self.conv3.in_channel))
conv3_weights = np.transpose(conv3_weights, axes=(0, 2, 1))
self.conv3.weight = Tensor(conv3_weights,
is_parameter=True,
requires_grad=True)
def test_slice_backward():
# shape of tensor to test
shape = (2, 4, 3)
# Test 1
a = Tensor.randn(*shape) #mytorch tensor
a.requires_grad = True
a_torch = get_same_torch_tensor(a) #pytorch tensor
b = a[1, 2, 0]
b_torch = a_torch[1, 2, 0]
(b**2).sum().backward()
(b_torch**2).sum().backward()
assert check_grad(a, a_torch, eps=eps)
# Test 2
a = Tensor.randn(*shape) #mytorch tensor
a.requires_grad = True
a_torch = get_same_torch_tensor(a) #pytorch tensor
b = a[0, 2, :]
b_torch = a_torch[0, 2, :]
(b**2).sum().backward()
(b_torch**2).sum().backward()
assert check_grad(a, a_torch, eps=eps)
# Test 3
a = Tensor.randn(*shape) #mytorch tensor
a.requires_grad = True
a_torch = get_same_torch_tensor(a) #pytorch tensor
b = a[:, 3, :]
b_torch = a_torch[:, 3, :]
(b**2).sum().backward()
(b_torch**2).sum().backward()
assert check_grad(a, a_torch, eps=eps)
# Test 4
a = Tensor.randn(*shape) #mytorch tensor
a.requires_grad = True
a_torch = get_same_torch_tensor(a) #pytorch tensor
b = a[:, :, 1]
b_torch = a_torch[:, :, 1]
(b**2).sum().backward()
(b_torch**2).sum().backward()
assert check_grad(a, a_torch, eps=eps)
# Test 5
a = Tensor.randn(*shape) #mytorch tensor
a.requires_grad = True
a_torch = get_same_torch_tensor(a) #pytorch tensor
b = a[:, 1:3, :]
b_torch = a_torch[:, 1:3, :]
(b**2).sum().backward()
(b_torch**2).sum().backward()
assert check_grad(a, a_torch, eps=eps)
return True
def test_big_model_step():
np.random.seed(11785)
# run a big model
model = Sequential(Linear(10, 15), ReLU(), Dropout(p=0.2),
Linear(15, 20), ReLU(), Dropout(p=0.1))
x, y = generate_dataset_for_mytorch_model(model, 4)
x, y = Tensor(x), Tensor(y)
criterion = CrossEntropyLoss()
optimizer = Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08)
# check output correct
out = model(x)
test_out = load_numpy_array('autograder/hw1_bonus_autograder/outputs/big_output.npy')
if not assertions_all(out.data, test_out, "test_big_model_step_out", 1e-5, 1e-6):
return False
# run backward
loss = criterion(out, y)
loss.backward()
# check params are correct (sorry this is ugly)
assert model[0].weight.grad is not None, "Linear layer must have gradient."
assert model[0].weight.grad.grad is None, "Final gradient tensor must not have its own gradient"
assert model[0].weight.grad.grad_fn is None, "Final gradient tensor must not have its own grad function"
assert model[0].weight.requires_grad, "Weight tensor must have requires_grad==True"
assert model[0].weight.is_parameter, "Weight tensor must be marked as a parameter tensor"
assert model[3].weight.grad is not None, "Linear layer must have gradient."
assert model[3].weight.grad.grad is None, "Final gradient tensor must not have its own gradient"
assert model[3].weight.grad.grad_fn is None, "Final gradient tensor must not have its own grad function"
assert model[3].weight.requires_grad, "Weight tensor must have requires_grad==True"
assert model[3].weight.is_parameter, "Weight tensor must be marked as a parameter tensor"
# check gradient for linear layer at idx 0 is correct
test_grad = load_numpy_array('autograder/hw1_bonus_autograder/outputs/big_grad.npy')
if not assertions_all(model[0].weight.grad.data, test_grad, "test_big_model_grad_0", 1e-5, 1e-6):
return False
# check gradient for linear layer at idx 3 is correct
test_grad = load_numpy_array('autograder/hw1_bonus_autograder/outputs/big_grad_3.npy')
if not assertions_all(model[3].weight.grad.data, test_grad, "test_big_model_grad_3", 1e-5, 1e-6):
return False
# weight update with adam
optimizer.step()
# check updated weight values
assert model[0].weight.requires_grad, "Weight tensor must have requires_grad==True"
assert model[0].weight.is_parameter, "Weight tensor must be marked as a parameter tensor"
test_weights_3 = load_numpy_array('autograder/hw1_bonus_autograder/outputs/big_weight_update_3.npy')
test_weights_0 = load_numpy_array('autograder/hw1_bonus_autograder/outputs/big_weight_update_0.npy')
return assertions_all(model[0].weight.data, test_weights_0, "test_big_weight_update_0", 1e-5, 1e-6) and \
assertions_all(model[3].weight.data, test_weights_3, "test_big_weight_update_3", 1e-5, 1e-6)
def __init__(self, in_features, out_features):
super().__init__()
self.in_features = in_features
self.out_features = out_features
# Randomly initializing layer weights
k = 1 / in_features
weight = k * (np.random.rand(out_features, in_features) - 0.5)
bias = k * (np.random.rand(out_features) - 0.5)
self.weight = Tensor(weight, requires_grad=True, is_parameter=True)
self.bias = Tensor(bias, requires_grad=True, is_parameter=True)
def __init__(self, in_channel, out_channel, kernel_size, stride=1):
super().__init__()
self.in_channel = in_channel
self.out_channel = out_channel
self.kernel_size = kernel_size
self.stride = stride
# Initializing weights and bias (not a very good initialization strategy)
weight = np.random.normal(0, 1.0,
(out_channel, in_channel, kernel_size))
self.weight = Tensor(weight, requires_grad=True, is_parameter=True)
bias = np.zeros(out_channel)
self.bias = Tensor(bias, requires_grad=True, is_parameter=True)
def test_conv_forward_back_just_1_layer():
in_c = 2
out_c = 3
kernel = 2
stride = 2
width = 5
batch_size = 1
# setup weights and biases
conv = Conv1d(in_c, out_c, kernel, stride)
# conv.weight = Tensor(np.random.randint(5, size=conv.weight.shape)+.0,
# requires_grad = True)
test_weight = np.asarray([[[1, 2], [2, 1]], [[0, 1], [1, 0]],
[[3, 2], [1, 0]]]) + .0
conv.weight = Tensor(test_weight, requires_grad=True)
# conv.bias = Tensor(np.random.randint(2, size=conv.out_channel)+.0,
# requires_grad = True)
conv.bias = Tensor(np.zeros(conv.out_channel) + .0, requires_grad=True)
conv_torch = nn.Conv1d(in_c, out_c, kernel_size=kernel, stride=stride)
conv_torch.weight = nn.Parameter(torch.tensor(conv.weight.data))
conv_torch.bias = nn.Parameter(torch.tensor(conv.bias.data))
print(f"weight:\n {conv.weight}")
print(f"bias:\n {conv.bias}")
# setup input
# x = Tensor(np.random.randint(5, size=(batch_size, in_c, width)),\
# requires_grad = True)
x = Tensor(np.asarray([[[1, 0, 1, 0, 1], [0, 1, 0, 1, 0]]]),
requires_grad=True)
x_torch = get_same_torch_tensor(x).double()
print(f"x:\n {x}")
# calculate output
o = conv(x)
o_torch = conv_torch(x_torch)
print(f"out:\n {o}")
# backward
o.backward()
o_torch.sum().backward()
print(f"grad_x:\n {x.grad}")
print(f"grad_w:\n {conv.weight.grad}")
print(f"grad_b:\n {conv.bias.grad}")
# check everything
assert check_val_and_grad(x, x_torch)
assert check_val_and_grad(o, o_torch)
assert check_val_and_grad(conv.weight, conv_torch.weight)
assert check_val_and_grad(conv.bias, conv_torch.bias)
def test_concat_backward():
# shape of tensor to test
tensor_shapes = [[(1, 4, 3), (1, 8, 3), (1, 5, 3)], [(2, 3, 4), (1, 3, 4)],
[(6, 7, 8, 9), (6, 7, 8, 1), (6, 7, 8, 2)],
[(1, 2, 3), (1, 2, 4), (1, 2, 3), (1, 2, 4)]]
cat_dims = [1, 0, 3, 2]
for tensor_shapes_cur, d_cur in zip(tensor_shapes, cat_dims):
# get mytorch and torch tensor: 'a'
a = [Tensor.randn(*shape_i) for shape_i in tensor_shapes_cur]
for i in range(len(a)):
a[i].requires_grad = True
a_torch = [get_same_torch_tensor(a_i) for a_i in a]
c = cat(a, d_cur)
c_torch = torch.cat(a_torch, dim=d_cur)
l = (c**2).sum()
l_torch = (c_torch**2).sum()
l.backward()
l_torch.backward()
for a_i, a_torch_i in zip(a, a_torch):
assert check_grad(a_i, a_torch_i, eps=eps)
return True
def validate(model, val_x, val_y):
"""Problem 3.3: Validation routine, tests on val data, scores accuracy
Relevant Args:
val_x (np.array): validation data (5000, 784)
val_y (np.array): validation labels (5000,)
Returns:
float: Accuracy = correct / total
"""
#TODO: implement validation based on pseudocode
model.eval()
num_samples = val_x.shape[0]
#val_batches = get_batch(val_x,val_y)
batches = list(
zip(np.array_split(val_x, num_samples // 100),
np.array_split(val_y, num_samples // 100)))
num_correct = 0
for i, (batch_data, batch_labels) in enumerate(batches):
#if(i*BATCH_SIZE>=num_samples):
# break
out = model(Tensor(batch_data))
batch_preds = np.argmax(out.data, axis=1)
#print('\nPrediction on Batch:',batch_preds[:10])
#print(f'\n Bach True label: {batch_labels}')
num_correct += (batch_preds == batch_labels).sum()
accuracy = (num_correct / len(val_x) * 100)
return accuracy
def test_distributed_scanning_mlp():
cnn = CNN_DistributedScanningMLP()
weights = np.load(os.path.join('autograder', 'hw2_autograder', 'weights', 'mlp_weights_part_c.npy'), allow_pickle=True)
weights = tuple(w.T for w in weights)
cnn.init_weights(weights)
data = np.loadtxt(os.path.join('autograder', 'hw2_autograder', 'data', 'data.asc')).T.reshape(1, 24, -1)
data = Tensor(data, requires_grad=False, is_parameter=False, is_leaf=True)
expected_result = np.load(os.path.join('autograder', 'hw2_autograder', 'ref_result', 'res_c.npy'), allow_pickle=True)
result = cnn(data)
# check that model is correctly configured
check_model_param_settings(cnn)
# if passes tests, return true.
# If exception anywhere (failure or crash), return false
try:
# check that output is correct
assert type(result.data) == type(expected_result), "Incorrect output type."
assert result.data.shape == expected_result.shape, "Incorrect output shape."
assert np.allclose(result.data, expected_result), "Incorrect output values."
except Exception as e:
traceback.print_exc()
return False
return True
def test_rnn_unit_backward():
input_sizes, hidden_sizes, data_lens = get_params()
for input_size, hidden_size, data_len in zip(input_sizes, hidden_sizes,
data_lens):
in_mytorch = Tensor.randn(data_len[0], input_size)
in_torch = get_same_torch_tensor(in_mytorch)
in_mytorch.requires_grad = True
in_torch.requires_grad = True
model_mytorch = RNNUnit(input_size, hidden_size)
model_torch = nn.RNNCell(input_size, hidden_size).double()
transfer_weights_rnn_unit(model_torch, model_mytorch)
resm = model_mytorch(in_mytorch)
rest = model_torch(in_torch)
lm = (resm**2).sum()
lt = (rest**2).sum()
lm.backward()
lt.backward()
assert compare_rnn_unit_param_grad(model_torch, model_mytorch)
assert check_grad(resm, rest, eps=eps)
return True
def test_time_iterator_forward():
input_sizes, hidden_sizes, data_lens = get_params()
for input_size, hidden_size, data_len in zip(input_sizes, hidden_sizes,
data_lens):
seq_mytorch = [
Tensor.randn(data_len[i], input_size) for i in range(len(data_len))
]
seq_torch = [get_same_torch_tensor(i) for i in seq_mytorch]
mpack = mpack_sequence(seq_mytorch)
tpack = nn.utils.rnn.pack_sequence(seq_torch, enforce_sorted=False)
model_mytorch = RNN(input_size, hidden_size)
model_torch = nn.RNN(input_size,
hidden_size,
num_layers=1,
batch_first=False).double()
transfer_weights(model_torch, model_mytorch)
resm, hm = model_mytorch(mpack)
rest, ht = model_torch(tpack)
assert check_val(resm.data, rest.data, eps=eps)
t_idx = list(np.argsort(data_len)[::-1])
# Torch returns data which can be bi-directional
assert check_val(hm, ht[0][t_idx], eps=eps)
return True
def test_slice_forward():
# shape of tensor to test
shape = (2, 4, 3)
# get mytorch and torch tensor: 'a'
a = Tensor.randn(*shape) #mytorch tensor
a.requires_grad = True
a_torch = get_same_torch_tensor(a) #pytorch tensor
# Test 1
b = a[1, 2, 0]
b_torch = a_torch[1, 2, 0]
assert check_val(b, b_torch, eps=eps)
# Test 2
b = a[0, 2, :]
b_torch = a_torch[0, 2, :]
assert check_val(b, b_torch, eps=eps)
# Test 3
b = a[:, 3, :]
b_torch = a_torch[:, 3, :]
assert check_val(b, b_torch, eps=eps)
# Test 4
b = a[:, :, 1]
b_torch = a_torch[:, :, 1]
assert check_val(b, b_torch, eps=eps)
# Test 5
b = a[:, 1:3, :]
b_torch = a_torch[:, 1:3, :]
assert check_grad(b, b_torch, eps=eps)
return True
def test_simple_scanning_mlp():
cnn = CNN_SimpleScanningMLP()
# Load and init weights
weights = np.load(os.path.join('autograder', 'hw2_autograder', 'weights', 'mlp_weights_part_b.npy'), allow_pickle = True)
weights = tuple(w.T for w in weights)
cnn.init_weights(weights)
# load data and expected answer
data = np.loadtxt(os.path.join('autograder', 'hw2_autograder', 'data', 'data.asc')).T.reshape(1, 24, -1)
data = Tensor(data, requires_grad=False, is_parameter=False, is_leaf=True)
expected_result = np.load(os.path.join('autograder', 'hw2_autograder', 'ref_result', 'res_b.npy'), allow_pickle=True)
# get forward output and check
result = cnn(data)
# check that model is correctly configured
check_model_param_settings(cnn)
# now check correct results
try:
# check that output is correct
assert type(result.data) == type(expected_result), f"Incorrect output type: {result.data}, expected: {expected_result}"
assert result.data.shape == expected_result.shape, f"Incorrect output shape: {result.data.shape}, expected: {expected_result.shape}"
assert np.allclose(result.data, expected_result), f"Incorrect output values: {result.data}, expected: {expected_result}"
except Exception as e:
traceback.print_exc()
return False
return True
def test_unpack_sequence_backward():
test_shapes = [[(4, 1), (5, 1)], [(4, 3), (10, 3), (2, 3)]]
a = True
for shapes in test_shapes:
# get mytorch and torch tensor: 'a'
seq1 = [Tensor.randn(*shape) for shape in shapes]
for t in seq1:
t.requires_grad = True
seq2 = [get_same_torch_tensor(t) for t in seq1]
# run mytorch and torch forward: 'c = cat (a, b)'
c_temp = pack_sequence(seq1)
c_temp2 = unpack_sequence(c_temp)
c = pack_sequence(c_temp2)
c_torch = torch.nn.utils.rnn.pack_sequence(seq2, enforce_sorted=False)
l = (c.data**2).sum()
l_torch = (c_torch.data**2).sum()
l.backward()
l_torch.backward()
for a1, a2 in zip(seq1, seq2):
assert check_grad(a1, a2, eps=eps)
#compare_ps(c_torch, c.data, "test_pack_sequence_backward")
return True
def test1():
a = Tensor.randn(1, 2, 3)
a.requires_grad = True
a_torch = get_same_torch_tensor(a)
b = Tensor.randn(1, 2, 3)
b.requires_grad = True
b_torch = get_same_torch_tensor(b)
c = a + b
c_torch = a_torch + b_torch
c_torch.sum().backward()
c.backward()
assert check_val_and_grad(a, a_torch)
assert check_val_and_grad(b, b_torch)
assert check_val_and_grad(c, c_torch)
def conv1d_forward_correctness(num_layers=1):
'''
CNN: scanning with a MLP with stride
'''
scores_dict = [0]
############################################################################################
############################# Initialize parameters ###################################
############################################################################################
in_c = np.random.randint(5, 15)
channels = [np.random.randint(5, 15) for i in range(num_layers + 1)]
kernel = [np.random.randint(3, 7) for i in range(num_layers)]
stride = [np.random.randint(3, 5) for i in range(num_layers)]
width = np.random.randint(60, 80)
batch_size = np.random.randint(1, 4)
x = np.random.randn(batch_size, channels[0], width)
#############################################################################################
################################# Create Models ########################################
#############################################################################################
test_layers = [
Conv1d(channels[i], channels[i + 1], kernel[i], stride[i])
for i in range(num_layers)
]
test_model = Sequential(*test_layers)
torch_layers = [
nn.Conv1d(channels[i], channels[i + 1], kernel[i], stride=stride[i])
for i in range(num_layers)
]
torch_model = nn.Sequential(*torch_layers)
for torch_layer, test_layer in zip(torch_model, test_model.layers):
torch_layer.weight = nn.Parameter(torch.tensor(test_layer.weight.data))
torch_layer.bias = nn.Parameter(torch.tensor(test_layer.bias.data))
#############################################################################################
######################### Get the correct results from PyTorch #########################
#############################################################################################
x1 = Variable(torch.tensor(x), requires_grad=True)
y1 = torch_model(x1)
torch_y = y1.detach().numpy()
#############################################################################################
################### Get fwd results from TestModel and compare ##########################
#############################################################################################
y2 = test_model(Tensor(x))
test_y = y2.data
# check that model is correctly configured
check_model_param_settings(test_model)
if not assertions(test_y, torch_y, 'type', 'y'): return scores_dict
if not assertions(test_y, torch_y, 'shape', 'y'): return scores_dict
if not assertions(test_y, torch_y, 'closeness', 'y'): return scores_dict
scores_dict[0] = 1
return scores_dict
def forward(self, x):
"""
Args:
x (Tensor): (batch_size, in_features)
Returns:
Tensor: (batch_size, out_features)
"""
return Tensor.matmul(x,self.weight.T()) + self.bias
def test_unsqueeze():
# shape of tensor to test
shape = (2, 2, 3)
# get mytorch and torch tensor: 'a'
a = Tensor.randn(*shape) #mytorch tensor
a.requires_grad = True
a_torch = get_same_torch_tensor(a) #pytorch tensor
b = Tensor.unsqueeze(a)
b_torch = torch.Tensor.unsqueeze(a_torch, 0)
assert check_val(b, b_torch, eps=eps)
b = Tensor.unsqueeze(a, 2)
b_torch = torch.Tensor.unsqueeze(a_torch, 2)
assert check_val(b, b_torch, eps=eps)
return True
def test_dropout_forward():
np.random.seed(11785)
# run on small model forward only
x = Tensor.randn(5, 10)
model = Sequential(Linear(10, 5), ReLU(), Dropout(p=0.6))
my_output = model(x)
test_output = load_numpy_array('autograder/hw1_bonus_autograder/outputs/dropout_forward.npy')
return assertions_all(my_output.data, test_output, "test_dropout_forward", 1e-5, 1e-6)
