def forward_(mytorch_model, mytorch_criterion, pytorch_model, pytorch_criterion, x, y): """ Calls forward on both mytorch and pytorch models. x: ndrray (batch_size, in_features) y: ndrray (batch_size,) Returns (passed, (mytorch x, mytorch y, pytorch x, pytorch y)), where passed is whether the test passed """ # forward pytorch_x = Variable(torch.tensor(x).double(), requires_grad=True) pytorch_y = pytorch_model(pytorch_x) if not pytorch_criterion is None: pytorch_y = pytorch_criterion(pytorch_y, torch.LongTensor(y)) mytorch_x = Tensor(x, requires_grad=True) mytorch_y = mytorch_model(mytorch_x) if not mytorch_criterion is None: mytorch_y = mytorch_criterion(mytorch_y, Tensor(y)) # check that model is correctly configured check_model_param_settings(mytorch_model) # forward check if not assertions_all(mytorch_y.data, pytorch_y.detach().numpy(), 'y'): return False, (mytorch_x, mytorch_y, pytorch_x, pytorch_y) return True, (mytorch_x, mytorch_y, pytorch_x, pytorch_y)
def test5(): a = Tensor(1, requires_grad=True) a_torch = get_same_torch_tensor(a) b = Tensor(2, requires_grad=True) b_torch = get_same_torch_tensor(b) c = Tensor(3, requires_grad=True) c_torch = get_same_torch_tensor(c) # d = a + a * b z1 = a * b z1_torch = a_torch * b_torch d = a + z1 d_torch = a_torch + z1_torch # e = (d + c) + 3 z2 = d + c z2_torch = d_torch + c_torch e = z2 + Tensor(3) e_torch = z2_torch + 3 e.backward() e_torch.sum().backward() assert check_val_and_grad(a, a_torch) assert check_val_and_grad(b, b_torch) assert check_val_and_grad(c, c_torch) assert check_val_and_grad(z1, z1_torch) assert check_val_and_grad(d, d_torch) assert check_val_and_grad(z2, z2_torch) assert check_val_and_grad(e, e_torch)
def test_dropout_forward_backward(): np.random.seed(11785) # run on small model, forward backward (no step) model = Sequential(Linear(10, 20), ReLU(), Dropout(p=0.6)) x, y = generate_dataset_for_mytorch_model(model, 5) x, y = Tensor(x), Tensor(y) criterion = CrossEntropyLoss() out = model(x) test_out = load_numpy_array( 'autograder/hw1_bonus_autograder/outputs/backward_output.npy') if not assertions_all(out.data, test_out, "test_dropout_forward_backward_output", 1e-5, 1e-6): return False loss = criterion(out, y) loss.backward() assert model[0].weight.grad is not None, "Linear layer must have gradient." assert model[ 0].weight.grad.grad is None, "Final gradient tensor must not have its own gradient" assert model[ 0].weight.grad.grad_fn is None, "Final gradient tensor must not have its own grad function" assert model[ 0].weight.requires_grad, "Weight tensor must have requires_grad==True" assert model[ 0].weight.is_parameter, "Weight tensor must be marked as a parameter tensor" test_grad = load_numpy_array( 'autograder/hw1_bonus_autograder/outputs/backward_grad.npy') return assertions_all(model[0].weight.grad.data, test_grad, "test_dropout_forward_backward_grad", 1e-5, 1e-6)
def testbroadcast(): """Tests addition WITH broadcasting matches torch's""" # shape of tensor to test # get mytorch and torch tensor: 'a' a = Tensor.randn(3, 4) a.requires_grad = True a_torch = get_same_torch_tensor(a) # get mytorch and torch tensor: 'b' b = Tensor.randn(4) b.requires_grad = True b_torch = get_same_torch_tensor(b) # run mytorch and torch forward: 'c = a + b' c = a + b c_torch = a_torch + b_torch # run mytorch and torch addition backward c.backward() c_torch.sum().backward() #print(b.grad); print(b_torch.grad) # check that c matches assert check_val_and_grad(c, c_torch) # check that dc/da and dc/db respectively match assert check_val_and_grad(a, a_torch) assert check_val_and_grad(b, b_torch)
def test6(): a = Tensor.randn(2, 3) a.requires_grad = True a_torch = get_same_torch_tensor(a) b = Tensor.randn(2, 3) b.requires_grad = True b_torch = get_same_torch_tensor(b) c = a / b c_torch = a_torch / b_torch d = a - b d_torch = a_torch - b_torch e = c + d e_torch = c_torch + d_torch e.backward() e_torch.sum().backward() assert check_val_and_grad(a, a_torch) assert check_val_and_grad(b, b_torch) assert check_val_and_grad(c, c_torch) assert check_val_and_grad(d, d_torch) assert check_val_and_grad(e, e_torch)
def train(model, optimizer, criterion, train_x, train_y, val_x, val_y, num_epochs=3): """Problem 3.2: Training routine that runs for `num_epochs` epochs. Returns: val_accuracies (list): (num_epochs,) """ val_accuracies = [] # TODO: Implement me! (Pseudocode on writeup) model.train() for epoch in range(num_epochs): shuffler = np.random.permutation(len(train_y)) train_x = train_x[shuffler] train_y = train_y[shuffler] batches = split_data_into_batches(train_x, train_y, 100) for i, (batch_data, batch_labels) in enumerate(batches): optimizer.zero_grad() # clear any previous gradients out = model(Tensor(batch_data)) loss = criterion(out, Tensor(batch_labels)) loss.backward() optimizer.step() # update weights with new gradients if i % 100 == 0: accuracy = validate(model, val_x, val_y) val_accuracies.append(accuracy) model.train() return val_accuracies
def __init__(self, input_size, hidden_size, nonlinearity='tanh'): super(RNNUnit, self).__init__() # Initializing parameters self.weight_ih = Tensor(np.random.randn(hidden_size, input_size), requires_grad=True, is_parameter=True) self.bias_ih = Tensor(np.zeros(hidden_size), requires_grad=True, is_parameter=True) self.weight_hh = Tensor(np.random.randn(hidden_size, hidden_size), requires_grad=True, is_parameter=True) self.bias_hh = Tensor(np.zeros(hidden_size), requires_grad=True, is_parameter=True) self.hidden_size = hidden_size # Setting the Activation Unit if nonlinearity == 'tanh': self.act = Tanh() elif nonlinearity == 'relu': self.act = ReLU()
def test7(): # a = 3 a = Tensor(3., requires_grad=False) a_torch = get_same_torch_tensor(a) # b = 4 b = Tensor(4., requires_grad=False) b_torch = get_same_torch_tensor(b) # c = 5 c = Tensor(5., requires_grad=True) c_torch = get_same_torch_tensor(c) # out = a * b + 3 * c z1 = a * b z1_torch = a_torch * b_torch z2 = Tensor(3) * c z2_torch = 3 * c_torch out = z1 + z2 out_torch = z1_torch + z2_torch out_torch.sum().backward() out.backward() assert check_val_and_grad(a, a_torch) assert check_val_and_grad(b, b_torch) assert check_val_and_grad(c, c_torch) assert check_val_and_grad(z1, z1_torch) assert check_val_and_grad(z2, z2_torch) assert check_val_and_grad(out, out_torch)
def train(model, optimizer, criterion, train_x, train_y, val_x, val_y, num_epochs=3): """Problem 3.2: Training routine that runs for `num_epochs` epochs. Returns: val_accuracies (list): (num_epochs,) """ model.train() store_validation_accuracy = [] store_loss = [] Avg_loss = [] for j in range(num_epochs): p = np.random.permutation(len(train_x)) shuffled_x = train_x[p] shuffled_y = train_y[p] xx = np.split(shuffled_x,len(shuffled_x)/BATCH_SIZE) yy = np.split(shuffled_y,len(shuffled_y)/BATCH_SIZE) for i, (batch_data,batch_labels) in enumerate(zip(xx,yy)): optimizer.zero_grad() # clear any previous gradients out = model(Tensor(batch_data)) loss = criterion(out,Tensor(batch_labels)) store_loss.append(loss.data) loss.backward() optimizer.step() if i % 100 == 0: Avg_loss.append(np.mean(np.array(store_loss))) store_loss = [] accuracy = validate(model,val_x,val_y) store_validation_accuracy.append(accuracy) model.train() print(Avg_loss) return store_validation_accuracy
def test_mul(): """Tests that mytorch's elementwise multiplication matches torch's""" # shape of tensor to test shape = (1, 2, 3) # get mytorch and torch tensor: 'a' a = Tensor.randn(*shape) a.requires_grad = True a_torch = get_same_torch_tensor(a) # get mytorch and torch tensor: 'b' b = Tensor.randn(*shape) b.requires_grad = True b_torch = get_same_torch_tensor(b) # run mytorch and torch forward: 'c = a * b' ctx = ContextManager() c = Mul.forward(ctx, a, b) c_torch = a_torch * b_torch # run mytorch and torch multiplication backward back = Mul.backward(ctx, Tensor.ones(*shape)) c_torch.sum().backward() # check that c matches assert check_val_and_grad(c, c_torch) # check that dc/da and dc/db respectively match assert check_val(back[0], a_torch.grad) assert check_val(back[1], b_torch.grad) # ensure * is overridden c_using_override = a * b assert check_val(c_using_override, c_torch) return True
def forward_(mytorch_model, mytorch_criterion, pytorch_model, pytorch_criterion, x, y): """ Calls forward on both mytorch and pytorch models. x: ndrray (batch_size, in_features) y: ndrray (batch_size,) Returns (passed, (mytorch x, mytorch y, pytorch x, pytorch y)), where passed is whether the test passed """ # forward pytorch_x = Variable(torch.tensor(x).double(), requires_grad=True) pytorch_y = pytorch_model(pytorch_x) if not pytorch_criterion is None: pytorch_y = pytorch_criterion(pytorch_y, torch.LongTensor(y)) mytorch_x = Tensor(x, requires_grad=True) mytorch_y = mytorch_model(mytorch_x) if not mytorch_criterion is None: mytorch_y = mytorch_criterion(mytorch_y, Tensor(y)) # forward check assert assertions_all(mytorch_y.data, pytorch_y.detach().numpy(), 'y'), "Forward Failed" return (mytorch_x, mytorch_y, pytorch_x, pytorch_y)
def init_weights(self, weights): """Use the 3 weight matrices of linear MLP to init the weights of the CNN. Args: weights (tuple(np.array)): shapes ((8, 192), (16, 8), (4, 16)) Think of each as a Linear.weight.data, shaped (out_features, in_features) """ w1, w2, w3 = weights # TODO: Convert the linear weights into Conv1d weights # Make sure to not add nodes to the comp graph! w1 = w1[0:2:, 0:48] w2 = w2[0:8:, 0:4] conv1_weights = np.reshape( w1, (self.conv1.out_channel, self.conv1.kernel_size, self.conv1.in_channel)) conv1_weights = np.transpose(conv1_weights, axes=(0, 2, 1)) self.conv1.weight = Tensor(conv1_weights, is_parameter=True, requires_grad=True) conv2_weights = np.reshape( w2, (self.conv2.out_channel, self.conv2.kernel_size, self.conv2.in_channel)) conv2_weights = np.transpose(conv2_weights, axes=(0, 2, 1)) self.conv2.weight = Tensor(conv2_weights, is_parameter=True, requires_grad=True) conv3_weights = np.reshape( w3, (self.conv3.out_channel, self.conv3.kernel_size, self.conv3.in_channel)) conv3_weights = np.transpose(conv3_weights, axes=(0, 2, 1)) self.conv3.weight = Tensor(conv3_weights, is_parameter=True, requires_grad=True)
def test_slice_backward(): # shape of tensor to test shape = (2, 4, 3) # Test 1 a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor b = a[1, 2, 0] b_torch = a_torch[1, 2, 0] (b**2).sum().backward() (b_torch**2).sum().backward() assert check_grad(a, a_torch, eps=eps) # Test 2 a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor b = a[0, 2, :] b_torch = a_torch[0, 2, :] (b**2).sum().backward() (b_torch**2).sum().backward() assert check_grad(a, a_torch, eps=eps) # Test 3 a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor b = a[:, 3, :] b_torch = a_torch[:, 3, :] (b**2).sum().backward() (b_torch**2).sum().backward() assert check_grad(a, a_torch, eps=eps) # Test 4 a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor b = a[:, :, 1] b_torch = a_torch[:, :, 1] (b**2).sum().backward() (b_torch**2).sum().backward() assert check_grad(a, a_torch, eps=eps) # Test 5 a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor b = a[:, 1:3, :] b_torch = a_torch[:, 1:3, :] (b**2).sum().backward() (b_torch**2).sum().backward() assert check_grad(a, a_torch, eps=eps) return True
def test_big_model_step(): np.random.seed(11785) # run a big model model = Sequential(Linear(10, 15), ReLU(), Dropout(p=0.2), Linear(15, 20), ReLU(), Dropout(p=0.1)) x, y = generate_dataset_for_mytorch_model(model, 4) x, y = Tensor(x), Tensor(y) criterion = CrossEntropyLoss() optimizer = Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08) # check output correct out = model(x) test_out = load_numpy_array('autograder/hw1_bonus_autograder/outputs/big_output.npy') if not assertions_all(out.data, test_out, "test_big_model_step_out", 1e-5, 1e-6): return False # run backward loss = criterion(out, y) loss.backward() # check params are correct (sorry this is ugly) assert model[0].weight.grad is not None, "Linear layer must have gradient." assert model[0].weight.grad.grad is None, "Final gradient tensor must not have its own gradient" assert model[0].weight.grad.grad_fn is None, "Final gradient tensor must not have its own grad function" assert model[0].weight.requires_grad, "Weight tensor must have requires_grad==True" assert model[0].weight.is_parameter, "Weight tensor must be marked as a parameter tensor" assert model[3].weight.grad is not None, "Linear layer must have gradient." assert model[3].weight.grad.grad is None, "Final gradient tensor must not have its own gradient" assert model[3].weight.grad.grad_fn is None, "Final gradient tensor must not have its own grad function" assert model[3].weight.requires_grad, "Weight tensor must have requires_grad==True" assert model[3].weight.is_parameter, "Weight tensor must be marked as a parameter tensor" # check gradient for linear layer at idx 0 is correct test_grad = load_numpy_array('autograder/hw1_bonus_autograder/outputs/big_grad.npy') if not assertions_all(model[0].weight.grad.data, test_grad, "test_big_model_grad_0", 1e-5, 1e-6): return False # check gradient for linear layer at idx 3 is correct test_grad = load_numpy_array('autograder/hw1_bonus_autograder/outputs/big_grad_3.npy') if not assertions_all(model[3].weight.grad.data, test_grad, "test_big_model_grad_3", 1e-5, 1e-6): return False # weight update with adam optimizer.step() # check updated weight values assert model[0].weight.requires_grad, "Weight tensor must have requires_grad==True" assert model[0].weight.is_parameter, "Weight tensor must be marked as a parameter tensor" test_weights_3 = load_numpy_array('autograder/hw1_bonus_autograder/outputs/big_weight_update_3.npy') test_weights_0 = load_numpy_array('autograder/hw1_bonus_autograder/outputs/big_weight_update_0.npy') return assertions_all(model[0].weight.data, test_weights_0, "test_big_weight_update_0", 1e-5, 1e-6) and \ assertions_all(model[3].weight.data, test_weights_3, "test_big_weight_update_3", 1e-5, 1e-6)
def __init__(self, in_features, out_features): super().__init__() self.in_features = in_features self.out_features = out_features # Randomly initializing layer weights k = 1 / in_features weight = k * (np.random.rand(out_features, in_features) - 0.5) bias = k * (np.random.rand(out_features) - 0.5) self.weight = Tensor(weight, requires_grad=True, is_parameter=True) self.bias = Tensor(bias, requires_grad=True, is_parameter=True)
def __init__(self, in_channel, out_channel, kernel_size, stride=1): super().__init__() self.in_channel = in_channel self.out_channel = out_channel self.kernel_size = kernel_size self.stride = stride # Initializing weights and bias (not a very good initialization strategy) weight = np.random.normal(0, 1.0, (out_channel, in_channel, kernel_size)) self.weight = Tensor(weight, requires_grad=True, is_parameter=True) bias = np.zeros(out_channel) self.bias = Tensor(bias, requires_grad=True, is_parameter=True)
def test_conv_forward_back_just_1_layer(): in_c = 2 out_c = 3 kernel = 2 stride = 2 width = 5 batch_size = 1 # setup weights and biases conv = Conv1d(in_c, out_c, kernel, stride) # conv.weight = Tensor(np.random.randint(5, size=conv.weight.shape)+.0, # requires_grad = True) test_weight = np.asarray([[[1, 2], [2, 1]], [[0, 1], [1, 0]], [[3, 2], [1, 0]]]) + .0 conv.weight = Tensor(test_weight, requires_grad=True) # conv.bias = Tensor(np.random.randint(2, size=conv.out_channel)+.0, # requires_grad = True) conv.bias = Tensor(np.zeros(conv.out_channel) + .0, requires_grad=True) conv_torch = nn.Conv1d(in_c, out_c, kernel_size=kernel, stride=stride) conv_torch.weight = nn.Parameter(torch.tensor(conv.weight.data)) conv_torch.bias = nn.Parameter(torch.tensor(conv.bias.data)) print(f"weight:\n {conv.weight}") print(f"bias:\n {conv.bias}") # setup input # x = Tensor(np.random.randint(5, size=(batch_size, in_c, width)),\ # requires_grad = True) x = Tensor(np.asarray([[[1, 0, 1, 0, 1], [0, 1, 0, 1, 0]]]), requires_grad=True) x_torch = get_same_torch_tensor(x).double() print(f"x:\n {x}") # calculate output o = conv(x) o_torch = conv_torch(x_torch) print(f"out:\n {o}") # backward o.backward() o_torch.sum().backward() print(f"grad_x:\n {x.grad}") print(f"grad_w:\n {conv.weight.grad}") print(f"grad_b:\n {conv.bias.grad}") # check everything assert check_val_and_grad(x, x_torch) assert check_val_and_grad(o, o_torch) assert check_val_and_grad(conv.weight, conv_torch.weight) assert check_val_and_grad(conv.bias, conv_torch.bias)
def test_concat_backward(): # shape of tensor to test tensor_shapes = [[(1, 4, 3), (1, 8, 3), (1, 5, 3)], [(2, 3, 4), (1, 3, 4)], [(6, 7, 8, 9), (6, 7, 8, 1), (6, 7, 8, 2)], [(1, 2, 3), (1, 2, 4), (1, 2, 3), (1, 2, 4)]] cat_dims = [1, 0, 3, 2] for tensor_shapes_cur, d_cur in zip(tensor_shapes, cat_dims): # get mytorch and torch tensor: 'a' a = [Tensor.randn(*shape_i) for shape_i in tensor_shapes_cur] for i in range(len(a)): a[i].requires_grad = True a_torch = [get_same_torch_tensor(a_i) for a_i in a] c = cat(a, d_cur) c_torch = torch.cat(a_torch, dim=d_cur) l = (c**2).sum() l_torch = (c_torch**2).sum() l.backward() l_torch.backward() for a_i, a_torch_i in zip(a, a_torch): assert check_grad(a_i, a_torch_i, eps=eps) return True
def validate(model, val_x, val_y): """Problem 3.3: Validation routine, tests on val data, scores accuracy Relevant Args: val_x (np.array): validation data (5000, 784) val_y (np.array): validation labels (5000,) Returns: float: Accuracy = correct / total """ #TODO: implement validation based on pseudocode model.eval() num_samples = val_x.shape[0] #val_batches = get_batch(val_x,val_y) batches = list( zip(np.array_split(val_x, num_samples // 100), np.array_split(val_y, num_samples // 100))) num_correct = 0 for i, (batch_data, batch_labels) in enumerate(batches): #if(i*BATCH_SIZE>=num_samples): # break out = model(Tensor(batch_data)) batch_preds = np.argmax(out.data, axis=1) #print('\nPrediction on Batch:',batch_preds[:10]) #print(f'\n Bach True label: {batch_labels}') num_correct += (batch_preds == batch_labels).sum() accuracy = (num_correct / len(val_x) * 100) return accuracy
def test_distributed_scanning_mlp(): cnn = CNN_DistributedScanningMLP() weights = np.load(os.path.join('autograder', 'hw2_autograder', 'weights', 'mlp_weights_part_c.npy'), allow_pickle=True) weights = tuple(w.T for w in weights) cnn.init_weights(weights) data = np.loadtxt(os.path.join('autograder', 'hw2_autograder', 'data', 'data.asc')).T.reshape(1, 24, -1) data = Tensor(data, requires_grad=False, is_parameter=False, is_leaf=True) expected_result = np.load(os.path.join('autograder', 'hw2_autograder', 'ref_result', 'res_c.npy'), allow_pickle=True) result = cnn(data) # check that model is correctly configured check_model_param_settings(cnn) # if passes tests, return true. # If exception anywhere (failure or crash), return false try: # check that output is correct assert type(result.data) == type(expected_result), "Incorrect output type." assert result.data.shape == expected_result.shape, "Incorrect output shape." assert np.allclose(result.data, expected_result), "Incorrect output values." except Exception as e: traceback.print_exc() return False return True
def test_rnn_unit_backward(): input_sizes, hidden_sizes, data_lens = get_params() for input_size, hidden_size, data_len in zip(input_sizes, hidden_sizes, data_lens): in_mytorch = Tensor.randn(data_len[0], input_size) in_torch = get_same_torch_tensor(in_mytorch) in_mytorch.requires_grad = True in_torch.requires_grad = True model_mytorch = RNNUnit(input_size, hidden_size) model_torch = nn.RNNCell(input_size, hidden_size).double() transfer_weights_rnn_unit(model_torch, model_mytorch) resm = model_mytorch(in_mytorch) rest = model_torch(in_torch) lm = (resm**2).sum() lt = (rest**2).sum() lm.backward() lt.backward() assert compare_rnn_unit_param_grad(model_torch, model_mytorch) assert check_grad(resm, rest, eps=eps) return True
def test_time_iterator_forward(): input_sizes, hidden_sizes, data_lens = get_params() for input_size, hidden_size, data_len in zip(input_sizes, hidden_sizes, data_lens): seq_mytorch = [ Tensor.randn(data_len[i], input_size) for i in range(len(data_len)) ] seq_torch = [get_same_torch_tensor(i) for i in seq_mytorch] mpack = mpack_sequence(seq_mytorch) tpack = nn.utils.rnn.pack_sequence(seq_torch, enforce_sorted=False) model_mytorch = RNN(input_size, hidden_size) model_torch = nn.RNN(input_size, hidden_size, num_layers=1, batch_first=False).double() transfer_weights(model_torch, model_mytorch) resm, hm = model_mytorch(mpack) rest, ht = model_torch(tpack) assert check_val(resm.data, rest.data, eps=eps) t_idx = list(np.argsort(data_len)[::-1]) # Torch returns data which can be bi-directional assert check_val(hm, ht[0][t_idx], eps=eps) return True
def test_slice_forward(): # shape of tensor to test shape = (2, 4, 3) # get mytorch and torch tensor: 'a' a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor # Test 1 b = a[1, 2, 0] b_torch = a_torch[1, 2, 0] assert check_val(b, b_torch, eps=eps) # Test 2 b = a[0, 2, :] b_torch = a_torch[0, 2, :] assert check_val(b, b_torch, eps=eps) # Test 3 b = a[:, 3, :] b_torch = a_torch[:, 3, :] assert check_val(b, b_torch, eps=eps) # Test 4 b = a[:, :, 1] b_torch = a_torch[:, :, 1] assert check_val(b, b_torch, eps=eps) # Test 5 b = a[:, 1:3, :] b_torch = a_torch[:, 1:3, :] assert check_grad(b, b_torch, eps=eps) return True
def test_simple_scanning_mlp(): cnn = CNN_SimpleScanningMLP() # Load and init weights weights = np.load(os.path.join('autograder', 'hw2_autograder', 'weights', 'mlp_weights_part_b.npy'), allow_pickle = True) weights = tuple(w.T for w in weights) cnn.init_weights(weights) # load data and expected answer data = np.loadtxt(os.path.join('autograder', 'hw2_autograder', 'data', 'data.asc')).T.reshape(1, 24, -1) data = Tensor(data, requires_grad=False, is_parameter=False, is_leaf=True) expected_result = np.load(os.path.join('autograder', 'hw2_autograder', 'ref_result', 'res_b.npy'), allow_pickle=True) # get forward output and check result = cnn(data) # check that model is correctly configured check_model_param_settings(cnn) # now check correct results try: # check that output is correct assert type(result.data) == type(expected_result), f"Incorrect output type: {result.data}, expected: {expected_result}" assert result.data.shape == expected_result.shape, f"Incorrect output shape: {result.data.shape}, expected: {expected_result.shape}" assert np.allclose(result.data, expected_result), f"Incorrect output values: {result.data}, expected: {expected_result}" except Exception as e: traceback.print_exc() return False return True
def test_unpack_sequence_backward(): test_shapes = [[(4, 1), (5, 1)], [(4, 3), (10, 3), (2, 3)]] a = True for shapes in test_shapes: # get mytorch and torch tensor: 'a' seq1 = [Tensor.randn(*shape) for shape in shapes] for t in seq1: t.requires_grad = True seq2 = [get_same_torch_tensor(t) for t in seq1] # run mytorch and torch forward: 'c = cat (a, b)' c_temp = pack_sequence(seq1) c_temp2 = unpack_sequence(c_temp) c = pack_sequence(c_temp2) c_torch = torch.nn.utils.rnn.pack_sequence(seq2, enforce_sorted=False) l = (c.data**2).sum() l_torch = (c_torch.data**2).sum() l.backward() l_torch.backward() for a1, a2 in zip(seq1, seq2): assert check_grad(a1, a2, eps=eps) #compare_ps(c_torch, c.data, "test_pack_sequence_backward") return True
def test1(): a = Tensor.randn(1, 2, 3) a.requires_grad = True a_torch = get_same_torch_tensor(a) b = Tensor.randn(1, 2, 3) b.requires_grad = True b_torch = get_same_torch_tensor(b) c = a + b c_torch = a_torch + b_torch c_torch.sum().backward() c.backward() assert check_val_and_grad(a, a_torch) assert check_val_and_grad(b, b_torch) assert check_val_and_grad(c, c_torch)
def conv1d_forward_correctness(num_layers=1): ''' CNN: scanning with a MLP with stride ''' scores_dict = [0] ############################################################################################ ############################# Initialize parameters ################################### ############################################################################################ in_c = np.random.randint(5, 15) channels = [np.random.randint(5, 15) for i in range(num_layers + 1)] kernel = [np.random.randint(3, 7) for i in range(num_layers)] stride = [np.random.randint(3, 5) for i in range(num_layers)] width = np.random.randint(60, 80) batch_size = np.random.randint(1, 4) x = np.random.randn(batch_size, channels[0], width) ############################################################################################# ################################# Create Models ######################################## ############################################################################################# test_layers = [ Conv1d(channels[i], channels[i + 1], kernel[i], stride[i]) for i in range(num_layers) ] test_model = Sequential(*test_layers) torch_layers = [ nn.Conv1d(channels[i], channels[i + 1], kernel[i], stride=stride[i]) for i in range(num_layers) ] torch_model = nn.Sequential(*torch_layers) for torch_layer, test_layer in zip(torch_model, test_model.layers): torch_layer.weight = nn.Parameter(torch.tensor(test_layer.weight.data)) torch_layer.bias = nn.Parameter(torch.tensor(test_layer.bias.data)) ############################################################################################# ######################### Get the correct results from PyTorch ######################### ############################################################################################# x1 = Variable(torch.tensor(x), requires_grad=True) y1 = torch_model(x1) torch_y = y1.detach().numpy() ############################################################################################# ################### Get fwd results from TestModel and compare ########################## ############################################################################################# y2 = test_model(Tensor(x)) test_y = y2.data # check that model is correctly configured check_model_param_settings(test_model) if not assertions(test_y, torch_y, 'type', 'y'): return scores_dict if not assertions(test_y, torch_y, 'shape', 'y'): return scores_dict if not assertions(test_y, torch_y, 'closeness', 'y'): return scores_dict scores_dict[0] = 1 return scores_dict
def forward(self, x): """ Args: x (Tensor): (batch_size, in_features) Returns: Tensor: (batch_size, out_features) """ return Tensor.matmul(x,self.weight.T()) + self.bias
def test_unsqueeze(): # shape of tensor to test shape = (2, 2, 3) # get mytorch and torch tensor: 'a' a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor b = Tensor.unsqueeze(a) b_torch = torch.Tensor.unsqueeze(a_torch, 0) assert check_val(b, b_torch, eps=eps) b = Tensor.unsqueeze(a, 2) b_torch = torch.Tensor.unsqueeze(a_torch, 2) assert check_val(b, b_torch, eps=eps) return True
def test_dropout_forward(): np.random.seed(11785) # run on small model forward only x = Tensor.randn(5, 10) model = Sequential(Linear(10, 5), ReLU(), Dropout(p=0.6)) my_output = model(x) test_output = load_numpy_array('autograder/hw1_bonus_autograder/outputs/dropout_forward.npy') return assertions_all(my_output.data, test_output, "test_dropout_forward", 1e-5, 1e-6)