def test_matmul(): """Tests that mytorch's matmul multiplication matches torch's""" # shape of tensor to test shape1 = (3, 4) shape2 = (4, 5) shape = (3, 5) # get mytorch and torch tensor: 'a' a = Tensor.randn(*shape1) a.requires_grad = True a_torch = get_same_torch_tensor(a) # get mytorch and torch tensor: 'b' b = Tensor.randn(*shape2) b.requires_grad = True b_torch = get_same_torch_tensor(b) # run mytorch and torch forward: 'c = a * b' ctx = ContextManager() c = matmul.forward(ctx, a, b) c_torch = torch.matmul(a_torch, b_torch) # run mytorch and torch multiplication backward back = matmul.backward(ctx, Tensor.ones(*shape)) c_torch.sum().backward() # check that c matches assert check_val_and_grad(c, c_torch) # check that dc/da and dc/db respectively match assert check_val(back[0], a_torch.grad) assert check_val(back[1], b_torch.grad) return True
def test6(): a = Tensor.randn(2, 3) a.requires_grad = True a_torch = get_same_torch_tensor(a) b = Tensor.randn(2, 3) b.requires_grad = True b_torch = get_same_torch_tensor(b) c = a / b c_torch = a_torch / b_torch d = a - b d_torch = a_torch - b_torch e = c + d e_torch = c_torch + d_torch e.backward() e_torch.sum().backward() assert check_val_and_grad(a, a_torch) assert check_val_and_grad(b, b_torch) assert check_val_and_grad(c, c_torch) assert check_val_and_grad(d, d_torch) assert check_val_and_grad(e, e_torch)
def test_mul(): """Tests that mytorch's elementwise multiplication matches torch's""" # shape of tensor to test shape = (1, 2, 3) # get mytorch and torch tensor: 'a' a = Tensor.randn(*shape) a.requires_grad = True a_torch = get_same_torch_tensor(a) # get mytorch and torch tensor: 'b' b = Tensor.randn(*shape) b.requires_grad = True b_torch = get_same_torch_tensor(b) # run mytorch and torch forward: 'c = a * b' ctx = ContextManager() c = Mul.forward(ctx, a, b) c_torch = a_torch * b_torch # run mytorch and torch multiplication backward back = Mul.backward(ctx, Tensor.ones(*shape)) c_torch.sum().backward() # check that c matches assert check_val_and_grad(c, c_torch) # check that dc/da and dc/db respectively match assert check_val(back[0], a_torch.grad) assert check_val(back[1], b_torch.grad) # ensure * is overridden c_using_override = a * b assert check_val(c_using_override, c_torch) return True
def testbroadcast(): """Tests addition WITH broadcasting matches torch's""" # shape of tensor to test # get mytorch and torch tensor: 'a' a = Tensor.randn(3, 4) a.requires_grad = True a_torch = get_same_torch_tensor(a) # get mytorch and torch tensor: 'b' b = Tensor.randn(4) b.requires_grad = True b_torch = get_same_torch_tensor(b) # run mytorch and torch forward: 'c = a + b' c = a + b c_torch = a_torch + b_torch # run mytorch and torch addition backward c.backward() c_torch.sum().backward() # check that c matches assert check_val_and_grad(c, c_torch) # check that dc/da and dc/db respectively match assert check_val_and_grad(a, a_torch) assert check_val_and_grad(b, b_torch)
def test_slice_backward(): # shape of tensor to test shape = (2, 4, 3) # Test 1 a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor b = a[1, 2, 0] b_torch = a_torch[1, 2, 0] (b**2).sum().backward() (b_torch**2).sum().backward() assert check_grad(a, a_torch, eps=eps) # Test 2 a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor b = a[0, 2, :] b_torch = a_torch[0, 2, :] (b**2).sum().backward() (b_torch**2).sum().backward() assert check_grad(a, a_torch, eps=eps) # Test 3 a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor b = a[:, 3, :] b_torch = a_torch[:, 3, :] (b**2).sum().backward() (b_torch**2).sum().backward() assert check_grad(a, a_torch, eps=eps) # Test 4 a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor b = a[:, :, 1] b_torch = a_torch[:, :, 1] (b**2).sum().backward() (b_torch**2).sum().backward() assert check_grad(a, a_torch, eps=eps) # Test 5 a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor b = a[:, 1:3, :] b_torch = a_torch[:, 1:3, :] (b**2).sum().backward() (b_torch**2).sum().backward() assert check_grad(a, a_torch, eps=eps) return True
def test_concat_backward(): # shape of tensor to test tensor_shapes = [[(1, 4, 3), (1, 8, 3), (1, 5, 3)], [(2, 3, 4), (1, 3, 4)], [(6, 7, 8, 9), (6, 7, 8, 1), (6, 7, 8, 2)], [(1, 2, 3), (1, 2, 4), (1, 2, 3), (1, 2, 4)]] cat_dims = [1, 0, 3, 2] for tensor_shapes_cur, d_cur in zip(tensor_shapes, cat_dims): # get mytorch and torch tensor: 'a' a = [Tensor.randn(*shape_i) for shape_i in tensor_shapes_cur] for i in range(len(a)): a[i].requires_grad = True a_torch = [get_same_torch_tensor(a_i) for a_i in a] c = cat(a, d_cur) c_torch = torch.cat(a_torch, dim=d_cur) l = (c**2).sum() l_torch = (c_torch**2).sum() l.backward() l_torch.backward() for a_i, a_torch_i in zip(a, a_torch): assert check_grad(a_i, a_torch_i, eps=eps) return True
def test_slice_forward(): # shape of tensor to test shape = (2, 4, 3) # get mytorch and torch tensor: 'a' a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor # Test 1 b = a[1, 2, 0] b_torch = a_torch[1, 2, 0] assert check_val(b, b_torch, eps=eps) # Test 2 b = a[0, 2, :] b_torch = a_torch[0, 2, :] assert check_val(b, b_torch, eps=eps) # Test 3 b = a[:, 3, :] b_torch = a_torch[:, 3, :] assert check_val(b, b_torch, eps=eps) # Test 4 b = a[:, :, 1] b_torch = a_torch[:, :, 1] assert check_val(b, b_torch, eps=eps) # Test 5 b = a[:, 1:3, :] b_torch = a_torch[:, 1:3, :] assert check_grad(b, b_torch, eps=eps) return True
def test_time_iterator_forward(): input_sizes, hidden_sizes, data_lens = get_params() for input_size, hidden_size, data_len in zip(input_sizes, hidden_sizes, data_lens): seq_mytorch = [ Tensor.randn(data_len[i], input_size) for i in range(len(data_len)) ] seq_torch = [get_same_torch_tensor(i) for i in seq_mytorch] mpack = mpack_sequence(seq_mytorch) tpack = nn.utils.rnn.pack_sequence(seq_torch, enforce_sorted=False) model_mytorch = RNN(input_size, hidden_size) model_torch = nn.RNN(input_size, hidden_size, num_layers=1, batch_first=False).double() transfer_weights(model_torch, model_mytorch) resm, hm = model_mytorch(mpack) rest, ht = model_torch(tpack) assert check_val(resm.data, rest.data, eps=eps) t_idx = list(np.argsort(data_len)[::-1]) # Torch returns data which can be bi-directional assert check_val(hm, ht[0][t_idx], eps=eps) return True
def test_rnn_unit_backward(): input_sizes, hidden_sizes, data_lens = get_params() for input_size, hidden_size, data_len in zip(input_sizes, hidden_sizes, data_lens): in_mytorch = Tensor.randn(data_len[0], input_size) in_torch = get_same_torch_tensor(in_mytorch) in_mytorch.requires_grad = True in_torch.requires_grad = True model_mytorch = RNNUnit(input_size, hidden_size) model_torch = nn.RNNCell(input_size, hidden_size).double() transfer_weights_rnn_unit(model_torch, model_mytorch) resm = model_mytorch(in_mytorch) rest = model_torch(in_torch) lm = (resm**2).sum() lt = (rest**2).sum() lm.backward() lt.backward() assert compare_rnn_unit_param_grad(model_torch, model_mytorch) assert check_grad(resm, rest, eps=eps) return True
def test_unpack_sequence_backward(): test_shapes = [[(4, 1), (5, 1)], [(4, 3), (10, 3), (2, 3)]] a = True for shapes in test_shapes: # get mytorch and torch tensor: 'a' seq1 = [Tensor.randn(*shape) for shape in shapes] for t in seq1: t.requires_grad = True seq2 = [get_same_torch_tensor(t) for t in seq1] # run mytorch and torch forward: 'c = cat (a, b)' c_temp = pack_sequence(seq1) c_temp2 = unpack_sequence(c_temp) c = pack_sequence(c_temp2) c_torch = torch.nn.utils.rnn.pack_sequence(seq2, enforce_sorted=False) l = (c.data**2).sum() l_torch = (c_torch.data**2).sum() l.backward() l_torch.backward() for a1, a2 in zip(seq1, seq2): assert check_grad(a1, a2, eps=eps) #compare_ps(c_torch, c.data, "test_pack_sequence_backward") return True
def test1(): a = Tensor.randn(1, 2, 3) a.requires_grad = True a_torch = get_same_torch_tensor(a) b = Tensor.randn(1, 2, 3) b.requires_grad = True b_torch = get_same_torch_tensor(b) c = a + b c_torch = a_torch + b_torch c_torch.sum().backward() c.backward() assert check_val_and_grad(a, a_torch) assert check_val_and_grad(b, b_torch) assert check_val_and_grad(c, c_torch)
def test_dropout_forward(): np.random.seed(11785) # run on small model forward only x = Tensor.randn(5, 10) model = Sequential(Linear(10, 5), ReLU(), Dropout(p=0.6)) my_output = model(x) test_output = load_numpy_array('autograder/hw1_bonus_autograder/outputs/dropout_forward.npy') return assertions_all(my_output.data, test_output, "test_dropout_forward", 1e-5, 1e-6)
def test_sub(): """Tests that mytorch subtraction matches torch's subtraction""" # shape of tensor to test shape = (1, 2, 3) # get mytorch and torch tensor: 'a' a = Tensor.randn(*shape) a.requires_grad = True a_torch = get_same_torch_tensor(a) # get mytorch and torch tensor: 'b' b = Tensor.randn(*shape) b.requires_grad = True b_torch = get_same_torch_tensor(b) # run mytorch and torch forward: 'c = a - b' ctx = ContextManager() c = Sub.forward(ctx, a, b) c_torch = a_torch - b_torch # run mytorch and torch subtraction backward back = Sub.backward(ctx, Tensor.ones(*shape)) c_torch.sum().backward() # check that c matches assert check_val_and_grad(c, c_torch) # check that dc/da and dc/db respectively match assert check_val(back[0], a_torch.grad) # print(f'Sub back[1] == {back[1]} and back[0] == {back[0]} and b_torch.grad == {b_torch.grad}') assert check_val(back[1], b_torch.grad) # ensure - is overridden c_using_override = a - b assert check_val(c_using_override, c_torch) return True
def test_unpack_sequence_forward(): test_shapes = [[(4, 1), (5, 1), (2, 1), (2, 1), (3, 1)], [(4, 3), (10, 3), (2, 3)]] for shapes in test_shapes: # get mytorch and torch tensor: 'a' seq1 = [Tensor.randn(*shape) for shape in shapes] # run mytorch and torch forward: 'c = cat (a, b)' c = pack_sequence(seq1) seq2 = unpack_sequence(c) for s1, s2 in zip(seq1, seq2): assert assertions_all(s1.data, s2.data, 'Unpack Forward') return True
def test_unsqueeze(): # shape of tensor to test shape = (2, 2, 3) # get mytorch and torch tensor: 'a' a = Tensor.randn(*shape) #mytorch tensor a.requires_grad = True a_torch = get_same_torch_tensor(a) #pytorch tensor b = Tensor.unsqueeze(a) b_torch = torch.Tensor.unsqueeze(a_torch, 0) assert check_val(b, b_torch, eps=eps) b = Tensor.unsqueeze(a, 2) b_torch = torch.Tensor.unsqueeze(a_torch, 2) assert check_val(b, b_torch, eps=eps) return True
def test_debdas(): predicted = Tensor.randn(4, 20) predicted.requires_grad = True predicted_torch = get_same_torch_tensor(predicted) target = Tensor(np.random.randint(20, size=(4, ))) target.requires_grad = True targets = to_one_hot(target, 20) targets_torch = get_same_torch_tensor(targets) p_std = predicted - Tensor(np.max(predicted.data)) p_std_torch = predicted_torch - torch.max(predicted_torch) p_exp = p_std.exp() p_exp_torch = torch.exp(p_std_torch) p_softmax = p_exp / p_exp.sumAxis(1) p_softmax_torch = p_exp_torch / torch.sum(p_exp_torch, 1, keepdim=True) p_log_softmax = p_softmax.log() p_log_softmax_torch = torch.log(p_softmax_torch) log_lik = targets * p_log_softmax log_lik_torch = targets_torch * p_log_softmax_torch log_lik_sum = log_lik.sumAxis(None) log_lik_sum_torch = torch.sum(log_lik_torch) ce = tensor.Tensor(-1) * log_lik_sum / tensor.Tensor(4) ce_torch = -1 * log_lik_sum_torch / 4 ce_torch.sum().backward() ce.backward() #assert check_val_and_grad(predicted, predicted_torch) assert check_val_and_grad(targets, targets_torch) assert check_val_and_grad(p_std, p_std_torch) assert check_val_and_grad(p_exp, p_exp_torch) assert check_val_and_grad(p_softmax, p_softmax_torch) assert check_val_and_grad(p_log_softmax, p_log_softmax_torch) assert check_val_and_grad(log_lik, log_lik_torch) assert check_val_and_grad(log_lik_sum, log_lik_sum_torch) assert check_val_and_grad(ce, ce_torch)
def test_rnn_unit_forward(): input_sizes, hidden_sizes, data_lens = get_params() for input_size, hidden_size, data_len in zip(input_sizes, hidden_sizes, data_lens): in_mytorch = Tensor.randn(data_len[0], input_size) in_torch = get_same_torch_tensor(in_mytorch) model_mytorch = RNNUnit(input_size, hidden_size) model_torch = nn.RNNCell(input_size, hidden_size).double() transfer_weights_rnn_unit(model_torch, model_mytorch) resm = model_mytorch(in_mytorch) rest = model_torch(in_torch) assert check_val(resm, rest, eps=eps) return True
def test_time_iterator_backward(): input_sizes, hidden_sizes, data_lens = get_params() for input_size, hidden_size, data_len in zip(input_sizes, hidden_sizes, data_lens): seq_mytorch = [ Tensor.randn(data_len[i], input_size) for i in range(len(data_len)) ] seq_torch = [get_same_torch_tensor(i) for i in seq_mytorch] mpack = mpack_sequence(seq_mytorch) tpack = nn.utils.rnn.pack_sequence(seq_torch, enforce_sorted=False) model_mytorch = RNN(input_size, hidden_size) model_torch = nn.RNN(input_size, hidden_size, num_layers=1, batch_first=False).double() transfer_weights(model_torch, model_mytorch) resm, hm = model_mytorch(mpack) rest, ht = model_torch(tpack) lm = (resm.data**2).sum() lt = (rest.data**2).sum() lm.backward() lt.backward() assert compare_rnn_param_grad(model_torch, model_mytorch) for ma, pa in zip(seq_mytorch, seq_torch): assert check_grad(ma, pa) return True
def test_pack_sequence_forward(): test_shapes = [[(4, 1), (5, 1)], [(4, 3), (10, 3), (2, 3)]] a = True for shapes in test_shapes: # get mytorch and torch tensor: 'a' seq1 = [Tensor.randn(*shape) for shape in shapes] seq2 = [get_same_torch_tensor(t) for t in seq1] # run mytorch and torch forward: 'c = cat (a, b)' c = pack_sequence(seq1) c_torch = torch.nn.utils.rnn.pack_sequence(seq2, enforce_sorted=False) assert check_val(c.data, c_torch.data) #compare_ps(c_torch, c.data, "test_pack_sequence_forward") assert compare_ndarrays(c.batch_sizes, c_torch.batch_sizes, test_name='Testing batch_sizes') assert compare_ndarrays(c.sorted_indices, c_torch.sorted_indices, test_name='Testing sorted_indices') return True