def test_sum_with_grad(self):
t1 = Tensor([1, 2, 3], requires_grad=True)
t2 = t1.sum()
t2.backward(Tensor(3))
assert t1.grad.data.tolist() == [3, 3, 3]
def load_data() -> ('Tensor', 'Tensor'):
x_data = np.linspace(0, 10, 10) + np.random.uniform(-1.5, 1.5, 10)
y_label = np.linspace(0, 10, 10) + np.random.uniform(-1.5, 1.5, 10)
x_data = Tensor(x_data.reshape(-1, 1), requires_grad=True)
y_label = Tensor(y_label.reshape(-1, 1), requires_grad=True)
return x_data, y_label
def test_mean_with_grad(self):
""" Test mean of all elements of tensor, and input data to backward function
"""
tensor1 = Tensor([1, 2, 3], requires_grad=True)
tensor2 = tensor1.mean()
tensor2.backward(Tensor(3.))
assert tensor1.grad.data.tolist() == [1, 1, 1]
def test_log(self):
""" Test log function for tensors
"""
tensor1 = Tensor([1, np.e, np.e**2], requires_grad=True)
tensor2 = Log()(tensor1)
tensor2.backward(Tensor([1, np.e, np.e**2]))
assert tensor2.data.tolist() == [0, 1, 2]
assert tensor1.grad.data.tolist() == [1, 1, 1]
def test_relu(self):
""" Test relu activation function for tensors
"""
tensor1 = Tensor([-2, 4, -6], requires_grad=True)
tensor2 = Relu()(tensor1)
tensor2.backward(Tensor([1, 1, 1]))
assert tensor2.data.tolist() == [0, 4, 0]
assert np.round(tensor1.grad.data, 2).tolist() == [0, 1, 0]
def test_softmax(self):
""" Test relu activation function for tensors
"""
tensor1 = Tensor([[-2, 4, -6]], requires_grad=True)
tensor2 = Softmax()(tensor1)
tensor2.backward(Tensor([[1, 1, 1]]))
assert np.round(tensor2.data, 2).tolist() == [[0, 1, 0]]
assert np.round(tensor1.grad.data, 2).tolist() == [[0, 0, 0]]
def test_sigmoid(self):
""" Test relu activation function for tensors
"""
tensor1 = Tensor([-100, 0, 10], requires_grad=True)
tensor2 = Sigmoid()(tensor1)
tensor2.backward(Tensor([1, 1, 1]))
assert np.round(tensor2.data, 2).tolist() == [0, 0.5, 1]
assert np.round(tensor1.grad.data, 2).tolist() == [0, 0.25, 0]
def test_simple_mul(self):
t1 = Tensor([1, 2, 3], requires_grad=True)
t2 = Tensor([4, 5, 6], requires_grad=True)
t3 = t1 * t2
t3.backward(Tensor([-1, -2, -3]))
assert t3.data.tolist() == [4, 10, 18]
assert t1.grad.data.tolist() == [-4, -10, -18]
assert t2.grad.data.tolist() == [-1, -4, -9]
def test_tanh(self):
""" Test tanh activation functions for tensors
"""
tensor1 = Tensor([0, 1, 2], requires_grad=True)
tensor2 = Tanh()(tensor1)
tensor2.backward(Tensor([1, 1, 1]))
assert np.round(tensor2.data,2).tolist() == [0, 0.76, 0.96]
assert np.round(tensor1.grad.data,2).tolist() == [1, 0.42, 0.07]
def test_simple_add(self):
t1 = Tensor([1, 2, 3], requires_grad=True)
t2 = Tensor([4, 5, 6], requires_grad=True)
t3 = t1 + t2
t3.backward(Tensor([-1, -2, -3]))
assert t3.data.tolist() == [5, 7, 9]
assert t1.grad.data.tolist() == [-1, -2, -3]
assert t2.grad.data.tolist() == [-1, -2, -3]
def test_broadcast_sub2(self):
t1 = Tensor([[1, 2, 3], [4, 5, 6]], requires_grad=True) # (2,3)
t2 = Tensor([[7, 8, 9]], requires_grad=True) # (1,3)
t3 = t1 - t2
assert t3.data.tolist() == [[-6, -6, -6], [-3, -3, -3]]
t3.backward(Tensor([[1, 1, 1], [1, 1, 1]]))
assert t1.grad.data.tolist() == [[1, 1, 1], [1, 1, 1]]
assert t2.grad.data.tolist() == [[-2, -2, -2]]
def test_broadcast_add2(self):
t1 = Tensor([[1, 2, 3], [4, 5, 6]],
requires_grad=True) # shape = (2, 3)
t2 = Tensor([[7, 8, 9]], requires_grad=True) # shape = (1, 3)
t3 = t1 + t2 # shape (2, 3)
t3.backward(Tensor([[1, 1, 1], [1, 1, 1]]))
assert t3.data.tolist() == [[8, 10, 12], [11, 13, 15]]
assert t1.grad.data.tolist() == [[1, 1, 1], [1, 1, 1]]
assert t2.grad.data.tolist() == [[2, 2, 2]]
def test_imul(self):
"""Test imul of tensor (tensor*=another_tensor) with the same shape
"""
tensor1 = Tensor([2, 4, 6, 8], requires_grad=True)
tensor2 = Tensor([5, 8, 1, 2])
tensor2 *= tensor1
assert tensor2.data.tolist() == [10, 32, 6, 16]
tensor2 *= tensor1
assert tensor2.data.tolist() == [20, 128, 36, 128]
def test_isub(self):
"""Test isub of tensor (tensor-=another_tensor) with the same shape
"""
tensor1 = Tensor([2, 4, 6, 8], requires_grad=True)
tensor2 = Tensor([5, 8, 1, 2])
tensor2 -= tensor1
assert tensor2.data.tolist() == [3, 4, -5, -6]
tensor2 -= 1
assert tensor2.data.tolist() == [2, 3, -6, -7]
def test_idiv(self):
"""Test idiv of tensor (tensor/=another_tensor) with the same shape
"""
tensor1 = Tensor([2, 2, 2, 2], requires_grad=True)
tensor2 = Tensor([4, 8, 12, 16])
tensor2 /= tensor1
assert tensor2.data.tolist() == [2, 4, 6, 8]
tensor2 /= tensor1
assert tensor2.data.tolist() == [1, 2, 3, 4]
def test_broadcast_mul2(self):
t1 = Tensor([[1, 2, 3], [4, 5, 6]], requires_grad=True) # (2,3)
t2 = Tensor([[7, 8, 9]], requires_grad=True) # (1,3)
t3 = t1 * t2
assert t3.data.tolist() == [[7, 16, 27], [28, 40, 54]]
t3.backward(Tensor([[1, 1, 1], [1, 1, 1]]))
assert t1.grad.data.tolist() == [[7, 8, 9], [7, 8, 9]]
assert t2.grad.data.tolist() == [[5, 7, 9]]
def test_sub(self):
"""Test substraction of tensor with the same shape
"""
tensor1 = Tensor([2, 4, 6, 8], requires_grad=True)
tensor2 = Tensor([5, 8, 1, 2], requires_grad=True)
tensor3 = tensor1 - tensor2
tensor3.backward(Tensor([-1., -2., -3., -4.]))
assert tensor3.data.tolist() == [-3, -4, 5, 6]
assert tensor1.grad.data.tolist() == [-1, -2, -3, -4]
assert tensor2.grad.data.tolist() == [1, 2, 3, 4]
def test_broadcast_sub_diff_shapes(self):
""" Test sub of tensors with same dim but different shape
"""
tensor1 = Tensor([[1, 2, 3], [4, 5, 6]], requires_grad=True)
tensor2 = Tensor([[7, 8, 9]], requires_grad=True)
tensor3 = tensor1 - tensor2
tensor3.backward(Tensor([[-1, -1, -1], [-1, -1, -1]]))
assert tensor3.data.tolist() == [[-6, -6, -6], [-3, -3, -3]]
assert tensor1.grad.data.tolist() == [[-1, -1, -1], [-1, -1, -1]]
assert tensor2.grad.data.tolist() == [[2, 2, 2]]
def test_simple_mul(self):
""" Test product between two tensors with the same shape
"""
tensor1 = Tensor([2, 4, 6], requires_grad=True)
tensor2 = Tensor([4, 5, 6], requires_grad=True)
tensor3 = tensor1 * tensor2
tensor3.backward(Tensor([-1, -2, -3]))
assert tensor3.data.tolist() == [8, 20, 36]
assert tensor1.grad.data.tolist() == [-4, -10, -18]
assert tensor2.grad.data.tolist() == [-2, -8, -18]
def test_simple_mul(self):
t1 = Tensor([1, 2, 3], requires_grad=True)
t2 = Tensor([4, 5, 6], requires_grad=True)
t3 = t1 * t2
self.assertEqual(t3.data.tolist(), [4, 10, 18])
t3.backward(Tensor([-1, -2, -3]))
self.assertEqual(t1.grad.data.tolist(), [-4, -10, -18])
self.assertEqual(t2.grad.data.tolist(), [-1, -4, -9])
def test_broadcast_mul1(self):
t1 = Tensor([[1, 2, 3], [4, 5, 6]], requires_grad=True) # (2, 3)
t2 = Tensor([7, 8, 9], requires_grad=True) # (3,)
t3 = t1 * t2 # (2, 3)
self.assertEqual(t3.data.tolist(), [[7, 16, 27], [28, 40, 54]])
t3.backward(Tensor([[1, 1, 1], [1, 1, 1]]))
self.assertEqual(t1.grad.data.tolist(), [[7, 8, 9], [7, 8, 9]])
self.assertEqual(t2.grad.data.tolist(), [5, 7, 9])
def test_broadcast_add2(self):
t1 = Tensor([[1, 2, 3], [4, 5, 6]], requires_grad=True) # (2, 3)
t2 = Tensor([[7, 8, 9]], requires_grad=True) # (1, 3)
t3 = t1 + t2 # (2, 3)
self.assertEqual(t3.data.tolist(), [[8, 10, 12], [11, 13, 15]])
t3.backward(Tensor([[1, 1, 1], [1, 1, 1]]))
self.assertEqual(t1.grad.data.tolist(), [[1, 1, 1], [1, 1, 1]])
self.assertEqual(t2.grad.data.tolist(), [[2, 2, 2]])
def test_simple_add(self):
t1 = Tensor([1, 2, 3], requires_grad=True)
t2 = Tensor([4, 5, 6], requires_grad=True)
t3 = t1 + t2
self.assertEqual(t3.data.tolist(), [5, 7, 9])
t3.backward(Tensor([-1, -2, -3]))
self.assertEqual(t1.grad.data.tolist(), [-1, -2, -3])
self.assertEqual(t2.grad.data.tolist(), [-1, -2, -3])
def test_broadcast_mul_diff_shape(self):
""" Test product of tensors with different shapes (broadcasting)
"""
tensor1 = Tensor([[1, 2, 3], [4, 5, 6]], requires_grad=True)
tensor2 = Tensor([[7, 8, 9]], requires_grad=True)
tensor3 = tensor1 * tensor2
tensor3.backward(Tensor([[1, 1, 1], [1, 1, 1]]))
assert tensor3.data.tolist() == [[7, 16, 27], [28, 40, 54]]
assert tensor1.grad.data.tolist() == [[7., 8., 9.], [7., 8., 9.]]
assert tensor2.grad.data.tolist() == [[5., 7., 9.]]
def test_broadcast_sub(self):
""" Test sub of tensors with 1 tensor of different dim (broadcasting)
"""
tensor1 = Tensor([[1, 2, 3], [4, 5, 6]], requires_grad=True)
tensor2 = Tensor([7, 8, 9], requires_grad=True)
tensor3 = tensor1 - tensor2
tensor3.backward(Tensor([[-1, -1, -1], [-1, -1, -1]]))
assert tensor3.data.tolist() == [[-6, -6, -6], [-3, -3, -3]]
assert tensor1.grad.data.tolist() == [[-1, -1, -1], [-1, -1, -1]]
assert tensor2.grad.data.tolist() == [2, 2, 2]
def test_simple_div(self):
""" Test div between two tensors with the same shape
"""
tensor1 = Tensor([2, 2.5, 3], requires_grad=True)
tensor2 = Tensor([4, 5, 6], requires_grad=True)
tensor3 = tensor2 / tensor1
tensor3.backward(Tensor([-1, -2, -3]))
assert tensor3.data.tolist() == [2, 2, 2]
assert tensor1.grad.data.tolist() == [1, 1.6, 2]
assert tensor2.grad.data.tolist() == [-0.5, -0.8, -1]
def test_simple_matmul(self):
t1 = Tensor([[1, 2], [3, 4], [5, 6]], requires_grad=True) # (3, 2)
t2 = Tensor([[10], [20]], requires_grad=True) # (2, 1)
t3 = t1 @ t2 # (3, 1)
assert t3.data.tolist() == [[50], [110], [170]]
grad = Tensor([[1], [2], [3]])
t3.backward(grad)
# Just copying the formula from the code in tensor.py ;D
np.testing.assert_allclose(t1.grad.data, grad.data @ t2.data.T)
np.testing.assert_allclose(t2.grad.data, t1.data.T @ grad.data)
def test_broadcast_div_diff_shape(self):
""" Test div of tensors with different shapes
"""
tensor1 = Tensor([[1, 3, 4], [1, 3, 4]], requires_grad=True) # (2, 3)
tensor2 = Tensor([[2, 3, 4]], requires_grad=True) # (3,)
tensor3 = tensor2 / tensor1 # shape (2, 3)
tensor3.backward(Tensor([[1, 1, 1], [1, 1, 1]]))
assert tensor3.data.tolist() == [[2, 1, 1], [2, 1, 1]]
assert np.round(tensor1.grad.data, 2).tolist() == [[-2, -0.33, -0.25],
[-2, -0.33, -0.25]]
assert np.round(tensor2.grad.data, 2).tolist() == [[2, 0.67, 0.5]]
def test_log_torch(self):
""" Test log function for tensors and compare with torch
"""
tensor1 = Tensor([1, np.e, np.e**2], requires_grad=True)
tensor2 = Log()(tensor1)
torch_tensor1 = tensor([1, np.e, np.e**2], dtype = float, requires_grad=True)
torch_tensor2 = log(torch_tensor1)
tensor2.backward(Tensor([1, np.e, np.e**2]))
torch_tensor2.backward(tensor([1, np.e, np.e**2]))
assert torch_tensor2.data.tolist() == tensor2.data.tolist()
assert np.round(torch_tensor1.grad.data,2).tolist() == np.round(tensor1.grad.data,2).tolist()
def test_mean_with_grad_torch(self):
""" Test mean of all elements of tensor, and input data to backward function
and compare with torch
"""
tensor1 = Tensor([1, 2, 3], requires_grad=True)
tensor2 = tensor1.mean()
tensor_torch1 = tensor([1, 2, 3], dtype=float, requires_grad=True)
tensor_torch2 = tensor_torch1.mean()
tensor2.backward(Tensor(3.))
tensor_torch2.backward(tensor(3.))
assert tensor1.grad.data.tolist() == tensor1.grad.data.tolist()