def test_tensor_mul():
a = tensor(data=[1, 2, 3])
b = tensor(data=[-1, 3, 1])
amb = a * b
assert amb.tolist() == [-1, 6, 3]
assert not amb.requires_grad
def test_softmax3d_backward():
a = tor4.tensor(
data=[[[0, 1, -1.0], [1, -2, 3]], [[1, 4, -2], [0, 0, -3]]],
requires_grad=True,
)
a_sm = nn.functional.softmax(a, dim=1)
a_sm.backward(
tor4.tensor([[[-5, 3, 0.0], [0, 0, 1]], [[3, 0, -3], [1, 2, 3]]]))
assert a_sm.requires_grad
assert np.allclose(
a_sm.tolist(),
[
[[0.2689, 0.9526, 0.018], [0.7311, 0.0474, 0.982]],
[[0.7311, 0.982, 0.7311], [0.2689, 0.018, 0.2689]],
],
atol=1e-4,
rtol=1e-4,
)
assert np.allclose(
a.grad.tolist(),
[
[[-0.9831, 0.1355, -0.0177], [0.9831, -0.1355, 0.0177]],
[[0.3932, -0.0353, -1.1797], [-0.3932, 0.0353, 1.1797]],
],
atol=1e-4,
rtol=1e-4,
)
def test_conv2d_backward5():
a = tor4.tensor(
[
[
[[0, 1, 2], [3, 4, 5.0]],
[[0, 1, 2], [3, 4, 5.0]],
[[0, 1, 2], [3, 4, 5.0]],
]
],
requires_grad=True,
)
w = tor4.tensor(
[[[[1, 0], [1, 1]]], [[[0, 1], [0, 1.0]]], [[[1, 0], [1, 1]]]],
requires_grad=True,
)
aw = nn.functional.conv2d(a, w, groups=3)
aw.backward(tor4.tensor([[[[2, -1]], [[1, 3]], [[3, -2.0]]]]))
assert aw.tolist() == [[[[7, 10]], [[5, 7]], [[7, 10]]]]
assert np.allclose(
a.grad.sum(1).tolist(), [[[5, -2, 3], [5, 3, 0]]], rtol=1e-4, atol=1e-4
)
# assert np.allclose(a.grad.tolist(), [[[[2, -1, 0], [2, 1, -1]], [[0, 1, 3], [0, 1, 3.]], [[3, -2, 0], [3, 1, -2.]]]], rtol=1e-4, atol=1e-4)
assert np.allclose(
w.grad.tolist(),
[[[[-1, 0], [2, 3]]], [[[3, 7], [15, 19]]], [[[-2, -1], [1, 2]]]],
rtol=1e-4,
atol=1e-4,
)
def test_tensor_matmul():
a = tensor(data=[1, 2, 3])
b = tensor(data=[4, 5, 6])
amb = a @ b
assert amb.tolist() == 32
assert not amb.requires_grad
def test_tesnor_neg_backward():
a = tensor(data=[1, 2, 3.0], requires_grad=True)
na = -a
na.backward(tensor([1, 1, 1]))
assert na.tolist() == [-1, -2, -3]
assert na.requires_grad
assert a.grad.tolist() == [-1, -1, -1]
def test_view_backward():
a = tor4.tensor([1.0, 2, 3, 4, 5, 6, 7, 8, 9], requires_grad=True)
b = a.view(-1, 3)
b.backward(tor4.tensor([[1, 1, 1], [2, 2, 2], [3, 4, 5.0]]))
assert b.shape == (3, 3)
assert b.tolist() == [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
assert a.grad.tolist() == [1, 1, 1, 2, 2, 2, 3, 4, 5]
def test_view_backward2():
a = tor4.tensor([[[1, 2], [3, 4], [5, 6.0]]], requires_grad=True)
b = a.view(2, 3)
b.backward(tor4.tensor([[0, 1, 0], [6, 3, 1.0]]))
assert b.shape == (2, 3)
assert b.tolist() == [[1, 2, 3], [4, 5, 6.0]]
assert a.grad.tolist() == [[[0, 1], [0, 6], [3, 1]]]
def test_tensor_imul_backward():
a = tensor(data=[1, 2, 3.0], requires_grad=True)
b = tensor(data=[-1, 3, 1])
try:
a *= b
raise AssertionError()
except RuntimeError:
assert True
def test_tensor_sum_backward2():
a = tensor(data=[-1, 1, 2.0], requires_grad=True)
a_sum = a.sum()
a_sum.backward(tensor(3))
assert a_sum.tolist() == 2
assert a_sum.requires_grad
assert a.grad.tolist() == [3, 3, 3]
def test_tensor_sum_keepdim1_backward():
a = tensor(data=[[-1, 1, 2], [1, 2, 3.0]], requires_grad=True)
a_sum = a.sum(dim=1, keepdim=True)
a_sum.backward(tensor(data=[[2], [3]]))
assert a_sum.tolist() == [[2], [6]]
assert a_sum.requires_grad
assert a.grad.tolist() == [[2, 2, 2], [3, 3, 3]]
def test_tensor_sum_keepdim2_backward():
a = tensor(data=[[[-1], [1], [2]], [[1], [2], [3.0]]], requires_grad=True)
a_sum = a.sum(dim=1, keepdim=True)
a_sum.backward(tensor(data=[[[2]], [[3]]]))
assert a_sum.tolist() == [[[2]], [[6]]]
assert a_sum.requires_grad
assert a.grad.tolist() == [[[2], [2], [2]], [[3], [3], [3]]]
def test_sigmoid_backward():
a = tor4.tensor(data=[0.0, 0, 0], requires_grad=True)
a_sigmoid = a.sigmoid()
a_sigmoid.backward(tor4.tensor(data=[1, 1, 1]))
assert a_sigmoid.tolist() == [0.5, 0.5, 0.5]
assert a_sigmoid.requires_grad
assert a.grad.tolist() == [0.25, 0.25, 0.25]
def test_softmax_backward2():
a = tor4.tensor(data=[0, 0, 0.0], requires_grad=True)
a_sm = nn.functional.softmax(a, dim=-1)
a_sm.backward(tor4.tensor([0, 1, -1.0]))
assert a_sm.requires_grad
assert a_sm.tolist() == [1 / 3, 1 / 3, 1 / 3]
assert a.grad.tolist() == [0, 1 / 3, -1 / 3]
def test_logsoftmax_backward():
a = tor4.tensor([0.0, 0, 0], requires_grad=True)
lsm = nn.functional.log_softmax(a, dim=-1)
lsm.backward(tor4.tensor([1, 1, 2.0]))
assert np.allclose(lsm.tolist(), np.log([1 / 3] * 3))
assert lsm.requires_grad
assert np.allclose(a.grad.tolist(), [-1 / 3, -1 / 3, 2 / 3])
def test_tensor_add():
a = tensor([1, 2, 3])
b = tensor([4, 5, 6])
apb = a + b
assert apb.tolist() == [5, 7, 9]
assert not apb.requires_grad
assert apb.is_leaf
def test_conv2d_backward3():
a = tor4.tensor([[[[0, 1, 2], [3, 4, 5.0]]]])
w = tor4.tensor([[[[1, 0], [1, 1]]], [[[0, 1], [0, 1.0]]]], requires_grad=True)
try:
nn.functional.conv2d(a, w, dilation=2)
raise AssertionError()
except RuntimeError:
assert True
def test_tesnor_rdiv_scalar_backward():
a = tensor(data=[1.0, 2.0, -3.0], requires_grad=True)
ad2 = 1 / a
ad2.backward(tensor(data=[1.0, 1.0, 1.0]))
assert ad2.tolist() == [1, 1 / 2, -1 / 3]
assert ad2.requires_grad
assert a.requires_grad
assert a.grad.tolist() == [-1, -1 / 4, -1 / 9]
def test_tesnor_div_scalar_backward():
a = tensor(data=[1.0, 2.0, 3.0], requires_grad=True)
ad2 = a / 2
ad2.backward(tensor(data=[1.0, 1.0, 1.0]))
assert ad2.tolist() == [1 / 2, 1, 3 / 2]
assert ad2.requires_grad
assert a.requires_grad
assert a.grad.tolist() == [1 / 2, 1 / 2, 1 / 2]
def test_tensor_iadd_backward():
a = tensor([1, 2, 3.0], requires_grad=True)
b = tensor([-1, 0, 1])
try:
a += b
raise AssertionError()
except RuntimeError:
assert True
def test_tensor_pow_scalar_backward():
a = tensor(data=[1, 2, 3.0], requires_grad=True)
ap2 = a ** 2
ap2.backward(tensor(data=[1, 1, 1]))
assert ap2.tolist() == [1, 4, 9]
assert ap2.requires_grad
assert a.requires_grad
assert a.grad.tolist() == [2, 4, 6]
def test_tensor_from_scalar():
a = tensor(data=1)
assert a.tolist() == 1
assert not a.requires_grad
assert a.is_leaf
a = tensor(data=2.0, requires_grad=True)
assert a.tolist() == 2.0
assert a.requires_grad
assert a.is_leaf
def test_tensor_matmul2():
a = tensor(data=[[1, 2, 3], [3, 2, 1]])
b = tensor(data=[[1, 1], [1, 1], [1, 1]])
amb = a @ b
assert amb.tolist() == [
[6, 6],
[6, 6],
]
assert not amb.requires_grad
assert amb.shape == (2, 2)
def test_max4_backward():
a = tor4.tensor(data=[[10, 3, -1], [-3, 4, 10.0]], requires_grad=True)
a_max, arg_max = a.max(dim=1, keepdim=True)
a_max.backward(tor4.tensor([[1], [1]]))
assert a_max.tolist() == [[10], [10]]
assert a_max.requires_grad
assert arg_max.tolist() == [[0], [2]]
assert not arg_max.requires_grad
assert a.grad.tolist() == [[1, 0, 0], [0, 0, 1]]
assert a.requires_grad
def test_max3_backward():
a = tor4.tensor(data=[[1, 3, -1], [-3, 4, 1.0]], requires_grad=True)
a_max, arg_max = a.max(dim=0)
a_max.backward(tor4.tensor([1, 1, 1]))
assert a_max.tolist() == [1, 4, 1]
assert a_max.requires_grad
assert arg_max.tolist() == [0, 1, 1]
assert not arg_max.requires_grad
assert a.grad.tolist() == [[1, 0, 0], [0, 1, 1]]
assert a.requires_grad
def test_dropout2d_eval_backward():
a = tor4.tensor(data=[[[1, 2, 3], [1, 2, 3.0]]],
dtype="float32",
requires_grad=True)
a_drop = nn.functional.dropout2d(a, p=0.8, training=False)
a_drop.backward(tor4.tensor(data=[[[2, 2, 2], [2, 2, 2.0]]]))
mask = a_drop.detach().numpy() != 0
assert mask.all()
assert a_drop.requires_grad
assert a.grad.tolist() == [[[2, 2, 2], [2, 2, 2.0]]]
def test_conv2d_backward31():
a = tor4.arange(9, dtype=tor4.float32, requires_grad=True)
b = a.view(1, 1, 3, 3)
w = tor4.tensor([[[[1, 0], [1, 1]]], [[[0, 1], [0, 1.0]]]], requires_grad=True)
aw = nn.functional.conv2d(b, w, dilation=2)
aw.backward(tor4.tensor([[[[2]], [[1.0]]]]))
assert aw.tolist() == [[[[14]], [[10]]]]
assert np.allclose(a.grad.tolist(), [2, 0, 1, 0, 0, 0, 2, 0, 3])
assert np.allclose(w.grad.tolist(), [[[[0, 4], [12, 16]]], [[[0, 2], [6, 8]]]])
def test_dropout_eval_backward():
a = tor4.tensor(data=[1, 2, 3, 4, 5, 6, 7, 8.0],
dtype="float32",
requires_grad=True)
a_drop = nn.functional.dropout(a, p=0.8, training=False)
a_drop.backward(tor4.tensor(data=[10, 10, 10.0, 10, 10, 10, 10, 10]))
mask = a_drop.detach().numpy() != 0
assert mask.all()
assert a_drop.requires_grad
assert a.grad.tolist() == [10, 10, 10, 10, 10, 10, 10, 10]
def test_logsoftmax_backward1():
a = tor4.tensor([1.0, 2, 3], requires_grad=True)
lsm = nn.functional.log_softmax(a, dim=-1)
lsm.backward(tor4.tensor([1, 1, 2.0]))
assert np.allclose(lsm.tolist(), [-2.4076, -1.4076, -0.4076],
rtol=1e-4,
atol=1e-4)
assert lsm.requires_grad
assert np.allclose(a.grad.tolist(), [0.6399, 0.0211, -0.661],
atol=1e-4,
rtol=1e-4)
def test_tensor_mul_broadcast2_backward():
a = tensor(data=[[1, 2, 3], [1, 1, 2]])
b = tensor(data=[[-1, 3, 1.0]], requires_grad=True)
amb = a * b
amb.backward(tensor([[1, 1, 1], [1, 1, 1]]))
assert amb.tolist() == [[-1, 6, 3], [-1, 3, 2]]
assert not a.requires_grad
assert b.requires_grad
assert amb.requires_grad
assert a.grad is None
assert b.grad.tolist() == [[2, 3, 5]]
def test_tensor_mul_broadcast3_backward():
a = tensor(data=[[[1, 2, 3], [1, 1, 2]], [[1, 2, 3], [1, 1, 2]]])
b = tensor(data=[[1], [0.0]], requires_grad=True)
amb = a * b
amb.backward(tensor([[[1, 1, 1], [1, 1, 1]], [[1, 1, 1], [1, 1, 1]]]))
assert amb.tolist() == [[[1, 2, 3], [0, 0, 0]], [[1, 2, 3], [0, 0, 0]]]
assert not a.requires_grad
assert b.requires_grad
assert amb.requires_grad
assert a.grad is None
assert b.grad.tolist() == [[12], [8]]
