def test_sigmoid():
a = gtensor(dm('-2 3'))
k = gtensor(dm('2 3'))
b = H.sigmoid(a)
(b * k).sum().backward()
grad = dm('0.20998717 0.13552998')
np.testing.assert_allclose(a.grad, grad)
def test_vector_vector():
t1 = gtensor(dm('1 2'))
t2 = gtensor(dm('3 4'))
(t1 + t2).sum().backward()
grad = dm('1 1')
np.testing.assert_allclose(t1.grad, grad)
np.testing.assert_allclose(t2.grad, grad)
def test_conv2d():
a = [
[1, 2, 3, 4],
[2, 1, 3, 4],
[3, 2, 1, 4],
[4, 3, 2, 1],
]
b = [
[1, 1, 1],
[1, 2, 1],
[1, 1, 3],
]
k = [
[1, 2],
[3, 4],
]
a = dm(a, dtype=np.double).reshape(1, 1, 4, 4)
b = dm(b, dtype=np.double).reshape(1, 1, 3, 3)
k = dm(k, dtype=np.double).reshape(1, 1, 2, 2)
ta = gtensor(a)
tb = gtensor(b)
tk = gtensor(k)
tc = H.conv2d(ta, tb)
(tc * tk).sum().backward()
c = dm([[[[21., 35.], [27., 24.]]]])
ga = dm([[[[1., 3., 3., 2.], [4., 11., 12., 6.], [4., 13., 16., 10.],
[3., 7., 13., 12.]]]])
gb = dm([[[[15., 23., 36.], [21., 17., 30.], [31., 21., 19.]]]])
np.testing.assert_allclose(tc.data, c)
np.testing.assert_allclose(ta.grad, ga)
np.testing.assert_allclose(tb.grad, gb)
def test_matrix_matrix():
t1 = gtensor(dm('1 2 3; 3 4 5'))
t2 = gtensor(dm('3 4 5; 5 6 7'))
(t1 + t2).sum().backward()
grad = dm('1 1 1; 1 1 1')
np.testing.assert_allclose(t1.grad, grad)
np.testing.assert_allclose(t2.grad, grad)
def test_any():
import torch
import torch.nn.functional as F
stride = 3
padding = 5
a = np.random.rand(2, 2, 5, 5)
w = np.random.rand(3, 2, 3, 3)
b = np.random.rand(3)
ta = torch.tensor(a, requires_grad=True)
tw = torch.tensor(w, requires_grad=True)
tb = torch.tensor(b, requires_grad=True)
ha = gtensor(a)
hw = gtensor(w)
hb = gtensor(b)
tc = F.conv2d(ta, tw, stride=stride, padding=padding, bias=tb)
hc = H.conv2d(ha, hw, stride=stride, padding=padding, bias=hb)
k = np.random.rand(*tc.shape)
tk = torch.tensor(k, requires_grad=True)
hk = gtensor(k)
(tc * tk).sum().backward()
(hc * hk).sum().backward()
gta = ta.grad.numpy()
gtw = tw.grad.numpy()
gtb = tb.grad.numpy()
np.testing.assert_allclose(tc.clone().detach().numpy(), hc.data)
np.testing.assert_allclose(gta, ha.grad)
np.testing.assert_allclose(gtw, hw.grad)
np.testing.assert_allclose(gtb, hb.grad)
def test_matrix_vector():
t1 = gtensor(dm('1 2 3; 3 4 5'))
t2 = gtensor(dm('2 3 4'))
(t1 + t2).sum().backward()
grad1 = dm('1 1 1; 1 1 1')
grad2 = dm('2 2 2')
np.testing.assert_allclose(t1.grad, grad1)
np.testing.assert_allclose(t2.grad, grad2)
def test_vector_vector():
t1 = gtensor(dm('1 2'))
t2 = gtensor(dm('3 4'))
(t1 / t2).sum().backward()
grad1 = dm('4 3') / 12
grad2 = dm('-8 -9') / 72
np.testing.assert_allclose(t1.grad, grad1)
np.testing.assert_allclose(t2.grad, grad2)
def test_matrix_vector():
t1 = gtensor(dm('1 2 3; 3 4 5'))
t2 = gtensor(dm('2 3 4'))
(t1 / t2).sum().backward()
grad1 = dm('6 4 3; 6 4 3') / 12
grad2 = dm('-6 -4 -3') / 6
np.testing.assert_allclose(t1.grad, grad1)
np.testing.assert_allclose(t2.grad, grad2)
def test_matrix_matrix():
t1 = gtensor(dm('1 2 3; 3 4 5'))
t2 = gtensor(dm('2 2 6; 3 2 10'))
(t1 / t2).sum().backward()
grad1 = (1 / t2.data)
grad2 = dm('-30 -60 -10; -40 -120 -6') / 120
np.testing.assert_allclose(t1.grad, grad1)
np.testing.assert_allclose(t2.grad, grad2)
def test_vector_vector():
# (3,) @ (3,)
t1 = gtensor(dm('4 9 3'))
t2 = gtensor(dm('8 2 5'))
(t1 @ t2).sum().backward()
grad1 = dm('8 2 5')
grad2 = dm('4 9 3')
np.testing.assert_allclose(t1.grad, grad1)
np.testing.assert_allclose(t2.grad, grad2)
def test_vector_matrix():
# (2, 3) @ (3,)
t1 = gtensor(dm('4 9'))
t2 = gtensor(dm('1 2 3; 3 4 5'))
(t1 @ t2).sum().backward()
grad1 = dm('6 12')
grad2 = dm('4 4 4; 9 9 9')
np.testing.assert_allclose(t1.grad, grad1)
np.testing.assert_allclose(t2.grad, grad2)
def test_axis():
x = dm('1 2 3; 4 5 6')
hx = gtensor(x)
k = dm('3 6')
hk = gtensor(k)
hs = hx.std(axis=1)
(hs * hk).sum().backward()
grad = dm('-1.5 0 1.5; -3 0 3')
np.testing.assert_allclose(hx.grad, grad)
def test_matmul_matrix():
# (2, 3) @ (3, 4)
t1 = gtensor(dm('1 2 3; 3 4 5'))
t2 = gtensor(dm('3 4 5 6; 5 6 7 8; 1 2 3 4'))
(t1 @ t2).sum().backward()
grad1 = dm('18 26 10; 18 26 10')
grad2 = dm('4 4 4 4; 6 6 6 6; 8 8 8 8')
np.testing.assert_allclose(t1.grad, grad1)
np.testing.assert_allclose(t2.grad, grad2)
def test_relu():
a = gtensor(dm('1 -2; -3 4'))
k = gtensor(dm('8 3; 2 1'))
b = H.relu(a)
ret = (b * k + b).sum()
ret.backward()
grad = dm('9 0; 0 2')
np.testing.assert_allclose(a.grad, grad)
def test_elu():
a = gtensor(dm('1 -2; -3 4'))
k = gtensor(dm('8 3; 2 1'))
b = H.elu(a)
ret = (b * k + b).sum()
ret.backward()
grad = dm([
[9., 0.54134113],
[0.14936121, 2.],
])
np.testing.assert_allclose(a.grad, grad)
def test_conv2d_with_padding():
a = [
[1, 2, 3, 4],
[2, 1, 3, 4],
[3, 2, 1, 4],
[4, 3, 2, 1],
]
b = [
[1, 1, 1],
[1, 2, 1],
[1, 1, 3],
]
k = [
[1, 2, 3, 4],
[3, 4, 5, 2],
[6, 1, 2, 1],
[2, 3, 1, 6],
]
a = dm(a, dtype=np.double).reshape(1, 1, 4, 4)
b = dm(b, dtype=np.double).reshape(1, 1, 3, 3)
k = dm(k, dtype=np.double).reshape(1, 1, 4, 4)
ta = gtensor(a)
tb = gtensor(b)
tk = gtensor(k)
tc = H.conv2d(ta, tb, padding=1)
(tc * tk).sum().backward()
c = dm([[[
[9., 20., 28., 18.],
[17., 21., 35., 23.],
[24., 27., 24., 19.],
[16., 18., 15., 9.],
]]])
ga = dm([[[
[11., 20., 23., 18.],
[20., 33., 33., 25.],
[25., 34., 35., 28.],
[14., 30., 17., 20.],
]]])
gb = dm([[[
[44., 94., 66.],
[74., 114., 89.],
[55., 91., 71.],
]]])
np.testing.assert_allclose(tc.data, c)
np.testing.assert_allclose(ta.grad, ga)
np.testing.assert_allclose(tb.grad, gb)
def test_cross_entropy():
logit = [
[6, -9, -11, -11, 13],
[2, 7, -8, 11, 2],
[0, 17, 9, 11, -5],
]
target = [2, 3, 4]
logit = np.array(logit, dtype=np.double)
target = np.array(target, dtype=np.longlong)
tlogit = gtensor(logit)
tloss = H.cross_entropy(tlogit, target)
tloss.backward()
loss = np.array(15.34070468499769)
grad = dm([
[
3.03683731e-04, 9.28975581e-11, -3.33333333e-01, 1.25723173e-11,
3.33029649e-01
],
[
4.03869206e-05, 5.99395047e-03, 1.83356336e-09, -6.07472615e-03,
4.03869206e-05
],
[
1.37610652e-08, 3.32397880e-01, 1.11507066e-04, 8.23931970e-04,
-3.33333333e-01
],
])
np.testing.assert_allclose(tloss.data, loss)
np.testing.assert_allclose(tlogit.grad, grad)
def test_2d_1d():
t1 = gtensor(dm('1 2; 4 5'))
shape = (4,)
t2 = t1.reshape(shape)
t2.sum().backward()
grad = dm('1 1; 1 1')
np.testing.assert_allclose(t1.grad, grad)
def test_nd_nd(): shape = (1,2,3,4) t1 = gtensor(np.random.rand(*shape)) t2 = t1.reshape(8, 3) t2.sum().backward() grad = np.ones(shape) np.testing.assert_allclose(t1.grad, grad)
def test_conv2d_with_stride():
a = [
[1, 2, 3, 4, 5],
[2, 1, 3, 4, 5],
[3, 2, 1, 4, 5],
[4, 3, 2, 1, 5],
[5, 2, 1, 3, 4],
]
b = [
[1, 1, 1],
[1, 2, 1],
[1, 1, 3],
]
k = [
[1, 2],
[3, 4],
]
a = dm(a, dtype=np.double).reshape(1, 1, 5, 5)
b = dm(b, dtype=np.double).reshape(1, 1, 3, 3)
k = dm(k, dtype=np.double).reshape(1, 1, 2, 2)
ta = gtensor(a)
tb = gtensor(b)
tk = gtensor(k)
tc = H.conv2d(ta, tb, stride=2)
(tc * tk).sum().backward()
c = dm([[[
[21., 48.],
[28., 35.],
]]])
ga = dm([[[
[1., 1., 3., 2., 2.],
[1., 2., 3., 4., 2.],
[4., 4., 12., 6., 10.],
[3., 6., 7., 8., 4.],
[3., 3., 13., 4., 12.],
]]])
gb = dm([[[
[20., 32., 36.],
[28., 22., 39.],
[24., 28., 30.],
]]])
np.testing.assert_allclose(tc.data, c)
np.testing.assert_allclose(ta.grad, ga)
np.testing.assert_allclose(tb.grad, gb)
def test_axis_none():
x = dm('1 2 3; 4 5 6')
hx = gtensor(x)
k = 3
hs = hx.std()
(hs * k).sum().backward()
grad = (x - x.mean()) / 5 / x.std(ddof=1) * k
np.testing.assert_allclose(hx.grad, grad)
def test_high_dim():
d = [[[0, 8, 0, 5], [7, 1, 7, 1], [9, 1, 6, 9]],
[[7, 0, 6, 2], [8, 5, 2, 5], [8, 1, 4, 6]]]
a = gtensor(dm(d))
a.max(axis=1).sum().backward()
print(a.grad)
g = [[[0., 1., 0., 0.], [0., 0., 1., 0.], [1., 0., 0., 1.]],
[[0., 0., 1., 0.], [1., 1., 0., 0.], [0., 0., 0., 1.]]]
grad = dm(g)
np.testing.assert_allclose(a.grad, grad)
def test_max_pool2d():
kernel_size = 3
stride = 2
padding = 0
a = [
[1, 2, 3, 2, 1],
[2, 1, 4, 4, 2],
[3, 2, 5, 4, 1],
[1, 3, 2, 2, 2],
[2, 1, 3, 4, 1],
]
k = [
[1, 2],
[3, 4],
]
c = [
[5, 5],
[5, 5],
]
a = dm(a, dtype=np.double).reshape(1, 1, 5, 5)
k = dm(k, dtype=np.double).reshape(1, 1, 2, 2)
c = dm(c, dtype=np.double).reshape(1, 1, 2, 2)
ta = gtensor(a)
tk = gtensor(k)
tc = H.max_pool2d(ta, kernel_size, stride=stride, padding=padding)
(tc * tk).sum().backward()
ga = [
[0, 0, 0, 0, 0],
[0, 0, 0, 0, 0],
[0, 0, 10, 0, 0],
[0, 0, 0, 0, 0],
[0, 0, 0, 0, 0],
]
ga = dm(ga, dtype=np.double).reshape(1, 1, 5, 5)
np.testing.assert_allclose(tc.data, c)
np.testing.assert_allclose(ta.grad, ga)
def test_any():
import torch
import torch.nn.functional as F
kernel_size = 3
stride = 2
padding = 1
a = np.random.rand(2, 2, 8, 8)
ta = torch.tensor(a, requires_grad=True)
ga = gtensor(a)
tc = F.max_pool2d(ta, kernel_size, stride=stride, padding=padding)
gc = H.max_pool2d(ga, kernel_size, stride=stride, padding=padding)
k = np.random.rand(*tc.shape)
tk = torch.tensor(k, requires_grad=True)
gk = gtensor(k)
(tc * tk).sum().backward()
(gc * gk).sum().backward()
c = tc.detach().numpy()
g = ta.grad.numpy()
np.testing.assert_allclose(gc.data, c)
np.testing.assert_allclose(ga.grad, g)
def test_multi_axes():
a = [[[4., -10., 7., 24.], [-1., 4., -6., 11.], [8., 11., -8., 8.]],
[[3., -10., 2., -5.], [10., 18., -5., -15.], [-1., 10., -13., 17.]]]
a = np.array(a)
ha = gtensor(a)
# k = [
# [[ 8., 6., 8., 10.],
# [ 9., 7., 1., 8.],
# [ 3., 2., 6., 6.]],
# [[ 7., 7., 1., 5.],
# [ 3., 5., 5., 3.],
# [ 7., 9., 5., 10.]]
# ]
k = [1., 2., 4.]
k = np.array(k)
hk = gtensor(k)
(H.mean(ha, axis=(0, 2)) * hk).sum().backward()
grad = [[[0.125, 0.125, 0.125, 0.125], [0.25, 0.25, 0.25, 0.25],
[0.5, 0.5, 0.5, 0.5]],
[[0.125, 0.125, 0.125, 0.125], [0.25, 0.25, 0.25, 0.25],
[0.5, 0.5, 0.5, 0.5]]]
grad = np.array(grad)
np.testing.assert_allclose(ha.grad, grad)
def test_axis_none():
t1 = gtensor(dm('1 2 3; 4 5 6'))
k = 3
(H.mean(t1, axis=None) * k).backward()
grad = dm('.5 .5 .5; .5 .5 .5')
np.testing.assert_allclose(t1.grad, grad)
def test_vector():
t1 = gtensor(dm('1 2 3; 4 5 6'))
t2 = t1[1]
t2.sum().backward()
grad = dm('0 0 0; 1 1 1')
np.testing.assert_allclose(t1.grad, grad)
def test_axis():
t1 = gtensor(dm('1 2 3; 4 5 6'))
k = gtensor(dm('3 6'))
(H.mean(t1, axis=1) * k).sum().backward()
grad = dm('1 1 1; 2 2 2')
np.testing.assert_allclose(t1.grad, grad)
def test_sub():
# vector sub vector
t1 = gtensor(dm('1 2'))
t2 = gtensor(dm('3 4'))
(t1 - t2).sum().backward()
grad = dm('1 1')
np.testing.assert_allclose(t1.grad, grad)
np.testing.assert_allclose(t2.grad, -grad)
# matrix sub matrix
t1 = gtensor(dm('1 2 3; 3 4 5'))
t2 = gtensor(dm('3 4 5; 5 6 7'))
(t1 - t2).sum().backward()
grad = dm('1 1 1; 1 1 1')
np.testing.assert_allclose(t1.grad, grad)
np.testing.assert_allclose(t2.grad, -grad)
# matrix sub vector
t1 = gtensor(dm('1 2 3; 3 4 5'))
t2 = gtensor(dm('2 3 4'))
(t1 - t2).sum().backward()
grad1 = dm('1 1 1; 1 1 1')
grad2 = dm('2 2 2')
np.testing.assert_allclose(t1.grad, grad1)
np.testing.assert_allclose(t2.grad, -grad2)
# vector sub matrix
t1 = gtensor(dm('2 3 4'))
t2 = gtensor(dm('1 2 3; 3 4 5'))
(t1 - t2).sum().backward()
grad1 = dm('2 2 2')
grad2 = dm('1 1 1; 1 1 1')
np.testing.assert_allclose(t1.grad, grad1)
np.testing.assert_allclose(t2.grad, -grad2)
# matrix sub scalar
t1 = gtensor(dm('1 2 3; 3 4 5'))
t2 = 2
(t1 - t2).sum().backward()
grad = dm('1 1 1; 1 1 1')
np.testing.assert_allclose(t1.grad, grad)
# scalar sub matrix
t1 = 2
t2 = gtensor(dm('1 2 3; 3 4 5'))
(t1 - t2).sum().backward()
grad = dm('1 1 1; 1 1 1')
np.testing.assert_allclose(t2.grad, -grad)
# vector sub scalar
t1 = gtensor(dm('1 2 3'))
t2 = 2
(t1 - t2).sum().backward()
grad = dm('1 1 1')
np.testing.assert_allclose(t1.grad, grad)
# scalar sub vector
t1 = 2
t2 = gtensor(dm('1 2 3'))
(t1 - t2).sum().backward()
grad = dm('1 1 1')
np.testing.assert_allclose(t2.grad, -grad)
def test_mul():
# vector mul vector
t1 = gtensor(dm('1 2'))
t2 = gtensor(dm('3 4'))
(t1 * t2).sum().backward()
np.testing.assert_allclose(t1.grad, t2.data)
np.testing.assert_allclose(t2.grad, t1.data)
# matrix mul matrix
t1 = gtensor(dm('1 2 3; 3 4 5'))
t2 = gtensor(dm('3 4 5; 5 6 7'))
(t1 * t2).sum().backward()
np.testing.assert_allclose(t1.grad, t2.data)
np.testing.assert_allclose(t2.grad, t1.data)
# matrix mul vector
t1 = gtensor(dm('1 2 3; 3 4 5'))
t2 = gtensor(dm('2 3 4'))
(t1 * t2).sum().backward()
grad1 = dm('2 3 4; 2 3 4')
grad2 = dm('4 6 8')
np.testing.assert_allclose(t1.grad, grad1)
np.testing.assert_allclose(t2.grad, grad2)
# vector mul matrix
t1 = gtensor(dm('2 3 4'))
t2 = gtensor(dm('1 2 3; 3 4 5'))
(t1 * t2).sum().backward()
grad1 = dm('4 6 8')
grad2 = dm('2 3 4; 2 3 4')
np.testing.assert_allclose(t1.grad, grad1)
np.testing.assert_allclose(t2.grad, grad2)
# matrix mul scalar
t1 = gtensor(dm('1 2 3; 3 4 5'))
t2 = 2
(t1 * t2).sum().backward()
grad = dm('2 2 2; 2 2 2')
np.testing.assert_allclose(t1.grad, grad)
# scalar mul matrix
t1 = 2
t2 = gtensor(dm('1 2 3; 3 4 5'))
(t1 * t2).sum().backward()
grad = dm('2 2 2; 2 2 2')
np.testing.assert_allclose(t2.grad, grad)
# vector mul scalar
t1 = gtensor(dm('1 2 3'))
t2 = 2
(t1 * t2).sum().backward()
grad = dm('2 2 2')
np.testing.assert_allclose(t1.grad, grad)
# scalar mul vector
t1 = 2
t2 = gtensor(dm('1 2 3'))
(t1 * t2).sum().backward()
grad = dm('2 2 2')
np.testing.assert_allclose(t2.grad, grad)
