def test_sigmoid(): a = gtensor(dm('-2 3')) k = gtensor(dm('2 3')) b = H.sigmoid(a) (b * k).sum().backward() grad = dm('0.20998717 0.13552998') np.testing.assert_allclose(a.grad, grad)
def test_vector_vector(): t1 = gtensor(dm('1 2')) t2 = gtensor(dm('3 4')) (t1 + t2).sum().backward() grad = dm('1 1') np.testing.assert_allclose(t1.grad, grad) np.testing.assert_allclose(t2.grad, grad)
def test_conv2d(): a = [ [1, 2, 3, 4], [2, 1, 3, 4], [3, 2, 1, 4], [4, 3, 2, 1], ] b = [ [1, 1, 1], [1, 2, 1], [1, 1, 3], ] k = [ [1, 2], [3, 4], ] a = dm(a, dtype=np.double).reshape(1, 1, 4, 4) b = dm(b, dtype=np.double).reshape(1, 1, 3, 3) k = dm(k, dtype=np.double).reshape(1, 1, 2, 2) ta = gtensor(a) tb = gtensor(b) tk = gtensor(k) tc = H.conv2d(ta, tb) (tc * tk).sum().backward() c = dm([[[[21., 35.], [27., 24.]]]]) ga = dm([[[[1., 3., 3., 2.], [4., 11., 12., 6.], [4., 13., 16., 10.], [3., 7., 13., 12.]]]]) gb = dm([[[[15., 23., 36.], [21., 17., 30.], [31., 21., 19.]]]]) np.testing.assert_allclose(tc.data, c) np.testing.assert_allclose(ta.grad, ga) np.testing.assert_allclose(tb.grad, gb)
def test_matrix_matrix(): t1 = gtensor(dm('1 2 3; 3 4 5')) t2 = gtensor(dm('3 4 5; 5 6 7')) (t1 + t2).sum().backward() grad = dm('1 1 1; 1 1 1') np.testing.assert_allclose(t1.grad, grad) np.testing.assert_allclose(t2.grad, grad)
def test_any(): import torch import torch.nn.functional as F stride = 3 padding = 5 a = np.random.rand(2, 2, 5, 5) w = np.random.rand(3, 2, 3, 3) b = np.random.rand(3) ta = torch.tensor(a, requires_grad=True) tw = torch.tensor(w, requires_grad=True) tb = torch.tensor(b, requires_grad=True) ha = gtensor(a) hw = gtensor(w) hb = gtensor(b) tc = F.conv2d(ta, tw, stride=stride, padding=padding, bias=tb) hc = H.conv2d(ha, hw, stride=stride, padding=padding, bias=hb) k = np.random.rand(*tc.shape) tk = torch.tensor(k, requires_grad=True) hk = gtensor(k) (tc * tk).sum().backward() (hc * hk).sum().backward() gta = ta.grad.numpy() gtw = tw.grad.numpy() gtb = tb.grad.numpy() np.testing.assert_allclose(tc.clone().detach().numpy(), hc.data) np.testing.assert_allclose(gta, ha.grad) np.testing.assert_allclose(gtw, hw.grad) np.testing.assert_allclose(gtb, hb.grad)
def test_matrix_vector(): t1 = gtensor(dm('1 2 3; 3 4 5')) t2 = gtensor(dm('2 3 4')) (t1 + t2).sum().backward() grad1 = dm('1 1 1; 1 1 1') grad2 = dm('2 2 2') np.testing.assert_allclose(t1.grad, grad1) np.testing.assert_allclose(t2.grad, grad2)
def test_vector_vector(): t1 = gtensor(dm('1 2')) t2 = gtensor(dm('3 4')) (t1 / t2).sum().backward() grad1 = dm('4 3') / 12 grad2 = dm('-8 -9') / 72 np.testing.assert_allclose(t1.grad, grad1) np.testing.assert_allclose(t2.grad, grad2)
def test_matrix_vector(): t1 = gtensor(dm('1 2 3; 3 4 5')) t2 = gtensor(dm('2 3 4')) (t1 / t2).sum().backward() grad1 = dm('6 4 3; 6 4 3') / 12 grad2 = dm('-6 -4 -3') / 6 np.testing.assert_allclose(t1.grad, grad1) np.testing.assert_allclose(t2.grad, grad2)
def test_matrix_matrix(): t1 = gtensor(dm('1 2 3; 3 4 5')) t2 = gtensor(dm('2 2 6; 3 2 10')) (t1 / t2).sum().backward() grad1 = (1 / t2.data) grad2 = dm('-30 -60 -10; -40 -120 -6') / 120 np.testing.assert_allclose(t1.grad, grad1) np.testing.assert_allclose(t2.grad, grad2)
def test_vector_vector(): # (3,) @ (3,) t1 = gtensor(dm('4 9 3')) t2 = gtensor(dm('8 2 5')) (t1 @ t2).sum().backward() grad1 = dm('8 2 5') grad2 = dm('4 9 3') np.testing.assert_allclose(t1.grad, grad1) np.testing.assert_allclose(t2.grad, grad2)
def test_vector_matrix(): # (2, 3) @ (3,) t1 = gtensor(dm('4 9')) t2 = gtensor(dm('1 2 3; 3 4 5')) (t1 @ t2).sum().backward() grad1 = dm('6 12') grad2 = dm('4 4 4; 9 9 9') np.testing.assert_allclose(t1.grad, grad1) np.testing.assert_allclose(t2.grad, grad2)
def test_axis(): x = dm('1 2 3; 4 5 6') hx = gtensor(x) k = dm('3 6') hk = gtensor(k) hs = hx.std(axis=1) (hs * hk).sum().backward() grad = dm('-1.5 0 1.5; -3 0 3') np.testing.assert_allclose(hx.grad, grad)
def test_matmul_matrix(): # (2, 3) @ (3, 4) t1 = gtensor(dm('1 2 3; 3 4 5')) t2 = gtensor(dm('3 4 5 6; 5 6 7 8; 1 2 3 4')) (t1 @ t2).sum().backward() grad1 = dm('18 26 10; 18 26 10') grad2 = dm('4 4 4 4; 6 6 6 6; 8 8 8 8') np.testing.assert_allclose(t1.grad, grad1) np.testing.assert_allclose(t2.grad, grad2)
def test_relu(): a = gtensor(dm('1 -2; -3 4')) k = gtensor(dm('8 3; 2 1')) b = H.relu(a) ret = (b * k + b).sum() ret.backward() grad = dm('9 0; 0 2') np.testing.assert_allclose(a.grad, grad)
def test_elu(): a = gtensor(dm('1 -2; -3 4')) k = gtensor(dm('8 3; 2 1')) b = H.elu(a) ret = (b * k + b).sum() ret.backward() grad = dm([ [9., 0.54134113], [0.14936121, 2.], ]) np.testing.assert_allclose(a.grad, grad)
def test_conv2d_with_padding(): a = [ [1, 2, 3, 4], [2, 1, 3, 4], [3, 2, 1, 4], [4, 3, 2, 1], ] b = [ [1, 1, 1], [1, 2, 1], [1, 1, 3], ] k = [ [1, 2, 3, 4], [3, 4, 5, 2], [6, 1, 2, 1], [2, 3, 1, 6], ] a = dm(a, dtype=np.double).reshape(1, 1, 4, 4) b = dm(b, dtype=np.double).reshape(1, 1, 3, 3) k = dm(k, dtype=np.double).reshape(1, 1, 4, 4) ta = gtensor(a) tb = gtensor(b) tk = gtensor(k) tc = H.conv2d(ta, tb, padding=1) (tc * tk).sum().backward() c = dm([[[ [9., 20., 28., 18.], [17., 21., 35., 23.], [24., 27., 24., 19.], [16., 18., 15., 9.], ]]]) ga = dm([[[ [11., 20., 23., 18.], [20., 33., 33., 25.], [25., 34., 35., 28.], [14., 30., 17., 20.], ]]]) gb = dm([[[ [44., 94., 66.], [74., 114., 89.], [55., 91., 71.], ]]]) np.testing.assert_allclose(tc.data, c) np.testing.assert_allclose(ta.grad, ga) np.testing.assert_allclose(tb.grad, gb)
def test_cross_entropy(): logit = [ [6, -9, -11, -11, 13], [2, 7, -8, 11, 2], [0, 17, 9, 11, -5], ] target = [2, 3, 4] logit = np.array(logit, dtype=np.double) target = np.array(target, dtype=np.longlong) tlogit = gtensor(logit) tloss = H.cross_entropy(tlogit, target) tloss.backward() loss = np.array(15.34070468499769) grad = dm([ [ 3.03683731e-04, 9.28975581e-11, -3.33333333e-01, 1.25723173e-11, 3.33029649e-01 ], [ 4.03869206e-05, 5.99395047e-03, 1.83356336e-09, -6.07472615e-03, 4.03869206e-05 ], [ 1.37610652e-08, 3.32397880e-01, 1.11507066e-04, 8.23931970e-04, -3.33333333e-01 ], ]) np.testing.assert_allclose(tloss.data, loss) np.testing.assert_allclose(tlogit.grad, grad)
def test_2d_1d(): t1 = gtensor(dm('1 2; 4 5')) shape = (4,) t2 = t1.reshape(shape) t2.sum().backward() grad = dm('1 1; 1 1') np.testing.assert_allclose(t1.grad, grad)
def test_nd_nd(): shape = (1,2,3,4) t1 = gtensor(np.random.rand(*shape)) t2 = t1.reshape(8, 3) t2.sum().backward() grad = np.ones(shape) np.testing.assert_allclose(t1.grad, grad)
def test_conv2d_with_stride(): a = [ [1, 2, 3, 4, 5], [2, 1, 3, 4, 5], [3, 2, 1, 4, 5], [4, 3, 2, 1, 5], [5, 2, 1, 3, 4], ] b = [ [1, 1, 1], [1, 2, 1], [1, 1, 3], ] k = [ [1, 2], [3, 4], ] a = dm(a, dtype=np.double).reshape(1, 1, 5, 5) b = dm(b, dtype=np.double).reshape(1, 1, 3, 3) k = dm(k, dtype=np.double).reshape(1, 1, 2, 2) ta = gtensor(a) tb = gtensor(b) tk = gtensor(k) tc = H.conv2d(ta, tb, stride=2) (tc * tk).sum().backward() c = dm([[[ [21., 48.], [28., 35.], ]]]) ga = dm([[[ [1., 1., 3., 2., 2.], [1., 2., 3., 4., 2.], [4., 4., 12., 6., 10.], [3., 6., 7., 8., 4.], [3., 3., 13., 4., 12.], ]]]) gb = dm([[[ [20., 32., 36.], [28., 22., 39.], [24., 28., 30.], ]]]) np.testing.assert_allclose(tc.data, c) np.testing.assert_allclose(ta.grad, ga) np.testing.assert_allclose(tb.grad, gb)
def test_axis_none(): x = dm('1 2 3; 4 5 6') hx = gtensor(x) k = 3 hs = hx.std() (hs * k).sum().backward() grad = (x - x.mean()) / 5 / x.std(ddof=1) * k np.testing.assert_allclose(hx.grad, grad)
def test_high_dim(): d = [[[0, 8, 0, 5], [7, 1, 7, 1], [9, 1, 6, 9]], [[7, 0, 6, 2], [8, 5, 2, 5], [8, 1, 4, 6]]] a = gtensor(dm(d)) a.max(axis=1).sum().backward() print(a.grad) g = [[[0., 1., 0., 0.], [0., 0., 1., 0.], [1., 0., 0., 1.]], [[0., 0., 1., 0.], [1., 1., 0., 0.], [0., 0., 0., 1.]]] grad = dm(g) np.testing.assert_allclose(a.grad, grad)
def test_max_pool2d(): kernel_size = 3 stride = 2 padding = 0 a = [ [1, 2, 3, 2, 1], [2, 1, 4, 4, 2], [3, 2, 5, 4, 1], [1, 3, 2, 2, 2], [2, 1, 3, 4, 1], ] k = [ [1, 2], [3, 4], ] c = [ [5, 5], [5, 5], ] a = dm(a, dtype=np.double).reshape(1, 1, 5, 5) k = dm(k, dtype=np.double).reshape(1, 1, 2, 2) c = dm(c, dtype=np.double).reshape(1, 1, 2, 2) ta = gtensor(a) tk = gtensor(k) tc = H.max_pool2d(ta, kernel_size, stride=stride, padding=padding) (tc * tk).sum().backward() ga = [ [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 10, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], ] ga = dm(ga, dtype=np.double).reshape(1, 1, 5, 5) np.testing.assert_allclose(tc.data, c) np.testing.assert_allclose(ta.grad, ga)
def test_any(): import torch import torch.nn.functional as F kernel_size = 3 stride = 2 padding = 1 a = np.random.rand(2, 2, 8, 8) ta = torch.tensor(a, requires_grad=True) ga = gtensor(a) tc = F.max_pool2d(ta, kernel_size, stride=stride, padding=padding) gc = H.max_pool2d(ga, kernel_size, stride=stride, padding=padding) k = np.random.rand(*tc.shape) tk = torch.tensor(k, requires_grad=True) gk = gtensor(k) (tc * tk).sum().backward() (gc * gk).sum().backward() c = tc.detach().numpy() g = ta.grad.numpy() np.testing.assert_allclose(gc.data, c) np.testing.assert_allclose(ga.grad, g)
def test_multi_axes(): a = [[[4., -10., 7., 24.], [-1., 4., -6., 11.], [8., 11., -8., 8.]], [[3., -10., 2., -5.], [10., 18., -5., -15.], [-1., 10., -13., 17.]]] a = np.array(a) ha = gtensor(a) # k = [ # [[ 8., 6., 8., 10.], # [ 9., 7., 1., 8.], # [ 3., 2., 6., 6.]], # [[ 7., 7., 1., 5.], # [ 3., 5., 5., 3.], # [ 7., 9., 5., 10.]] # ] k = [1., 2., 4.] k = np.array(k) hk = gtensor(k) (H.mean(ha, axis=(0, 2)) * hk).sum().backward() grad = [[[0.125, 0.125, 0.125, 0.125], [0.25, 0.25, 0.25, 0.25], [0.5, 0.5, 0.5, 0.5]], [[0.125, 0.125, 0.125, 0.125], [0.25, 0.25, 0.25, 0.25], [0.5, 0.5, 0.5, 0.5]]] grad = np.array(grad) np.testing.assert_allclose(ha.grad, grad)
def test_axis_none(): t1 = gtensor(dm('1 2 3; 4 5 6')) k = 3 (H.mean(t1, axis=None) * k).backward() grad = dm('.5 .5 .5; .5 .5 .5') np.testing.assert_allclose(t1.grad, grad)
def test_vector(): t1 = gtensor(dm('1 2 3; 4 5 6')) t2 = t1[1] t2.sum().backward() grad = dm('0 0 0; 1 1 1') np.testing.assert_allclose(t1.grad, grad)
def test_axis(): t1 = gtensor(dm('1 2 3; 4 5 6')) k = gtensor(dm('3 6')) (H.mean(t1, axis=1) * k).sum().backward() grad = dm('1 1 1; 2 2 2') np.testing.assert_allclose(t1.grad, grad)
def test_sub(): # vector sub vector t1 = gtensor(dm('1 2')) t2 = gtensor(dm('3 4')) (t1 - t2).sum().backward() grad = dm('1 1') np.testing.assert_allclose(t1.grad, grad) np.testing.assert_allclose(t2.grad, -grad) # matrix sub matrix t1 = gtensor(dm('1 2 3; 3 4 5')) t2 = gtensor(dm('3 4 5; 5 6 7')) (t1 - t2).sum().backward() grad = dm('1 1 1; 1 1 1') np.testing.assert_allclose(t1.grad, grad) np.testing.assert_allclose(t2.grad, -grad) # matrix sub vector t1 = gtensor(dm('1 2 3; 3 4 5')) t2 = gtensor(dm('2 3 4')) (t1 - t2).sum().backward() grad1 = dm('1 1 1; 1 1 1') grad2 = dm('2 2 2') np.testing.assert_allclose(t1.grad, grad1) np.testing.assert_allclose(t2.grad, -grad2) # vector sub matrix t1 = gtensor(dm('2 3 4')) t2 = gtensor(dm('1 2 3; 3 4 5')) (t1 - t2).sum().backward() grad1 = dm('2 2 2') grad2 = dm('1 1 1; 1 1 1') np.testing.assert_allclose(t1.grad, grad1) np.testing.assert_allclose(t2.grad, -grad2) # matrix sub scalar t1 = gtensor(dm('1 2 3; 3 4 5')) t2 = 2 (t1 - t2).sum().backward() grad = dm('1 1 1; 1 1 1') np.testing.assert_allclose(t1.grad, grad) # scalar sub matrix t1 = 2 t2 = gtensor(dm('1 2 3; 3 4 5')) (t1 - t2).sum().backward() grad = dm('1 1 1; 1 1 1') np.testing.assert_allclose(t2.grad, -grad) # vector sub scalar t1 = gtensor(dm('1 2 3')) t2 = 2 (t1 - t2).sum().backward() grad = dm('1 1 1') np.testing.assert_allclose(t1.grad, grad) # scalar sub vector t1 = 2 t2 = gtensor(dm('1 2 3')) (t1 - t2).sum().backward() grad = dm('1 1 1') np.testing.assert_allclose(t2.grad, -grad)
def test_mul(): # vector mul vector t1 = gtensor(dm('1 2')) t2 = gtensor(dm('3 4')) (t1 * t2).sum().backward() np.testing.assert_allclose(t1.grad, t2.data) np.testing.assert_allclose(t2.grad, t1.data) # matrix mul matrix t1 = gtensor(dm('1 2 3; 3 4 5')) t2 = gtensor(dm('3 4 5; 5 6 7')) (t1 * t2).sum().backward() np.testing.assert_allclose(t1.grad, t2.data) np.testing.assert_allclose(t2.grad, t1.data) # matrix mul vector t1 = gtensor(dm('1 2 3; 3 4 5')) t2 = gtensor(dm('2 3 4')) (t1 * t2).sum().backward() grad1 = dm('2 3 4; 2 3 4') grad2 = dm('4 6 8') np.testing.assert_allclose(t1.grad, grad1) np.testing.assert_allclose(t2.grad, grad2) # vector mul matrix t1 = gtensor(dm('2 3 4')) t2 = gtensor(dm('1 2 3; 3 4 5')) (t1 * t2).sum().backward() grad1 = dm('4 6 8') grad2 = dm('2 3 4; 2 3 4') np.testing.assert_allclose(t1.grad, grad1) np.testing.assert_allclose(t2.grad, grad2) # matrix mul scalar t1 = gtensor(dm('1 2 3; 3 4 5')) t2 = 2 (t1 * t2).sum().backward() grad = dm('2 2 2; 2 2 2') np.testing.assert_allclose(t1.grad, grad) # scalar mul matrix t1 = 2 t2 = gtensor(dm('1 2 3; 3 4 5')) (t1 * t2).sum().backward() grad = dm('2 2 2; 2 2 2') np.testing.assert_allclose(t2.grad, grad) # vector mul scalar t1 = gtensor(dm('1 2 3')) t2 = 2 (t1 * t2).sum().backward() grad = dm('2 2 2') np.testing.assert_allclose(t1.grad, grad) # scalar mul vector t1 = 2 t2 = gtensor(dm('1 2 3')) (t1 * t2).sum().backward() grad = dm('2 2 2') np.testing.assert_allclose(t2.grad, grad)