def test_nested_value_2(): npr.seed(11) for ii in xrange(NUM_TRIALS): npX = npr.randn(10, 20) X = kayak.Parameter(npX) Y = kayak.MatSum(X, axis=1) Z = kayak.MatSum(Y) assert np.all(close_float(Y.value.ravel(), np.sum(npX, axis=1))) assert close_float(Z.value, np.sum(npX))
def test_keepdims_grad_1(): npr.seed(10) for ii in xrange(NUM_TRIALS): npX = npr.randn(10, 20) X = kayak.Parameter(npX) Y = kayak.MatSum(X, axis=0, keepdims=False) Z = kayak.MatSum(Y) assert Z.grad(X).shape == npX.shape assert np.all(close_float(Z.grad(X), np.ones(npX.shape))) assert kayak.util.checkgrad(X, Z) < MAX_GRAD_DIFF
def test_keepdims_value_1(): npr.seed(9) for ii in xrange(NUM_TRIALS): npX = npr.randn(10, 20) X = kayak.Parameter(npX) Y = kayak.MatSum(X, axis=0, keepdims=False) Z = kayak.MatSum(Y) assert Y.shape == np.sum(npX, axis=0, keepdims=False).shape assert np.all(close_float(Y.value, np.sum(npX, axis=0, keepdims=False))) assert close_float(Z.value, np.sum(npX))
def test_keepdims_grad_2(): npr.seed(10) for ii in xrange(NUM_TRIALS): npW = npr.randn(5, 10, 20) npX = npr.randn(5, 10, 20) W = kayak.Parameter(npW) X = kayak.Parameter(npX) Y = W * X Z = kayak.MatSum(Y, axis=2, keepdims=False) S = kayak.MatSum(Z) assert S.grad(W).shape == npW.shape # assert np.all(close_float(Z.grad(X), np.ones(npX.shape))) assert kayak.util.checkgrad(X, S) < MAX_GRAD_DIFF
def test_batcher_clears_shape_cache(): batcher = kayak.Batcher(2, 3) X = kayak.Inputs(np.array([[1, 2, 3], [2, 3, 4], [3, 4, 5]]), batcher) Y = kayak.MatSum(X, axis=1) correct_shapes = [(2, 1), (1, 1)] for ii, batch in enumerate(batcher): assert Y.shape == correct_shapes[ii]
def test_batcher_clears_value_cache(): batcher = kayak.Batcher(1, 2) X = kayak.Inputs(np.array([[1, 2, 3], [2, 3, 4]]), batcher) Y = kayak.MatSum(X) correct_vals = [6, 9] for ii, batch in enumerate(batcher): assert Y.value == correct_vals[ii]
def test_graph_diamond(): npr.seed(2) N = 10 D = 5 H1 = 6 H2 = 7 X = kayak.Inputs(npr.randn(N,D)) W1 = kayak.Parameter(npr.randn(D,H1)) W2a = kayak.Parameter(npr.randn(H1,H2)) W2b = kayak.Parameter(npr.randn(H1,H2)) W3 = kayak.Parameter(npr.randn(H2,1)) U1 = kayak.SoftReLU(kayak.MatMult(X, W1)) U2a = kayak.SoftReLU(kayak.MatMult(U1, W2a)) U2b = kayak.SoftReLU(kayak.MatMult(U1, W2b)) U3a = kayak.SoftReLU(kayak.MatMult(U2a, W3)) U3b = kayak.SoftReLU(kayak.MatMult(U2b, W3)) out = kayak.MatSum(kayak.MatAdd(U3a, U3b)) out.value print kayak.util.checkgrad(W1, out) print kayak.util.checkgrad(W2a, out) print kayak.util.checkgrad(W2b, out) print kayak.util.checkgrad(W3, out) assert kayak.util.checkgrad(W1, out) < MAX_GRAD_DIFF assert kayak.util.checkgrad(W2a, out) < MAX_GRAD_DIFF assert kayak.util.checkgrad(W2b, out) < MAX_GRAD_DIFF assert kayak.util.checkgrad(W3, out) < MAX_GRAD_DIFF
def test_dropout_clears_value_cache(): X = kayak.Inputs(np.random.randn(10,10)) Y = kayak.Dropout(X) Z = kayak.MatSum(Y, axis=1) val1 = Z.value Y.draw_new_mask() val2 = Z.value assert not np.all(val1 == val2) assert np.all(Z.value == Z.value)
def test_vector_value_1(): npr.seed(3) for ii in xrange(NUM_TRIALS): npX = npr.randn(10, 1) X = kayak.Parameter(npX) Y = kayak.MatSum(X) # Verify the sum. assert close_float(Y.value, np.sum(npX))
def test_tensor_value_1(): npr.seed(13) for ii in xrange(NUM_TRIALS): npX = npr.randn(10, 20, 30) X = kayak.Parameter(npX) Y = kayak.MatSum(X) assert X.shape == npX.shape assert close_float(Y.value, np.sum(npX))
def test_tensor_value_4(): npr.seed(16) for ii in xrange(NUM_TRIALS): npX = npr.randn(10, 20, 30) X = kayak.Parameter(npX) Y = kayak.MatSum(X, axis=0) assert np.all( close_float(Y.value, np.expand_dims(np.sum(npX, axis=0), axis=0)))
def test_matrix_value(): npr.seed(7) for ii in xrange(NUM_TRIALS): npX = npr.randn(10, 20) X = kayak.Parameter(npX) Y = kayak.MatSum(X) # Verify the value. assert close_float(Y.value, np.sum(npX))
def test_scalar_value(): npr.seed(1) for ii in xrange(NUM_TRIALS): npX = npr.randn() X = kayak.Parameter(npX) Y = kayak.MatSum(X) # Verify that a scalar is reproduced. assert close_float(Y.value, npX)
def test_logsoftmax_grad_2(): npr.seed(4) for ii in xrange(NUM_TRIALS): np_X = npr.randn(5, 6) X = kayak.Parameter(np_X) Y = kayak.LogSoftMax(X, axis=0) Z = kayak.MatSum(Y) assert kayak.util.checkgrad(X, Z) < MAX_GRAD_DIFF
def test_alldropout_grad(): npr.seed(5) np_X = npr.randn(10,20) X = kayak.Parameter(np_X) Y = kayak.Dropout(X, drop_prob=1.0) Z = kayak.MatSum(Y) Z.value assert Z.grad(X).shape == np_X.shape assert kayak.util.checkgrad(X, Z) < MAX_GRAD_DIFF
def test_softmax_grad_1(): npr.seed(3) for ii in xrange(NUM_TRIALS): np_X = npr.randn(5, 6) X = kayak.Parameter(np_X) Y = kayak.SoftMax(X) Z = kayak.MatSum(Y * Y) assert kayak.util.checkgrad(X, Z) < MAX_GRAD_DIFF
def test_scalar_grad(): npr.seed(2) for ii in xrange(NUM_TRIALS): npX = npr.randn() X = kayak.Parameter(npX) Y = kayak.MatSum(X) # Verify that the gradient is one. Y.value assert Y.grad(X) == 1.0 assert kayak.util.checkgrad(X, Y) < MAX_GRAD_DIFF
def test_matrix_grad(): npr.seed(6) for ii in xrange(NUM_TRIALS): np_pred = npr.randn(10,20) np_targ = npr.randn(10,20) pred = kayak.Parameter(np_pred) targ = kayak.Targets(np_targ) out = kayak.MatSum(kayak.LogMultinomialLoss(pred, targ)) assert kayak.util.checkgrad(pred, out) < MAX_GRAD_DIFF
def test_elemmult_grad_8(): npr.seed(15) for ii in xrange(NUM_TRIALS): np_A = npr.randn(5,6) A = kayak.Parameter(np_A) D = kayak.ElemMult(A, A) E = kayak.MatSum(D) assert E.grad(A).shape == np_A.shape assert kayak.util.checkgrad(A, E) < MAX_GRAD_DIFF
def test_neg_grad_1(): npr.seed(8) for ii in xrange(NUM_TRIALS): np_A = npr.randn(5, 6) A = kayak.Parameter(np_A) C = -A D = kayak.MatSum(C) D.value assert kayak.util.checkgrad(A, D) < MAX_GRAD_DIFF
def test_indexing_grad_2(): npr.seed(3) for ii in xrange(NUM_TRIALS): np_X = npr.randn(6, 2, 7, 3) inds = npr.permutation(7)[:5] X = kayak.Parameter(np_X) Y = kayak.Take(X, inds,axis=2) Z = kayak.MatSum(Y) Z.value assert_less(kayak.util.checkgrad(X, Z), MAX_GRAD_DIFF)
def test_softrelu_grad(): npr.seed(2) for ii in xrange(NUM_TRIALS): np_X = npr.randn(6, 5) X = kayak.Parameter(np_X) Y = kayak.SoftReLU(X) Z = kayak.MatSum(Y) Z.value assert np.all(Z.grad(X) >= 0.0) assert kayak.util.checkgrad(X, Z) < MAX_GRAD_DIFF
def test_logistic_grad(): npr.seed(2) for ii in xrange(NUM_TRIALS): np_X = npr.randn(6, 5) X = kayak.Parameter(np_X) Y = kayak.Logistic(X) Z = kayak.MatSum(Y) Z.value assert np.all(Z.grad(X) >= 0.0) assert_less(kayak.util.checkgrad(X, Z), MAX_GRAD_DIFF)
def test_elemabs_grad_2(): npr.seed(9) for ii in xrange(NUM_TRIALS): np_A = npr.randn(1) A = kayak.Parameter(np_A) D = kayak.ElemAbs(A) E = kayak.MatSum(D) E.value assert kayak.util.checkgrad(A, E) < MAX_GRAD_DIFF
def test_elemabs_grad_1(): npr.seed(3) for ii in xrange(NUM_TRIALS): np_A = npr.randn(5,6) A = kayak.Parameter(np_A) C = kayak.ElemAbs(A) D = kayak.MatSum(C) D.value assert kayak.util.checkgrad(A, D) < MAX_GRAD_DIFF
def test_logsoftmax_grad_3(): npr.seed(5) for ii in xrange(NUM_TRIALS): np_X = npr.randn(5, 6) np_T = npr.randint(0, 10, np_X.shape) X = kayak.Parameter(np_X) T = kayak.Targets(np_T) Y = kayak.LogSoftMax(X) Z = kayak.MatSum(kayak.LogMultinomialLoss(Y, T)) assert kayak.util.checkgrad(X, Z) < MAX_GRAD_DIFF
def __init__(self, maxnum, reduced_dims): self.threshold = 1e-2 dummyword = np.zeros((maxnum, 1)) W1 = np.random.randn(reduced_dims, maxnum) * 0.1 W2 = np.random.randn(maxnum, reduced_dims) * 0.1 self.input = ky.Parameter(dummyword) self.W1 = ky.Parameter(W1) self.W2 = ky.Parameter(W2) self.output = ky.MatMult(self.W1, self.input) self.recons = ky.MatMult(self.W2, self.output) self.loss = ky.MatSum(ky.L2Loss(self.recons, self.input)) #self.totloss = ky.MatAdd(self.loss,ky.L2Norm(self.W2,weight=1e-2),ky.L2Norm(self.W1,weight = 1e-2)) self.totloss = self.loss
def test_matrix_grad(): npr.seed(8) for ii in xrange(NUM_TRIALS): npX = npr.randn(10, 20) X = kayak.Parameter(npX) Y = kayak.MatSum(X) # Verify the value. Y.value assert Y.grad(X).shape == npX.shape assert np.all(close_float(Y.grad(X), np.ones(npX.shape))) assert kayak.util.checkgrad(X, Y) < MAX_GRAD_DIFF
def check_tensormult(A_shape, B_shape, axes): np_A = npr.randn(*A_shape) np_B = npr.randn(*B_shape) A = kayak.Parameter(np_A) B = kayak.Parameter(np_B) C = kayak.TensorMult(A, B, axes) D = kayak.Parameter(npr.randn(*C.shape)) L = kayak.MatSum(kayak.ElemMult(C, D)) assert np.all(close_float(C.value, np.tensordot(np_A, np_B, axes))) assert kayak.util.checkgrad(A, L) < MAX_GRAD_DIFF assert kayak.util.checkgrad(B, L) < MAX_GRAD_DIFF
def test_matmult_grad_2(): npr.seed(4) for ii in xrange(NUM_TRIALS): np_A = npr.randn(5, 5) A = kayak.Parameter(np_A) C = kayak.MatMult(A, A) D = kayak.MatSum(C) D.value assert D.grad(A).shape == (5, 5) assert kayak.util.checkgrad(A, D) < MAX_GRAD_DIFF