def __init__(self, maxnum, reduced_dims):
self.threshold = 1e-2
dummyword = np.zeros((maxnum, 1))
W1 = np.random.randn(reduced_dims, maxnum) * 0.1
W2 = np.random.randn(maxnum, reduced_dims) * 0.1
self.input = ky.Parameter(dummyword)
self.W1 = ky.Parameter(W1)
self.W2 = ky.Parameter(W2)
self.output = ky.MatMult(self.W1, self.input)
self.recons = ky.MatMult(self.W2, self.output)
self.loss = ky.MatSum(ky.L2Loss(self.recons, self.input))
#self.totloss = ky.MatAdd(self.loss,ky.L2Norm(self.W2,weight=1e-2),ky.L2Norm(self.W1,weight = 1e-2))
self.totloss = self.loss
def test_reshape_2():
npr.seed(2)
np_A = npr.randn(5,10)
A = kayak.Parameter(np_A)
B = kayak.Reshape(A, (2,25))
C = kayak.Parameter(npr.randn(25,5))
D = kayak.MatMult(B, C)
out = kayak.MatSum(D)
out.value
assert out.grad(A).shape == np_A.shape
assert kayak.util.checkgrad(A, out) < MAX_GRAD_DIFF
def test_matadd_values_3():
npr.seed(3)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5, 6)
np_B = npr.randn(1, 6)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
C = kayak.MatAdd(A, B)
assert C.shape == (5, 6)
assert np.all(close_float(C.value, np_A + np_B))
def test_convolve1d_1():
npr.seed(3)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5, 6)
np_B = npr.randn(6, 7)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
C = kayak.Convolve1d(A, B, ncolors=1)
# If the filters are the same size as the data
assert C.value.shape == (5, 7)
def test_matadd_values_7():
npr.seed(7)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5, 6)
np_B = npr.randn(5, 6)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
D = kayak.MatAdd(A, B, A)
assert D.shape == (5, 6)
assert np.all(close_float(D.value, 2 * np_A + np_B))
def test_transpose_3():
npr.seed(3)
np_A = npr.randn(5, 10)
A = kayak.Parameter(np_A)
B = kayak.Transpose(A)
C = kayak.Parameter(npr.randn(5, 5))
D = kayak.MatMult(B, C)
out = kayak.MatSum(D)
out.value
assert out.grad(A).shape == np_A.shape
assert kayak.util.checkgrad(A, out) < MAX_GRAD_DIFF
def test_stacking_grad():
npr.seed(2)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(6,10)
np_B = npr.randn(6,5)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
Y = kayak.Hstack(A, B)
Z = kayak.MatSum(Y)
Z.value
assert_less(kayak.util.checkgrad(A, Z), MAX_GRAD_DIFF)
assert_less(kayak.util.checkgrad(B, Z), MAX_GRAD_DIFF)
def test_keepdims_grad_2():
npr.seed(10)
for ii in xrange(NUM_TRIALS):
npW = npr.randn(5, 10, 20)
npX = npr.randn(5, 10, 20)
W = kayak.Parameter(npW)
X = kayak.Parameter(npX)
Y = W * X
Z = kayak.MatMean(Y, axis=2, keepdims=False)
S = kayak.MatMean(Z)
assert S.grad(W).shape == npW.shape
assert_less(kayak.util.checkgrad(X, S), MAX_GRAD_DIFF)
def test_matmult_values_3():
npr.seed(3)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5, 6)
np_B = npr.randn(6, 7)
np_C = npr.randn(7, 8)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
C = kayak.Parameter(np_C)
D = kayak.MatMult(A, B, C)
assert D.value.shape == (5, 8)
assert np.all(close_float(D.value, np.dot(np_A, np.dot(np_B, np_C))))
def test_elemmult_grad_1():
npr.seed(8)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5,6)
np_B = npr.randn(5,6)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
C = kayak.ElemMult(A, B)
D = kayak.MatSum(C)
D.value
assert kayak.util.checkgrad(A, D) < MAX_GRAD_DIFF
assert kayak.util.checkgrad(B, D) < MAX_GRAD_DIFF
def test_keepdims_grad_2():
npr.seed(10)
for ii in xrange(NUM_TRIALS):
npW = npr.randn(5, 10, 20)
npX = npr.randn(5, 10, 20)
W = kayak.Parameter(npW)
X = kayak.Parameter(npX)
Y = W * X
Z = kayak.MatSum(Y, axis=2, keepdims=False)
S = kayak.MatSum(Z)
assert S.grad(W).shape == npW.shape
# assert np.all(close_float(Z.grad(X), np.ones(npX.shape)))
assert kayak.util.checkgrad(X, S) < MAX_GRAD_DIFF
def test_elemmult_values_5():
npr.seed(2)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5,1)
np_B = npr.randn(1,6)
np_C = npr.randn(1,1)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
C = kayak.Parameter(np_C)
D = kayak.ElemMult(A, B, C)
assert D.shape == (5,6)
assert np.all( close_float(D.value, np_A*np_B*np_C))
def test_matadd_values_2():
npr.seed(2)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5,6)
np_B = npr.randn(5,6)
np_C = npr.randn(5,6)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
C = kayak.Parameter(np_C)
D = A+B+C
assert D.shape == np_A.shape
assert np.all( close_float(D.value, np_A+np_B+np_C))
def test_identity():
npr.seed(1)
np_A = npr.randn(6, 7)
A = kayak.Parameter(np_A)
B = kayak.Identity(A)
assert np.all(close_float(B.value, np_A))
assert np.all(close_float(B.grad(A), np.ones((6, 7))))
def test_elemmult_grad_7():
npr.seed(14)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5,6)
np_B = npr.randn(1,1)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
D = kayak.ElemMult(A, B, A)
E = kayak.MatSum(D)
E.value
assert E.grad(A).shape == np_A.shape
assert E.grad(B).shape == np_B.shape
assert kayak.util.checkgrad(A, E) < MAX_GRAD_DIFF
assert kayak.util.checkgrad(B, E) < MAX_GRAD_DIFF
def test_matadd_grad_7():
npr.seed(14)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5,6)
np_B = npr.randn(5,6)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
D = A+B+A
E = kayak.MatSum(D)
E.value
assert E.grad(A).shape == np_A.shape
assert E.grad(B).shape == np_B.shape
assert kayak.util.checkgrad(A, E) < MAX_GRAD_DIFF
assert kayak.util.checkgrad(B, E) < MAX_GRAD_DIFF
def test_matadd_grad_6():
npr.seed(13)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5,6)
np_B = npr.randn(1,1)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
C = A+B
D = kayak.MatSum(C)
D.value
assert D.grad(A).shape == np_A.shape
assert D.grad(B).shape == np_B.shape
assert kayak.util.checkgrad(A, D) < MAX_GRAD_DIFF
assert kayak.util.checkgrad(B, D) < MAX_GRAD_DIFF
def test_convolve1d_grad_1():
npr.seed(3)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5, 6)
np_B = npr.randn(6, 7)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
C = kayak.Convolve1d(A, B)
D = kayak.MatSum(C)
D.value
assert_equals(D.grad(A).shape, (5, 6))
assert_equals(D.grad(B).shape, (6, 7))
assert_less(kayak.util.checkgrad(A, D), MAX_GRAD_DIFF)
assert_less(kayak.util.checkgrad(B, D), MAX_GRAD_DIFF)
def test_matmult_grad_1():
npr.seed(3)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5, 6)
np_B = npr.randn(6, 7)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
C = kayak.MatMult(A, B)
D = kayak.MatSum(C)
D.value
assert D.grad(A).shape == (5, 6)
assert D.grad(B).shape == (6, 7)
assert kayak.util.checkgrad(A, D) < MAX_GRAD_DIFF
assert kayak.util.checkgrad(B, D) < MAX_GRAD_DIFF
def test_alldropout_values():
npr.seed(2)
# Drop everything out.
np_X = npr.randn(10,20)
X = kayak.Parameter(np_X)
Y = kayak.Dropout(X, drop_prob=1.0)
assert np.all(Y.value == 0.0)
def test_cache_utility():
npr.seed(3)
num_layers = 17
num_dims = 3
X = kayak.Inputs(npr.randn(10, num_dims))
W1 = kayak.Parameter(npr.randn(num_dims, num_dims))
W2 = kayak.Parameter(npr.randn(num_dims, num_dims))
Z = kayak.MatMult(X, W1)
for jj in xrange(num_layers):
Z = kayak.SoftReLU(kayak.MatAdd(kayak.MatMult(Z, W2),
kayak.MatMult(Z, W2)))
out = kayak.MatSum(Z)
assert kayak.util.checkgrad(W1, out) < 1e-4
def test_tanh_values():
npr.seed(1)
for ii in xrange(NUM_TRIALS):
np_X = npr.randn(6, 5)
X = kayak.Parameter(np_X)
Y = kayak.TanH(X)
assert np.all(close_float(np.tanh(np_X), Y.value))
def test_matmult_grad_vect_mat():
npr.seed(5)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(6, )
np_B = npr.randn(6, 7)
np_C = npr.randn(7, )
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
C = kayak.Parameter(np_C)
D = kayak.MatMult(A, B)
E = kayak.MatSum(kayak.ElemMult(C, D))
assert E.grad(A).shape == (6, )
assert E.grad(B).shape == (6, 7)
assert kayak.util.checkgrad(A, E) < MAX_GRAD_DIFF
assert kayak.util.checkgrad(B, E) < MAX_GRAD_DIFF
def test_logistic_values():
npr.seed(1)
for ii in xrange(NUM_TRIALS):
np_X = npr.randn(6, 5)
X = kayak.Parameter(np_X)
Y = kayak.Logistic(X)
assert np.all(close_float(1.0 / (1.0 + np.exp(-np_X)), Y.value))
def test_matadd_grad_2():
npr.seed(9)
for ii in xrange(NUM_TRIALS):
np_A = npr.randn(5,6)
np_B = npr.randn(5,6)
np_C = npr.randn(5,6)
A = kayak.Parameter(np_A)
B = kayak.Parameter(np_B)
C = kayak.Parameter(np_C)
D = A+B+C
E = kayak.MatSum(D)
E.value
assert kayak.util.checkgrad(A, E) < MAX_GRAD_DIFF
assert kayak.util.checkgrad(B, E) < MAX_GRAD_DIFF
assert kayak.util.checkgrad(C, E) < MAX_GRAD_DIFF
def test_vector_value_1():
npr.seed(3)
for ii in xrange(NUM_TRIALS):
npX = npr.randn(10, 1)
X = kayak.Parameter(npX)
Y = kayak.MatSum(X)
# Verify the sum.
assert close_float(Y.value, np.sum(npX))
def test_reshape_1():
npr.seed(1)
np_A = npr.randn(5,10)
A = kayak.Parameter(np_A)
B = kayak.Reshape(A, (25,2))
B.value
assert B.shape == (25,2)
def test_indexing_values():
npr.seed(1)
for ii in xrange(NUM_TRIALS):
np_X = npr.randn(6,10)
inds = npr.permutation(10)[:5]
X = kayak.Parameter(np_X)
Y = kayak.Take(X, inds,axis=1)
assert(np.array_equal(Y.value, np.take(np_X, inds,axis=1)))
def test_softrelu_values():
npr.seed(1)
for ii in xrange(NUM_TRIALS):
np_X = npr.randn(6, 5)
X = kayak.Parameter(np_X)
Y = kayak.SoftReLU(X)
assert np.all(Y.value >= 0.0)
assert np.all(close_float(np.log(1.0 + np.exp(np_X)), Y.value))
def test_matrix_value():
npr.seed(7)
for ii in xrange(NUM_TRIALS):
npX = npr.randn(10, 20)
X = kayak.Parameter(npX)
Y = kayak.MatSum(X)
# Verify the value.
assert close_float(Y.value, np.sum(npX))