def test_numeric_gradient(self):
l = SoftMaxLayer()
x = np.random.rand(3)
in_delta = np.random.rand(3)
for i, d in enumerate(in_delta):
aux_delta = np.zeros(in_delta.size)
aux_delta[i] = in_delta[i]
l.forward(x)
delta = l.backward(aux_delta)
gradient = l.numeric_gradient(x)
assert_almost_equal(in_delta[i] * gradient[i, :], delta, decimal=5)
def test_forward_backward(self):
l = SoftMaxLayer()
y = l.forward(np.array([5, 5, 6]))
self.assertEqual(y.shape, (3, ))
assert_almost_equal(y,
np.array([0.2119416, 0.2119416, 0.5761169]),
decimal=5)
assert_array_equal(1, np.sum(y))
d = l.backward(np.array([2, 3, 6]))
self.assertEqual(d.shape, (3, ))
#assert_almost_equal(d,np.array([-1.792177 , -1.5802354, 1.2406412]))
return
while True:
J, dJdy = trainer.learn_throughtime(
v,
zip(train, target),
CrossEntropyLoss(),
# NegativeLogLikelihoodLoss(),
opt,
epochs,
window_size)
v.save('vanilla.net')
str = ''
x = to_one_hot_vect(char_to_ix['c'], vocab_size)
for i in range(200):
y = sm.forward(v.forward(x))
str += ix_to_char[np.random.choice(range(vocab_size), p=y.ravel())]
x = to_one_hot_vect(np.argmax(y), vocab_size)
print str
# print [ix_to_char[np.argmax(t)] for t in train]
# print [ix_to_char[np.argmax(t)] for t in target]
#
# while True:
# # J, dJdy = trainer.learn_throghtime(
# # v,
# # zip(train,target),
# # NegativeLogLikelihoodLoss(),
# # # GradientDescent(learning_rate=0.0001),
# # # GradientDescentMomentum(learning_rate=0.0001,momentum=0.001),
# # AdaGrad(learning_rate=0.001),