def test_numeric_gradient(self): l = SoftMaxLayer() x = np.random.rand(3) in_delta = np.random.rand(3) for i, d in enumerate(in_delta): aux_delta = np.zeros(in_delta.size) aux_delta[i] = in_delta[i] l.forward(x) delta = l.backward(aux_delta) gradient = l.numeric_gradient(x) assert_almost_equal(in_delta[i] * gradient[i, :], delta, decimal=5)
def test_forward_backward(self): l = SoftMaxLayer() y = l.forward(np.array([5, 5, 6])) self.assertEqual(y.shape, (3, )) assert_almost_equal(y, np.array([0.2119416, 0.2119416, 0.5761169]), decimal=5) assert_array_equal(1, np.sum(y)) d = l.backward(np.array([2, 3, 6])) self.assertEqual(d.shape, (3, )) #assert_almost_equal(d,np.array([-1.792177 , -1.5802354, 1.2406412])) return
while True: J, dJdy = trainer.learn_throughtime( v, zip(train, target), CrossEntropyLoss(), # NegativeLogLikelihoodLoss(), opt, epochs, window_size) v.save('vanilla.net') str = '' x = to_one_hot_vect(char_to_ix['c'], vocab_size) for i in range(200): y = sm.forward(v.forward(x)) str += ix_to_char[np.random.choice(range(vocab_size), p=y.ravel())] x = to_one_hot_vect(np.argmax(y), vocab_size) print str # print [ix_to_char[np.argmax(t)] for t in train] # print [ix_to_char[np.argmax(t)] for t in target] # # while True: # # J, dJdy = trainer.learn_throghtime( # # v, # # zip(train,target), # # NegativeLogLikelihoodLoss(), # # # GradientDescent(learning_rate=0.0001), # # # GradientDescentMomentum(learning_rate=0.0001,momentum=0.001), # # AdaGrad(learning_rate=0.001),