print("Expected loss for random predictions: %f" % np.log(vocabulary_size))
print("Actual loss: %f" %
model_test_forward.calculate_loss(X_train[:1000], y_train[:1000]))
print()
print("#######################")
print("# Test GRADIENT CHECK #")
print("#######################")
# To avoid performing millions of expensive calculations we use
# a smaller vocabulary size for checking.
grad_check_vocab_size = 100
np.random.seed(10) # re-seed the generator
model_test_grad_check = RNNNumpy(grad_check_vocab_size, 10, bptt_truncate=1000)
model_test_grad_check.gradient_check([0, 1, 2, 3], [1, 2, 3, 4])
print()
print("##########################")
print("# Test a single SGD STEP #")
print("##########################")
np.random.seed(10)
model_test_sgd_step = RNNNumpy(vocabulary_size)
model_test_sgd_step.sgd_step(X_train[10], y_train[10], 0.005)
print()
# Train on a small subset of the data to see what happens
print("####################################")
print("# Test TRAINING on a small dataset #")
print("\n -------------aaaaaaaaaa")
# gives the indices of the highest probability predictions for each word:
predictions = model.predict(X_train[10])
print(predictions.shape)
print(predictions)
print("\n -------------bbbbbbbbb")
# Limit to 1000 examples to save time
print("Expected Loss for random predictions: %f" % np.log(vocabulary_size))
print("Actual loss: %f" % model.calculate_loss(X_train[:1000], y_train[:1000]))
# To avoid performing millions of expensive calculations we use a smaller vocabulary size for checking.
grad_check_vocab_size = 100
np.random.seed(10)
modelcheck = RNNNumpy(grad_check_vocab_size, 10, bptt_truncate=1000)
modelcheck.gradient_check([0, 1, 2, 3], [1, 2, 3, 4])
print("\n -------------ccccccccc")
# get a sense of how long it would take to train our network:
np.random.seed(10)
model = RNNNumpy(vocabulary_size)
t1 = time.time()
model.sgd_step(X_train[10], y_train[10],
0.005) #do 1 step of SGD to test updating of para
t2 = time.time()
print("SGD Step time with RNNNumpy: %f milliseconds" % ((t2 - t1) * 1000.))
# Train on a small subset of the data to see what happens
# np.random.seed(10)
# model = RNNNumpy(vocabulary_size)
# losses = train_with_sgd(model, X_train[:100], y_train[:100], nepoch=10, evaluate_loss_after=1)
