# X_train - every words of each sentence except for the last one X_train = np.asarray([[word_to_index[w] for w in sent[:-1]] for sent in tokenized_sentences]) # y_train - every words except for the first one y_train = np.asarray([[word_to_index[w] for w in sent[1:]] for sent in tokenized_sentences]) print() print("############################") print("# Test FORWARD PROPAGATION #") print("############################") model_test_forward_prop = RNNNumpy(vocabulary_size) # Using 10th training example o, s = model_test_forward_prop.forward_propagation(X_train[10]) print(o.shape) # (45, 8000) print(o) # Try calculating a prediction by forward-propagating # with the current weight values # even though they obviously would be very far from optimal predictions = model_test_forward_prop.predict(X_train[10]) print(predictions.shape) print(predictions) # According to the tutorial: Since we have (vocabulary_size) words, so each word # should be predicted, on average, with probability 1/C, which would yield # a loss of L = -1/N * N * log(1/C) = log(C) print("Expected loss for random predictions: %f" % np.log(vocabulary_size))
for sent in tokenized_sentences]) print(X_train[0]) print(y_train[0]) # Print an training data example x_example, y_example = X_train[10], y_train[10] print("x:\n%s\n%s" % (" ".join([index_to_word[x] for x in x_example]), x_example)) print("\ny:\n%s\n%s" % (" ".join([index_to_word[x] for x in y_example]), y_example)) #### Build RNN Numpy model ######## np.random.seed(10) model = RNNNumpy(vocabulary_size) out, s = model.forward_propagation(X_train[10]) print("Size of forward_propagation is:") print(out.shape) print(out) #give the prob of the next words print("\n -------------aaaaaaaaaa") # gives the indices of the highest probability predictions for each word: predictions = model.predict(X_train[10]) print(predictions.shape) print(predictions) print("\n -------------bbbbbbbbb") # Limit to 1000 examples to save time print("Expected Loss for random predictions: %f" % np.log(vocabulary_size)) print("Actual loss: %f" % model.calculate_loss(X_train[:1000], y_train[:1000]))
import preprocess from rnn_numpy import RNNNumpy import numpy as np X_train,y_train,vocabulary_size = preprocess.create_train_data() np.random.seed(10) model = RNNNumpy(vocabulary_size) output, hidden_states = model.forward_propagation(X_train[10]) print output.shape print output predictions = model.predict(X_train[10]) print predictions.shape print predictions # Limit to 1000 examples to save time print "Expected Loss for random predictions: %f" % np.log(vocabulary_size) print "Actual loss: %f" % model.calculate_loss(X_train[:1000], y_train[:1000])
index_to_word = [x[0] for x in vocab] index_to_word.append(unknown_token) word_to_index = dict([(w,i) for i,w in enumerate(index_to_word)]) print "Using vocabulary size %d." % vocabulary_size print "The least frequent word in our vocabulary is '%s' and appeared %d times." % (vocab[-1][0], vocab[-1][1]) # Replace all words not in our vocabulary with the unknown token for i, sent in enumerate(tokenized_sentences): tokenized_sentences[i] = [w if w in word_to_index else unknown_token for w in sent] # Create the training data X_train = np.asarray([[word_to_index[w] for w in sent[:-1]] for sent in tokenized_sentences]) y_train = np.asarray([[word_to_index[w] for w in sent[1:]] for sent in tokenized_sentences]) np.random.seed(10) model = RNNNumpy(vocabulary_size) o, s = model.forward_propagation(X_train[10]) print o.shape print o # model = RNNNumpy(vocabulary_size, hidden_dim=_HIDDEN_DIM) # t1 = time.time() # model.sgd_step(X_train[10], y_train[10], _LEARNING_RATE) # t2 = time.time() # print "SGD Step time: %f milliseconds" % ((t2 - t1) * 1000.) # if _MODEL_FILE != None: # load_model_parameters_numpy(_MODEL_FILE, model) # train_with_sgd(model, X_train, y_train, nepoch=_NEPOCH, learning_rate=_LEARNING_RATE)