#### YOUR CODE HERE #### ## # Pare down to a smaller dataset, for speed (optional) ntrain = len(Y_train) ntrain = 5000 X = X_train[:ntrain] Y = Y_train[:ntrain] nepoch = 5 idxiter_random = random.permutation(len(Y)) for i in range(2,nepoch): permut = random.permutation(len(Y)) idxiter_random = concatenate((idxiter_random,permut),axis=0) curve = model.train_sgd(X,Y,idxiter_random,None,400,1000) #minibatch_sgd(X,Y,0.1) #clf3.train_sgd(X_train,y_train,idxiter,None,400,1000) ## Evaluate cross-entropy loss on the dev set, ## then convert to perplexity for your writeup dev_loss = model.compute_mean_loss(X_dev, Y_dev) print dev_loss ## DO NOT CHANGE THIS CELL ## # Report your numbers, after computing dev_loss above. def adjust_loss(loss, funk): return (loss + funk * log(funk))/(1 - funk) print "Unadjusted: %.03f" % exp(dev_loss) print "Adjusted for missing vocab: %.03f" % exp(adjust_loss(dev_loss, fraction_lost))
random.seed(10) L0 = zeros((vocabsize, hdim)) # replace with random init, L0 = 0.1 * random.randn(*L0.shape) # or do in RNNLM.__init__() # test parameters; you probably want to change these model = RNNLM(L0, U0 = L0, alpha=0.1, rseed=10, bptt=3) ntrain = len(Y_train) X = X_train[:ntrain] Y = Y_train[:ntrain] k = 5 indices = range(ntrain) def idxiter_batches(): num_batches = ntrain / k for i in xrange(num_batches): yield random.choice(indices, k) model_output = model.train_sgd(X=X, y=Y, idxiter=idxiter_batches(), printevery=100, costevery=10000) dev_loss = model.compute_mean_loss(X_dev, Y_dev) ## DO NOT CHANGE THIS CELL ## # Report your numbers, after computing dev_loss above. def adjust_loss(loss, funk, q, mode='basic'): if mode == 'basic': # remove freebies only: score if had no UUUNKKK return (loss + funk*log(funk))/(1 - funk) else: # remove freebies, replace with best prediction on remaining return loss + funk*log(funk) - funk*log(q) # q = best unigram frequency from omitted vocab # this is the best expected loss out of that set q = vocab.freq[vocabsize] / sum(vocab.freq[vocabsize:]) print "Unadjusted: %.03f" % exp(dev_loss)
#random.seed(10) nepoch = 1 N = nepoch * len(Y_train) k = 5 # minibatch size fraction_lost = 0.07923163705 #idx=[] #print X_train.size #for i in range(N/k): # idx.append(random.choice(len(Y_train),k)) if method == "RNNLM": L0 = zeros((vocabsize, hdim)) # replace with random init, # or do in RNNLM.__init__() model = RNNLM(L0, U0 = L0, alpha=0.1, bptt=3) idx = epochiter(len(Y_train), nepoch) model.train_sgd(X = X_train, y = Y_train, idxiter = idx, printevery = 500, costevery = 500) dev_loss = model.compute_mean_loss(X_dev, Y_dev) if not os.path.exists("model/" + method): os.makedirs("model/" + method) print "Unadjusted: %.03f" % exp(dev_loss) print "Adjusted for missing vocab: %.03f" % exp(adjust_loss(dev_loss, fraction_lost)) save("model/" + method + "/rnnlm.L.npy", model.sparams.L) save("model/" + method + "/rnnlm.U.npy", model.params.U) save("model/" + method + "/rnnlm.H.npy", model.params.H) print "RNNLM" elif method == "RNNPT": #random.seed(10)