#### YOUR CODE HERE ####
##
# Pare down to a smaller dataset, for speed (optional)
ntrain = len(Y_train)
ntrain = 5000
X = X_train[:ntrain]
Y = Y_train[:ntrain]
nepoch = 5
idxiter_random = random.permutation(len(Y))
for i in range(2,nepoch):
permut = random.permutation(len(Y))
idxiter_random = concatenate((idxiter_random,permut),axis=0)
curve = model.train_sgd(X,Y,idxiter_random,None,400,1000) #minibatch_sgd(X,Y,0.1)
#clf3.train_sgd(X_train,y_train,idxiter,None,400,1000)
## Evaluate cross-entropy loss on the dev set,
## then convert to perplexity for your writeup
dev_loss = model.compute_mean_loss(X_dev, Y_dev)
print dev_loss
## DO NOT CHANGE THIS CELL ##
# Report your numbers, after computing dev_loss above.
def adjust_loss(loss, funk):
return (loss + funk * log(funk))/(1 - funk)
print "Unadjusted: %.03f" % exp(dev_loss)
print "Adjusted for missing vocab: %.03f" % exp(adjust_loss(dev_loss, fraction_lost))
random.seed(10)
L0 = zeros((vocabsize, hdim)) # replace with random init,
L0 = 0.1 * random.randn(*L0.shape) # or do in RNNLM.__init__()
# test parameters; you probably want to change these
model = RNNLM(L0, U0 = L0, alpha=0.1, rseed=10, bptt=3)
ntrain = len(Y_train)
X = X_train[:ntrain]
Y = Y_train[:ntrain]
k = 5
indices = range(ntrain)
def idxiter_batches():
num_batches = ntrain / k
for i in xrange(num_batches):
yield random.choice(indices, k)
model_output = model.train_sgd(X=X, y=Y, idxiter=idxiter_batches(), printevery=100, costevery=10000)
dev_loss = model.compute_mean_loss(X_dev, Y_dev)
## DO NOT CHANGE THIS CELL ##
# Report your numbers, after computing dev_loss above.
def adjust_loss(loss, funk, q, mode='basic'):
if mode == 'basic':
# remove freebies only: score if had no UUUNKKK
return (loss + funk*log(funk))/(1 - funk)
else:
# remove freebies, replace with best prediction on remaining
return loss + funk*log(funk) - funk*log(q)
# q = best unigram frequency from omitted vocab
# this is the best expected loss out of that set
q = vocab.freq[vocabsize] / sum(vocab.freq[vocabsize:])
print "Unadjusted: %.03f" % exp(dev_loss)
#random.seed(10)
nepoch = 1
N = nepoch * len(Y_train)
k = 5 # minibatch size
fraction_lost = 0.07923163705
#idx=[]
#print X_train.size
#for i in range(N/k):
# idx.append(random.choice(len(Y_train),k))
if method == "RNNLM":
L0 = zeros((vocabsize, hdim)) # replace with random init,
# or do in RNNLM.__init__()
model = RNNLM(L0, U0 = L0, alpha=0.1, bptt=3)
idx = epochiter(len(Y_train), nepoch)
model.train_sgd(X = X_train, y = Y_train, idxiter = idx, printevery = 500, costevery = 500)
dev_loss = model.compute_mean_loss(X_dev, Y_dev)
if not os.path.exists("model/" + method):
os.makedirs("model/" + method)
print "Unadjusted: %.03f" % exp(dev_loss)
print "Adjusted for missing vocab: %.03f" % exp(adjust_loss(dev_loss, fraction_lost))
save("model/" + method + "/rnnlm.L.npy", model.sparams.L)
save("model/" + method + "/rnnlm.U.npy", model.params.U)
save("model/" + method + "/rnnlm.H.npy", model.params.H)
print "RNNLM"
elif method == "RNNPT":
#random.seed(10)
