costP,_ = self.costAndGrad(data) L[i,j] -= epsilon numGrad = (costP - cost)/epsilon err = np.abs(dL[j][i] - numGrad) err2+=err count+=1 if 1e-5 > err2 / count: print "Grad Check Passed for dL" else: print "Grad Check Failed for dL: Sum of Error = %.9f" % (err2/count) if __name__ == '__main__': import tree as treeM train = treeM.load_trees() numW = len(treeM.load_word_to_index_map()) wvecDim = 4 outputDim = 42 rnn = RNN(wvecDim, outputDim, numW, mbSize=4) rnn.initParams() mbData = train[:4] print "Numerical gradient check..." rnn.check_grad(mbData)
def run(args=None): usage = "usage : %prog [options]" parser = optparse.OptionParser(usage=usage) parser.add_option("--test",action="store_true",dest="test",default=False) # Optimizer parser.add_option("--minibatch",dest="minibatch",type="int",default=30) parser.add_option("--optimizer",dest="optimizer",type="string", default="adagrad") parser.add_option("--epochs",dest="epochs",type="int",default=50) parser.add_option("--step",dest="step",type="float",default=1e-2) parser.add_option("--middleDim",dest="middleDim",type="int",default=10) parser.add_option("--outputDim",dest="outputDim",type="int",default=5) parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30) parser.add_option("--outFile",dest="outFile",type="string", default="models/test.bin") parser.add_option("--inFile",dest="inFile",type="string", default="models/test.bin") parser.add_option("--data",dest="data",type="string",default="train") parser.add_option("--model",dest="model",type="string",default="RNN") (opts, args) = parser.parse_args(args) # make this false if you dont care about your accuracies per epoch, makes things faster! evaluate_accuracy_while_training = True # Testing if opts.test: test(opts.inFile, opts.data, opts.model) return print "Loading data..." train_accuracies = [] dev_accuracies = [] # load training data trees = tr.load_trees(TRAIN_DATA_FILE) opts.numWords = len(tr.load_word_to_index_map()) if (opts.model=='RNTN'): nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) elif(opts.model=='RNN'): nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) elif(opts.model=='RNN2'): nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2' % opts.model nn.initParams() sgd = optimizer.SGD(nn, alpha=opts.step, minibatch=opts.minibatch, optimizer=opts.optimizer) dev_trees = tr.load_trees(DEV_DATA_FILE) for e in range(opts.epochs): start = time.time() print "Running epoch %d" % e sgd.run(trees) end = time.time() print "Time per epoch : %f" % (end-start) # save the net to the output file #f = open(opts.outFile, 'wb') #pickle.dump(opts, f, -1) #pickle.dump(sgd.costt, f, -1) #pickle.dump(nn.stack, f, -1) #np.save(f, nn.stack) #f.close() joblib.dump(opts, opts.outFile + "_opts") joblib.dump(sgd.costt, opts.outFile + "_cost") joblib.dump(nn.stack, opts.outFile + "_stack") if evaluate_accuracy_while_training: print "testing on training set..." train_accuracies.append(test(opts.outFile, "train", opts.model, trees)) print "testing on dev set..." dev_accuracies.append(test(opts.outFile, "dev", opts.model, dev_trees)) # clear the fprop flags in trees and dev_trees for tree in trees: tr.traverse(tree.root, func=tr.clear_fprop) for tree in dev_trees: tr.traverse(tree.root, func=tr.clear_fprop) print "fprop in trees cleared" if False: # don't do this for now #if evaluate_accuracy_while_training: #print train_accuracies #print dev_accuracies # Plot train/dev_accuracies here x = range(opts.epochs) figure(figsize=(6,4)) plot(x, train_accuracies, color='b', marker='o', linestyle='-', label="training") plot(x, dev_accuracies, color='g', marker='o', linestyle='-', label="dev") title("Accuracy vs num epochs.") xlabel("Epochs") ylabel("Accuracy") #ylim(ymin=0, ymax=max(1.1*max(train_accuracies),3*min(train_accuracies))) legend() savefig("train_dev_acc.png")
L[i, j] += epsilon costP, _ = self.costAndGrad(data) L[i, j] -= epsilon numGrad = (costP - cost) / epsilon err = np.abs(dL[j][i] - numGrad) err2 += err count += 1 if 1e-5 > err2 / count: print "Grad Check Passed for dL" else: print "Grad Check Failed for dL: Sum of Error = %.9f" % (err2 / count) if __name__ == '__main__': import tree as treeM train = treeM.load_trees() numW = len(treeM.load_word_to_index_map()) wvecDim = 4 outputDim = 42 rnn = RNN(wvecDim, outputDim, numW, mbSize=4) rnn.initParams() mbData = train[:4] print "Numerical gradient check..." rnn.check_grad(mbData)