def test(netFile, dataSet, model='RNN', trees=None): if trees == None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s" % netFile with open(netFile, 'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model == 'RNTN'): nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'RNN'): nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'RNN2'): nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN3'): nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'DCNN'): nn = DCNN(opts.wvecDim, opts.ktop, opts.m1, opts.m2, opts.n1, opts.n2, 0, opts.outputDim, opts.numWords, 2, opts.minibatch, rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..." % model cost, correct, guess, total = nn.costAndGrad(trees, test=True) correct_sum = 0 for i in xrange(0, len(correct)): correct_sum += (guess[i] == correct[i]) # TODO # Plot the confusion matrix? conf_arr = np.zeros((5, 5)) for i in xrange(0, len(correct)): current_correct = correct[i] current_guess = guess[i] conf_arr[current_correct][current_guess] += 1.0 makeconf(conf_arr, model, dataSet) print "Cost %f, Acc %f" % (cost, correct_sum / float(total)) return correct_sum / float(total)
def test(netFile, dataSet, model='RNN', trees=None): if trees == None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s" % netFile opts = None with open(netFile, 'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model == 'RNTN'): nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'RNN'): nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'RNN2'): nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN3'): nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'DCNN'): nn = DCNN(opts.wvecDim, opts.ktop, opts.m1, opts.m2, opts.n1, opts.n2, 0, opts.outputDim, opts.numWords, 2, opts.minibatch, rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..." % model cost, correct, guess, total = nn.costAndGrad(trees, test=True) correct_sum = 0 for i in xrange(0, len(correct)): correct_sum += (guess[i] == correct[i]) cm = confusion_matrix(correct, guess) makeconf(cm) plt.savefig("plots/" + opts.model + "/confusion_matrix_" + model + "wvecDim_" + str(opts.wvecDim) + "_middleDim_" + str(opts.middleDim) + ".png") print "Cost %f, Acc %f" % (cost, correct_sum / float(total)) return correct_sum / float(total)
def test(netFile, dataSet, model='RNN', trees=None, confusion_matrix_file=None, acti=None): if trees == None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s" % netFile with open(netFile, 'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model == 'RNTN'): nn = RNTN(wvecDim=opts.wvecDim, outputDim=opts.outputDim, numWords=opts.numWords, mbSize=opts.minibatch, rho=opts.rho, acti=acti) elif (model == 'RNN'): nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) else: raise '%s is not a valid neural network so far only RNTN, RNN' % opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..." % model cost, correct, guess, total = nn.costAndGrad(trees, test=True) correct_sum = 0 for i in xrange(0, len(correct)): correct_sum += (guess[i] == correct[i]) correctSent = 0 for tree in trees: sentLabel = tree.root.label sentPrediction = tree.root.prediction if sentLabel == sentPrediction: correctSent += 1 # Generate confusion matrix #if confusion_matrix_file is not None: # cm = confusion_matrix(correct, guess) # makeconf(cm, confusion_matrix_file) print "%s: Cost %f, Acc %f, Sentence-Level: Acc %f" % ( dataSet, cost, correct_sum / float(total), correctSent / float(len(trees))) return (correct_sum / float(total), correctSent / float(len(trees)))
def test(netFile, dataSet, model="RNN", trees=None): if trees == None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s" % netFile with open(netFile, "r") as fid: opts = pickle.load(fid) _ = pickle.load(fid) if model == "RNTN": nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif model == "RNN": nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif model == "RNN2": nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif opts.model == "RNN3": nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif model == "DCNN": nn = DCNN( opts.wvecDim, opts.ktop, opts.m1, opts.m2, opts.n1, opts.n2, 0, opts.outputDim, opts.numWords, 2, opts.minibatch, rho=1e-4, ) trees = cnn.tree2matrix(trees) else: raise "%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN" % opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..." % model cost, correct, guess, total = nn.costAndGrad(trees, test=True) correct_sum = 0 for i in xrange(0, len(correct)): correct_sum += guess[i] == correct[i] # TODO # Plot the confusion matrix? print "Cost %f, Acc %f" % (cost, correct_sum / float(total)) return correct_sum / float(total)
def test(netFile,dataSet, model='RNN', trees=None,e=100): if trees==None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s"%netFile with open(netFile,'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model=='RNTN'): nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout) elif(model=='RNN'): nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout) elif(model=='RNN2'): nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout) elif(model=='RNN2TANH'): nn = RNN2TANH(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout) elif(model=='RNN3'): nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout) elif(model=='DCNN'): nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, and DCNN'%opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..."%model cost,correct, guess, total = nn.costAndGrad(trees,test=True) correct_sum = 0 for i in xrange(0,len(correct)): correct_sum+=(guess[i]==correct[i]) if e%10==0: labels = range(max(set(correct))+1) correct = np.array(correct) guess = np.array(guess) conf_arr = [] for i in labels: sub_arr = [] for j in labels: sub_arr.append(sum((correct == i) & (guess==j))) conf_arr.append(sub_arr) makeconf(conf_arr,'temp/'+model+'_conf_mat_'+dataSet+'_'+str(e)+'.') print "Cost %f, Acc %f"%(cost,correct_sum/float(total)), print "Pos F1 %f"%(evaluateF1(correct, guess, 2)), "Neg F1 %f"%(evaluateF1(correct, guess, 0)) return correct_sum/float(total)
def test(netFile,dataSet, model='RNN', trees=None): if trees==None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s"%netFile with open(netFile,'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model=='RNTN'): nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) elif(model=='RNN'): nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) elif(model=='RNN2'): nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch) elif(opts.model=='RNN3'): nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch) elif(model=='DCNN'): nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN'%opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..."%model cost,correct, guess, total = nn.costAndGrad(trees,test=True) correct_sum = 0 for i in xrange(0,len(correct)): correct_sum+=(guess[i]==correct[i]) # TODO # Plot the confusion matrix? conf_arr = np.zeros((5,5)) for i in xrange(0,len(correct)): current_correct = correct[i] current_guess = guess[i] conf_arr[current_correct][current_guess] += 1.0 makeconf(conf_arr, model, dataSet) print "Cost %f, Acc %f"%(cost,correct_sum/float(total)) return correct_sum/float(total)
def test(netFile,dataSet, model='RNN', trees=None): if trees==None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s"%netFile opts = None with open(netFile,'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model=='RNTN'): nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) elif(model=='RNN'): nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) elif(model=='RNN2'): nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch) elif(opts.model=='RNN3'): nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch) elif(model=='DCNN'): nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN'%opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..."%model cost,correct, guess, total = nn.costAndGrad(trees,test=True) correct_sum = 0 for i in xrange(0,len(correct)): correct_sum+=(guess[i]==correct[i]) cm = confusion_matrix(correct, guess) makeconf(cm) plt.savefig("plots/" + opts.model + "/confusion_matrix_" + model + "wvecDim_" + str(opts.wvecDim) + "_middleDim_" + str(opts.middleDim) + ".png") print "Cost %f, Acc %f"%(cost,correct_sum/float(total)) return correct_sum/float(total)
def test(netFile,dataSet, model='RNN', trees=None, confusion_matrix_file=None, acti=None): if trees==None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s"%netFile with open(netFile,'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model=='RNTN'): nn = RNTN(wvecDim=opts.wvecDim,outputDim=opts.outputDim,numWords=opts.numWords,mbSize=opts.minibatch,rho=opts.rho, acti=acti) elif(model=='RNN'): nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) else: raise '%s is not a valid neural network so far only RNTN, RNN'%opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..."%model cost,correct, guess, total = nn.costAndGrad(trees,test=True) correct_sum = 0 for i in xrange(0,len(correct)): correct_sum+=(guess[i]==correct[i]) correctSent = 0 for tree in trees: sentLabel = tree.root.label sentPrediction = tree.root.prediction if sentLabel == sentPrediction: correctSent += 1 # Generate confusion matrix #if confusion_matrix_file is not None: # cm = confusion_matrix(correct, guess) # makeconf(cm, confusion_matrix_file) print "%s: Cost %f, Acc %f, Sentence-Level: Acc %f"%(dataSet,cost,correct_sum/float(total),correctSent/float(len(trees))) return (correct_sum/float(total), correctSent/float(len(trees)))
def run(args=None): usage = "usage : %prog [options]" parser = optparse.OptionParser(usage=usage) parser.add_option("--test",action="store_true",dest="test",default=False) # Optimizer parser.add_option("--minibatch",dest="minibatch",type="int",default=30) parser.add_option("--optimizer",dest="optimizer",type="string", default="adagrad") parser.add_option("--epochs",dest="epochs",type="int",default=50) parser.add_option("--step",dest="step",type="float",default=1e-2) parser.add_option("--middleDim",dest="middleDim",type="int",default=10) parser.add_option("--outputDim",dest="outputDim",type="int",default=5) parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30) # for DCNN only parser.add_option("--ktop",dest="ktop",type="int",default=5) parser.add_option("--m1",dest="m1",type="int",default=10) parser.add_option("--m2",dest="m2",type="int",default=7) parser.add_option("--n1",dest="n1",type="int",default=6) parser.add_option("--n2",dest="n2",type="int",default=12) parser.add_option("--outFile",dest="outFile",type="string", default="models/test.bin") parser.add_option("--inFile",dest="inFile",type="string", default="models/test.bin") parser.add_option("--data",dest="data",type="string",default="train") parser.add_option("--model",dest="model",type="string",default="RNN") (opts,args)=parser.parse_args(args) # make this false if you dont care about your accuracies per epoch, makes things faster! evaluate_accuracy_while_training = True # Testing if opts.test: test(opts.inFile,0,opts.data,opts.model) return print "Loading data..." train_accuracies = [] dev_accuracies = [] # load training data trees = tr.loadTrees('train') opts.numWords = len(tr.loadWordMap()) if (opts.model=='RNTN'): nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) elif(opts.model=='RNN'): nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) elif(opts.model=='RNN2'): nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch) elif(opts.model=='RNN3'): nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch) elif(opts.model=='DCNN'): nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN'%opts.model assert opts.inFile is not None, "Must give pre-trained model to continue training" print "Training netFile %s"%opts.inFile with open(opts.inFile,'r') as fin: opts = pickle.load(fin) _ = pickle.load(fin) nn.initParams() nn.fromFile(fin) sgd = optimizer.SGD(nn,alpha=opts.step,minibatch=opts.minibatch, optimizer=opts.optimizer) dev_trees = tr.loadTrees("dev") for e in range(opts.epochs): start = time.time() print "Running epoch %d"%e sgd.run(trees) end = time.time() print "Time per epoch : %f"%(end-start) outfile_name = opts.outFile#.split('.')[0] + '_epochs_' +str(e) + '.' + opts.outFile.split('.')[1] with open(outfile_name,'w') as fid: pickle.dump(opts,fid) pickle.dump(sgd.costt,fid) nn.toFile(fid) if evaluate_accuracy_while_training: print "testing on training set real quick" train_accuracies.append(test(outfile_name,e,"train",opts.model,trees)) print "testing on dev set real quick" dev_accuracies.append(test(outfile_name,e,"dev",opts.model,dev_trees)) # clear the fprop flags in trees and dev_trees for tree in trees: tr.leftTraverse(tree.root,nodeFn=tr.clearFprop) for tree in dev_trees: tr.leftTraverse(tree.root,nodeFn=tr.clearFprop) print "fprop in trees cleared" if evaluate_accuracy_while_training: pdb.set_trace() print train_accuracies print dev_accuracies # TODO: # Plot train/dev_accuracies here? plt.plot(range(opts.epochs), train_accuracies) plt.title('Train accuracies') plt.xlabel('Epochs') plt.ylabel('Accuracies') plt.savefig('images/RNN2_train_accuracies.png') plt.plot(range(opts.epochs), dev_accuracies) plt.title('Dev accuracies') plt.xlabel('Epochs') plt.ylabel('Accuracies') plt.savefig('images/RNN2_dev_accuracies.png')
def run(args=None): usage = "usage : %prog [options]" parser = optparse.OptionParser(usage=usage) parser.add_option("--test", action="store_true", dest="test", default=False) # Optimizer parser.add_option("--minibatch", dest="minibatch", type="int", default=30) parser.add_option("--optimizer", dest="optimizer", type="string", default="adagrad") parser.add_option("--epochs", dest="epochs", type="int", default=50) parser.add_option("--step", dest="step", type="float", default=1e-2) parser.add_option("--middleDim", dest="middleDim", type="int", default=10) parser.add_option("--outputDim", dest="outputDim", type="int", default=5) parser.add_option("--wvecDim", dest="wvecDim", type="int", default=30) # for DCNN only parser.add_option("--ktop", dest="ktop", type="int", default=5) parser.add_option("--m1", dest="m1", type="int", default=10) parser.add_option("--m2", dest="m2", type="int", default=7) parser.add_option("--n1", dest="n1", type="int", default=6) parser.add_option("--n2", dest="n2", type="int", default=12) parser.add_option("--outFile", dest="outFile", type="string", default="models/test.bin") parser.add_option("--inFile", dest="inFile", type="string", default="models/test.bin") parser.add_option("--data", dest="data", type="string", default="train") parser.add_option("--model", dest="model", type="string", default="RNN") (opts, args) = parser.parse_args(args) # make this false if you dont care about your accuracies per epoch, makes things faster! evaluate_accuracy_while_training = True # Testing if opts.test: test(opts.inFile, 0, opts.data, opts.model) return print "Loading data..." train_accuracies = [] dev_accuracies = [] # load training data trees = tr.loadTrees('train') opts.numWords = len(tr.loadWordMap()) if (opts.model == 'RNTN'): nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN'): nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN2'): nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN3'): nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'DCNN'): nn = DCNN(opts.wvecDim, opts.ktop, opts.m1, opts.m2, opts.n1, opts.n2, 0, opts.outputDim, opts.numWords, 2, opts.minibatch, rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model assert opts.inFile is not None, "Must give pre-trained model to continue training" print "Training netFile %s" % opts.inFile with open(opts.inFile, 'r') as fin: opts = pickle.load(fin) _ = pickle.load(fin) nn.initParams() nn.fromFile(fin) sgd = optimizer.SGD(nn, alpha=opts.step, minibatch=opts.minibatch, optimizer=opts.optimizer) dev_trees = tr.loadTrees("dev") for e in range(opts.epochs): start = time.time() print "Running epoch %d" % e sgd.run(trees) end = time.time() print "Time per epoch : %f" % (end - start) outfile_name = opts.outFile #.split('.')[0] + '_epochs_' +str(e) + '.' + opts.outFile.split('.')[1] with open(outfile_name, 'w') as fid: pickle.dump(opts, fid) pickle.dump(sgd.costt, fid) nn.toFile(fid) if evaluate_accuracy_while_training: print "testing on training set real quick" train_accuracies.append( test(outfile_name, e, "train", opts.model, trees)) print "testing on dev set real quick" dev_accuracies.append( test(outfile_name, e, "dev", opts.model, dev_trees)) # clear the fprop flags in trees and dev_trees for tree in trees: tr.leftTraverse(tree.root, nodeFn=tr.clearFprop) for tree in dev_trees: tr.leftTraverse(tree.root, nodeFn=tr.clearFprop) print "fprop in trees cleared" if evaluate_accuracy_while_training: pdb.set_trace() print train_accuracies print dev_accuracies # TODO: # Plot train/dev_accuracies here? plt.plot(range(opts.epochs), train_accuracies) plt.title('Train accuracies') plt.xlabel('Epochs') plt.ylabel('Accuracies') plt.savefig('images/RNN2_train_accuracies.png') plt.plot(range(opts.epochs), dev_accuracies) plt.title('Dev accuracies') plt.xlabel('Epochs') plt.ylabel('Accuracies') plt.savefig('images/RNN2_dev_accuracies.png')