def test(netFile, dataSet, model='RNN', trees=None): if trees == None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s" % netFile with open(netFile, 'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model == 'RNTN'): nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'RNN'): nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'RNN2'): nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN3'): nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'DCNN'): nn = DCNN(opts.wvecDim, opts.ktop, opts.m1, opts.m2, opts.n1, opts.n2, 0, opts.outputDim, opts.numWords, 2, opts.minibatch, rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..." % model cost, correct, guess, total = nn.costAndGrad(trees, test=True) correct_sum = 0 for i in xrange(0, len(correct)): correct_sum += (guess[i] == correct[i]) # TODO # Plot the confusion matrix? conf_arr = np.zeros((5, 5)) for i in xrange(0, len(correct)): current_correct = correct[i] current_guess = guess[i] conf_arr[current_correct][current_guess] += 1.0 makeconf(conf_arr, model, dataSet) print "Cost %f, Acc %f" % (cost, correct_sum / float(total)) return correct_sum / float(total)
def test(netFile, dataSet, model='RNN', trees=None): if trees == None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s" % netFile opts = None with open(netFile, 'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model == 'RNTN'): nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'RNN'): nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'RNN2'): nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN3'): nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'DCNN'): nn = DCNN(opts.wvecDim, opts.ktop, opts.m1, opts.m2, opts.n1, opts.n2, 0, opts.outputDim, opts.numWords, 2, opts.minibatch, rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..." % model cost, correct, guess, total = nn.costAndGrad(trees, test=True) correct_sum = 0 for i in xrange(0, len(correct)): correct_sum += (guess[i] == correct[i]) cm = confusion_matrix(correct, guess) makeconf(cm) plt.savefig("plots/" + opts.model + "/confusion_matrix_" + model + "wvecDim_" + str(opts.wvecDim) + "_middleDim_" + str(opts.middleDim) + ".png") print "Cost %f, Acc %f" % (cost, correct_sum / float(total)) return correct_sum / float(total)
def test(net_file, data_set, label_method, model='RNN', trees=None): if trees is None: trees = tree.load_all(data_set, label_method) assert net_file is not None, "Must give model to test" print "Testing netFile %s" % net_file with open(net_file, 'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if model == 'RNTN': nn = RNTN(opts.wvec_dim, opts.output_dim, opts.num_words, opts.minibatch) elif model == 'RNN': nn = RNN(opts.wvec_dim, opts.output_dim, opts.num_words, opts.minibatch) elif opts.model == 'TreeLSTM': nn = TreeLSTM(opts.wvec_dim, opts.mem_dim, opts.output_dim, opts.num_words, opts.minibatch, rho=opts.rho) elif opts.model == 'TreeTLSTM': nn = TreeTLSTM(opts.wvec_dim, opts.mem_dim, opts.output_dim, opts.num_words, opts.minibatch, rho=opts.rho) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model nn.init_params() nn.from_file(fid) print "Testing %s..." % model cost, correct, guess = nn.cost_and_grad(trees, test=True) correct_sum = 0 for i in xrange(0, len(correct)): correct_sum += (guess[i] == correct[i]) confusion = [[0 for i in range(nn.output_dim)] for j in range(nn.output_dim)] for i, j in zip(correct, guess): confusion[i][j] += 1 # makeconf(confusion) pre, rec, f1, support = metrics.precision_recall_fscore_support( correct, guess) #print "Cost %f, Acc %f" % (cost, correct_sum / float(len(correct))) #return correct_sum / float(len(correct)) f1 = (100 * sum(f1[1:] * support[1:]) / sum(support[1:])) print "Cost %f, F1 %f, Acc %f" % (cost, f1, correct_sum / float(len(correct))) return f1
def test(netFile, dataSet, model='RNN', trees=None, confusion_matrix_file=None, acti=None): if trees == None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s" % netFile with open(netFile, 'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model == 'RNTN'): nn = RNTN(wvecDim=opts.wvecDim, outputDim=opts.outputDim, numWords=opts.numWords, mbSize=opts.minibatch, rho=opts.rho, acti=acti) elif (model == 'RNN'): nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) else: raise '%s is not a valid neural network so far only RNTN, RNN' % opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..." % model cost, correct, guess, total = nn.costAndGrad(trees, test=True) correct_sum = 0 for i in xrange(0, len(correct)): correct_sum += (guess[i] == correct[i]) correctSent = 0 for tree in trees: sentLabel = tree.root.label sentPrediction = tree.root.prediction if sentLabel == sentPrediction: correctSent += 1 # Generate confusion matrix #if confusion_matrix_file is not None: # cm = confusion_matrix(correct, guess) # makeconf(cm, confusion_matrix_file) print "%s: Cost %f, Acc %f, Sentence-Level: Acc %f" % ( dataSet, cost, correct_sum / float(total), correctSent / float(len(trees))) return (correct_sum / float(total), correctSent / float(len(trees)))
def run(args=None): usage = "usage : %prog [options]" parser = optparse.OptionParser(usage=usage) parser.add_option("--test", action="store_true", dest="test", default=False) # Optimizer parser.add_option("--minibatch", dest="minibatch", type="int", default=30) parser.add_option("--optimizer", dest="optimizer", type="string", default="adagrad") parser.add_option("--epochs", dest="epochs", type="int", default=50) parser.add_option("--step", dest="step", type="float", default=1e-2) parser.add_option("--middleDim", dest="middleDim", type="int", default=10) parser.add_option("--outputDim", dest="outputDim", type="int", default=5) parser.add_option("--wvecDim", dest="wvecDim", type="int", default=30) # for DCNN only parser.add_option("--ktop", dest="ktop", type="int", default=5) parser.add_option("--m1", dest="m1", type="int", default=10) parser.add_option("--m2", dest="m2", type="int", default=7) parser.add_option("--n1", dest="n1", type="int", default=6) parser.add_option("--n2", dest="n2", type="int", default=12) parser.add_option("--outFile", dest="outFile", type="string", default="models/test.bin") parser.add_option("--inFile", dest="inFile", type="string", default="models/test.bin") parser.add_option("--data", dest="data", type="string", default="train") parser.add_option("--model", dest="model", type="string", default="RNN") (opts, args) = parser.parse_args(args) # make this false if you dont care about your accuracies per epoch, makes things faster! evaluate_accuracy_while_training = True # Testing if opts.test: test(opts.inFile, opts.data, opts.model) return print "Loading data..." train_accuracies = [] dev_accuracies = [] # load training data trees = tr.loadTrees('train') opts.numWords = len(tr.loadWordMap()) if (opts.model == 'RNTN'): nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN'): nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN2'): nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN3'): nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'DCNN'): nn = DCNN(opts.wvecDim, opts.ktop, opts.m1, opts.m2, opts.n1, opts.n2, 0, opts.outputDim, opts.numWords, 2, opts.minibatch, rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model nn.initParams() sgd = optimizer.SGD(nn, alpha=opts.step, minibatch=opts.minibatch, optimizer=opts.optimizer) dev_trees = tr.loadTrees("dev") for e in range(opts.epochs): start = time.time() print "Running epoch %d" % e sgd.run(trees) end = time.time() print "Time per epoch : %f" % (end - start) with open(opts.outFile, 'w') as fid: pickle.dump(opts, fid) pickle.dump(sgd.costt, fid) nn.toFile(fid) if evaluate_accuracy_while_training: print "testing on training set real quick" train_accuracies.append( test(opts.outFile, "train", opts.model, trees)) print "testing on dev set real quick" dev_accuracies.append( test(opts.outFile, "dev", opts.model, dev_trees)) # clear the fprop flags in trees and dev_trees for tree in trees: tr.leftTraverse(tree.root, nodeFn=tr.clearFprop) for tree in dev_trees: tr.leftTraverse(tree.root, nodeFn=tr.clearFprop) print "fprop in trees cleared" if evaluate_accuracy_while_training: pdb.set_trace() print train_accuracies print dev_accuracies
def run(args=None): usage = "usage : %prog [options]" parser = optparse.OptionParser(usage=usage) parser.add_option("--test", action="store_true", dest="test", default=False) # Optimizer parser.add_option("--minibatch", dest="minibatch", type="int", default=30) parser.add_option("--optimizer", dest="optimizer", type="string", default="adagrad") parser.add_option("--epochs", dest="epochs", type="int", default=50) parser.add_option("--step", dest="step", type="float", default=5e-2) parser.add_option("--rho", dest="rho", type="float", default=1e-3) # Dimension parser.add_option("--wvecDim", dest="wvec_dim", type="int", default=30) parser.add_option("--memDim", dest="mem_dim", type="int", default=30) parser.add_option("--outFile", dest="out_file", type="string", default="models/test.bin") parser.add_option("--inFile", dest="in_file", type="string", default="models/test.bin") parser.add_option("--data", dest="data", type="string", default="train") parser.add_option("--model", dest="model", type="string", default="RNN") parser.add_option("--label", dest="label_method", type="string", default="rating") (opts, args) = parser.parse_args(args) evaluate_accuracy_while_training = True if opts.label_method == 'rating': label_method = tree.rating_label opts.output_dim = 5 elif opts.label_method == 'aspect': label_method = tree.aspect_label opts.output_dim = 5 elif opts.label_method == 'pair': label_method = tree.pair_label opts.output_dim = 25 else: raise '%s is not a valid labelling method.' % opts.label_method # Testing if opts.test: test(opts.in_file, opts.data, label_method, opts.model) return print "Loading data..." train_accuracies = [] dev_accuracies = [] # load training data trees = tree.load_trees('./data/train.json', label_method) training_word_map = tree.load_word_map() opts.num_words = len(training_word_map) tree.convert_trees(trees, training_word_map) labels = [each.label for each in trees] count = np.zeros(opts.output_dim) for label in labels: count[label] += 1 # weight = 10 / (count ** 0.1) weight = np.ones(opts.output_dim) if opts.model == 'RNTN': nn = RNTN(opts.wvec_dim, opts.output_dim, opts.num_words, opts.minibatch, rho=opts.rho) elif opts.model == 'RNN': nn = RNN(opts.wvec_dim, opts.output_dim, opts.num_words, opts.minibatch, rho=opts.rho, weight=weight) elif opts.model == 'TreeLSTM': nn = TreeLSTM(opts.wvec_dim, opts.mem_dim, opts.output_dim, opts.num_words, opts.minibatch, rho=opts.rho) elif opts.model == 'TreeTLSTM': nn = TreeTLSTM(opts.wvec_dim, opts.mem_dim, opts.output_dim, opts.num_words, opts.minibatch, rho=opts.rho) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model nn.init_params() sgd = optimizer.SGD(nn, alpha=opts.step, minibatch=opts.minibatch, optimizer=opts.optimizer) dev_trees = tree.load_trees('./data/dev.json', label_method) tree.convert_trees(dev_trees, training_word_map) for e in range(opts.epochs): start = time.time() print "Running epoch %d" % e sgd.run(trees, e) end = time.time() print "Time per epoch : %f" % (end - start) with open(opts.out_file, 'w') as fid: pickle.dump(opts, fid) pickle.dump(sgd.costt, fid) nn.to_file(fid) if evaluate_accuracy_while_training: # pdb.set_trace() print "testing on training set real quick" train_accuracies.append(test(opts.out_file, "train", label_method, opts.model, trees)) print "testing on dev set real quick" dev_accuracies.append(test(opts.out_file, "dev", label_method, opts.model, dev_trees)) if evaluate_accuracy_while_training: print train_accuracies print dev_accuracies plt.plot(train_accuracies, label='train') plt.plot(dev_accuracies, label='dev') plt.legend(loc=2) plt.axvline(x=np.argmax(dev_accuracies), linestyle='--') plt.show()
def run(args=None): usage = "usage : %prog [options]" parser = optparse.OptionParser(usage=usage) parser.add_option("--test", action="store_true", dest="test", default=False) # Optimizer parser.add_option("--minibatch", dest="minibatch", type="int", default=30) parser.add_option("--optimizer", dest="optimizer", type="string", default="adagrad") parser.add_option("--epochs", dest="epochs", type="int", default=50) parser.add_option("--step", dest="step", type="float", default=1e-2) parser.add_option("--middleDim", dest="middleDim", type="int", default=10) parser.add_option("--outputDim", dest="outputDim", type="int", default=5) parser.add_option("--wvecDim", dest="wvecDim", type="int", default=30) # By @tiagokv, just to ease the first assignment test parser.add_option("--wvecDimBatch", dest="wvecDimBatch", type="string", default="") # for DCNN only parser.add_option("--ktop", dest="ktop", type="int", default=5) parser.add_option("--m1", dest="m1", type="int", default=10) parser.add_option("--m2", dest="m2", type="int", default=7) parser.add_option("--n1", dest="n1", type="int", default=6) parser.add_option("--n2", dest="n2", type="int", default=12) parser.add_option("--outFile", dest="outFile", type="string", default="models/test.bin") parser.add_option("--inFile", dest="inFile", type="string", default="models/test.bin") parser.add_option("--data", dest="data", type="string", default="train") parser.add_option("--model", dest="model", type="string", default="RNN") (opts, args) = parser.parse_args(args) # make this false if you dont care about your accuracies per epoch, makes things faster! evaluate_accuracy_while_training = True # Testing if opts.test: test(opts.inFile, opts.data, opts.model) return print "Loading data..." train_accuracies = [] dev_accuracies = [] # load training data trees = tr.loadTrees('train') opts.numWords = len(tr.loadWordMap()) if (opts.model == 'RNTN'): nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN'): nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN2'): nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN3'): nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'DCNN'): nn = DCNN(opts.wvecDim, opts.ktop, opts.m1, opts.m2, opts.n1, opts.n2, 0, opts.outputDim, opts.numWords, 2, opts.minibatch, rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model nn.initParams() sgd = optimizer.SGD(nn, alpha=opts.step, minibatch=opts.minibatch, optimizer=opts.optimizer) # assuring folder for plots exists if (os.path.isdir('plots') == False): os.makedirs('test') if (os.path.isdir('plots/' + opts.model) == False): os.makedirs('plots/' + opts.model) dev_trees = tr.loadTrees("dev") for e in range(opts.epochs): start = time.time() print "Running epoch %d" % e sgd.run(trees) end = time.time() print "Time per epoch : %f" % (end - start) with open(opts.outFile, 'w') as fid: pickle.dump(opts, fid) pickle.dump(sgd.costt, fid) nn.toFile(fid) if evaluate_accuracy_while_training: print "testing on training set real quick" train_accuracies.append( test(opts.outFile, "train", opts.model, trees)) print "testing on dev set real quick" dev_accuracies.append( test(opts.outFile, "dev", opts.model, dev_trees)) # clear the fprop flags in trees and dev_trees for tree in trees: tr.leftTraverse(tree.root, nodeFn=tr.clearFprop) for tree in dev_trees: tr.leftTraverse(tree.root, nodeFn=tr.clearFprop) print "fprop in trees cleared" if evaluate_accuracy_while_training: #pdb.set_trace() plt.figure() #Lets set up the plot plt.title('Accuracy in set per epochs') plt.plot(range(opts.epochs), train_accuracies, label='train') plt.plot(range(opts.epochs), dev_accuracies, label='dev') with open('dev_accu' + opts.model, 'a') as fid: fid.write( str(opts.wvecDim) + ',' + str(opts.middleDim) + ',' + str(dev_accuracies[-1]) + ';') #plt.axis([0,opts.epochs,0,1]) plt.xlabel('epochs') plt.ylabel('accuracy') plt.legend(loc=2, borderaxespad=0.) #always save with middleDim, even if it's a one-layer RNN plt.savefig('plots/' + opts.model + '/accuracy_wvec_' + str(opts.wvecDim) + '_middleDim_' + str(opts.middleDim) + ' .png') print 'image saved at %s' % os.getcwd()
def run(args=None): usage = "usage : %prog [options]" parser = optparse.OptionParser(usage=usage) parser.add_option("--test", action="store_true", dest="test", default=False) # Optimizer parser.add_option("--minibatch", dest="minibatch", type="int", default=30) parser.add_option("--optimizer", dest="optimizer", type="string", default="adagrad") parser.add_option("--epochs", dest="epochs", type="int", default=50) parser.add_option("--step", dest="step", type="float", default=1e-2) parser.add_option("--init", dest="init", type="float", default=0.01) parser.add_option("--outputDim", dest="outputDim", type="int", default=5) parser.add_option("--wvecDim", dest="wvecDim", type="int", default=30) parser.add_option("--rho", dest="rho", type="float", default=1e-6) parser.add_option("--outFile", dest="outFile", type="string", default="models/test.bin") parser.add_option("--inFile", dest="inFile", type="string", default="models/test.bin") parser.add_option("--data", dest="data", type="string", default="train") parser.add_option("--model", dest="model", type="string", default="RNTN") parser.add_option("--maxTrain", dest="maxTrain", type="int", default=-1) parser.add_option("--activation", dest="acti", type="string", default="tanh") parser.add_option("--partial", action="store_true", dest="partial", default=False) parser.add_option("--w2v", dest="w2vmodel", type="string") (opts, args) = parser.parse_args(args) # make this false if you dont care about your accuracies per epoch, makes things faster! evaluate_accuracy_while_training = True # Testing if opts.test: cmfile = opts.inFile + ".confusion_matrix-" + opts.data + ".png" test(opts.inFile, opts.data, opts.model, acti=opts.acti) return print "Loading data..." embedding = None wordMap = None if opts.w2vmodel is not None: print "Loading pre-trained word2vec model from %s" % opts.w2vmodel w2v = models.Word2Vec.load(opts.w2vmodel) embedding, wordMap = readW2v(w2v, opts.wvecDim) train_accuracies = [] train_rootAccuracies = [] dev_accuracies = [] dev_rootAccuracies = [] # load training data trees = tr.loadTrees('train', wordMap=wordMap)[:opts.maxTrain] #train.full.15 if opts.maxTrain > -1: print "Training only on %d trees" % opts.maxTrain opts.numWords = len(tr.loadWordMap()) if opts.partial == True: print "Only partial feedback" if (opts.model == 'RNTN'): nn = RNTN(wvecDim=opts.wvecDim, outputDim=opts.outputDim, numWords=opts.numWords, mbSize=opts.minibatch, rho=opts.rho, acti=opts.acti, init=opts.init, partial=opts.partial) else: raise '%s is not a valid neural network so far only RNTN, RNN' % opts.model nn.initParams(embedding=embedding) sgd = optimizer.SGD(nn, alpha=opts.step, minibatch=opts.minibatch, optimizer=opts.optimizer) dev_trees = tr.loadTrees("dev") #dev.full.15 for e in range(opts.epochs): start = time.time() print "Running epoch %d" % e sgd.run(trees) end = time.time() print "Time per epoch : %f" % (end - start) with open(opts.outFile, 'w') as fid: pickle.dump(opts, fid) pickle.dump(sgd.costt, fid) nn.toFile(fid) if evaluate_accuracy_while_training: print "testing on training set" acc, sacc = test(opts.outFile, "train", opts.model, trees, acti=opts.acti) train_accuracies.append(acc) train_rootAccuracies.append(sacc) print "testing on dev set" dacc, dsacc = test(opts.outFile, "dev", opts.model, dev_trees, acti=opts.acti) dev_accuracies.append(dacc) dev_rootAccuracies.append(dsacc) # clear the fprop flags and dev_trees for tree in trees: tr.leftTraverse(tree.root, nodeFn=tr.clearFprop) for tree in dev_trees: tr.leftTraverse(tree.root, nodeFn=tr.clearFprop) print "fprop in trees cleared" if evaluate_accuracy_while_training: pdb.set_trace() print train_accuracies print dev_accuracies print "on sentence-level:" print train_rootAccuracies print dev_rootAccuracies # Plot train/dev_accuracies plt.figure() plt.plot(range(len(train_accuracies)), train_accuracies, label='Train') plt.plot(range(len(dev_accuracies)), dev_accuracies, label='Dev') plt.xlabel("Epoch") plt.ylabel("Accuracy") plt.legend() # plot.show() plt.savefig(opts.outFile + ".accuracy_plot.png") # Plot train/dev_accuracies plt.figure() plt.plot(range(len(train_rootAccuracies)), train_rootAccuracies, label='Train') plt.plot(range(len(dev_rootAccuracies)), dev_rootAccuracies, label='Dev') plt.xlabel("Epoch") plt.ylabel("Accuracy") plt.legend() # plot.show() plt.savefig(opts.outFile + ".sent.accuracy_plot.png")