def test(netFile, dataSet, model='RNN', trees=None): if trees == None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s" % netFile with open(netFile, 'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model == 'RNTN'): nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'RNN'): nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'RNN2'): nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN3'): nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'DCNN'): nn = DCNN(opts.wvecDim, opts.ktop, opts.m1, opts.m2, opts.n1, opts.n2, 0, opts.outputDim, opts.numWords, 2, opts.minibatch, rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..." % model cost, correct, guess, total = nn.costAndGrad(trees, test=True) correct_sum = 0 for i in xrange(0, len(correct)): correct_sum += (guess[i] == correct[i]) # TODO # Plot the confusion matrix? conf_arr = np.zeros((5, 5)) for i in xrange(0, len(correct)): current_correct = correct[i] current_guess = guess[i] conf_arr[current_correct][current_guess] += 1.0 makeconf(conf_arr, model, dataSet) print "Cost %f, Acc %f" % (cost, correct_sum / float(total)) return correct_sum / float(total)
def test(netFile, dataSet, model='RNN', trees=None): if trees == None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s" % netFile opts = None with open(netFile, 'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model == 'RNTN'): nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'RNN'): nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'RNN2'): nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (opts.model == 'RNN3'): nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif (model == 'DCNN'): nn = DCNN(opts.wvecDim, opts.ktop, opts.m1, opts.m2, opts.n1, opts.n2, 0, opts.outputDim, opts.numWords, 2, opts.minibatch, rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..." % model cost, correct, guess, total = nn.costAndGrad(trees, test=True) correct_sum = 0 for i in xrange(0, len(correct)): correct_sum += (guess[i] == correct[i]) cm = confusion_matrix(correct, guess) makeconf(cm) plt.savefig("plots/" + opts.model + "/confusion_matrix_" + model + "wvecDim_" + str(opts.wvecDim) + "_middleDim_" + str(opts.middleDim) + ".png") print "Cost %f, Acc %f" % (cost, correct_sum / float(total)) return correct_sum / float(total)
def test(netFile, dataSet, model='RNN', trees=None, confusion_matrix_file=None, acti=None): if trees == None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s" % netFile with open(netFile, 'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model == 'RNTN'): nn = RNTN(wvecDim=opts.wvecDim, outputDim=opts.outputDim, numWords=opts.numWords, mbSize=opts.minibatch, rho=opts.rho, acti=acti) elif (model == 'RNN'): nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) else: raise '%s is not a valid neural network so far only RNTN, RNN' % opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..." % model cost, correct, guess, total = nn.costAndGrad(trees, test=True) correct_sum = 0 for i in xrange(0, len(correct)): correct_sum += (guess[i] == correct[i]) correctSent = 0 for tree in trees: sentLabel = tree.root.label sentPrediction = tree.root.prediction if sentLabel == sentPrediction: correctSent += 1 # Generate confusion matrix #if confusion_matrix_file is not None: # cm = confusion_matrix(correct, guess) # makeconf(cm, confusion_matrix_file) print "%s: Cost %f, Acc %f, Sentence-Level: Acc %f" % ( dataSet, cost, correct_sum / float(total), correctSent / float(len(trees))) return (correct_sum / float(total), correctSent / float(len(trees)))
def test(netFile, dataSet, model="RNN", trees=None): if trees == None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s" % netFile with open(netFile, "r") as fid: opts = pickle.load(fid) _ = pickle.load(fid) if model == "RNTN": nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif model == "RNN": nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch) elif model == "RNN2": nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif opts.model == "RNN3": nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch) elif model == "DCNN": nn = DCNN( opts.wvecDim, opts.ktop, opts.m1, opts.m2, opts.n1, opts.n2, 0, opts.outputDim, opts.numWords, 2, opts.minibatch, rho=1e-4, ) trees = cnn.tree2matrix(trees) else: raise "%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN" % opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..." % model cost, correct, guess, total = nn.costAndGrad(trees, test=True) correct_sum = 0 for i in xrange(0, len(correct)): correct_sum += guess[i] == correct[i] # TODO # Plot the confusion matrix? print "Cost %f, Acc %f" % (cost, correct_sum / float(total)) return correct_sum / float(total)
def test(netFile,dataSet, model='RNN', trees=None,e=100): if trees==None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s"%netFile with open(netFile,'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model=='RNTN'): nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout) elif(model=='RNN'): nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout) elif(model=='RNN2'): nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout) elif(model=='RNN2TANH'): nn = RNN2TANH(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout) elif(model=='RNN3'): nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout) elif(model=='DCNN'): nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, and DCNN'%opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..."%model cost,correct, guess, total = nn.costAndGrad(trees,test=True) correct_sum = 0 for i in xrange(0,len(correct)): correct_sum+=(guess[i]==correct[i]) if e%10==0: labels = range(max(set(correct))+1) correct = np.array(correct) guess = np.array(guess) conf_arr = [] for i in labels: sub_arr = [] for j in labels: sub_arr.append(sum((correct == i) & (guess==j))) conf_arr.append(sub_arr) makeconf(conf_arr,'temp/'+model+'_conf_mat_'+dataSet+'_'+str(e)+'.') print "Cost %f, Acc %f"%(cost,correct_sum/float(total)), print "Pos F1 %f"%(evaluateF1(correct, guess, 2)), "Neg F1 %f"%(evaluateF1(correct, guess, 0)) return correct_sum/float(total)
def test(netFile,dataSet, model='RNN', trees=None): if trees==None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s"%netFile with open(netFile,'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model=='RNTN'): nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) elif(model=='RNN'): nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) elif(model=='RNN2'): nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch) elif(opts.model=='RNN3'): nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch) elif(model=='DCNN'): nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN'%opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..."%model cost,correct, guess, total = nn.costAndGrad(trees,test=True) correct_sum = 0 for i in xrange(0,len(correct)): correct_sum+=(guess[i]==correct[i]) # TODO # Plot the confusion matrix? conf_arr = np.zeros((5,5)) for i in xrange(0,len(correct)): current_correct = correct[i] current_guess = guess[i] conf_arr[current_correct][current_guess] += 1.0 makeconf(conf_arr, model, dataSet) print "Cost %f, Acc %f"%(cost,correct_sum/float(total)) return correct_sum/float(total)
def test(netFile,dataSet, model='RNN', trees=None): if trees==None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s"%netFile opts = None with open(netFile,'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model=='RNTN'): nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) elif(model=='RNN'): nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) elif(model=='RNN2'): nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch) elif(opts.model=='RNN3'): nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch) elif(model=='DCNN'): nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4) trees = cnn.tree2matrix(trees) else: raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN'%opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..."%model cost,correct, guess, total = nn.costAndGrad(trees,test=True) correct_sum = 0 for i in xrange(0,len(correct)): correct_sum+=(guess[i]==correct[i]) cm = confusion_matrix(correct, guess) makeconf(cm) plt.savefig("plots/" + opts.model + "/confusion_matrix_" + model + "wvecDim_" + str(opts.wvecDim) + "_middleDim_" + str(opts.middleDim) + ".png") print "Cost %f, Acc %f"%(cost,correct_sum/float(total)) return correct_sum/float(total)
def test(netFile,dataSet, model='RNN', trees=None, confusion_matrix_file=None, acti=None): if trees==None: trees = tr.loadTrees(dataSet) assert netFile is not None, "Must give model to test" print "Testing netFile %s"%netFile with open(netFile,'r') as fid: opts = pickle.load(fid) _ = pickle.load(fid) if (model=='RNTN'): nn = RNTN(wvecDim=opts.wvecDim,outputDim=opts.outputDim,numWords=opts.numWords,mbSize=opts.minibatch,rho=opts.rho, acti=acti) elif(model=='RNN'): nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch) else: raise '%s is not a valid neural network so far only RNTN, RNN'%opts.model nn.initParams() nn.fromFile(fid) print "Testing %s..."%model cost,correct, guess, total = nn.costAndGrad(trees,test=True) correct_sum = 0 for i in xrange(0,len(correct)): correct_sum+=(guess[i]==correct[i]) correctSent = 0 for tree in trees: sentLabel = tree.root.label sentPrediction = tree.root.prediction if sentLabel == sentPrediction: correctSent += 1 # Generate confusion matrix #if confusion_matrix_file is not None: # cm = confusion_matrix(correct, guess) # makeconf(cm, confusion_matrix_file) print "%s: Cost %f, Acc %f, Sentence-Level: Acc %f"%(dataSet,cost,correct_sum/float(total),correctSent/float(len(trees))) return (correct_sum/float(total), correctSent/float(len(trees)))