def test(netFile, dataSet, model='RNN', trees=None):
if trees == None:
trees = tr.loadTrees(dataSet)
assert netFile is not None, "Must give model to test"
print "Testing netFile %s" % netFile
with open(netFile, 'r') as fid:
opts = pickle.load(fid)
_ = pickle.load(fid)
if (model == 'RNTN'):
nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords,
opts.minibatch)
elif (model == 'RNN'):
nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords,
opts.minibatch)
elif (model == 'RNN2'):
nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim,
opts.numWords, opts.minibatch)
elif (opts.model == 'RNN3'):
nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim,
opts.numWords, opts.minibatch)
elif (model == 'DCNN'):
nn = DCNN(opts.wvecDim,
opts.ktop,
opts.m1,
opts.m2,
opts.n1,
opts.n2,
0,
opts.outputDim,
opts.numWords,
2,
opts.minibatch,
rho=1e-4)
trees = cnn.tree2matrix(trees)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model
nn.initParams()
nn.fromFile(fid)
print "Testing %s..." % model
cost, correct, guess, total = nn.costAndGrad(trees, test=True)
correct_sum = 0
for i in xrange(0, len(correct)):
correct_sum += (guess[i] == correct[i])
# TODO
# Plot the confusion matrix?
conf_arr = np.zeros((5, 5))
for i in xrange(0, len(correct)):
current_correct = correct[i]
current_guess = guess[i]
conf_arr[current_correct][current_guess] += 1.0
makeconf(conf_arr, model, dataSet)
print "Cost %f, Acc %f" % (cost, correct_sum / float(total))
return correct_sum / float(total)
def test(netFile, dataSet, model='RNN', trees=None):
if trees == None:
trees = tr.loadTrees(dataSet)
assert netFile is not None, "Must give model to test"
print "Testing netFile %s" % netFile
opts = None
with open(netFile, 'r') as fid:
opts = pickle.load(fid)
_ = pickle.load(fid)
if (model == 'RNTN'):
nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords,
opts.minibatch)
elif (model == 'RNN'):
nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords,
opts.minibatch)
elif (model == 'RNN2'):
nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim,
opts.numWords, opts.minibatch)
elif (opts.model == 'RNN3'):
nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim,
opts.numWords, opts.minibatch)
elif (model == 'DCNN'):
nn = DCNN(opts.wvecDim,
opts.ktop,
opts.m1,
opts.m2,
opts.n1,
opts.n2,
0,
opts.outputDim,
opts.numWords,
2,
opts.minibatch,
rho=1e-4)
trees = cnn.tree2matrix(trees)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model
nn.initParams()
nn.fromFile(fid)
print "Testing %s..." % model
cost, correct, guess, total = nn.costAndGrad(trees, test=True)
correct_sum = 0
for i in xrange(0, len(correct)):
correct_sum += (guess[i] == correct[i])
cm = confusion_matrix(correct, guess)
makeconf(cm)
plt.savefig("plots/" + opts.model + "/confusion_matrix_" + model +
"wvecDim_" + str(opts.wvecDim) + "_middleDim_" +
str(opts.middleDim) + ".png")
print "Cost %f, Acc %f" % (cost, correct_sum / float(total))
return correct_sum / float(total)
def test(netFile,
dataSet,
model='RNN',
trees=None,
confusion_matrix_file=None,
acti=None):
if trees == None:
trees = tr.loadTrees(dataSet)
assert netFile is not None, "Must give model to test"
print "Testing netFile %s" % netFile
with open(netFile, 'r') as fid:
opts = pickle.load(fid)
_ = pickle.load(fid)
if (model == 'RNTN'):
nn = RNTN(wvecDim=opts.wvecDim,
outputDim=opts.outputDim,
numWords=opts.numWords,
mbSize=opts.minibatch,
rho=opts.rho,
acti=acti)
elif (model == 'RNN'):
nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords,
opts.minibatch)
else:
raise '%s is not a valid neural network so far only RNTN, RNN' % opts.model
nn.initParams()
nn.fromFile(fid)
print "Testing %s..." % model
cost, correct, guess, total = nn.costAndGrad(trees, test=True)
correct_sum = 0
for i in xrange(0, len(correct)):
correct_sum += (guess[i] == correct[i])
correctSent = 0
for tree in trees:
sentLabel = tree.root.label
sentPrediction = tree.root.prediction
if sentLabel == sentPrediction:
correctSent += 1
# Generate confusion matrix
#if confusion_matrix_file is not None:
# cm = confusion_matrix(correct, guess)
# makeconf(cm, confusion_matrix_file)
print "%s: Cost %f, Acc %f, Sentence-Level: Acc %f" % (
dataSet, cost, correct_sum / float(total),
correctSent / float(len(trees)))
return (correct_sum / float(total), correctSent / float(len(trees)))
def test(netFile, dataSet, model="RNN", trees=None):
if trees == None:
trees = tr.loadTrees(dataSet)
assert netFile is not None, "Must give model to test"
print "Testing netFile %s" % netFile
with open(netFile, "r") as fid:
opts = pickle.load(fid)
_ = pickle.load(fid)
if model == "RNTN":
nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch)
elif model == "RNN":
nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch)
elif model == "RNN2":
nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch)
elif opts.model == "RNN3":
nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords, opts.minibatch)
elif model == "DCNN":
nn = DCNN(
opts.wvecDim,
opts.ktop,
opts.m1,
opts.m2,
opts.n1,
opts.n2,
0,
opts.outputDim,
opts.numWords,
2,
opts.minibatch,
rho=1e-4,
)
trees = cnn.tree2matrix(trees)
else:
raise "%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN" % opts.model
nn.initParams()
nn.fromFile(fid)
print "Testing %s..." % model
cost, correct, guess, total = nn.costAndGrad(trees, test=True)
correct_sum = 0
for i in xrange(0, len(correct)):
correct_sum += guess[i] == correct[i]
# TODO
# Plot the confusion matrix?
print "Cost %f, Acc %f" % (cost, correct_sum / float(total))
return correct_sum / float(total)
def test(netFile,dataSet, model='RNN', trees=None,e=100):
if trees==None:
trees = tr.loadTrees(dataSet)
assert netFile is not None, "Must give model to test"
print "Testing netFile %s"%netFile
with open(netFile,'r') as fid:
opts = pickle.load(fid)
_ = pickle.load(fid)
if (model=='RNTN'):
nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout)
elif(model=='RNN'):
nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout)
elif(model=='RNN2'):
nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout)
elif(model=='RNN2TANH'):
nn = RNN2TANH(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout)
elif(model=='RNN3'):
nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout)
elif(model=='DCNN'):
nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4)
trees = cnn.tree2matrix(trees)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, and DCNN'%opts.model
nn.initParams()
nn.fromFile(fid)
print "Testing %s..."%model
cost,correct, guess, total = nn.costAndGrad(trees,test=True)
correct_sum = 0
for i in xrange(0,len(correct)):
correct_sum+=(guess[i]==correct[i])
if e%10==0:
labels = range(max(set(correct))+1)
correct = np.array(correct)
guess = np.array(guess)
conf_arr = []
for i in labels:
sub_arr = []
for j in labels:
sub_arr.append(sum((correct == i) & (guess==j)))
conf_arr.append(sub_arr)
makeconf(conf_arr,'temp/'+model+'_conf_mat_'+dataSet+'_'+str(e)+'.')
print "Cost %f, Acc %f"%(cost,correct_sum/float(total)),
print "Pos F1 %f"%(evaluateF1(correct, guess, 2)), "Neg F1 %f"%(evaluateF1(correct, guess, 0))
return correct_sum/float(total)
def test(netFile,dataSet, model='RNN', trees=None):
if trees==None:
trees = tr.loadTrees(dataSet)
assert netFile is not None, "Must give model to test"
print "Testing netFile %s"%netFile
with open(netFile,'r') as fid:
opts = pickle.load(fid)
_ = pickle.load(fid)
if (model=='RNTN'):
nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(model=='RNN'):
nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(model=='RNN2'):
nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='RNN3'):
nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(model=='DCNN'):
nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4)
trees = cnn.tree2matrix(trees)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN'%opts.model
nn.initParams()
nn.fromFile(fid)
print "Testing %s..."%model
cost,correct, guess, total = nn.costAndGrad(trees,test=True)
correct_sum = 0
for i in xrange(0,len(correct)):
correct_sum+=(guess[i]==correct[i])
# TODO
# Plot the confusion matrix?
conf_arr = np.zeros((5,5))
for i in xrange(0,len(correct)):
current_correct = correct[i]
current_guess = guess[i]
conf_arr[current_correct][current_guess] += 1.0
makeconf(conf_arr, model, dataSet)
print "Cost %f, Acc %f"%(cost,correct_sum/float(total))
return correct_sum/float(total)
def test(netFile,dataSet, model='RNN', trees=None):
if trees==None:
trees = tr.loadTrees(dataSet)
assert netFile is not None, "Must give model to test"
print "Testing netFile %s"%netFile
opts = None
with open(netFile,'r') as fid:
opts = pickle.load(fid)
_ = pickle.load(fid)
if (model=='RNTN'):
nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(model=='RNN'):
nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(model=='RNN2'):
nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='RNN3'):
nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(model=='DCNN'):
nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4)
trees = cnn.tree2matrix(trees)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN'%opts.model
nn.initParams()
nn.fromFile(fid)
print "Testing %s..."%model
cost,correct, guess, total = nn.costAndGrad(trees,test=True)
correct_sum = 0
for i in xrange(0,len(correct)):
correct_sum+=(guess[i]==correct[i])
cm = confusion_matrix(correct, guess)
makeconf(cm)
plt.savefig("plots/" + opts.model + "/confusion_matrix_" + model + "wvecDim_" + str(opts.wvecDim) + "_middleDim_" + str(opts.middleDim) + ".png")
print "Cost %f, Acc %f"%(cost,correct_sum/float(total))
return correct_sum/float(total)
def test(netFile,dataSet, model='RNN', trees=None, confusion_matrix_file=None, acti=None):
if trees==None:
trees = tr.loadTrees(dataSet)
assert netFile is not None, "Must give model to test"
print "Testing netFile %s"%netFile
with open(netFile,'r') as fid:
opts = pickle.load(fid)
_ = pickle.load(fid)
if (model=='RNTN'):
nn = RNTN(wvecDim=opts.wvecDim,outputDim=opts.outputDim,numWords=opts.numWords,mbSize=opts.minibatch,rho=opts.rho, acti=acti)
elif(model=='RNN'):
nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
else:
raise '%s is not a valid neural network so far only RNTN, RNN'%opts.model
nn.initParams()
nn.fromFile(fid)
print "Testing %s..."%model
cost,correct, guess, total = nn.costAndGrad(trees,test=True)
correct_sum = 0
for i in xrange(0,len(correct)):
correct_sum+=(guess[i]==correct[i])
correctSent = 0
for tree in trees:
sentLabel = tree.root.label
sentPrediction = tree.root.prediction
if sentLabel == sentPrediction:
correctSent += 1
# Generate confusion matrix
#if confusion_matrix_file is not None:
# cm = confusion_matrix(correct, guess)
# makeconf(cm, confusion_matrix_file)
print "%s: Cost %f, Acc %f, Sentence-Level: Acc %f"%(dataSet,cost,correct_sum/float(total),correctSent/float(len(trees)))
return (correct_sum/float(total), correctSent/float(len(trees)))
def run(args=None):
usage = "usage : %prog [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--test",
action="store_true",
dest="test",
default=False)
# Optimizer
parser.add_option("--minibatch", dest="minibatch", type="int", default=30)
parser.add_option("--optimizer",
dest="optimizer",
type="string",
default="adagrad")
parser.add_option("--epochs", dest="epochs", type="int", default=50)
parser.add_option("--step", dest="step", type="float", default=1e-2)
parser.add_option("--middleDim", dest="middleDim", type="int", default=10)
parser.add_option("--outputDim", dest="outputDim", type="int", default=5)
parser.add_option("--wvecDim", dest="wvecDim", type="int", default=30)
# for DCNN only
parser.add_option("--ktop", dest="ktop", type="int", default=5)
parser.add_option("--m1", dest="m1", type="int", default=10)
parser.add_option("--m2", dest="m2", type="int", default=7)
parser.add_option("--n1", dest="n1", type="int", default=6)
parser.add_option("--n2", dest="n2", type="int", default=12)
parser.add_option("--outFile",
dest="outFile",
type="string",
default="models/test.bin")
parser.add_option("--inFile",
dest="inFile",
type="string",
default="models/test.bin")
parser.add_option("--data", dest="data", type="string", default="train")
parser.add_option("--model", dest="model", type="string", default="RNN")
(opts, args) = parser.parse_args(args)
# make this false if you dont care about your accuracies per epoch, makes things faster!
evaluate_accuracy_while_training = True
# Testing
if opts.test:
test(opts.inFile, opts.data, opts.model)
return
print "Loading data..."
train_accuracies = []
dev_accuracies = []
# load training data
trees = tr.loadTrees('train')
opts.numWords = len(tr.loadWordMap())
if (opts.model == 'RNTN'):
nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch)
elif (opts.model == 'RNN'):
nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch)
elif (opts.model == 'RNN2'):
nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords,
opts.minibatch)
elif (opts.model == 'RNN3'):
nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords,
opts.minibatch)
elif (opts.model == 'DCNN'):
nn = DCNN(opts.wvecDim,
opts.ktop,
opts.m1,
opts.m2,
opts.n1,
opts.n2,
0,
opts.outputDim,
opts.numWords,
2,
opts.minibatch,
rho=1e-4)
trees = cnn.tree2matrix(trees)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model
nn.initParams()
sgd = optimizer.SGD(nn,
alpha=opts.step,
minibatch=opts.minibatch,
optimizer=opts.optimizer)
dev_trees = tr.loadTrees("dev")
for e in range(opts.epochs):
start = time.time()
print "Running epoch %d" % e
sgd.run(trees)
end = time.time()
print "Time per epoch : %f" % (end - start)
with open(opts.outFile, 'w') as fid:
pickle.dump(opts, fid)
pickle.dump(sgd.costt, fid)
nn.toFile(fid)
if evaluate_accuracy_while_training:
print "testing on training set real quick"
train_accuracies.append(
test(opts.outFile, "train", opts.model, trees))
print "testing on dev set real quick"
dev_accuracies.append(
test(opts.outFile, "dev", opts.model, dev_trees))
# clear the fprop flags in trees and dev_trees
for tree in trees:
tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
for tree in dev_trees:
tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
print "fprop in trees cleared"
if evaluate_accuracy_while_training:
pdb.set_trace()
print train_accuracies
print dev_accuracies
def run( args = None):
usage = "usage : %prog [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--test",action="store_true",dest="test",default=False)
# Optimizer
parser.add_option("--minibatch",dest="minibatch",type="int",default=30)
parser.add_option("--optimizer",dest="optimizer",type="string",default="adagrad")
parser.add_option("--epochs",dest="epochs",type="int",default=50)
parser.add_option("--step",dest="step",type="float",default=1e-2)
parser.add_option("--middleDim",dest="middleDim",type="int",default=10)
parser.add_option("--outputDim",dest="outputDim",type="int",default=5)
parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30)
# for DCNN only
parser.add_option("--ktop",dest="ktop",type="int",default=5)
parser.add_option("--m1",dest="m1",type="int",default=10)
parser.add_option("--m2",dest="m2",type="int",default=7)
parser.add_option("--n1",dest="n1",type="int",default=6)
parser.add_option("--n2",dest="n2",type="int",default=12)
parser.add_option("--outFile",dest="outFile",type="string",default="models/test.bin")
parser.add_option("--inFile",dest="inFile",type="string",default="models/test.bin")
parser.add_option("--data",dest="data",type="string",default="train")
parser.add_option("--model",dest="model",type="string",default="RNN")
(opts,args)=parser.parse_args(args)
# make this false if you dont care about your accuracies per epoch, makes things faster!
evaluate_accuracy_while_training = True
# Testing
if opts.test:
test(opts.inFile, opts.data, opts.model)
return
print "Loading data..."
train_accuracies = []
dev_accuracies = []
# load training data
trees = tr.loadTrees('train')
opts.numWords = len(tr.loadWordMap())
if (opts.model=='RNTN'):
nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='RNN'):
nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='RNN2'):
nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='RNN3'):
nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='DCNN'):
nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4)
trees = cnn.tree2matrix(trees)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN'%opts.model
nn.initParams()
sgd = optimizer.SGD(nn, alpha=opts.step, minibatch=opts.minibatch, optimizer=opts.optimizer)
dev_trees = tr.loadTrees("dev")
for e in range(opts.epochs):
start = time.time()
print "Running epoch %d"%e
sgd.run(trees)
end = time.time()
print "Time per epoch : %f" %(end-start)
with open(opts.outFile,'w') as fid:
pickle.dump(opts,fid)
pickle.dump(sgd.costt,fid)
nn.toFile(fid)
if evaluate_accuracy_while_training:
print "testing on training set real quick"
train_accuracies.append(test(opts.outFile,"train",opts.model,trees))
print "testing on dev set real quick"
dev_accuracies.append(test(opts.outFile,"dev",opts.model,dev_trees))
# clear the fprop flags in trees and dev_trees
for tree in trees:
tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
for tree in dev_trees:
tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
print "fprop in trees cleared"
if evaluate_accuracy_while_training:
pdb.set_trace()
print train_accuracies
print dev_accuracies
def run(args=None):
usage = "usage : %prog [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--test",
action="store_true",
dest="test",
default=False)
# Optimizer
parser.add_option("--minibatch", dest="minibatch", type="int", default=30)
parser.add_option("--optimizer",
dest="optimizer",
type="string",
default="adagrad")
parser.add_option("--epochs", dest="epochs", type="int", default=50)
parser.add_option("--step", dest="step", type="float", default=1e-2)
parser.add_option("--middleDim", dest="middleDim", type="int", default=10)
parser.add_option("--outputDim", dest="outputDim", type="int", default=5)
parser.add_option("--wvecDim", dest="wvecDim", type="int", default=30)
# By @tiagokv, just to ease the first assignment test
parser.add_option("--wvecDimBatch",
dest="wvecDimBatch",
type="string",
default="")
# for DCNN only
parser.add_option("--ktop", dest="ktop", type="int", default=5)
parser.add_option("--m1", dest="m1", type="int", default=10)
parser.add_option("--m2", dest="m2", type="int", default=7)
parser.add_option("--n1", dest="n1", type="int", default=6)
parser.add_option("--n2", dest="n2", type="int", default=12)
parser.add_option("--outFile",
dest="outFile",
type="string",
default="models/test.bin")
parser.add_option("--inFile",
dest="inFile",
type="string",
default="models/test.bin")
parser.add_option("--data", dest="data", type="string", default="train")
parser.add_option("--model", dest="model", type="string", default="RNN")
(opts, args) = parser.parse_args(args)
# make this false if you dont care about your accuracies per epoch, makes things faster!
evaluate_accuracy_while_training = True
# Testing
if opts.test:
test(opts.inFile, opts.data, opts.model)
return
print "Loading data..."
train_accuracies = []
dev_accuracies = []
# load training data
trees = tr.loadTrees('train')
opts.numWords = len(tr.loadWordMap())
if (opts.model == 'RNTN'):
nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch)
elif (opts.model == 'RNN'):
nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch)
elif (opts.model == 'RNN2'):
nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords,
opts.minibatch)
elif (opts.model == 'RNN3'):
nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords,
opts.minibatch)
elif (opts.model == 'DCNN'):
nn = DCNN(opts.wvecDim,
opts.ktop,
opts.m1,
opts.m2,
opts.n1,
opts.n2,
0,
opts.outputDim,
opts.numWords,
2,
opts.minibatch,
rho=1e-4)
trees = cnn.tree2matrix(trees)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model
nn.initParams()
sgd = optimizer.SGD(nn,
alpha=opts.step,
minibatch=opts.minibatch,
optimizer=opts.optimizer)
# assuring folder for plots exists
if (os.path.isdir('plots') == False): os.makedirs('test')
if (os.path.isdir('plots/' + opts.model) == False):
os.makedirs('plots/' + opts.model)
dev_trees = tr.loadTrees("dev")
for e in range(opts.epochs):
start = time.time()
print "Running epoch %d" % e
sgd.run(trees)
end = time.time()
print "Time per epoch : %f" % (end - start)
with open(opts.outFile, 'w') as fid:
pickle.dump(opts, fid)
pickle.dump(sgd.costt, fid)
nn.toFile(fid)
if evaluate_accuracy_while_training:
print "testing on training set real quick"
train_accuracies.append(
test(opts.outFile, "train", opts.model, trees))
print "testing on dev set real quick"
dev_accuracies.append(
test(opts.outFile, "dev", opts.model, dev_trees))
# clear the fprop flags in trees and dev_trees
for tree in trees:
tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
for tree in dev_trees:
tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
print "fprop in trees cleared"
if evaluate_accuracy_while_training:
#pdb.set_trace()
plt.figure()
#Lets set up the plot
plt.title('Accuracy in set per epochs')
plt.plot(range(opts.epochs), train_accuracies, label='train')
plt.plot(range(opts.epochs), dev_accuracies, label='dev')
with open('dev_accu' + opts.model, 'a') as fid:
fid.write(
str(opts.wvecDim) + ',' + str(opts.middleDim) + ',' +
str(dev_accuracies[-1]) + ';')
#plt.axis([0,opts.epochs,0,1])
plt.xlabel('epochs')
plt.ylabel('accuracy')
plt.legend(loc=2, borderaxespad=0.)
#always save with middleDim, even if it's a one-layer RNN
plt.savefig('plots/' + opts.model + '/accuracy_wvec_' +
str(opts.wvecDim) + '_middleDim_' + str(opts.middleDim) +
' .png')
print 'image saved at %s' % os.getcwd()
def run(args=None):
usage = "usage : %prog [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--test",action="store_true",dest="test",default=False)
# Optimizer
parser.add_option("--minibatch",dest="minibatch",type="int",default=30)
parser.add_option("--optimizer",dest="optimizer",type="string",
default="adagrad")
parser.add_option("--epochs",dest="epochs",type="int",default=50)
parser.add_option("--step",dest="step",type="float",default=1e-2)
parser.add_option("--middleDim",dest="middleDim",type="int",default=10)
parser.add_option("--outputDim",dest="outputDim",type="int",default=5)
parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30)
# By @tiagokv, just to ease the first assignment test
parser.add_option("--wvecDimBatch",dest="wvecDimBatch",type="string",default="")
# for DCNN only
parser.add_option("--ktop",dest="ktop",type="int",default=5)
parser.add_option("--m1",dest="m1",type="int",default=10)
parser.add_option("--m2",dest="m2",type="int",default=7)
parser.add_option("--n1",dest="n1",type="int",default=6)
parser.add_option("--n2",dest="n2",type="int",default=12)
parser.add_option("--outFile",dest="outFile",type="string",
default="models/test.bin")
parser.add_option("--inFile",dest="inFile",type="string",
default="models/test.bin")
parser.add_option("--data",dest="data",type="string",default="train")
parser.add_option("--model",dest="model",type="string",default="RNN")
(opts,args)=parser.parse_args(args)
# make this false if you dont care about your accuracies per epoch, makes things faster!
evaluate_accuracy_while_training = True
# Testing
if opts.test:
test(opts.inFile,opts.data,opts.model)
return
print "Loading data..."
train_accuracies = []
dev_accuracies = []
# load training data
trees = tr.loadTrees('train')
opts.numWords = len(tr.loadWordMap())
if (opts.model=='RNTN'):
nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='RNN'):
nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='RNN2'):
nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='RNN3'):
nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='DCNN'):
nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4)
trees = cnn.tree2matrix(trees)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN'%opts.model
nn.initParams()
sgd = optimizer.SGD(nn,alpha=opts.step,minibatch=opts.minibatch,
optimizer=opts.optimizer)
# assuring folder for plots exists
if( os.path.isdir('plots') == False ): os.makedirs('test')
if( os.path.isdir('plots/' + opts.model ) == False ): os.makedirs('plots/' + opts.model)
dev_trees = tr.loadTrees("dev")
for e in range(opts.epochs):
start = time.time()
print "Running epoch %d"%e
sgd.run(trees)
end = time.time()
print "Time per epoch : %f"%(end-start)
with open(opts.outFile,'w') as fid:
pickle.dump(opts,fid)
pickle.dump(sgd.costt,fid)
nn.toFile(fid)
if evaluate_accuracy_while_training:
print "testing on training set real quick"
train_accuracies.append(test(opts.outFile,"train",opts.model,trees))
print "testing on dev set real quick"
dev_accuracies.append(test(opts.outFile,"dev",opts.model,dev_trees))
# clear the fprop flags in trees and dev_trees
for tree in trees:
tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
for tree in dev_trees:
tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
print "fprop in trees cleared"
if evaluate_accuracy_while_training:
#pdb.set_trace()
plt.figure()
#Lets set up the plot
plt.title('Accuracy in set per epochs')
plt.plot(range(opts.epochs),train_accuracies,label='train')
plt.plot(range(opts.epochs),dev_accuracies,label='dev')
with open('dev_accu' + opts.model,'a') as fid:
fid.write(str(opts.wvecDim) + ',' + str(opts.middleDim) + ',' + str(dev_accuracies[-1]) + ';')
#plt.axis([0,opts.epochs,0,1])
plt.xlabel('epochs')
plt.ylabel('accuracy')
plt.legend(loc=2, borderaxespad=0.)
#always save with middleDim, even if it's a one-layer RNN
plt.savefig('plots/' + opts.model + '/accuracy_wvec_' + str(opts.wvecDim) + '_middleDim_' + str(opts.middleDim) + ' .png')
print 'image saved at %s' % os.getcwd()
def run(args=None):
usage = "usage : %prog [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--test",action="store_true",dest="test",default=False)
# Optimizer
parser.add_option("--minibatch",dest="minibatch",type="int",default=30)
parser.add_option("--optimizer",dest="optimizer",type="string",
default="adagrad")
parser.add_option("--epochs",dest="epochs",type="int",default=50)
parser.add_option("--step",dest="step",type="float",default=1e-2)
parser.add_option("--init",dest="init",type="float",default=0.01)
parser.add_option("--outputDim",dest="outputDim",type="int",default=5)
parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30)
parser.add_option("--rho",dest="rho",type="float",default=1e-6)
parser.add_option("--outFile",dest="outFile",type="string",
default="models/test.bin")
parser.add_option("--inFile",dest="inFile",type="string",
default="models/test.bin")
parser.add_option("--data",dest="data",type="string",default="train")
parser.add_option("--model",dest="model",type="string",default="RNTN")
parser.add_option("--maxTrain",dest="maxTrain", type="int", default=-1)
parser.add_option("--activation",dest="acti", type="string", default="tanh")
parser.add_option("--partial",action="store_true",dest="partial",default=False)
parser.add_option("--w2v",dest="w2vmodel", type="string")
(opts,args)=parser.parse_args(args)
# make this false if you dont care about your accuracies per epoch, makes things faster!
evaluate_accuracy_while_training = True
# Testing
if opts.test:
cmfile = opts.inFile + ".confusion_matrix-" + opts.data + ".png"
test(opts.inFile,opts.data,opts.model,acti=opts.acti)
return
print "Loading data..."
embedding = None
wordMap = None
if opts.w2vmodel is not None:
print "Loading pre-trained word2vec model from %s" % opts.w2vmodel
w2v = models.Word2Vec.load(opts.w2vmodel)
embedding, wordMap = readW2v(w2v,opts.wvecDim)
train_accuracies = []
train_rootAccuracies = []
dev_accuracies = []
dev_rootAccuracies = []
# load training data
trees = tr.loadTrees('train',wordMap=wordMap)[:opts.maxTrain] #train.full.15
if opts.maxTrain > -1:
print "Training only on %d trees" % opts.maxTrain
opts.numWords = len(tr.loadWordMap())
if opts.partial==True:
print "Only partial feedback"
if (opts.model=='RNTN'):
nn = RNTN(wvecDim=opts.wvecDim,outputDim=opts.outputDim,numWords=opts.numWords,
mbSize=opts.minibatch,rho=opts.rho, acti=opts.acti, init=opts.init, partial=opts.partial)
else:
raise '%s is not a valid neural network so far only RNTN, RNN'%opts.model
nn.initParams(embedding=embedding)
sgd = optimizer.SGD(nn,alpha=opts.step,minibatch=opts.minibatch,
optimizer=opts.optimizer)
dev_trees = tr.loadTrees("dev") #dev.full.15
for e in range(opts.epochs):
start = time.time()
print "Running epoch %d"%e
sgd.run(trees)
end = time.time()
print "Time per epoch : %f"%(end-start)
with open(opts.outFile,'w') as fid:
pickle.dump(opts,fid)
pickle.dump(sgd.costt,fid)
nn.toFile(fid)
if evaluate_accuracy_while_training:
print "testing on training set"
acc, sacc = test(opts.outFile,"train",opts.model,trees, acti=opts.acti)
train_accuracies.append(acc)
train_rootAccuracies.append(sacc)
print "testing on dev set"
dacc, dsacc = test(opts.outFile,"dev",opts.model,dev_trees, acti=opts.acti)
dev_accuracies.append(dacc)
dev_rootAccuracies.append(dsacc)
# clear the fprop flags and dev_trees
for tree in trees:
tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
for tree in dev_trees:
tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
print "fprop in trees cleared"
if evaluate_accuracy_while_training:
pdb.set_trace()
print train_accuracies
print dev_accuracies
print "on sentence-level:"
print train_rootAccuracies
print dev_rootAccuracies
# Plot train/dev_accuracies
plt.figure()
plt.plot(range(len(train_accuracies)), train_accuracies, label='Train')
plt.plot(range(len(dev_accuracies)), dev_accuracies, label='Dev')
plt.xlabel("Epoch")
plt.ylabel("Accuracy")
plt.legend()
# plot.show()
plt.savefig(opts.outFile + ".accuracy_plot.png")
# Plot train/dev_accuracies
plt.figure()
plt.plot(range(len(train_rootAccuracies)), train_rootAccuracies, label='Train')
plt.plot(range(len(dev_rootAccuracies)), dev_rootAccuracies, label='Dev')
plt.xlabel("Epoch")
plt.ylabel("Accuracy")
plt.legend()
# plot.show()
plt.savefig(opts.outFile + ".sent.accuracy_plot.png")
def run(args=None):
usage = "usage : %prog [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--test",action="store_true",dest="test",default=False)
# Optimizer
parser.add_option("--minibatch",dest="minibatch",type="int",default=30)
parser.add_option("--optimizer",dest="optimizer",type="string",
default="adagrad")
parser.add_option("--epochs",dest="epochs",type="int",default=50)
parser.add_option("--step",dest="step",type="float",default=1e-2)
parser.add_option("--rho",dest="rho",type="float",default=1e-4)
parser.add_option("--middleDim",dest="middleDim",type="int",default=10)
parser.add_option("--outputDim",dest="outputDim",type="int",default=3)
parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30)
# for DCNN only
parser.add_option("--ktop",dest="ktop",type="int",default=5)
parser.add_option("--m1",dest="m1",type="int",default=10)
parser.add_option("--m2",dest="m2",type="int",default=7)
parser.add_option("--n1",dest="n1",type="int",default=6)
parser.add_option("--n2",dest="n2",type="int",default=12)
parser.add_option("--outFile",dest="outFile",type="string",
default="models/test.bin")
parser.add_option("--inFile",dest="inFile",type="string",
default="models/test.bin")
parser.add_option("--data",dest="data",type="string",default="train")
parser.add_option("--model",dest="model",type="string",default="RNN")
parser.add_option("--pretrain",dest="pretrain",default=False)
parser.add_option("--dropout",dest="dropout",default=False)
(opts,args)=parser.parse_args(args)
# make this false if you dont care about your accuracies per epoch
evaluate_accuracy_while_training = True
# Testing
if opts.test:
test(opts.inFile,opts.data,opts.model,e=1000)
return
print "Loading data..."
train_accuracies = []
dev_accuracies = []
# load training data
trees = tr.loadTrees('train')
opts.numWords = len(tr.loadWordMap())
if (opts.model=='RNTN'):
nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout,opts.rho)
elif(opts.model=='RNN'):
nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout,opts.rho)
elif(opts.model=='RNN2'):
nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout,opts.rho)
elif(opts.model=='RNN2TANH'):
nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout,opts.rho)
elif(opts.model=='RNN3'):
nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch,opts.pretrain,opts.dropout,opts.rho)
elif(opts.model=='DCNN'):
nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4)
trees = cnn.tree2matrix(trees)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, and DCNN'%opts.model
nn.initParams()
sgd = optimizer.SGD(nn,alpha=opts.step,minibatch=opts.minibatch,
optimizer=opts.optimizer)
for e in range(opts.epochs):
start = time.time()
print "Running epoch %d"%e
sgd.run(trees)
end = time.time()
print "Time per epoch : %f"%(end-start)
with open(opts.outFile,'w') as fid:
pickle.dump(opts,fid)
pickle.dump(sgd.costt,fid)
nn.toFile(fid)
if evaluate_accuracy_while_training:
print "testing on training set real quick"
train_accuracies.append(test(opts.outFile,"train",opts.model,trees,e))
print "testing on dev set real quick"
dev_accuracies.append(test(opts.outFile,"dev",opts.model,e=e))
if e%10==0:
if evaluate_accuracy_while_training:
print train_accuracies
print dev_accuracies
plt.figure()
plt.plot(train_accuracies,label = 'Train')
plt.plot(dev_accuracies,label = 'Dev')
plt.legend()
plt.savefig('temp/train_dev_accuracies_'+str(opts.model)+'_middle_'+str(opts.middleDim)+'.png')
if evaluate_accuracy_while_training:
print train_accuracies
print dev_accuracies
plt.figure()
plt.plot(train_accuracies,label = 'Train')
plt.plot(dev_accuracies,label = 'Dev')
plt.legend()
plt.savefig('temp/train_dev_accuracies_'+str(opts.model)+'_middle_'+str(opts.middleDim)+'.png')
def run(args=None):
usage = "usage : %prog [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--test",
action="store_true",
dest="test",
default=False)
# Optimizer
parser.add_option("--minibatch", dest="minibatch", type="int", default=30)
parser.add_option("--optimizer",
dest="optimizer",
type="string",
default="adagrad")
parser.add_option("--epochs", dest="epochs", type="int", default=50)
parser.add_option("--step", dest="step", type="float", default=1e-2)
parser.add_option("--init", dest="init", type="float", default=0.01)
parser.add_option("--outputDim", dest="outputDim", type="int", default=5)
parser.add_option("--wvecDim", dest="wvecDim", type="int", default=30)
parser.add_option("--rho", dest="rho", type="float", default=1e-6)
parser.add_option("--outFile",
dest="outFile",
type="string",
default="models/test.bin")
parser.add_option("--inFile",
dest="inFile",
type="string",
default="models/test.bin")
parser.add_option("--data", dest="data", type="string", default="train")
parser.add_option("--model", dest="model", type="string", default="RNTN")
parser.add_option("--maxTrain", dest="maxTrain", type="int", default=-1)
parser.add_option("--activation",
dest="acti",
type="string",
default="tanh")
parser.add_option("--partial",
action="store_true",
dest="partial",
default=False)
parser.add_option("--w2v", dest="w2vmodel", type="string")
(opts, args) = parser.parse_args(args)
# make this false if you dont care about your accuracies per epoch, makes things faster!
evaluate_accuracy_while_training = True
# Testing
if opts.test:
cmfile = opts.inFile + ".confusion_matrix-" + opts.data + ".png"
test(opts.inFile, opts.data, opts.model, acti=opts.acti)
return
print "Loading data..."
embedding = None
wordMap = None
if opts.w2vmodel is not None:
print "Loading pre-trained word2vec model from %s" % opts.w2vmodel
w2v = models.Word2Vec.load(opts.w2vmodel)
embedding, wordMap = readW2v(w2v, opts.wvecDim)
train_accuracies = []
train_rootAccuracies = []
dev_accuracies = []
dev_rootAccuracies = []
# load training data
trees = tr.loadTrees('train',
wordMap=wordMap)[:opts.maxTrain] #train.full.15
if opts.maxTrain > -1:
print "Training only on %d trees" % opts.maxTrain
opts.numWords = len(tr.loadWordMap())
if opts.partial == True:
print "Only partial feedback"
if (opts.model == 'RNTN'):
nn = RNTN(wvecDim=opts.wvecDim,
outputDim=opts.outputDim,
numWords=opts.numWords,
mbSize=opts.minibatch,
rho=opts.rho,
acti=opts.acti,
init=opts.init,
partial=opts.partial)
else:
raise '%s is not a valid neural network so far only RNTN, RNN' % opts.model
nn.initParams(embedding=embedding)
sgd = optimizer.SGD(nn,
alpha=opts.step,
minibatch=opts.minibatch,
optimizer=opts.optimizer)
dev_trees = tr.loadTrees("dev") #dev.full.15
for e in range(opts.epochs):
start = time.time()
print "Running epoch %d" % e
sgd.run(trees)
end = time.time()
print "Time per epoch : %f" % (end - start)
with open(opts.outFile, 'w') as fid:
pickle.dump(opts, fid)
pickle.dump(sgd.costt, fid)
nn.toFile(fid)
if evaluate_accuracy_while_training:
print "testing on training set"
acc, sacc = test(opts.outFile,
"train",
opts.model,
trees,
acti=opts.acti)
train_accuracies.append(acc)
train_rootAccuracies.append(sacc)
print "testing on dev set"
dacc, dsacc = test(opts.outFile,
"dev",
opts.model,
dev_trees,
acti=opts.acti)
dev_accuracies.append(dacc)
dev_rootAccuracies.append(dsacc)
# clear the fprop flags and dev_trees
for tree in trees:
tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
for tree in dev_trees:
tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
print "fprop in trees cleared"
if evaluate_accuracy_while_training:
pdb.set_trace()
print train_accuracies
print dev_accuracies
print "on sentence-level:"
print train_rootAccuracies
print dev_rootAccuracies
# Plot train/dev_accuracies
plt.figure()
plt.plot(range(len(train_accuracies)), train_accuracies, label='Train')
plt.plot(range(len(dev_accuracies)), dev_accuracies, label='Dev')
plt.xlabel("Epoch")
plt.ylabel("Accuracy")
plt.legend()
# plot.show()
plt.savefig(opts.outFile + ".accuracy_plot.png")
# Plot train/dev_accuracies
plt.figure()
plt.plot(range(len(train_rootAccuracies)),
train_rootAccuracies,
label='Train')
plt.plot(range(len(dev_rootAccuracies)),
dev_rootAccuracies,
label='Dev')
plt.xlabel("Epoch")
plt.ylabel("Accuracy")
plt.legend()
# plot.show()
plt.savefig(opts.outFile + ".sent.accuracy_plot.png")