def test(netFile, dataSet, model='RNN', trees=None):
if trees == None:
if dataSet == "train":
trees = tr.load_trees(TRAIN_DATA_FILE)
elif dataSet == "dev":
trees = tr.load_trees(DEV_DATA_FILE)
assert netFile is not None, "Must give model to test"
print "Testing netFile %s" % netFile
#f = open(netFile, 'rb')
#opts = pickle.load(f)
#_ = pickle.load(f)
opts = joblib.load(netFile + "_opts")
_ = joblib.load(netFile + "_cost")
if (model=='RNTN'):
nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(model=='RNN'):
nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(model=='RNN2'):
nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, and RNN2' % opts.model
nn.initParams()
#nn.stack = pickle.load(f)
#nn.stack = np.load(f)
nn.stack = joblib.load(netFile + "_stack")
#f.close()
print "Testing %s..." % model
cost, correct, guess, total = nn.costAndGrad(trees, test=True)
correct_sum = 0
for i in xrange(0, len(correct)):
correct_sum += (guess[i] == correct[i])
# confusion matrix
conf_arr = np.zeros((opts.outputDim, opts.outputDim))
for i in xrange(len(correct)):
curr_correct = correct[i]
curr_guess = guess[i]
conf_arr[curr_correct][curr_guess] += 1.0
#makeconf(conf_arr)
print "Cost %f, Acc %f" % (cost, correct_sum / float(total))
return correct_sum / float(total)
def fit(self, trees, export_filename='models/RNTN.pickle', verbose=False):
import sgd
self.word_map = tr.load_word_map()
self.num_words = len(self.word_map)
self.init_params()
self.optimizer = sgd.SGD(self, self.learning_rate, self.batch_size,
self.optimizer_algorithm)
test_trees = tr.load_trees('test')
with open("log.csv", "a", newline='') as csvfile:
csvwriter = csv.writer(csvfile)
fieldnames = ["Timestamp", "Vector size", "Learning rate",
"Batch size", "Regularization", "Epoch",
"Train cost", "Train accuracy",
"Test cost", "Test accuracy"]
if csvfile.tell() == 0:
csvwriter.writerow(fieldnames)
for epoch in range(self.max_epochs):
print("Running epoch {} ...".format(epoch))
start = time.time()
self.optimizer.optimize(trees)
end = time.time()
print(" Time per epoch = {:.4f}".format(end-start))
# Save the model
self.save(export_filename)
# Test the model on train and test set
train_cost, train_result = self.test(trees)
train_accuracy = 100.0 * train_result.trace() / train_result.sum()
test_cost, test_result = self.test(test_trees)
test_accuracy = 100.0 * test_result.trace() / test_result.sum()
# Append data to CSV file
row = [datetime.now(), self.dim, self.learning_rate,
self.batch_size, self.reg, epoch,
train_cost, train_accuracy,
test_cost, test_accuracy]
csvwriter.writerow(row)
def run(args=None):
usage = "usage : %prog [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--test",action="store_true",dest="test",default=False)
# Optimizer
parser.add_option("--minibatch",dest="minibatch",type="int",default=30)
parser.add_option("--optimizer",dest="optimizer",type="string",
default="adagrad")
parser.add_option("--epochs",dest="epochs",type="int",default=50)
parser.add_option("--step",dest="step",type="float",default=1e-2)
parser.add_option("--middleDim",dest="middleDim",type="int",default=10)
parser.add_option("--outputDim",dest="outputDim",type="int",default=5)
parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30)
parser.add_option("--outFile",dest="outFile",type="string",
default="models/test.bin")
parser.add_option("--inFile",dest="inFile",type="string",
default="models/test.bin")
parser.add_option("--data",dest="data",type="string",default="train")
parser.add_option("--model",dest="model",type="string",default="RNN")
(opts, args) = parser.parse_args(args)
# make this false if you dont care about your accuracies per epoch, makes things faster!
evaluate_accuracy_while_training = True
# Testing
if opts.test:
test(opts.inFile, opts.data, opts.model)
return
print "Loading data..."
train_accuracies = []
dev_accuracies = []
# load training data
trees = tr.load_trees(TRAIN_DATA_FILE)
opts.numWords = len(tr.load_word_to_index_map())
if (opts.model=='RNTN'):
nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='RNN'):
nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='RNN2'):
nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, RNN2' % opts.model
nn.initParams()
sgd = optimizer.SGD(nn, alpha=opts.step, minibatch=opts.minibatch, optimizer=opts.optimizer)
dev_trees = tr.load_trees(DEV_DATA_FILE)
for e in range(opts.epochs):
start = time.time()
print "Running epoch %d" % e
sgd.run(trees)
end = time.time()
print "Time per epoch : %f" % (end-start)
# save the net to the output file
#f = open(opts.outFile, 'wb')
#pickle.dump(opts, f, -1)
#pickle.dump(sgd.costt, f, -1)
#pickle.dump(nn.stack, f, -1)
#np.save(f, nn.stack)
#f.close()
joblib.dump(opts, opts.outFile + "_opts")
joblib.dump(sgd.costt, opts.outFile + "_cost")
joblib.dump(nn.stack, opts.outFile + "_stack")
if evaluate_accuracy_while_training:
print "testing on training set..."
train_accuracies.append(test(opts.outFile, "train", opts.model, trees))
print "testing on dev set..."
dev_accuracies.append(test(opts.outFile, "dev", opts.model, dev_trees))
# clear the fprop flags in trees and dev_trees
for tree in trees:
tr.traverse(tree.root, func=tr.clear_fprop)
for tree in dev_trees:
tr.traverse(tree.root, func=tr.clear_fprop)
print "fprop in trees cleared"
if False: # don't do this for now
#if evaluate_accuracy_while_training:
#print train_accuracies
#print dev_accuracies
# Plot train/dev_accuracies here
x = range(opts.epochs)
figure(figsize=(6,4))
plot(x, train_accuracies, color='b', marker='o', linestyle='-', label="training")
plot(x, dev_accuracies, color='g', marker='o', linestyle='-', label="dev")
title("Accuracy vs num epochs.")
xlabel("Epochs")
ylabel("Accuracy")
#ylim(ymin=0, ymax=max(1.1*max(train_accuracies),3*min(train_accuracies)))
legend()
savefig("train_dev_acc.png")
L[i,j] += epsilon
costP,_ = self.costAndGrad(data)
L[i,j] -= epsilon
numGrad = (costP - cost)/epsilon
err = np.abs(dL[j][i] - numGrad)
err2+=err
count+=1
if 1e-5 > err2 / count:
print "Grad Check Passed for dL"
else:
print "Grad Check Failed for dL: Sum of Error = %.9f" % (err2/count)
if __name__ == '__main__':
import tree as treeM
train = treeM.load_trees()
numW = len(treeM.load_word_to_index_map())
wvecDim = 4
outputDim = 42
rnn = RNN(wvecDim, outputDim, numW, mbSize=4)
rnn.initParams()
mbData = train[:4]
print "Numerical gradient check..."
rnn.check_grad(mbData)
L[i, j] += epsilon
costP, _ = self.costAndGrad(data)
L[i, j] -= epsilon
numGrad = (costP - cost) / epsilon
err = np.abs(dL[j][i] - numGrad)
err2 += err
count += 1
if 1e-5 > err2 / count:
print "Grad Check Passed for dL"
else:
print "Grad Check Failed for dL: Sum of Error = %.9f" % (err2 /
count)
if __name__ == '__main__':
import tree as treeM
train = treeM.load_trees()
numW = len(treeM.load_word_to_index_map())
wvecDim = 4
outputDim = 42
rnn = RNN(wvecDim, outputDim, numW, mbSize=4)
rnn.initParams()
mbData = train[:4]
print "Numerical gradient check..."
rnn.check_grad(mbData)
def main():
# Parse arguments
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-d",
"--dim",
type=int,
default=25,
help="Vector dimension")
parser.add_argument("-k",
"--output-dim",
type=int,
default=5,
help="Number of output classes")
parser.add_argument("-e",
"--epochs",
type=int,
default=50,
help="Maximum number of epochs")
parser.add_argument("-f",
"--dataset",
type=str,
default="train",
choices=['train', 'dev', 'test'],
help="Dataset")
parser.add_argument("-l",
"--learning-rate",
type=float,
default=1e-2,
help="Learning rate")
parser.add_argument("-b",
"--batch-size",
type=int,
default=30,
help="Batch size")
parser.add_argument("-r",
"--reg",
type=float,
default=1e-6,
help="Regularization parameter")
parser.add_argument("-t",
"--test",
action="store_true",
help="Test a model")
parser.add_argument("-m",
"--model",
type=str,
default='models/RNTN.pickle',
help="Model file")
parser.add_argument("-o",
"--optimizer",
type=str,
default='adagrad',
help="Optimizer",
choices=['sgd', 'adagrad'])
args = parser.parse_args()
# Test
if args.test:
print("Testing...")
model = rntn.RNTN.load(args.model)
test_trees = tr.load_trees(args.dataset)
cost, result = model.test(test_trees)
accuracy = 100.0 * result.trace() / result.sum()
print("Cost = {:.2f}, Correct = {:.0f} / {:.0f}, Accuracy = {:.2f} %".
format(cost, result.trace(), result.sum(), accuracy))
else:
# Initialize the model
model = rntn.RNTN(dim=args.dim,
output_dim=args.output_dim,
batch_size=args.batch_size,
reg=args.reg,
learning_rate=args.learning_rate,
max_epochs=args.epochs,
optimizer=args.optimizer)
# Train
train_trees = tr.load_trees(args.dataset)
model.fit(train_trees, export_filename=args.model)