def train_dev(self,
xTrain,
sentiment,
xDev,
yDev,
learning_rate=0.001,
epochs=1):
sentiment = torch.Tensor(sentiment)
sentiment = sentiment.type(torch.LongTensor)
optimizer = torch.optim.SGD(self.parameters(), lr=learning_rate)
criterion = nn.NLLLoss()
dev_acc_list = []
training_loss = []
for iteration in range(1, epochs + 1):
avg_loss = 0
avg_count = 0
for i, review in enumerate(xTrain):
if i == 4845:
phoo = 2
hidden = self.initHidden()
self.zero_grad()
for j, word in enumerate(review):
word_tensor = torch.Tensor(word).unsqueeze(0)
output, hidden = self.forward(word_tensor, hidden)
loss = criterion(output, sentiment[i])
loss.backward()
# Print loss for the review
loss_value = loss.item()
training_loss.append(loss_value)
avg_loss += loss_value
avg_count += 1
# print(loss_value)
optimizer.step()
# Add parameters' gradients to their values, multiplied by learning rate
# for p in self.parameters():
# p.data.add_(p.grad.data, alpha=-learning_rate)
print(avg_loss / avg_count)
yPredict = self.predict(xDev)
dev_acc = nn_tools.Accuracy(yDev, yPredict)
dev_acc_list.append(dev_acc)
print("Loss over iteration")
print(training_loss)
print("Dev Accuracy")
print(dev_acc_list)
return
def ExecuteTuning(runSpecification, datasets, xTrainRaw, yTrain, yTrainList, xDevRaw, yDev, yDevList):
startTime = time.time()
# Look for the model we need
word2vec_name = "word2vec_tuning" + "_" + str(runSpecification['num_features']) + "_" + str(runSpecification['context'])
print(word2vec_name)
try:
print("LOADING Word2Vec Model")
word2vec_model = Word2Vec.load(word2vec_name)
except Exception as ex:
print(ex)
print("Could't load Word2Vec, BUILDING...")
word2vec_model = generate_word2vec(word2vec_name, datasets, runSpecification['num_features'], runSpecification['context'])
# Get doc2vec
xTrain = getFeatureList(xTrainRaw, word2vec_model, runSpecification['num_features'], num_splits=runSpecification['num_splits'])
xDev = getFeatureList(xDevRaw, word2vec_model, runSpecification['num_features'], num_splits=runSpecification['num_splits'])
# Step 4 Training NN model
# MODEL_PATH = "ff_" + str(runSpecification['num_features']) + "_" + str(runSpecification['context']) + ".pt"
MODEL_PATH = "rnn" + "_" + str(runSpecification['num_features']) + "_" + str(runSpecification['num_splits']) + "_" + str(runSpecification['hidden_size'])
try:
r_model = torch.load(MODEL_PATH)
except:
r_model = nn_recurrent_model.RNN(input_size=runSpecification['num_features'], hidden_size=runSpecification['hidden_size'],)
r_model.train(xTrain, yTrain, learning_rate = 0.01, epochs=10)
torch.save(r_model, MODEL_PATH)
# Step 5 Evaluate performance
yTrainingPredicted = r_model.predict(xTrain)
training_accuracy = nn_tools.Accuracy(yTrainList, yTrainingPredicted)
yValidatePredicted = r_model.predict(xDev)
dev_accuracy = nn_tools.Accuracy(yDevList, yValidatePredicted)
runSpecification['train_accuracy'] = training_accuracy
runSpecification['dev_accuracy'] = dev_accuracy
endTime = time.time()
runSpecification['runtime'] = endTime - startTime
return runSpecification
def evaluate(lstm,
word2vec_model,
clean_reviews,
sentiments,
classification_threshold=0.5):
lstm.eval()
with torch.no_grad():
sentiments_predicted = [
0 if lstm(
Utils.createInputBatch(clean_review, word2vec_model,
NUM_FEATURES)).item() <
classification_threshold else 1
for clean_review in tqdm(clean_reviews)
]
return nn_tools.Accuracy(sentiments, sentiments_predicted)
yDev_list = torch.Tensor([y_val for y_val in dataset_B["sentiment"]])
# Get word2vec
xTrain = getFeatureList(getCleanReviews(dataset_A, useSmall=None), word2vec_model, NUM_FEATURES, num_splits=4)
xDev = getFeatureList(getCleanReviews(dataset_B, useSmall=None), word2vec_model, NUM_FEATURES, num_splits=4)
# Step 4 Training NN model
MODEL_PATH = "recurrent_model.pt"
doTraining = False
if doTraining:
print("TRAINING recurrent nn model")
r_model = nn_recurrent_model.RNN(input_size=NUM_FEATURES)
r_model.train(xTrain, yTrain, learning_rate = 0.01, epochs=4)
torch.save(r_model, MODEL_PATH)
else:
r_model = torch.load(MODEL_PATH)
# Step 5 Evaluate performance
print()
print("EVALUATION")
yTrainingPredicted = r_model.predict(xTrain)
training_accuracy = nn_tools.Accuracy(yTrain_list, yTrainingPredicted)
print("Training Accuracy: " + str(training_accuracy))
yValidatePredicted = r_model.predict(xDev)
dev_accuracy = nn_tools.Accuracy(yDev_list, yValidatePredicted)
print("Development Accuracy: " + str(dev_accuracy))
# Step 6 Generate ROC curve
(modelFPRs, modelFNRs, thresholds) = nn_tools.TabulateModelPerformanceForROC(r_model, xDev, yDev_list)
def ExecuteTuning(runSpecification, datasets, xTrainRaw, yTrain, xDevRaw,
yDev):
startTime = time.time()
# Look for the model we need
doc2vec_name = "doc2vec_tuning" + "_" + str(
runSpecification['num_features']) + "_" + str(
runSpecification['context'])
print(doc2vec_name)
try:
print("LOADING Doc2Vec Model")
doc2vec_model = Doc2Vec.load(doc2vec_name)
except Exception as ex:
print(ex)
print("Could't load Doc2Vec, BUILDING...")
doc2vec_model = generate_doc2vec(doc2vec_name, datasets,
runSpecification['num_features'],
runSpecification['context'])
# Get doc2vec
xTrainDoc = getDocFeatureVec(xTrainRaw, doc2vec_model,
runSpecification['num_features'])
xTrainDoc = torch.tensor(xTrainDoc)
xDevDoc = getDocFeatureVec(xDevRaw, doc2vec_model,
runSpecification['num_features'])
xDevDoc = torch.tensor(xDevDoc)
# Step 4 Training NN model
# MODEL_PATH = "ff_" + str(runSpecification['num_features']) + "_" + str(runSpecification['context']) + ".pt"
MODEL_PATH = "ff2" + "_" + str(runSpecification['h1']) + "_" + str(
runSpecification['h2']) + "_" + str(runSpecification['learning_rate'])
doTraining = False
# try:
# l_model = torch.load(MODEL_PATH)
# except:
# l_model = nn_model.NeuralNetwork(input_nodes=runSpecification['num_features'], layer1=runSpecification["h1"], layer2=runSpecification['h2'])
# l_model.train_model_persample(xTrainDoc, yTrain, learning_rate=runSpecification["learning_rate"])
# torch.save(l_model, MODEL_PATH)
l_model = nn_model.NeuralNetwork(
input_nodes=runSpecification['num_features'],
layer1=runSpecification["h1"],
layer2=runSpecification['h2'])
l_model.train_model_persample(
xTrainDoc, yTrain, learning_rate=runSpecification["learning_rate"])
torch.save(l_model, MODEL_PATH)
# Step 5 Evaluate performance
yTrainingPredicted = l_model.predict(xTrainDoc)
training_accuracy = nn_tools.Accuracy(yTrain, yTrainingPredicted)
yValidatePredicted = l_model.predict(xDevDoc)
dev_accuracy = nn_tools.Accuracy(yDev, yValidatePredicted)
runSpecification['train_accuracy'] = training_accuracy
runSpecification['dev_accuracy'] = dev_accuracy
endTime = time.time()
runSpecification['runtime'] = endTime - startTime
return runSpecification
xDevDoc = torch.tensor(xDevDoc)
xDevDoc = xDevDoc.to(device)
# Step 4 Training NN model
try:
assert (load_nn)
l_model = torch.load(ffnn_name)
except:
print("TRAINING nn model")
l_model = nn_model.NeuralNetwork(input_nodes=num_features)
l_model.to(device)
nn_model_instructor.train(l_model, xTrainDoc, yTrain)
torch.save(l_model, ffnn_name)
# Step 5 Evaluate performance
print()
print("EVALUATION")
yTrainingPredicted = nn_model_instructor.predict(l_model, xTrainDoc)
training_accuracy = nn_tools.Accuracy(yTrain, yTrainingPredicted)
print("Training Accuracy: " + str(training_accuracy))
# yValidatePredicted = l_model.predict(xDevDoc)
# dev_accuracy = nn_tools.Accuracy(yDev, yValidatePredicted)
# print("Development Accuracy: " + str(dev_accuracy))
# # Step 6 Generate ROC curve
# (modelFPRs, modelFNRs, thresholds) = nn_tools.TabulateModelPerformanceForROC(l_model, xDevDoc, yDev)
# print(modelFPRs)
# print(modelFNRs)
# print(thresholds)
def train_model_persample_dev(self,
xTrain,
yTrain,
xDev,
yDev,
max_epoch=51,
convergence=0.000001,
learning_rate=1,
min_epochs=50):
#Set training mode
self.train(mode=False)
# Setup training values
converged = False
epoch = 1
lastLoss = None
optimizer = torch.optim.SGD(self.parameters(), lr=learning_rate)
lossFunction = torch.nn.MSELoss(reduction='mean')
loss_across_epoch = []
dev_acc_list = []
while not converged and epoch < max_epoch:
# Do the forward pass
for i in range(len(xTrain)):
# sample = xTrain[i].unsqueeze(0)
sample = xTrain[i].unsqueeze(0)
yTrainPredicted = self(sample)
trainLoss = lossFunction(yTrainPredicted, yTrain[i])
# Reset the gradients in the network to zero
optimizer.zero_grad()
# Backprop the errors from the loss on this iteration
trainLoss.backward()
# Do a weight update step
optimizer.step()
loss = trainLoss.item()
print("Current Loss: " + str(loss))
print(loss)
loss_across_epoch.append(loss)
if lastLoss is None:
lastLoss = loss
else:
if abs(lastLoss - loss) < convergence and epoch > min_epochs:
converged = True
epoch = epoch + 1
yPredict = self.predict(xDev)
dev_accuracy = nn_tools.Accuracy(yDev, yPredict)
dev_acc_list.append(dev_accuracy)
print("Total Epochs: " + str(epoch))
self.train(mode=True)
print("TRAINING LOSS")
print(loss_across_epoch)
print("Dev accuracy")
print(dev_acc_list)
# torch.save(l_model, MODEL_PATH)
l_model = nn_model.NeuralNetwork(input_nodes=runSpecification['num_features'],
layer1=runSpecification["h1"],
layer2=runSpecification['h2'])
# l_model.train_model_persample(xTrainDoc, yTrain, learning_rate=runSpecification["learning_rate"])
l_model.train_model_persample_dev(
xTrainDoc,
yTrain,
xDevDoc,
yDev,
learning_rate=runSpecification["learning_rate"])
# Step 5 Evaluate performance
yTrainingPredicted = l_model.predict(xTrainDoc)
training_accuracy = nn_tools.Accuracy(yTrain, yTrainingPredicted)
yValidatePredicted = l_model.predict(xDevDoc)
print("Prediction Output")
print(yValidatePredicted)
dev_accuracy = nn_tools.Accuracy(yDev, yValidatePredicted)
yTestPredicted = l_model.predict(xTestDoc)
test_accuracy = nn_tools.Accuracy(yTest, yTestPredicted)
runSpecification['train_accuracy'] = training_accuracy
runSpecification['dev_accuracy'] = dev_accuracy
runSpecification['test_accuracy'] = test_accuracy
print(training_accuracy)
print(dev_accuracy)
lstm, test_iterator)
rounded_predictions = torch.round(probability_estimates)
if ERROR_ANALYSIS:
print("Confusion matrix:",
nn_tools.ConfusionMatrix(true_labels, rounded_predictions))
# flip for false positives
with open("LSTMFalseNegatives.txt", mode="w") as file:
print("False negatives")
for i in range(len(true_labels)):
if true_labels[i] == 1 and rounded_predictions[i] == 0:
file.write(dev_data_pd["review"][i] + "\n\n")
else:
# accuracies
print("Dev set accuracy:",
nn_tools.Accuracy(true_labels, rounded_predictions))
if input("Display test set accuracy? y/n ").lower() == "y":
print(
"Test set accuracy:",
nn_tools.Accuracy(true_labels_test,
torch.round(probability_estimates_test)))
# get ROC data
FPRs, FNRs, thresholds = nn_tools.TabulateModelPerformanceForROCFromProbabilityEstimates(
true_labels, probability_estimates)
print("FPRs:", FPRs)
print("FNRs:", FNRs)
print("Thresholds:", thresholds)