def train_dev(self, xTrain, sentiment, xDev, yDev, learning_rate=0.001, epochs=1): sentiment = torch.Tensor(sentiment) sentiment = sentiment.type(torch.LongTensor) optimizer = torch.optim.SGD(self.parameters(), lr=learning_rate) criterion = nn.NLLLoss() dev_acc_list = [] training_loss = [] for iteration in range(1, epochs + 1): avg_loss = 0 avg_count = 0 for i, review in enumerate(xTrain): if i == 4845: phoo = 2 hidden = self.initHidden() self.zero_grad() for j, word in enumerate(review): word_tensor = torch.Tensor(word).unsqueeze(0) output, hidden = self.forward(word_tensor, hidden) loss = criterion(output, sentiment[i]) loss.backward() # Print loss for the review loss_value = loss.item() training_loss.append(loss_value) avg_loss += loss_value avg_count += 1 # print(loss_value) optimizer.step() # Add parameters' gradients to their values, multiplied by learning rate # for p in self.parameters(): # p.data.add_(p.grad.data, alpha=-learning_rate) print(avg_loss / avg_count) yPredict = self.predict(xDev) dev_acc = nn_tools.Accuracy(yDev, yPredict) dev_acc_list.append(dev_acc) print("Loss over iteration") print(training_loss) print("Dev Accuracy") print(dev_acc_list) return
def ExecuteTuning(runSpecification, datasets, xTrainRaw, yTrain, yTrainList, xDevRaw, yDev, yDevList): startTime = time.time() # Look for the model we need word2vec_name = "word2vec_tuning" + "_" + str(runSpecification['num_features']) + "_" + str(runSpecification['context']) print(word2vec_name) try: print("LOADING Word2Vec Model") word2vec_model = Word2Vec.load(word2vec_name) except Exception as ex: print(ex) print("Could't load Word2Vec, BUILDING...") word2vec_model = generate_word2vec(word2vec_name, datasets, runSpecification['num_features'], runSpecification['context']) # Get doc2vec xTrain = getFeatureList(xTrainRaw, word2vec_model, runSpecification['num_features'], num_splits=runSpecification['num_splits']) xDev = getFeatureList(xDevRaw, word2vec_model, runSpecification['num_features'], num_splits=runSpecification['num_splits']) # Step 4 Training NN model # MODEL_PATH = "ff_" + str(runSpecification['num_features']) + "_" + str(runSpecification['context']) + ".pt" MODEL_PATH = "rnn" + "_" + str(runSpecification['num_features']) + "_" + str(runSpecification['num_splits']) + "_" + str(runSpecification['hidden_size']) try: r_model = torch.load(MODEL_PATH) except: r_model = nn_recurrent_model.RNN(input_size=runSpecification['num_features'], hidden_size=runSpecification['hidden_size'],) r_model.train(xTrain, yTrain, learning_rate = 0.01, epochs=10) torch.save(r_model, MODEL_PATH) # Step 5 Evaluate performance yTrainingPredicted = r_model.predict(xTrain) training_accuracy = nn_tools.Accuracy(yTrainList, yTrainingPredicted) yValidatePredicted = r_model.predict(xDev) dev_accuracy = nn_tools.Accuracy(yDevList, yValidatePredicted) runSpecification['train_accuracy'] = training_accuracy runSpecification['dev_accuracy'] = dev_accuracy endTime = time.time() runSpecification['runtime'] = endTime - startTime return runSpecification
def evaluate(lstm, word2vec_model, clean_reviews, sentiments, classification_threshold=0.5): lstm.eval() with torch.no_grad(): sentiments_predicted = [ 0 if lstm( Utils.createInputBatch(clean_review, word2vec_model, NUM_FEATURES)).item() < classification_threshold else 1 for clean_review in tqdm(clean_reviews) ] return nn_tools.Accuracy(sentiments, sentiments_predicted)
yDev_list = torch.Tensor([y_val for y_val in dataset_B["sentiment"]]) # Get word2vec xTrain = getFeatureList(getCleanReviews(dataset_A, useSmall=None), word2vec_model, NUM_FEATURES, num_splits=4) xDev = getFeatureList(getCleanReviews(dataset_B, useSmall=None), word2vec_model, NUM_FEATURES, num_splits=4) # Step 4 Training NN model MODEL_PATH = "recurrent_model.pt" doTraining = False if doTraining: print("TRAINING recurrent nn model") r_model = nn_recurrent_model.RNN(input_size=NUM_FEATURES) r_model.train(xTrain, yTrain, learning_rate = 0.01, epochs=4) torch.save(r_model, MODEL_PATH) else: r_model = torch.load(MODEL_PATH) # Step 5 Evaluate performance print() print("EVALUATION") yTrainingPredicted = r_model.predict(xTrain) training_accuracy = nn_tools.Accuracy(yTrain_list, yTrainingPredicted) print("Training Accuracy: " + str(training_accuracy)) yValidatePredicted = r_model.predict(xDev) dev_accuracy = nn_tools.Accuracy(yDev_list, yValidatePredicted) print("Development Accuracy: " + str(dev_accuracy)) # Step 6 Generate ROC curve (modelFPRs, modelFNRs, thresholds) = nn_tools.TabulateModelPerformanceForROC(r_model, xDev, yDev_list)
def ExecuteTuning(runSpecification, datasets, xTrainRaw, yTrain, xDevRaw, yDev): startTime = time.time() # Look for the model we need doc2vec_name = "doc2vec_tuning" + "_" + str( runSpecification['num_features']) + "_" + str( runSpecification['context']) print(doc2vec_name) try: print("LOADING Doc2Vec Model") doc2vec_model = Doc2Vec.load(doc2vec_name) except Exception as ex: print(ex) print("Could't load Doc2Vec, BUILDING...") doc2vec_model = generate_doc2vec(doc2vec_name, datasets, runSpecification['num_features'], runSpecification['context']) # Get doc2vec xTrainDoc = getDocFeatureVec(xTrainRaw, doc2vec_model, runSpecification['num_features']) xTrainDoc = torch.tensor(xTrainDoc) xDevDoc = getDocFeatureVec(xDevRaw, doc2vec_model, runSpecification['num_features']) xDevDoc = torch.tensor(xDevDoc) # Step 4 Training NN model # MODEL_PATH = "ff_" + str(runSpecification['num_features']) + "_" + str(runSpecification['context']) + ".pt" MODEL_PATH = "ff2" + "_" + str(runSpecification['h1']) + "_" + str( runSpecification['h2']) + "_" + str(runSpecification['learning_rate']) doTraining = False # try: # l_model = torch.load(MODEL_PATH) # except: # l_model = nn_model.NeuralNetwork(input_nodes=runSpecification['num_features'], layer1=runSpecification["h1"], layer2=runSpecification['h2']) # l_model.train_model_persample(xTrainDoc, yTrain, learning_rate=runSpecification["learning_rate"]) # torch.save(l_model, MODEL_PATH) l_model = nn_model.NeuralNetwork( input_nodes=runSpecification['num_features'], layer1=runSpecification["h1"], layer2=runSpecification['h2']) l_model.train_model_persample( xTrainDoc, yTrain, learning_rate=runSpecification["learning_rate"]) torch.save(l_model, MODEL_PATH) # Step 5 Evaluate performance yTrainingPredicted = l_model.predict(xTrainDoc) training_accuracy = nn_tools.Accuracy(yTrain, yTrainingPredicted) yValidatePredicted = l_model.predict(xDevDoc) dev_accuracy = nn_tools.Accuracy(yDev, yValidatePredicted) runSpecification['train_accuracy'] = training_accuracy runSpecification['dev_accuracy'] = dev_accuracy endTime = time.time() runSpecification['runtime'] = endTime - startTime return runSpecification
xDevDoc = torch.tensor(xDevDoc) xDevDoc = xDevDoc.to(device) # Step 4 Training NN model try: assert (load_nn) l_model = torch.load(ffnn_name) except: print("TRAINING nn model") l_model = nn_model.NeuralNetwork(input_nodes=num_features) l_model.to(device) nn_model_instructor.train(l_model, xTrainDoc, yTrain) torch.save(l_model, ffnn_name) # Step 5 Evaluate performance print() print("EVALUATION") yTrainingPredicted = nn_model_instructor.predict(l_model, xTrainDoc) training_accuracy = nn_tools.Accuracy(yTrain, yTrainingPredicted) print("Training Accuracy: " + str(training_accuracy)) # yValidatePredicted = l_model.predict(xDevDoc) # dev_accuracy = nn_tools.Accuracy(yDev, yValidatePredicted) # print("Development Accuracy: " + str(dev_accuracy)) # # Step 6 Generate ROC curve # (modelFPRs, modelFNRs, thresholds) = nn_tools.TabulateModelPerformanceForROC(l_model, xDevDoc, yDev) # print(modelFPRs) # print(modelFNRs) # print(thresholds)
def train_model_persample_dev(self, xTrain, yTrain, xDev, yDev, max_epoch=51, convergence=0.000001, learning_rate=1, min_epochs=50): #Set training mode self.train(mode=False) # Setup training values converged = False epoch = 1 lastLoss = None optimizer = torch.optim.SGD(self.parameters(), lr=learning_rate) lossFunction = torch.nn.MSELoss(reduction='mean') loss_across_epoch = [] dev_acc_list = [] while not converged and epoch < max_epoch: # Do the forward pass for i in range(len(xTrain)): # sample = xTrain[i].unsqueeze(0) sample = xTrain[i].unsqueeze(0) yTrainPredicted = self(sample) trainLoss = lossFunction(yTrainPredicted, yTrain[i]) # Reset the gradients in the network to zero optimizer.zero_grad() # Backprop the errors from the loss on this iteration trainLoss.backward() # Do a weight update step optimizer.step() loss = trainLoss.item() print("Current Loss: " + str(loss)) print(loss) loss_across_epoch.append(loss) if lastLoss is None: lastLoss = loss else: if abs(lastLoss - loss) < convergence and epoch > min_epochs: converged = True epoch = epoch + 1 yPredict = self.predict(xDev) dev_accuracy = nn_tools.Accuracy(yDev, yPredict) dev_acc_list.append(dev_accuracy) print("Total Epochs: " + str(epoch)) self.train(mode=True) print("TRAINING LOSS") print(loss_across_epoch) print("Dev accuracy") print(dev_acc_list)
# torch.save(l_model, MODEL_PATH) l_model = nn_model.NeuralNetwork(input_nodes=runSpecification['num_features'], layer1=runSpecification["h1"], layer2=runSpecification['h2']) # l_model.train_model_persample(xTrainDoc, yTrain, learning_rate=runSpecification["learning_rate"]) l_model.train_model_persample_dev( xTrainDoc, yTrain, xDevDoc, yDev, learning_rate=runSpecification["learning_rate"]) # Step 5 Evaluate performance yTrainingPredicted = l_model.predict(xTrainDoc) training_accuracy = nn_tools.Accuracy(yTrain, yTrainingPredicted) yValidatePredicted = l_model.predict(xDevDoc) print("Prediction Output") print(yValidatePredicted) dev_accuracy = nn_tools.Accuracy(yDev, yValidatePredicted) yTestPredicted = l_model.predict(xTestDoc) test_accuracy = nn_tools.Accuracy(yTest, yTestPredicted) runSpecification['train_accuracy'] = training_accuracy runSpecification['dev_accuracy'] = dev_accuracy runSpecification['test_accuracy'] = test_accuracy print(training_accuracy) print(dev_accuracy)
lstm, test_iterator) rounded_predictions = torch.round(probability_estimates) if ERROR_ANALYSIS: print("Confusion matrix:", nn_tools.ConfusionMatrix(true_labels, rounded_predictions)) # flip for false positives with open("LSTMFalseNegatives.txt", mode="w") as file: print("False negatives") for i in range(len(true_labels)): if true_labels[i] == 1 and rounded_predictions[i] == 0: file.write(dev_data_pd["review"][i] + "\n\n") else: # accuracies print("Dev set accuracy:", nn_tools.Accuracy(true_labels, rounded_predictions)) if input("Display test set accuracy? y/n ").lower() == "y": print( "Test set accuracy:", nn_tools.Accuracy(true_labels_test, torch.round(probability_estimates_test))) # get ROC data FPRs, FNRs, thresholds = nn_tools.TabulateModelPerformanceForROCFromProbabilityEstimates( true_labels, probability_estimates) print("FPRs:", FPRs) print("FNRs:", FNRs) print("Thresholds:", thresholds)