def run_evaluation_test(questions, labels): correct_labels = [] prediction_labels = [] test_batch_X, test_batch_Y, test_batch_target_X, test_batch_target_Y, test_batch_seq = test_set.next( BATCH_SIZE, 100) correct_labels += (evaluation_helper.get_questions(test_batch_target_X)) rmse = evaluation_helper.rmse(correct_labels, prediction_labels) auc = evaluation_helper.auc(correct_labels, prediction_labels) pearson = evaluation_helper.pearson(correct_labels, prediction_labels) accuracy = evaluation_helper.accuracy(correct_labels, prediction_labels) print("RMSE is: ", rmse) print("AUC is: ", auc) print("Pearson coef is:", pearson) print("Accuracy is:", accuracy)
def run_test(self, hidden): with torch.no_grad(): prediction_labels = [] correct_labels = [] for i in range(num_batch_test): ### Forward pass questions, answers, questions_target, answers_target, batch_seq_len = test_data.next( BATCH_SIZE, sequence_length) ### Detach hidden layer from history so we don't backpropagate through every step ### and reuse the same weights hidden = detach_hidden(hidden) optimizer.zero_grad() tag_scores, hidden = model.forward(questions, answers, hidden) answers_target = torch.tensor(answers_target, dtype=torch.int64) # Preparing data for evaluation questions = (evaluation_helper.get_questions( np.asarray(questions_target))) preds = np.asarray(torch.sigmoid(tag_scores).detach()) prediction_labels += (evaluation_helper.get_predictions( preds, questions)) correct_labels += (evaluation_helper.get_labels( np.asarray(answers_target), questions)) # EVALUATION rmse_test = evaluation_helper.rmse(correct_labels, prediction_labels) auc_test = evaluation_helper.auc(correct_labels, prediction_labels) pearson_test = evaluation_helper.pearson(correct_labels, prediction_labels) accuracy_test = evaluation_helper.accuracy(correct_labels, prediction_labels) print("################ TEST SET ##################") evaluation_helper.print_results(rmse_test, auc_test, pearson_test, accuracy_test) # Comet reporting if LOG_COMET: experiment.log_metric("rmse_test", rmse_test) experiment.log_metric("auc_test", auc_test) experiment.log_metric("pearson_test", pearson_test) experiment.log_metric("accuracy_test", accuracy_test) return rmse_test, prediction_labels
def run_train(model, sess): print("New data, epoch", epoch_idx) prediction_labels = [] correct_labels = [] for step in range(num_batches_train): questions, answers, questions_target, answers_target, batch_seq = train_set.next( BATCH_SIZE, num_steps) _total_loss, _train_step, _predictions_series = model.run_model_train( questions, answers, questions_target, answers_target, batch_seq, sess) questions = evaluation_helper.get_questions(questions_target) prediction_labels += evaluation_helper.get_predictions( _predictions_series, questions) correct_labels += evaluation_helper.get_labels(answers_target, questions) # EVALUATION if step + 1 >= num_batches_train: time1 = time.time() rmse = evaluation_helper.rmse(correct_labels, prediction_labels) auc = evaluation_helper.auc(correct_labels, prediction_labels) pearson = evaluation_helper.pearson(correct_labels, prediction_labels) accuracy = evaluation_helper.accuracy(correct_labels, prediction_labels) print("Time of the run was %s seconds" % (time.time() - time1)) loss_list.append(_total_loss) graph_loss.append(_total_loss) graph_rmse_train.append(rmse) print("Step", step, "Loss", _total_loss) print("RMSE is: ", rmse) print("AUC is: ", auc) print("Accuracy is: ", accuracy) print("Pearson coef is:", pearson) if LOG_COMET: experiment.log_metric("rmse_train", rmse) experiment.log_metric("auc_train", auc) experiment.log_metric("pearson_train", pearson) experiment.log_metric("accuracy_train", accuracy)
def run_test(model): print("--------------------------------") print("Calculating test set predictions") questions = [] prediction_labels = [] correct_labels = [] for i in range(num_batches_test): test_batch_x, test_batch_y, test_batch_target_x, test_batch_target_y, test_batch_seq = test_set.next( BATCH_SIZE, num_steps) test_predictions = model.run_model_test(test_batch_x, test_batch_y, test_batch_seq) questions = (evaluation_helper.get_questions(test_batch_target_x)) prediction_labels += (evaluation_helper.get_predictions( test_predictions, questions)) correct_labels += (evaluation_helper.get_labels( test_batch_target_y, questions)) # EVALUATION rmse_test = evaluation_helper.rmse(correct_labels, prediction_labels) auc_test = evaluation_helper.auc(correct_labels, prediction_labels) pearson_test = evaluation_helper.pearson(correct_labels, prediction_labels) accuracy_test = evaluation_helper.accuracy(correct_labels, prediction_labels) if LOG_COMET: experiment.log_metric("rmse_test", rmse_test) experiment.log_metric("auc_test", auc_test) experiment.log_metric("pearson_test", pearson_test) experiment.log_metric("accuracy_test", accuracy_test) graph_rmse_test.append(rmse_test) print("RMSE for test set:%.5f" % rmse_test) print("AUC for test set:%.5f" % auc_test) print("Pearson coef is:", pearson_test) print("Accuracy is:", accuracy_test) print("--------------------------------") return rmse_test, auc_test, pearson_test, accuracy_test
def run_train(self, hidden): prediction_labels = [] correct_labels = [] for i in range(num_batch_train): ### Forward pass questions, answers, questions_target, answers_target, batch_seq_len = train_data.next( BATCH_SIZE, sequence_length) ### Detach hidden layer from history so we don't backpropagate through every step ### and reuse the same weights hidden = detach_hidden(hidden) optimizer.zero_grad() tag_scores, hidden = model.forward(questions, answers, hidden) ### Preparing data for backpropagation target: List[int] = [] for batch_num in range(len(questions_target)): for seq_num, target_id in enumerate( questions_target[batch_num]): target.append(batch_num * num_classes * sequence_length + seq_num * num_classes + int(target_id)) # Preparing logits targets = torch.tensor(target, dtype=torch.int64) tag_scores2 = tag_scores.view(-1) logits = torch.gather(tag_scores2, 0, targets) # Preparing correct answers answers_target = torch.tensor(answers_target, dtype=torch.int64) target_correctness = answers_target.view(-1) target_correctness = torch.tensor(target_correctness, dtype=torch.float) ### Backpropagation ### loss_function = nn.BCEWithLogitsLoss() loss = loss_function(logits, target_correctness) loss.backward() optimizer.step() # Preparing data for evaluation questions = (evaluation_helper.get_questions( np.asarray(questions_target))) preds = np.asarray(torch.sigmoid(tag_scores).detach()) prediction_labels += (evaluation_helper.get_predictions( preds, questions)) correct_labels += (evaluation_helper.get_labels( np.asarray(answers_target), questions)) ### EVALUATION ### rmse_train = evaluation_helper.rmse(correct_labels, prediction_labels) auc_train = evaluation_helper.auc(correct_labels, prediction_labels) pearson_train = evaluation_helper.pearson(correct_labels, prediction_labels) accuracy_train = evaluation_helper.accuracy(correct_labels, prediction_labels) print("==========================TRAIN SET==========================") evaluation_helper.print_results(rmse_train, auc_train, pearson_train, accuracy_train) # Comet reporting if LOG_COMET: experiment.log_metric("rmse_train", rmse_train) experiment.log_metric("auc_train", auc_train) experiment.log_metric("pearson_train", pearson_train) experiment.log_metric("accuracy_train", accuracy_train) return rmse_train
questions = np.array(features['place_asked']) answers = np.array(features['place_answered']) labels = np.equal(questions, answers) labels = list(map(lambda x: 1 if x else 0, labels)) features['correct'] = np.array(labels) ids = [i for i in range(1500)] filtered = features[features['place_asked'].isin(ids)] grouped = filtered.groupby('place_asked')['correct'].agg([['averages', 'mean']]) correct_labels = features['correct'] prediction_labels = [] print(grouped) for index, row in features.iterrows(): question_id = int(row['place_asked']) temp = grouped['averages'] prediction_labels.append(temp[question_id]) print(evaluation_helper.rmse(correct_labels,prediction_labels)) print(evaluation_helper.auc(correct_labels, prediction_labels)) print(evaluation_helper.pearson(correct_labels, prediction_labels)) print(evaluation_helper.accuracy(correct_labels, prediction_labels))