def run_evaluation(questions, labels):
correct_labels = []
prediction_labels = []
for label in labels:
correct_labels += label
prediction_labels += [
np.random.uniform(0, 1) for _ in range(0, len(label))
]
rmse = sqrt(metrics.mean_squared_error(correct_labels, prediction_labels))
auc = metrics.roc_auc_score(correct_labels, prediction_labels)
fpr, tpr, thresholds = metrics.roc_curve(correct_labels,
prediction_labels,
pos_label=1)
auc2 = metrics.auc(fpr, tpr)
pearson = pearsonr(correct_labels, prediction_labels)
r2 = r2_score(correct_labels, prediction_labels)
accuracy = evaluation_helper.accuracy(correct_labels, prediction_labels)
# pearson = pearson * pearson
print("RMSE is: ", rmse)
print("AUC is: ", auc)
print("AUC is: ", auc2)
print("Pearson coef is:", pearson)
print("Pearson coef is:", r2)
print("Accuracy is:", accuracy)
def run_evaluation_test(questions, labels):
correct_labels = []
prediction_labels = []
test_batch_X, test_batch_Y, test_batch_target_X, test_batch_target_Y, test_batch_seq = test_set.next(
BATCH_SIZE, 100)
correct_labels += (evaluation_helper.get_questions(test_batch_target_X))
rmse = evaluation_helper.rmse(correct_labels, prediction_labels)
auc = evaluation_helper.auc(correct_labels, prediction_labels)
pearson = evaluation_helper.pearson(correct_labels, prediction_labels)
accuracy = evaluation_helper.accuracy(correct_labels, prediction_labels)
print("RMSE is: ", rmse)
print("AUC is: ", auc)
print("Pearson coef is:", pearson)
print("Accuracy is:", accuracy)
def run_test(self, hidden):
with torch.no_grad():
prediction_labels = []
correct_labels = []
for i in range(num_batch_test):
### Forward pass
questions, answers, questions_target, answers_target, batch_seq_len = test_data.next(
BATCH_SIZE, sequence_length)
### Detach hidden layer from history so we don't backpropagate through every step
### and reuse the same weights
hidden = detach_hidden(hidden)
optimizer.zero_grad()
tag_scores, hidden = model.forward(questions, answers, hidden)
answers_target = torch.tensor(answers_target,
dtype=torch.int64)
# Preparing data for evaluation
questions = (evaluation_helper.get_questions(
np.asarray(questions_target)))
preds = np.asarray(torch.sigmoid(tag_scores).detach())
prediction_labels += (evaluation_helper.get_predictions(
preds, questions))
correct_labels += (evaluation_helper.get_labels(
np.asarray(answers_target), questions))
# EVALUATION
rmse_test = evaluation_helper.rmse(correct_labels,
prediction_labels)
auc_test = evaluation_helper.auc(correct_labels, prediction_labels)
pearson_test = evaluation_helper.pearson(correct_labels,
prediction_labels)
accuracy_test = evaluation_helper.accuracy(correct_labels,
prediction_labels)
print("################ TEST SET ##################")
evaluation_helper.print_results(rmse_test, auc_test, pearson_test,
accuracy_test)
# Comet reporting
if LOG_COMET:
experiment.log_metric("rmse_test", rmse_test)
experiment.log_metric("auc_test", auc_test)
experiment.log_metric("pearson_test", pearson_test)
experiment.log_metric("accuracy_test", accuracy_test)
return rmse_test, prediction_labels
def run_train(model, sess):
print("New data, epoch", epoch_idx)
prediction_labels = []
correct_labels = []
for step in range(num_batches_train):
questions, answers, questions_target, answers_target, batch_seq = train_set.next(
BATCH_SIZE, num_steps)
_total_loss, _train_step, _predictions_series = model.run_model_train(
questions, answers, questions_target, answers_target, batch_seq,
sess)
questions = evaluation_helper.get_questions(questions_target)
prediction_labels += evaluation_helper.get_predictions(
_predictions_series, questions)
correct_labels += evaluation_helper.get_labels(answers_target,
questions)
# EVALUATION
if step + 1 >= num_batches_train:
time1 = time.time()
rmse = evaluation_helper.rmse(correct_labels, prediction_labels)
auc = evaluation_helper.auc(correct_labels, prediction_labels)
pearson = evaluation_helper.pearson(correct_labels,
prediction_labels)
accuracy = evaluation_helper.accuracy(correct_labels,
prediction_labels)
print("Time of the run was %s seconds" % (time.time() - time1))
loss_list.append(_total_loss)
graph_loss.append(_total_loss)
graph_rmse_train.append(rmse)
print("Step", step, "Loss", _total_loss)
print("RMSE is: ", rmse)
print("AUC is: ", auc)
print("Accuracy is: ", accuracy)
print("Pearson coef is:", pearson)
if LOG_COMET:
experiment.log_metric("rmse_train", rmse)
experiment.log_metric("auc_train", auc)
experiment.log_metric("pearson_train", pearson)
experiment.log_metric("accuracy_train", accuracy)
def run_test(model):
print("--------------------------------")
print("Calculating test set predictions")
questions = []
prediction_labels = []
correct_labels = []
for i in range(num_batches_test):
test_batch_x, test_batch_y, test_batch_target_x, test_batch_target_y, test_batch_seq = test_set.next(
BATCH_SIZE, num_steps)
test_predictions = model.run_model_test(test_batch_x, test_batch_y,
test_batch_seq)
questions = (evaluation_helper.get_questions(test_batch_target_x))
prediction_labels += (evaluation_helper.get_predictions(
test_predictions, questions))
correct_labels += (evaluation_helper.get_labels(
test_batch_target_y, questions))
# EVALUATION
rmse_test = evaluation_helper.rmse(correct_labels, prediction_labels)
auc_test = evaluation_helper.auc(correct_labels, prediction_labels)
pearson_test = evaluation_helper.pearson(correct_labels, prediction_labels)
accuracy_test = evaluation_helper.accuracy(correct_labels,
prediction_labels)
if LOG_COMET:
experiment.log_metric("rmse_test", rmse_test)
experiment.log_metric("auc_test", auc_test)
experiment.log_metric("pearson_test", pearson_test)
experiment.log_metric("accuracy_test", accuracy_test)
graph_rmse_test.append(rmse_test)
print("RMSE for test set:%.5f" % rmse_test)
print("AUC for test set:%.5f" % auc_test)
print("Pearson coef is:", pearson_test)
print("Accuracy is:", accuracy_test)
print("--------------------------------")
return rmse_test, auc_test, pearson_test, accuracy_test
def run_train(self, hidden):
prediction_labels = []
correct_labels = []
for i in range(num_batch_train):
### Forward pass
questions, answers, questions_target, answers_target, batch_seq_len = train_data.next(
BATCH_SIZE, sequence_length)
### Detach hidden layer from history so we don't backpropagate through every step
### and reuse the same weights
hidden = detach_hidden(hidden)
optimizer.zero_grad()
tag_scores, hidden = model.forward(questions, answers, hidden)
### Preparing data for backpropagation
target: List[int] = []
for batch_num in range(len(questions_target)):
for seq_num, target_id in enumerate(
questions_target[batch_num]):
target.append(batch_num * num_classes * sequence_length +
seq_num * num_classes + int(target_id))
# Preparing logits
targets = torch.tensor(target, dtype=torch.int64)
tag_scores2 = tag_scores.view(-1)
logits = torch.gather(tag_scores2, 0, targets)
# Preparing correct answers
answers_target = torch.tensor(answers_target, dtype=torch.int64)
target_correctness = answers_target.view(-1)
target_correctness = torch.tensor(target_correctness,
dtype=torch.float)
### Backpropagation ###
loss_function = nn.BCEWithLogitsLoss()
loss = loss_function(logits, target_correctness)
loss.backward()
optimizer.step()
# Preparing data for evaluation
questions = (evaluation_helper.get_questions(
np.asarray(questions_target)))
preds = np.asarray(torch.sigmoid(tag_scores).detach())
prediction_labels += (evaluation_helper.get_predictions(
preds, questions))
correct_labels += (evaluation_helper.get_labels(
np.asarray(answers_target), questions))
### EVALUATION ###
rmse_train = evaluation_helper.rmse(correct_labels, prediction_labels)
auc_train = evaluation_helper.auc(correct_labels, prediction_labels)
pearson_train = evaluation_helper.pearson(correct_labels,
prediction_labels)
accuracy_train = evaluation_helper.accuracy(correct_labels,
prediction_labels)
print("==========================TRAIN SET==========================")
evaluation_helper.print_results(rmse_train, auc_train, pearson_train,
accuracy_train)
# Comet reporting
if LOG_COMET:
experiment.log_metric("rmse_train", rmse_train)
experiment.log_metric("auc_train", auc_train)
experiment.log_metric("pearson_train", pearson_train)
experiment.log_metric("accuracy_train", accuracy_train)
return rmse_train
questions = np.array(features['place_asked'])
answers = np.array(features['place_answered'])
labels = np.equal(questions, answers)
labels = list(map(lambda x: 1 if x else 0, labels))
features['correct'] = np.array(labels)
ids = [i for i in range(1500)]
filtered = features[features['place_asked'].isin(ids)]
grouped = filtered.groupby('place_asked')['correct'].agg([['averages', 'mean']])
correct_labels = features['correct']
prediction_labels = []
print(grouped)
for index, row in features.iterrows():
question_id = int(row['place_asked'])
temp = grouped['averages']
prediction_labels.append(temp[question_id])
print(evaluation_helper.rmse(correct_labels,prediction_labels))
print(evaluation_helper.auc(correct_labels, prediction_labels))
print(evaluation_helper.pearson(correct_labels, prediction_labels))
print(evaluation_helper.accuracy(correct_labels, prediction_labels))