def main(iterations, use_test=False):
"""
@param iterations is the number of epochs that the model runs for.
@return accuracy, guessing accuracy, F1, guessing F1
"""
x_train, y_train, x_val, y_val, x_test, y_test = load_data(use_test)
y_test += 1
print(
"Loaded {} training examples, {} validation examples, {} testing examples"
.format(len(x_train), len(x_val), len(x_test)))
model = train_model(x_train,
y_train,
x_val,
y_val,
iterations,
learning_rate=0.001)
preds = test_model(model, x_test)
# calculate f1 and accuracy
y_test = y_test.cpu()
acc = accuracy_score(y_test, preds)
f1 = f1_score(y_test, preds, average='macro')
prec = precision_score(y_test, preds, average='macro')
rec = recall_score(y_test, preds, average='macro')
# calculate the majority class f1 and accuracy
mode = stats.mode(y_train.cpu())[0][0][0]
majority_guess = [mode for _ in preds]
guess_acc = accuracy_score(y_test, majority_guess)
guess_f1 = f1_score(y_test, majority_guess, average='macro')
guess_prec = precision_score(y_test, majority_guess, average='macro')
guess_rec = recall_score(y_test, majority_guess, average='macro')
return acc, f1, prec, rec, guess_acc, guess_f1, guess_prec, guess_rec
def run_lr_pipeline(iterations, use_test, path='./model_lr.pth'):
x_train, y_train, x_val, y_val, x_test, y_test = load_data(use_test, path)
y_test += 1
print(
"Loaded {} training examples, {} validation examples, {} testing examples"
.format(len(x_train), len(x_val), len(x_test)))
model = train_model(x_train,
y_train,
x_val,
y_val,
iterations,
learning_rate=0.001)
preds = test_model(model, x_test)
preds = preds.cpu()
y_test = y_test.cpu()
# calculate f1 and accuracy
print(classification_report(y_test, preds))
acc = accuracy_score(y_test, preds)
f1 = f1_score(y_test, preds, average='macro')
prec = precision_score(y_test, preds, average='macro')
rec = recall_score(y_test, preds, average='macro')
print(acc)
print(f1)
print(prec)
print(rec)
print("\n\n")
return acc, f1, prec, rec