def plt_roc_curve(preds, labels, save_str='', save=False):
"""Plots the ROC curve of the classifier
Parameters
----------
preds : array_like
Classifier predictions
labels : array_like
True class labels
save_str : str
String to use to save fig and data, if save. Should not have
file extension, should not be full path - just name of .pickle
and .png files that will be saved
save : bool
If True, will save the fig and data
"""
# Thresholds to use for plt
thresholds = np.linspace(0, 1, 1000)
# Init arrays for storing FPR and TPR
fprs = np.zeros_like(thresholds)
tprs = np.zeros_like(thresholds)
for ii in range(np.size(thresholds)):
# Get TPR here
tprs[ii] = get_sens(preds=preds,
labels=labels,
threshold=thresholds[ii])
# Get FPR here
fprs[ii] = 1 - get_spec(
preds=preds, labels=labels, threshold=thresholds[ii])
# Make fig
plt.figure(figsize=(12, 6))
plt.rc("font", family="Times New Roman")
plt.tick_params(labelsize=20)
plt.plot(fprs, tprs, 'k-')
plt.plot(np.linspace(0, 1, 1000), np.linspace(0, 1, 1000), 'b--')
plt.xlabel('False Positive Rate', fontsize=24)
plt.ylabel('True Positive Rate', fontsize=24)
plt.tight_layout()
if save: # If saving
verify_path(os.path.join(get_proj_path(), 'output/roc-figs/'))
out_path = os.path.join(get_proj_path(), 'output/roc-figs/')
plt.savefig(os.path.join(out_path, '%s.png' % save_str), dpi=150)
plt.close()
save_pickle(np.array([fprs, tprs]),
os.path.join(out_path, '%s.pickle' % save_str))
# Train the model
model.fit(x=train_data,
y=train_labels,
epochs=__N_EPOCHS,
shuffle=True,
batch_size=2056,
verbose=False)
# Predict on the test set
test_preds = model.predict(x=test_data)
# Get metrics at 0.50 decision threshold
auc = roc_auc_score(y_true=test_labels, y_score=test_preds)
acc = get_acc(preds=test_preds[:, 1], labels=test_labels[:, 1])
sens = get_sens(preds=test_preds[:, 1], labels=test_labels[:, 1])
spec = get_spec(preds=test_preds[:, 1], labels=test_labels[:, 1])
# Get metrics at optimal decision threshold
opt_thresh = get_opt_thresh(preds=test_preds[:, 1],
labels=test_labels[:, 1],
n_thresholds=10000)
test_acc_opt = get_acc(preds=test_preds[:, 1],
labels=test_labels[:, 1],
threshold=opt_thresh)
test_sens_opt = get_sens(preds=test_preds[:, 1],
labels=test_labels[:, 1],
threshold=opt_thresh)
test_spec_opt = get_spec(preds=test_preds[:, 1],
labels=test_labels[:, 1],
threshold=opt_thresh)
g1_preds = dnn.predict(x=g1_d)
# Get and store ROC AUC
g1_auc = 100 * roc_auc_score(y_true=g1_labels, y_score=g1_preds)
auc_scores[run_idx] = g1_auc
# Get optimal decision threshold
opt_thresh = get_opt_thresh(preds=g1_preds[:, 1],
labels=g1_labels[:, 1])
# Store performance metrics
accs[run_idx] = 100 * get_acc(
preds=g1_preds[:, 1], labels=g1_labels[:, 1], threshold=opt_thresh)
sens[run_idx] = 100 * get_sens(
preds=g1_preds[:, 1], labels=g1_labels[:, 1], threshold=opt_thresh)
spec[run_idx] = 100 * get_spec(
preds=g1_preds[:, 1], labels=g1_labels[:, 1], threshold=opt_thresh)
# Plot ROC curve
plt_roc_curve(preds=g1_preds[:, 1],
labels=g1_labels[:, 1],
save_str='dnn_run_%d_roc' % run_idx,
save=True)
# Report AUC at this run
logger.info('\t\tAUC:\t%.2f' % g1_auc)
# Get the class predictions
class_preds = g1_preds * np.zeros_like(g1_preds)
class_preds[g1_preds >= opt_thresh] = 1
# Reset the model
clear_session()