def plt_roc_curve(preds, labels, save_str='', save=False):
"""Plots the ROC curve of the classifier
Parameters
----------
preds : array_like
Classifier predictions
labels : array_like
True class labels
save_str : str
String to use to save fig and data, if save. Should not have
file extension, should not be full path - just name of .pickle
and .png files that will be saved
save : bool
If True, will save the fig and data
"""
# Thresholds to use for plt
thresholds = np.linspace(0, 1, 1000)
# Init arrays for storing FPR and TPR
fprs = np.zeros_like(thresholds)
tprs = np.zeros_like(thresholds)
for ii in range(np.size(thresholds)):
# Get TPR here
tprs[ii] = get_sens(preds=preds,
labels=labels,
threshold=thresholds[ii])
# Get FPR here
fprs[ii] = 1 - get_spec(
preds=preds, labels=labels, threshold=thresholds[ii])
# Make fig
plt.figure(figsize=(12, 6))
plt.rc("font", family="Times New Roman")
plt.tick_params(labelsize=20)
plt.plot(fprs, tprs, 'k-')
plt.plot(np.linspace(0, 1, 1000), np.linspace(0, 1, 1000), 'b--')
plt.xlabel('False Positive Rate', fontsize=24)
plt.ylabel('True Positive Rate', fontsize=24)
plt.tight_layout()
if save: # If saving
verify_path(os.path.join(get_proj_path(), 'output/roc-figs/'))
out_path = os.path.join(get_proj_path(), 'output/roc-figs/')
plt.savefig(os.path.join(out_path, '%s.png' % save_str), dpi=150)
plt.close()
save_pickle(np.array([fprs, tprs]),
os.path.join(out_path, '%s.pickle' % save_str))
October 15th, 2020
"""
import os
import numpy as np
import matplotlib.pyplot as plt
from umbms import get_proj_path, verify_path
from umbms.loadsave import load_pickle
###############################################################################
__DATA_DIR = os.path.join(get_proj_path(), 'output/roc-figs/')
__OUT_DIR = os.path.join(get_proj_path(), 'output/figs/')
verify_path(__OUT_DIR)
###############################################################################
if __name__ == "__main__":
# Load ROC data obtained from testing on G1 after training on G1
cnn_roc = load_pickle(os.path.join(__DATA_DIR, 'cnn_run_8_roc.pickle'))
dnn_roc = load_pickle(os.path.join(__DATA_DIR, 'dnn_run_15_roc.pickle'))
logreg_roc = load_pickle(
os.path.join(__DATA_DIR, 'logreg_run_0_roc.pickle'))
# Define colours for the figure
mid_grey = [118, 113, 113]
mid_grey = [ii / 255 for ii in mid_grey]
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse
from matplotlib.cm import get_cmap
from sklearn.decomposition import PCA
from umbms import get_proj_path, verify_path, get_script_logger
from umbms.loadsave import load_pickle, save_pickle
from umbms.ai.preproc import to_td
###############################################################################
__DATA_DIR = os.path.join(get_proj_path(), 'data/umbmid/')
__OUT_DIR = os.path.join(get_proj_path(), 'output/figs/')
verify_path(__OUT_DIR)
fig_out = os.path.join(get_proj_path(), 'output/figs/')
verify_path(fig_out)
###############################################################################
def get_valid_session_ns(metadata, dataset):
"""Gets the session numbers in the metadata for more than 5 scans
Parameters
----------
metadata : array_like
Array of metadata info
dataset : array_like
(100 * auc, 100 * acc, 100 * sens, 100 * spec))
logger.info('\t\tResults here at optimal threshold:')
logger.info(
'\t\t\tAUC: %.3f\tAcc: %.3f\tSens: %.3f\tSpec: %.3f' %
(auc * 100, 100 * test_acc_opt, test_sens_opt, test_spec_opt))
# Store the results over each run for this adipose shell
adi_results[adi_id] = [
aucs_here, accs_here, sens_here, spec_here, accs_opt_thresh_here,
sens_opt_thresh_here, spec_opt_thresh_here
]
# Report the results to the logger
for adi_id in unique_adi_ids:
logger.info('CNN Gen-2 Results on Unseen Adi-ID: %s' % adi_id)
report_metrics(adi_results[adi_id][0], adi_results[adi_id][1],
adi_results[adi_id][2], adi_results[adi_id][3], logger)
# Save the results to a .pickle file
save_dir = os.path.join(get_proj_path(), 'output/g2-by-adi//')
verify_path(save_dir)
save_pickle(adi_results, os.path.join(save_dir,
'g2_by_adi_metrics.pickle'))
# Save the incorrect predictions
out_dir = os.path.join(get_proj_path(), 'output/by-adi-preds/')
verify_path(out_dir)
save_pickle(incor_preds, os.path.join(out_dir, 'byadi_incor_preds.pickle'))
save_pickle(cor_preds, os.path.join(out_dir, 'byadi_cor_preds.pickle'))
from umbms import get_proj_path, get_script_logger, verify_path from umbms.loadsave import load_pickle from umbms.ai.makesets import get_class_labels from umbms.ai.models import get_sino_dnn from umbms.ai.preproc import resize_features_for_keras, to_td from umbms.ai.augment import full_aug from umbms.ai.gencompare import correct_g1_ini_ant_ang ############################################################################### __DATA_DIR = os.path.join(get_proj_path(), 'data/umbmid/') __FIG_OUT_DIR = os.path.join(get_proj_path(), 'output/2006-g2-v-g1/') verify_path(__FIG_OUT_DIR) ############################################################################### # Number of epochs to train over __N_EPOCHS = 6000 # Set the bounds on the time-domain window that will be applied to all # data before training or testing __T_WIN_LOW = 5 __T_WIN_HIGH = 40 # Number of runs to use to identify best-performing number of epochs __N_RUNS = 10 ###############################################################################
logger.info('\t\tAUC:\t%.2f' % g1_auc)
# Get the class predictions
class_preds = g1_preds * np.zeros_like(g1_preds)
class_preds[g1_preds >= opt_thresh] = 1
# Store correct and incorrect predictions
for s_idx in range(np.size(g1_preds, axis=0)):
if class_preds[s_idx, 1] != g1_labels[s_idx, 1]:
incorrect_preds.append(g1_md[s_idx])
else:
correct_preds.append(g1_md[s_idx])
# Reset the model
clear_session()
# Report performance metrics to logger
logger.info('Average performance metrics')
logger.info('')
report_metrics(aucs=auc_scores, accs=accs, sens=sens, spec=spec,
logger=logger)
# Save the incorrect predictions
out_dir = os.path.join(get_proj_path(), 'output/incorrect-preds/')
verify_path(out_dir)
save_pickle(incorrect_preds,
os.path.join(out_dir, 'incorrect_preds.pickle'))
save_pickle(correct_preds,
os.path.join(out_dir, 'correct_preds.pickle'))
