def plt_roc_curve(preds, labels, save_str='', save=False): """Plots the ROC curve of the classifier Parameters ---------- preds : array_like Classifier predictions labels : array_like True class labels save_str : str String to use to save fig and data, if save. Should not have file extension, should not be full path - just name of .pickle and .png files that will be saved save : bool If True, will save the fig and data """ # Thresholds to use for plt thresholds = np.linspace(0, 1, 1000) # Init arrays for storing FPR and TPR fprs = np.zeros_like(thresholds) tprs = np.zeros_like(thresholds) for ii in range(np.size(thresholds)): # Get TPR here tprs[ii] = get_sens(preds=preds, labels=labels, threshold=thresholds[ii]) # Get FPR here fprs[ii] = 1 - get_spec( preds=preds, labels=labels, threshold=thresholds[ii]) # Make fig plt.figure(figsize=(12, 6)) plt.rc("font", family="Times New Roman") plt.tick_params(labelsize=20) plt.plot(fprs, tprs, 'k-') plt.plot(np.linspace(0, 1, 1000), np.linspace(0, 1, 1000), 'b--') plt.xlabel('False Positive Rate', fontsize=24) plt.ylabel('True Positive Rate', fontsize=24) plt.tight_layout() if save: # If saving verify_path(os.path.join(get_proj_path(), 'output/roc-figs/')) out_path = os.path.join(get_proj_path(), 'output/roc-figs/') plt.savefig(os.path.join(out_path, '%s.png' % save_str), dpi=150) plt.close() save_pickle(np.array([fprs, tprs]), os.path.join(out_path, '%s.pickle' % save_str))
October 15th, 2020 """ import os import numpy as np import matplotlib.pyplot as plt from umbms import get_proj_path, verify_path from umbms.loadsave import load_pickle ############################################################################### __DATA_DIR = os.path.join(get_proj_path(), 'output/roc-figs/') __OUT_DIR = os.path.join(get_proj_path(), 'output/figs/') verify_path(__OUT_DIR) ############################################################################### if __name__ == "__main__": # Load ROC data obtained from testing on G1 after training on G1 cnn_roc = load_pickle(os.path.join(__DATA_DIR, 'cnn_run_8_roc.pickle')) dnn_roc = load_pickle(os.path.join(__DATA_DIR, 'dnn_run_15_roc.pickle')) logreg_roc = load_pickle( os.path.join(__DATA_DIR, 'logreg_run_0_roc.pickle')) # Define colours for the figure mid_grey = [118, 113, 113] mid_grey = [ii / 255 for ii in mid_grey]
import matplotlib.pyplot as plt from matplotlib.patches import Ellipse from matplotlib.cm import get_cmap from sklearn.decomposition import PCA from umbms import get_proj_path, verify_path, get_script_logger from umbms.loadsave import load_pickle, save_pickle from umbms.ai.preproc import to_td ############################################################################### __DATA_DIR = os.path.join(get_proj_path(), 'data/umbmid/') __OUT_DIR = os.path.join(get_proj_path(), 'output/figs/') verify_path(__OUT_DIR) fig_out = os.path.join(get_proj_path(), 'output/figs/') verify_path(fig_out) ############################################################################### def get_valid_session_ns(metadata, dataset): """Gets the session numbers in the metadata for more than 5 scans Parameters ---------- metadata : array_like Array of metadata info dataset : array_like
(100 * auc, 100 * acc, 100 * sens, 100 * spec)) logger.info('\t\tResults here at optimal threshold:') logger.info( '\t\t\tAUC: %.3f\tAcc: %.3f\tSens: %.3f\tSpec: %.3f' % (auc * 100, 100 * test_acc_opt, test_sens_opt, test_spec_opt)) # Store the results over each run for this adipose shell adi_results[adi_id] = [ aucs_here, accs_here, sens_here, spec_here, accs_opt_thresh_here, sens_opt_thresh_here, spec_opt_thresh_here ] # Report the results to the logger for adi_id in unique_adi_ids: logger.info('CNN Gen-2 Results on Unseen Adi-ID: %s' % adi_id) report_metrics(adi_results[adi_id][0], adi_results[adi_id][1], adi_results[adi_id][2], adi_results[adi_id][3], logger) # Save the results to a .pickle file save_dir = os.path.join(get_proj_path(), 'output/g2-by-adi//') verify_path(save_dir) save_pickle(adi_results, os.path.join(save_dir, 'g2_by_adi_metrics.pickle')) # Save the incorrect predictions out_dir = os.path.join(get_proj_path(), 'output/by-adi-preds/') verify_path(out_dir) save_pickle(incor_preds, os.path.join(out_dir, 'byadi_incor_preds.pickle')) save_pickle(cor_preds, os.path.join(out_dir, 'byadi_cor_preds.pickle'))
from umbms import get_proj_path, get_script_logger, verify_path from umbms.loadsave import load_pickle from umbms.ai.makesets import get_class_labels from umbms.ai.models import get_sino_dnn from umbms.ai.preproc import resize_features_for_keras, to_td from umbms.ai.augment import full_aug from umbms.ai.gencompare import correct_g1_ini_ant_ang ############################################################################### __DATA_DIR = os.path.join(get_proj_path(), 'data/umbmid/') __FIG_OUT_DIR = os.path.join(get_proj_path(), 'output/2006-g2-v-g1/') verify_path(__FIG_OUT_DIR) ############################################################################### # Number of epochs to train over __N_EPOCHS = 6000 # Set the bounds on the time-domain window that will be applied to all # data before training or testing __T_WIN_LOW = 5 __T_WIN_HIGH = 40 # Number of runs to use to identify best-performing number of epochs __N_RUNS = 10 ###############################################################################
logger.info('\t\tAUC:\t%.2f' % g1_auc) # Get the class predictions class_preds = g1_preds * np.zeros_like(g1_preds) class_preds[g1_preds >= opt_thresh] = 1 # Store correct and incorrect predictions for s_idx in range(np.size(g1_preds, axis=0)): if class_preds[s_idx, 1] != g1_labels[s_idx, 1]: incorrect_preds.append(g1_md[s_idx]) else: correct_preds.append(g1_md[s_idx]) # Reset the model clear_session() # Report performance metrics to logger logger.info('Average performance metrics') logger.info('') report_metrics(aucs=auc_scores, accs=accs, sens=sens, spec=spec, logger=logger) # Save the incorrect predictions out_dir = os.path.join(get_proj_path(), 'output/incorrect-preds/') verify_path(out_dir) save_pickle(incorrect_preds, os.path.join(out_dir, 'incorrect_preds.pickle')) save_pickle(correct_preds, os.path.join(out_dir, 'correct_preds.pickle'))