def train_eval_binary_relevace(model, train_set, test_set, run_epochs, path_results, num_cls, cls_names, target_emitter, rundate, val_set=None, start_epoch=0, verbose=1, batch_size=4, experiment_notes="", show=False, prev_hist=None): """ Train and evaluate Keras model *_set[0] should have shape ([n samples], [x=time axis], [y=frequency axis]) containing the input spectrograms *_set[1] should have shape ([n samples],) containing the corresponding class labels If using in a loop, feed return values into start_epoch and prev_hist to keep accurate epoch count :param model: keras.engine.training.Model Compiled Keras model :param train_set: 2-tuple of array-likes Training set (see above) :param test_set: 2-tuple of array-likes Testing set (see above) :param run_epochs: int Number of epochs to run :param path_results: str Output directory :param num_cls: int Total number of classes :param cls_names: list of str List of class names ordered by corresponding class label :param target_emitter: str Positive class for the network :param rundate: date time The rundate of the current training session :param val_set: 2-tuple of array-likes Validation set (see above) :param start_epoch: int Current epoch of model before training :param verbose: int Keras verbosity mode (0, 1, or 2) :param batch_size: int :param experiment_notes: str Notes about experiment :param show: bool Whether to show plots :param prev_hist: dict of arrays Previous interval's training history :return count_epoch: int Current epoch of model after training :return curr_hist: dict of arrays Current interval's training history """ # Set up results directory if not os.path.exists(path_results + '/' + target_emitter): os.mkdir(path_results + '/' + target_emitter) os.mkdir(path_results + '/' + target_emitter + '/' + rundate) os.mkdir(path_results + '/' + target_emitter + '/' + rundate + '/models/') if not os.path.exists(path_results + '/' + target_emitter + '/' + rundate): os.mkdir(path_results + '/' + target_emitter + '/' + rundate) os.mkdir(path_results + '/' + target_emitter + '/' + rundate + '/models/') save_path = path_results + '/' + target_emitter + '/' + rundate # Save model summary orig_stdout = sys.stdout summ_file = open(save_path + '/models/modelsummary.txt', 'w') sys.stdout = summ_file print(model.summary()) sys.stdout = orig_stdout summ_file.close() # Save experiment notes orig_stdout = sys.stdout experiment_notes_file = open(save_path + '/models/_experiment_notes.txt', 'w') sys.stdout = experiment_notes_file print(experiment_notes) sys.stdout = orig_stdout experiment_notes_file.close() # Perform training count_epoch = start_epoch + run_epochs history = model.fit_generator( generator=batch_gen_binary(train_set[0], train_set[1], batch_size), steps_per_epoch=train_set[0].shape[0] // batch_size, epochs=count_epoch, initial_epoch=start_epoch, verbose=verbose, workers=1, validation_data=[ val_set[0][...].reshape(val_set[0].shape + (1, )), val_set[1] ]) # Save trained model and training info save_name = save_path + '/models/epochs' + str(start_epoch) + '-' + str( count_epoch) + '.h5' model.save(save_name) if prev_hist is None: acc = np.asarray(history.history['acc']) val_acc = np.asarray(history.history['val_acc']) loss = np.asarray(history.history['loss']) val_loss = np.asarray(history.history['val_loss']) else: acc = np.append(prev_hist['acc'], np.asarray(history.history['acc'])) val_acc = np.append(prev_hist['val_acc'], np.asarray(history.history['val_acc'])) loss = np.append(prev_hist['loss'], np.asarray(history.history['loss'])) val_loss = np.append(prev_hist['val_loss'], np.asarray(history.history['val_loss'])) curr_hist = { 'acc': acc, 'val_acc': val_acc, 'loss': loss, 'val_loss': val_loss } np.save(save_path + '/history_acc.npy', acc) np.save(save_path + '/history_valacc.npy', val_acc) np.save(save_path + '/history_loss.npy', loss) np.save(save_path + '/history_valloss.npy', val_loss) plt.figure(figsize=(10, 5)) plt.plot(acc, 'b', label="Training") plt.plot(val_acc, 'g', label="Validation") plt.xlabel("Epoch") plt.ylabel("Accuracy") plt.legend(loc='upper left') plt.savefig(save_path + '/accuracy.png', bbox_inches='tight') plt.figure(figsize=(10, 5)) plt.plot(loss, 'b', label="Training") plt.plot(val_loss, 'g', label="Validation") plt.xlabel("Epoch") plt.ylabel("Loss") plt.legend(loc='upper left') plt.savefig(save_path + '/loss.png', bbox_inches='tight') if show: plt.show() # Evaluate on test set print("Evaluating model on test set") test_loss_acc = model.evaluate(test_set[0][...].reshape(test_set[0].shape + (1, )), test_set[1], batch_size=batch_size, verbose=verbose) print("\nComputing scores on test set") scores = model.predict(test_set[0][...].reshape(test_set[0].shape + (1, )), batch_size=batch_size, verbose=verbose) arr_scores = np.asarray(scores) np.save(save_path + '/scores.npy', arr_scores) cm = confusion_matrix(test_set[1], np.argmax(scores, axis=-1)) plot_cm(cm, cls_names=cls_names, normalize=True, title="Loss: " + str(test_loss_acc[0]) + " | Acc: " + str(test_loss_acc[1]), path_save=save_path + '/cm.png', show=show) print('\n\n') return count_epoch, curr_hist
def train_eval_close_emitter(model, train_set, same_set, simi_set, run_epochs, path_results, num_cls, cls_names, val_set=None, start_epoch=0, verbose=1, batch_size=4, experiment_notes="", show=False, prev_hist=None): """ Train and evaluate Keras model *_set[0] should have shape ([n samples], [x=time axis], [y=frequency axis]) containing the input spectrograms *_set[1] should have shape ([n samples],) containing the corresponding class labels same/simi_set[2] should have shape ([n samples],) containing the corresponding true class names If using in a loop, feed return values into start_epoch and prev_hist to keep accurate epoch count :param model: keras.engine.training.Model Compiled Keras model :param train_set: 2-tuple of array-likes Training set (see above) :param same_set: 3-tuple of array-likes Testing set (see above) :param simi_set: 3-tuple of array-likes Testing set (see above) :param path_results str Output directory :param num_cls int Total number of classes :param cls_names list of str List of class names ordered by corresponding class label :param run_epochs: int Number of epochs to run :param val_set: 2-tuple of array-likes Validation set (see above) :param start_epoch: int Current epoch of model before training :param verbose: int Keras verbosity mode (0, 1, or 2) :param batch_size: int :param experiment_notes: str Notes about experiment :param show: bool Whether to show plots :param prev_hist: dict of arrays Previous interval's training history :return count_epoch: int Current epoch of model after training :return curr_hist: dict of arrays Current interval's training history """ # Set up results directory rundate = datetime.datetime.now().strftime('%b%d%y-%H%M%S') os.mkdir(path_results + '/' + rundate) os.mkdir(path_results + '/' + rundate + '/models') # Save model summary orig_stdout = sys.stdout summ_file = open(path_results + '/' + rundate + '/models/modelsummary.txt', 'w') sys.stdout = summ_file print(model.summary()) sys.stdout = orig_stdout summ_file.close() # Save experiment notes orig_stdout = sys.stdout experiment_notes_file = open( path_results + '/' + rundate + '/_experiment_notes.txt', 'w') sys.stdout = experiment_notes_file print(experiment_notes) sys.stdout = orig_stdout experiment_notes_file.close() # Perform training count_epoch = start_epoch + run_epochs history = model.fit_generator( generator=batch_gen(train_set[0], train_set[1], batch_size), steps_per_epoch=train_set[0].shape[0] // batch_size, epochs=count_epoch, initial_epoch=start_epoch, verbose=verbose, workers=1, validation_data=[ val_set[0][...].reshape(val_set[0].shape + (1, )), keras.utils.to_categorical(val_set[1], num_classes=num_cls) ]) # Save trained model and training info model.save(path_results + '/' + rundate + '/models/epochs' + str(start_epoch) + '-' + str(count_epoch) + '.h5') if prev_hist is None: acc = np.asarray(history.history['acc']) val_acc = np.asarray(history.history['val_acc']) loss = np.asarray(history.history['loss']) val_loss = np.asarray(history.history['val_loss']) else: acc = np.append(prev_hist['acc'], np.asarray(history.history['acc'])) val_acc = np.append(prev_hist['val_acc'], np.asarray(history.history['val_acc'])) loss = np.append(prev_hist['loss'], np.asarray(history.history['loss'])) val_loss = np.append(prev_hist['val_loss'], np.asarray(history.history['val_loss'])) curr_hist = { 'acc': acc, 'val_acc': val_acc, 'loss': loss, 'val_loss': val_loss } np.save(path_results + '/' + rundate + '/history_acc.npy', acc) np.save(path_results + '/' + rundate + '/history_valacc.npy', val_acc) np.save(path_results + '/' + rundate + '/history_loss.npy', loss) np.save(path_results + '/' + rundate + '/history_valloss.npy', val_loss) plt.figure(figsize=(10, 5)) plt.plot(acc, 'b', label="Training") plt.plot(val_acc, 'g', label="Validation") plt.xlabel("Epoch") plt.ylabel("Accuracy") plt.legend(loc='upper left') plt.savefig(path_results + '/' + rundate + '/accuracy.png', bbox_inches='tight') plt.figure(figsize=(10, 5)) plt.plot(loss, 'b', label="Training") plt.plot(val_loss, 'g', label="Validation") plt.xlabel("Epoch") plt.ylabel("Loss") plt.legend(loc='upper left') plt.savefig(path_results + '/' + rundate + '/loss.png', bbox_inches='tight') if show: plt.show() # Evaluate on validation set print("Evaluating model on validation set") val_loss_acc = model.evaluate( val_set[0][...].reshape(val_set[0].shape + (1, )), keras.utils.to_categorical(val_set[1], num_classes=num_cls), batch_size=batch_size, verbose=verbose) print("\nComputing scores on validation set") scores = model.predict(val_set[0][...].reshape(val_set[0].shape + (1, )), batch_size=batch_size, verbose=verbose) arr_scores = np.asarray(scores) np.save(path_results + '/' + rundate + '/val_scores.npy', arr_scores) cm = confusion_matrix(val_set[1], np.argmax(scores, axis=-1)) plot_cm(cm, cls_names=cls_names, normalize=True, title="Loss: " + str(val_loss_acc[0]) + " | Acc: " + str(val_loss_acc[1]), path_save=path_results + '/' + rundate + '/val_cm.png', show=show) print('\n\n') # Evaluate on same set print("Evaluating model on same set") same_loss_acc = model.evaluate( same_set[0][...].reshape(same_set[0].shape + (1, )), keras.utils.to_categorical(same_set[1], num_classes=num_cls), batch_size=batch_size, verbose=verbose) print("\nComputing scores on same set") same_scores = model.predict(same_set[0][...].reshape(same_set[0].shape + (1, )), batch_size=batch_size, verbose=verbose) same_arr_scores = np.asarray(same_scores) np.save(path_results + '/' + rundate + '/same_scores.npy', same_arr_scores) # Build same confusion matrix cm = cnf_matrix(same_set[1], np.argmax(same_scores, axis=-1), num_cls) same_cls_preds = cls_names.copy() same_cls_trues = [] list_same_cls = list(same_set[1]) same_classes = sorted(np.unique(same_set[1])) for cls in same_classes: true_emit = same_set[2][list_same_cls.index(cls)] same_cls_preds[cls] = same_cls_preds[cls] + "(" + str(true_emit) + ")" same_cls_trues.append(str(true_emit)) cm = cm[same_classes] same_cls_cm = cm[:, same_classes] same_cls_labels = [same_cls_preds[cls] for cls in same_classes] other_cls_cm = cm[:, list(set(range(num_cls)) - set(same_classes))] other_cls_labels = [ same_cls_preds[cls] for cls in list(set(range(num_cls)) - set(same_classes)) ] cm = np.concatenate([same_cls_cm, other_cls_cm], axis=1) same_cls_preds = same_cls_labels + other_cls_labels plot_cm(cm, cls_names=same_cls_preds, normalize=True, title="Loss: " + str(same_loss_acc[0]) + " | Acc: " + str(same_loss_acc[1]), path_save=path_results + '/' + rundate + '/same_cm.png', show=show, true_cls_names=same_cls_trues) print('\n\n') # Evaluate on similar set print("Evaluating model on simi set") simi_loss_acc = model.evaluate( simi_set[0][...].reshape(simi_set[0].shape + (1, )), keras.utils.to_categorical(simi_set[1], num_classes=num_cls), batch_size=batch_size, verbose=verbose) print("\nComputing scores on simi set") simi_scores = model.predict(simi_set[0][...].reshape(simi_set[0].shape + (1, )), batch_size=batch_size, verbose=verbose) simi_arr_scores = np.asarray(simi_scores) np.save(path_results + '/' + rundate + '/simi_scores.npy', simi_arr_scores) # Build simi confusion matrix cm = cnf_matrix(simi_set[1], np.argmax(simi_scores, axis=-1), num_cls) simi_cls_preds = cls_names.copy() simi_cls_trues = [] list_simi_cls = list(simi_set[1]) simi_classes = sorted(np.unique(simi_set[1])) for cls in simi_classes: true_emit = simi_set[2][list_simi_cls.index(cls)] simi_cls_preds[cls] = simi_cls_preds[cls] + "(" + str(true_emit) + ")" simi_cls_trues.append(str(true_emit)) cm = cm[simi_classes] simi_cls_cm = cm[:, simi_classes] simi_cls_labels = [simi_cls_preds[cls] for cls in simi_classes] other_cls_cm = cm[:, list(set(range(num_cls)) - set(simi_classes))] other_cls_labels = [ simi_cls_preds[cls] for cls in list(set(range(num_cls)) - set(simi_classes)) ] cm = np.concatenate([simi_cls_cm, other_cls_cm], axis=1) simi_cls_preds = simi_cls_labels + other_cls_labels plot_cm(cm, cls_names=simi_cls_preds, normalize=True, title="Loss: " + str(simi_loss_acc[0]) + " | Acc: " + str(simi_loss_acc[1]), path_save=path_results + '/' + rundate + '/simi_cm.png', show=show, true_cls_names=simi_cls_trues) print('\n\n') return count_epoch, curr_hist
def test_eval_close_emitter(model, val_set, same_set, simi_set, path_results, num_cls, cls_names, verbose=1, batch_size=4, experiment_notes="", show=False, cm_figsize=(10, 10), hist_figsize=(10, 5), bins=100, hist_xlim=(0, 1.05), threat_names=None): """ Evaluate a pretrained Keras model *_set[0] should have shape ([n samples], [x=time axis], [y=frequency axis]) containing the input spectrograms *_set[1] should have shape ([n samples],) containing the corresponding class labels same/simi_set[2] should have shape ([n samples],) containing the corresponding true class names If using in a loop, feed return values into start_epoch and prev_hist to keep accurate epoch count :param model: keras.engine.training.Model Compiled Keras model :param val_set: 2-tuple of array-likes Testing set (see above) :param same_set: 3-tuple of array-likes Testing set (see above) :param simi_set: 3-tuple of array-likes Testing set (see above) :param path_results str Output directory :param num_cls int Total number of classes :param cls_names list of str List of class names ordered by corresponding class label :param verbose: int Keras verbosity mode (0, 1, or 2) :param batch_size: int :param experiment_notes: str Notes about experiment :param show: bool Whether to show plots :param cm_figsize: 2-tuple of ints Matplotlib figure size for confusion matrix & ROC plot :param hist_figsize 2-tuple of ints Matplotlib figure size for histograms :param bins int Number of bins in score histograms :param hist_xlim 2-tuple of floats X-axis limits of histograms :param threat_names dict of strings key: Emitter number, value: Corresponding threat :return: None """ # Set up results directory rundate = datetime.datetime.now().strftime('%b%d%y-%H%M%S') os.mkdir(path_results + '/' + rundate) os.mkdir(path_results + '/' + rundate + '/models') # Save model summary orig_stdout = sys.stdout summ_file = open(path_results + '/' + rundate + '/models/modelsummary.txt', 'w') sys.stdout = summ_file print(model.summary()) sys.stdout = orig_stdout summ_file.close() # Save experiment notes orig_stdout = sys.stdout experiment_notes_file = open( path_results + '/' + rundate + '/_experiment_notes.txt', 'w') sys.stdout = experiment_notes_file print(experiment_notes) sys.stdout = orig_stdout experiment_notes_file.close() # Evaluate on validation set print("Evaluating model on validation set") val_loss_acc = model.evaluate( val_set[0][...].reshape(val_set[0].shape + (1, )), keras.utils.to_categorical(val_set[1], num_classes=num_cls), batch_size=batch_size, verbose=verbose) print("\nComputing scores on validation set") scores = model.predict(val_set[0][...].reshape(val_set[0].shape + (1, )), batch_size=batch_size, verbose=verbose) arr_scores = np.asarray(scores) np.save(path_results + '/' + rundate + '/val_scores.npy', arr_scores) if threat_names is not None: val_cm_cls_names = [threat_names[cls] for cls in cls_names] else: val_cm_cls_names = cls_names cm = confusion_matrix(val_set[1], np.argmax(scores, axis=-1)) plot_cm(cm, cls_names=val_cm_cls_names, normalize=True, title="Total Accuracy: " + "%.2f" % (val_loss_acc[1] * 100) + "%", path_save=path_results + '/' + rundate + '/val_cm.png', show=show) print('\n\n') # Evaluate on same set print("Evaluating model on same set") same_loss_acc = model.evaluate( same_set[0][...].reshape(same_set[0].shape + (1, )), keras.utils.to_categorical(same_set[1], num_classes=num_cls), batch_size=batch_size, verbose=verbose) print("\nComputing scores on same set") same_scores = model.predict(same_set[0][...].reshape(same_set[0].shape + (1, )), batch_size=batch_size, verbose=verbose) same_arr_scores = np.asarray(same_scores) np.save(path_results + '/' + rundate + '/same_scores.npy', same_arr_scores) # Build same confusion matrix cm = cnf_matrix(same_set[1], np.argmax(same_scores, axis=-1), num_cls) same_cls_preds = cls_names.copy() same_cls_trues = [] list_same_cls = list(same_set[1]) same_classes = sorted(np.unique(same_set[1])) for cls in same_classes: true_emit = same_set[2][list_same_cls.index(cls)] same_cls_preds[cls] = same_cls_preds[cls] + "(" + str(true_emit) + ")" same_cls_trues.append(str(true_emit)) cm = cm[same_classes] same_cls_cm = cm[:, same_classes] same_cls_labels = [same_cls_preds[cls] for cls in same_classes] other_cls_cm = cm[:, list(set(range(num_cls)) - set(same_classes))] other_cls_labels = [ same_cls_preds[cls] for cls in list(set(range(num_cls)) - set(same_classes)) ] cm = np.concatenate([same_cls_cm, other_cls_cm], axis=1) same_cls_preds = same_cls_labels + other_cls_labels plot_cm(cm, cls_names=same_cls_preds, normalize=True, title="Total Accuracy: " + "%.2f" % (same_loss_acc[1] * 100) + "%", path_save=path_results + '/' + rundate + '/same_cm.png', show=show, true_cls_names=same_cls_trues) print('\n\n') # Evaluate on similar set print("Evaluating model on simi set") simi_loss_acc = model.evaluate( simi_set[0][...].reshape(simi_set[0].shape + (1, )), keras.utils.to_categorical(simi_set[1], num_classes=num_cls), batch_size=batch_size, verbose=verbose) print("\nComputing scores on simi set") simi_scores = model.predict(simi_set[0][...].reshape(simi_set[0].shape + (1, )), batch_size=batch_size, verbose=verbose) simi_arr_scores = np.asarray(simi_scores) np.save(path_results + '/' + rundate + '/simi_scores.npy', simi_arr_scores) # Build simi confusion matrix cm = cnf_matrix(simi_set[1], np.argmax(simi_scores, axis=-1), num_cls) simi_cls_preds = cls_names.copy() simi_cls_trues = [] list_simi_cls = list(simi_set[1]) simi_classes = sorted(np.unique(simi_set[1])) for cls in simi_classes: true_emit = simi_set[2][list_simi_cls.index(cls)] simi_cls_preds[cls] = simi_cls_preds[cls] + "(" + str(true_emit) + ")" simi_cls_trues.append(str(true_emit)) cm = cm[simi_classes] simi_cls_cm = cm[:, simi_classes] simi_cls_labels = [simi_cls_preds[cls] for cls in simi_classes] other_cls_cm = cm[:, list(set(range(num_cls)) - set(simi_classes))] other_cls_labels = [ simi_cls_preds[cls] for cls in list(set(range(num_cls)) - set(simi_classes)) ] cm = np.concatenate([simi_cls_cm, other_cls_cm], axis=1) simi_cls_preds = simi_cls_labels + other_cls_labels plot_cm(cm, cls_names=simi_cls_preds, normalize=True, title="Total Accuracy: " + "%.2f" % (simi_loss_acc[1] * 100) + "%", path_save=path_results + '/' + rundate + '/simi_cm.png', show=show, true_cls_names=simi_cls_trues) # Score histograms # ASSUMES EVEN-CLASS, CLASS-NAME-SORTED, UNSHUFFLED SETS if arr_scores.shape[0] % num_cls != 0: raise ValueError( "Number of validation samples does not evenly divide into number of classes" ) else: samp_per_emit = arr_scores.shape[0] / num_cls fig = plt.figure(figsize=hist_figsize) for i in range(num_cls): cls_scores = arr_scores[int(i * samp_per_emit):int(i * samp_per_emit + samp_per_emit)] plt.hist(np.max(cls_scores, axis=1), range=(0, 1.05), bins=bins, alpha=1 / num_cls, label=cls_names[i]) plt.title("Validation Set Per-class Score Distributions") plt.xlim(hist_xlim) plt.ylim(0, arr_scores.shape[0] / num_cls) plt.legend(loc="upper left") plt.savefig(path_results + '/' + rundate + '/val_score_hist.png', bbox_inches='tight') plt.show() same_cls_names = np.unique(same_set[2]) same_num_cls = same_cls_names.shape[0] if same_arr_scores.shape[0] % same_num_cls != 0: raise ValueError( "Number of same set samples does not evenly divide into number of classes" ) else: same_samp_per_emit = same_arr_scores.shape[0] / same_num_cls fig = plt.figure(figsize=hist_figsize) for i in range(same_num_cls): same_cls_scores = same_arr_scores[int(i * same_samp_per_emit ):int(i * same_samp_per_emit + same_samp_per_emit)] plt.hist(np.max(same_cls_scores, axis=1), range=(0, 1.05), bins=bins, alpha=1 / same_num_cls, label=str(same_cls_names[i])) plt.title("Same Set Per-class Score Distributions") plt.xlim(hist_xlim) plt.ylim(0, same_arr_scores.shape[0] / same_num_cls) plt.legend(loc='upper left') plt.savefig(path_results + '/' + rundate + '/same_score_hist.png', bbox_inches='tight') plt.show() simi_cls_names = np.unique(simi_set[2]) simi_num_cls = simi_cls_names.shape[0] if simi_arr_scores.shape[0] % simi_num_cls != 0: raise ValueError( "Number of simi set samples does not evenly divide into number of classes" ) else: simi_samp_per_emit = simi_arr_scores.shape[0] / simi_num_cls fig = plt.figure(figsize=hist_figsize) for i in range(simi_num_cls): simi_cls_scores = simi_arr_scores[int(i * simi_samp_per_emit ):int(i * simi_samp_per_emit + simi_samp_per_emit)] plt.hist(np.max(simi_cls_scores, axis=1), range=(0, 1.05), bins=bins, alpha=1 / simi_num_cls, label=str(simi_cls_names[i])) plt.title("Similar Set Per-class Score Distributions") plt.xlim(hist_xlim) plt.ylim(0, simi_arr_scores.shape[0] / simi_num_cls) plt.legend(loc='upper left') plt.savefig(path_results + '/' + rundate + '/simi_score_hist.png', bbox_inches='tight') plt.show() # Plot combined same/similar ROC curves same_simi_scores = np.concatenate([same_arr_scores, simi_arr_scores], axis=0) same_targets = keras.utils.to_categorical(same_set[1], num_classes=num_cls) simi_targets = keras.utils.to_categorical(simi_set[1], num_classes=num_cls) same_simi_targets = np.concatenate([same_targets, simi_targets], axis=0) fpr = {} tpr = {} roc_auc = {} for cls in range(num_cls): fpr[cls], tpr[cls], _ = roc_curve(same_simi_targets[:, cls], same_simi_scores[:, cls]) roc_auc[cls] = auc(fpr[cls], tpr[cls]) fig = plt.figure(figsize=cm_figsize) for i in range(num_cls): if i in same_classes and i in simi_classes: plt.plot(fpr[i], tpr[i], '-.', lw=2, label='Emitter {0} (AUC={1:0.2f})' ''.format(cls_names[i], roc_auc[i])) elif i in same_classes: plt.plot(fpr[i], tpr[i], '-', lw=2, label='Emitter {0} (AUC={1:0.2f})' ''.format(cls_names[i], roc_auc[i])) elif i in simi_classes: plt.plot(fpr[i], tpr[i], '--', lw=2, label='Emitter {0} (AUC={1:0.2f})' ''.format(cls_names[i], roc_auc[i])) else: raise ValueError("Class not in same or similar set") plt.plot([0, 1], [0, 1], 'k:', lw=2) plt.xlim([-0.05, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Per-class ROC Curves') plt.legend(loc="lower right") plt.savefig(path_results + '/' + rundate + '/roc.png', bbox_inches='tight') plt.show()
def test_eval(model, test_set, path_results, num_cls, cls_names, verbose=1, batch_size=4, experiment_notes="", show=False, cm_figsize=(10, 10)): """ Evaluate a pretrained Keras model :param model: keras.engine.training.Model Compiled Keras model :param test_set: 2-tuple of array-likes Testing set (see above) :param path_results str Output directory :param num_cls int Total number of classes :param cls_names list of str List of class names ordered by corresponding class label :param verbose: int Keras verbosity mode (0, 1, or 2) :param batch_size: int :param experiment_notes: str Notes about experiment :param show: bool Whether to show plots :param cm_figsize: 2-tuple of ints Matplotlib figure size for confusion matrix :return: None """ # Set up results directory rundate = datetime.datetime.now().strftime('%b%d%y-%H%M%S') os.mkdir(path_results + '/' + rundate) os.mkdir(path_results + '/' + rundate + '/models') # Save model summary orig_stdout = sys.stdout summ_file = open(path_results + '/' + rundate + '/models/modelsummary.txt', 'w') sys.stdout = summ_file print(model.summary()) sys.stdout = orig_stdout summ_file.close() # Save experiment notes orig_stdout = sys.stdout experiment_notes_file = open( path_results + '/' + rundate + '/_experiment_notes.txt', 'w') sys.stdout = experiment_notes_file print(experiment_notes) sys.stdout = orig_stdout experiment_notes_file.close() # Evaluate on test set print("Evaluating model on test set") test_loss_acc = model.evaluate( test_set[0][...].reshape(test_set[0].shape + (1, )), keras.utils.to_categorical(test_set[1], num_classes=num_cls), batch_size=batch_size, verbose=verbose) print("\nComputing scores on test set") scores = model.predict(test_set[0][...].reshape(test_set[0].shape + (1, )), batch_size=batch_size, verbose=verbose) arr_scores = np.asarray(scores) np.save(path_results + '/' + rundate + '/scores.npy', arr_scores) cm = confusion_matrix(test_set[1], np.argmax(scores, axis=-1)) plot_cm(cm, cls_names=cls_names, normalize=True, title="Loss: " + str(test_loss_acc[0]) + " | Acc: " + str(test_loss_acc[1]), path_save=path_results + '/' + rundate + '/cm.png', show=show, figsize=cm_figsize)