def plot_confusion_matrix(labels: np.ndarray,
predictions: np.ndarray,
class_label_names: Optional[Dict[Union[str, int],
Union[str,
int]]] = None,
normalize: Optional[str] = None,
title_fontsize: Optional[int] = 12,
x_label_fontsize: Optional[int] = 12,
y_label_fontsize: Optional[int] = 12,
heatmap_color: Optional[str] = 'Greens') -> None:
"""Plot confusion matrix for a multiclass classification model.
Args:
labels: An array of true labels containing multiclass labels.
predictions: An array of predictions containing multiclass labels.
class_label_names: Dictionary of multiclass labels and corresponding target
names. The type of both class lable and target names can be either 'int'
or 'str'. E.g. {0: 'low_value', 1: 'mid_value', 2: 'high_value'}.
normalize: A parameter controlling whether to normalize the counts in the
matrix.
title_fontsize: Font size of the figure title.
x_label_fontsize: Font size of the x axis labels.
y_label_fontsize: Font size of the y axis labels.
heatmap_color: Color of the heatmap plot.
Returns:
Heatmap of confusion matrix.
"""
utils.assert_label_and_prediction_length_match(labels, predictions)
if class_label_names is None:
class_labels = list(set(labels))
target_names = ['%s' % l for l in class_labels]
else:
class_labels = list(class_label_names.keys())
target_names = list(class_label_names.values())
plot = ConfusionMatrixDisplay.from_predictions(y_true=labels,
y_pred=predictions,
labels=np.unique(labels),
display_labels=target_names,
normalize=normalize,
include_values=True,
cmap=heatmap_color)
plot.ax_.set_title('Confusion matrix', fontsize=title_fontsize)
plot.ax_.set_xlabel('Predicted label', fontsize=x_label_fontsize)
plot.ax_.set_ylabel('Actual label', fontsize=y_label_fontsize)
plt.show()
def display_confusion_matrix(y, y_pred, cm_filename):
from sklearn.metrics import confusion_matrix
from sklearn.metrics import ConfusionMatrixDisplay
import matplotlib.pyplot as plt
cmat = confusion_matrix(y.cpu().numpy(), y_pred.cpu().numpy())
for c in range(cmat.shape[0]):
cmat[c, c] = 0.0
_, ax = plt.subplots(1, 1, figsize=(16, 16))
ConfusionMatrixDisplay(cmat).plot(ax=ax)
if cm_filename is None:
plt.show()
else:
plt.savefig(cm_filename)
plt.close('all')
def evaluate_model(test_labels, params, scores, predictions):
'''
Preparing information to show.
:param test_labels: labels of test set
:param params: parameters
:param scores: scores of images per each class
:param predictions: predicted labels
:return: mean_accuracy, disp_confusion_matrix, indices_of_the_largest_error_images
'''
print_statement('Evaluation of the model:')
display_labels = get_ordered_list(test_labels)
mean_accuracy = sklearn.metrics.accuracy_score(test_labels, predictions)
confusion_matrix = sklearn.metrics.confusion_matrix(test_labels, predictions)
disp_confusion_matrix = ConfusionMatrixDisplay(confusion_matrix, display_labels).plot()
indices_of_the_largest_error_images = get_indices_of_the_largest_error_images(scores, test_labels, params)
return mean_accuracy, disp_confusion_matrix, indices_of_the_largest_error_images
def plot_confusion_matrix(fp_type, preds, ax):
"""Plots confusion matrices for visualising predictions given as argument
:param fp_type : specifies which of Handcrafted or Learned was used to generate
predictions
:param preds : predictions to visualise
:param ax : Axes object to plot to
"""
cm = confusion_matrix(ground_truth, preds, labels)
cm_plot = ConfusionMatrixDisplay(cm, labels)
plot_cm_custom(cm_plot, ax=ax, values_format='d')
ttl = ax.title
ttl.set_text(fp_type)
ttl.set_fontweight('bold')
ttl.set_position([0.5, -0.3])
def plot_confusion_matrix(y_true, y_predicted, labels=[], ax=None, output=None):
if ax is None:
_, ax = plt.subplots(1, 1, figsize=(8, 8))
cm = confusion_matrix(y_true, y_predicted)
cm = 100 * cm / cm.sum(axis=1)[:, np.newaxis]
# ax.xaxis.tick_top()
ConfusionMatrixDisplay(
confusion_matrix=cm,
display_labels=labels,
).plot(include_values=True, cmap=plt.cm.Blues, values_format='0.0f', ax=ax, xticks_rotation='vertical')
if output is not None:
plt.savefig(output)
def label_show_confusion_matrix(y_test, y_pred):
from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix
from matplotlib import pyplot as plt
labels = ['Clean', 'Leak']
# print("Report:", classification_report(y_test, y_pred))
fig = plt.figure()
ax = fig.add_subplot(111)
ConfusionMatrixDisplay(confusion_matrix(y_pred, y_test, labels=[0, 1]),
display_labels=labels).plot(values_format=".0f",
ax=ax,
cmap=plt.cm.Blues)
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()
def predict(self, img_path, model_path, visualize=False):
model = Classifier()
ckpt = torch.load(model_path)
model.load_state_dict(ckpt)
model = model.to(self.device)
model.eval()
images = [f for f in os.listdir(img_path) if f.endswith('.png')]
success = 0
y_true, y_pred = [], []
for imgname in images:
image = cv2.imread(os.path.join(img_path, imgname))
img = image.copy() / 255.0
img = np.transpose(img, (2, 0, 1))
img = np.expand_dims(img, axis=0)
img = torch.from_numpy(img).to(self.device).float()
pred_label = model(img).cpu().detach().numpy()
if ("Lateral" in imgname) and (pred_label < 0.5):
success += 1
elif ("AP" in imgname) and (pred_label > 0.5):
success += 1
true_label = "Lateral" if "Lateral" in imgname else "AP"
pred_label = "Lateral" if pred_label < 0.5 else "AP"
y_true.append(true_label)
y_pred.append(pred_label)
if visualize:
cv2.putText(image, "pred: {}".format(pred_label), (20, 20),
cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 255), 2)
cv2.putText(image, "true: {}".format(true_label), (20, 40),
cv2.FONT_HERSHEY_PLAIN, 1, (0, 255, 0), 2)
cv2.imshow('predicted', image)
cv2.waitKey(0)
print("[INFO] accuracy is : {:.2f} %".format(100 * success /
len(images)))
cm = confusion_matrix(y_true, y_pred)
print(
classification_report(y_true,
y_pred,
target_names=["AP", "Lateral"]))
#print("[INFO] confusion matrix:\n {}".format(cm))
disp = ConfusionMatrixDisplay(confusion_matrix=cm,
display_labels=("AP", "Lateral")).plot()
plt.show()
def plot_cfs_matrix(y_true,y_pred,labels): # if include_noise: # cm = confusion_matrix(y_true,y_pred,labels=["Trocar 1","Trocar 2","Trocar 3","Trocar 4","Incorrect Data"],normalize='true') # disp = ConfusionMatrixDisplay(confusion_matrix=cm, # display_labels=["Trocar 1","Trocar 2","Trocar 3","Trocar 4","Incorrect Data"]).plot() # else: # cm = confusion_matrix(y_true,y_pred,labels=["Trocar 1","Trocar 2","Trocar 3","Trocar 4"],normalize='true') # disp = ConfusionMatrixDisplay(confusion_matrix=cm, # display_labels=["Trocar 1","Trocar 2","Trocar 3","Trocar 4"]).plot() cm = confusion_matrix(y_true,y_pred,labels=labels,normalize='true') disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels).plot() # NOTE: Fill all variables here with default values of the plot_confusion_matrix plt.show()
def evaluate():
print("evaluating model!"
) # replace this with code to evaluate what must be evaluated
df = pd.read_csv("predictions.csv", encoding='iso-8859-1')
accuracy = accuracy_score(df['actual'], df['predictions']) * 100
accuracy = round(accuracy, 2)
print("Accuracy score on Test data for SVC is: ", accuracy, '%')
file = open('accuracy.txt', 'w')
file.write(str(accuracy))
file.close()
conf_mat = confusion_matrix(df['actual'],
df['predictions'],
labels=["male", "female"])
ConfusionMatrixDisplay(confusion_matrix=conf_mat,
display_labels=["male", "female"]).plot()
plt.title('Confusion-matrix')
plt.savefig('Confusion-matrix.png')
def evaluate(model: nn.Module,
test_loader: torch.utils.data.DataLoader,
loss_fn=None,
writer: SummaryWriter = None,
epoch: int = -1):
with torch.no_grad():
model.eval()
epoch_losses = []
epoch_accuracy = []
conf_matrix = np.zeros((2, 2), dtype='int32')
for step, (x, x_len, y) in enumerate(test_loader):
x, y = x.cuda(), y.cuda()
y_pred = model(x, x_len)
y_argmax = torch.argmax(y_pred, 1)
accuracy = y_argmax.eq(y).sum().item() / y.shape[0]
epoch_accuracy.append(accuracy)
conf_matrix += confusion_matrix(y.cpu().numpy(),
y_argmax.cpu().numpy())
if loss_fn:
loss_val = loss_fn(y_pred, y)
epoch_losses.append(loss_val.item())
print(' Test batch {} of {}'.format(step + 1, len(test_loader)),
file=sys.stderr)
print(
f'Test loss: {np.mean(epoch_losses):.4f}, accuracy: {np.mean(epoch_accuracy):.4f}'
)
if writer:
writer.add_scalar('Loss/test',
np.mean(epoch_losses),
global_step=epoch)
writer.add_scalar('Accuracy/test',
np.mean(epoch_accuracy),
global_step=epoch)
conf_plot = ConfusionMatrixDisplay(
confusion_matrix=conf_matrix,
display_labels=['ham', 'spam']).plot(cmap='Blues',
values_format='d')
writer.add_figure('ConfusionMatrix',
conf_plot.figure_,
global_step=epoch,
close=True)
return np.mean(epoch_accuracy)
def evaluation(X_Actual, Y_actual, classifier, classifier_name):
Y_predicted = classifier.predict(X_Actual)
cm = confusion_matrix(Y_actual, Y_predicted)
ConfusionMatrixDisplay(confusion_matrix=cm,
display_labels=classifier.classes_).plot()
plt.title(classifier_name)
#5 fold cross validation
x = cross_val_score(classifier, X_Actual, Y_actual, cv=5)
print("cross validation score:", x, "\n")
Yscores = cross_val_predict(classifier, X_Actual, Y_actual, cv=5,\
method="predict_proba")
Yscores = Yscores[:, -1]
#ROC curve
fpr, tpr, thresholds = roc_curve(Y_actual, Yscores)
#Metrics
tn, fp, fn, tp = cm.ravel()
sensitivity = tp / (tp + fn)
fpr_cm = fp / (fp + tn)
specificity = tn / (tn + fp)
G = math.sqrt(sensitivity * specificity)
LP = sensitivity / (1 - specificity)
LR = (1 - sensitivity) / specificity
#DP=(math.sqrt(3)/math.pi)*(math.log(sensitivity/(1-sensitivity))+math.log(specificity/(1-specificity)))
gamma = sensitivity - (1 - sensitivity)
BA = (1 / 2) * (sensitivity + specificity)
WBA = (3 / 4) * (sensitivity) + (1 / 4) * specificity
index_metrics = ["false alarm","hit rate","precision","accuracy","recall", \
"AUC","Kolmogorov–Smirnov","G-mean","Positive Likelihood Ratio",\
"Negative Likelihood Ratio", "Youden Index",\
"Balanced Accuracy", "Weighted Balance Accuracy"]
d = {classifier_name: [fpr_cm, sensitivity, precision_score(Y_actual, Y_predicted)\
,accuracy_score(Y_actual, Y_predicted),\
recall_score(Y_actual, Y_predicted),\
roc_auc_score(Y_actual, Yscores),\
stats.ks_2samp(Y_actual, Yscores)[0],G,LP,\
LR,gamma,BA,WBA]}
df_metrics = pd.DataFrame(d, index_metrics)
return fpr, tpr, classifier_name, df_metrics
def test_confusion_matrix_standard_format(pyplot):
cm = np.array([[10000000, 0], [123456, 12345678]])
plotted_text = ConfusionMatrixDisplay(cm, [False, True]).plot().text_
# Values should be shown as whole numbers 'd',
# except the first number which should be shown as 1e+07 (longer length)
# and the last number will be showns as 1.2e+07 (longer length)
test = [t.get_text() for t in plotted_text.ravel()]
assert test == ['1e+07', '0', '123456', '1.2e+07']
cm = np.array([[0.1, 10], [100, 0.525]])
plotted_text = ConfusionMatrixDisplay(cm, [False, True]).plot().text_
# Values should now formatted as '.2g', since there's a float in
# Values are have two dec places max, (e.g 100 becomes 1e+02)
test = [t.get_text() for t in plotted_text.ravel()]
assert test == ['0.1', '10', '1e+02', '0.53']
def show_confusion_matrix(confusion, index_range=(0, 11), kappas=None):
"""Diplay all confusion matrix within range
:param confusion: the list of confusion matrices
:type confusion: List: np.ndarray
:param index_range: the range of indices within the confusion param to display, defaults to (0, 10)
:type index_range: tuple(int), optional
"""
fig, axes = plt.subplots(nrows=len(confusion),
figsize=(7, len(confusion) * 10))
ax = axes.ravel()
for i in range(index_range[0], index_range[1]):
if kappas is not None:
#print('Kappa: ', kappas[i])
kappa = round(kappas[i], 2)
if i == 0:
# special case labels
cmd = ConfusionMatrixDisplay(
confusion[i], display_labels=[i for i in range(0, 10)])
cmd.plot(ax=ax[i])
if not kappas is None:
cmd.ax_.set_title(f'All classes\nKappa: {kappa}',
fontsize=20) # type: ignore
else:
cmd.ax_.set_title('All classes', fontsize=20)
else:
cmd = ConfusionMatrixDisplay(confusion[i],
display_labels=['True', 'False'])
cmd.plot(ax=ax[i])
label = label_def.get(i - 1, i)
if not kappas is None:
cmd.ax_.set_title(f'{label}\n Kappa: {kappa}',
fontsize=20) # type: ignore
else:
cmd.ax_.set_title(f'{label}', fontsize=20)
plt.subplots_adjust(wspace=0.40, hspace=0.1)
#axes.flat[-1].set_visible(False)
plt.show()
def run(args):
# Cancer types
cancer_types = pd.read_csv(filepath / 'data/combined_cancer_types',
sep='\t',
names=['CELL', 'CTYPE'])
# Concat preds from all runs
runs_dirs = [Path(p) for p in glob(str(trn_path / 'run_*'))]
prd_te_all = agg_preds_from_cls_runs(runs_dirs, phase='_te.csv')
prd_te_all = update_prd_table(prd_te_all, cancer_types=cancer_types)
# Save aggregated master table
prd_te_all.to_csv(trn_path / 'prd_te_all.csv', index=False)
# Confusion Matrix
conf = confusion_matrix(prd_te_all.y_true_cls,
prd_te_all.y_pred_cls,
normalize=None)
conf_plot = ConfusionMatrixDisplay(conf, display_labels=['NoResp', 'Resp'])
conf_plot.plot(include_values=True,
cmap=plt.cm.Blues,
ax=None,
xticks_rotation=None,
values_format='d')
plt.show()
plt.savefig(trn_path / 'conf_mat.png', dpi=100)
# Confusion Matrix (normalized)
conf = confusion_matrix(prd_te_all.y_true_cls,
prd_te_all.y_pred_cls,
normalize='all')
conf_plot = ConfusionMatrixDisplay(conf, display_labels=['NoResp', 'Resp'])
conf_plot.plot(include_values=True,
cmap=plt.cm.Blues,
ax=None,
xticks_rotation=None,
values_format='.2f')
plt.savefig(trn_path / 'conf_mat_norm.png', dpi=100)
def test(self,
img,
lbl,
classNames=None,
f1=False,
savefig=True,
save_path=""):
if '/' not in save_path and '\\' not in save_path:
save_path += '/'
prediction = self.model.predict(img, batch_size=self.batch)
print(lbl.argmax(axis=1), prediction.argmax(axis=1))
if f1:
print(
classification_report(lbl.argmax(axis=1),
prediction.argmax(axis=1),
target_names=classNames))
if savefig:
cm = confusion_matrix(lbl.argmax(axis=1),
prediction.argmax(axis=1))
cm_display = ConfusionMatrixDisplay(cm).plot()
cm_display.figure_.savefig(
save_path + 'Confusion Matrix {}.png'.format(self.name))
def test_confusion_matrix_display_validation(pyplot):
"""Check that we raise the proper error when validating parameters."""
X, y = make_classification(
n_samples=100, n_informative=5, n_classes=5, random_state=0
)
regressor = SVR().fit(X, y)
y_pred_regressor = regressor.predict(X)
y_pred_classifier = SVC().fit(X, y).predict(X)
err_msg = "ConfusionMatrixDisplay.from_estimator only supports classifiers"
with pytest.raises(ValueError, match=err_msg):
ConfusionMatrixDisplay.from_estimator(regressor, X, y)
err_msg = "Mix type of y not allowed, got types"
with pytest.raises(ValueError, match=err_msg):
# Force `y_true` to be seen as a regression problem
ConfusionMatrixDisplay.from_predictions(y + 0.5, y_pred_classifier)
with pytest.raises(ValueError, match=err_msg):
ConfusionMatrixDisplay.from_predictions(y, y_pred_regressor)
err_msg = "Found input variables with inconsistent numbers of samples"
with pytest.raises(ValueError, match=err_msg):
ConfusionMatrixDisplay.from_predictions(y, y_pred_classifier[::2])
def evaluation(y_pred, classifier, classifier_name):
cm = confusion_matrix(Ytrain, y_pred)
ConfusionMatrixDisplay(confusion_matrix=cm,
display_labels=classifier.classes_).plot()
#5 fold
Yscores = cross_val_predict(classifier,
Xtrain_norm,
Ytrain,
cv=None,
method="predict")
fpr, tpr, thresholds = roc_curve(Ytrain, Yscores)
plt.figure(figsize=(8, 6))
plot_roc_curve(fpr, tpr)
plt.show()
roc_auc_score(Ytrain, Yscores)
y_pred = classifier.predict(Xtest_norm)
print(classifier_name)
print("precision:", precision_score(Ytest, y_pred))
print("accuracy:", accuracy_score(Ytest, y_pred))
print("recall:", recall_score(Ytest, y_pred))
print("AUC:", roc_auc_score(Ytest, y_pred), "\n")
def plot_results(prediction, labels, output_dir, rec_device, n_classes):
acc = round(accuracy_score(prediction, labels) * 100, 2)
cm = confusion_matrix(prediction, labels, normalize='true')
fig, ax = plt.subplots(figsize=(15, 15))
plt.rcParams.update({'font.size': 18})
if n_classes == 10:
cmd = ConfusionMatrixDisplay(cm, display_labels=list(LABELS_10.keys()))
else:
cmd = ConfusionMatrixDisplay(cm, display_labels=list(LABELS_3.keys()))
cmd.plot(cmap='Blues', xticks_rotation=60, values_format='.2f', ax=ax)
plt.title(f'Acoustic Scene Clasiffication {rec_device}- Confusion Matrix - Accuracy = {acc}%', fontsize=18,
fontweight='bold')
plt.xlabel('Predicted label', fontsize=18, fontweight='bold')
plt.ylabel('True label', fontsize=18, fontweight='bold')
plt.setp(ax.get_xticklabels(), fontsize=18)
plt.setp(ax.get_yticklabels(), fontsize=18)
plt.tight_layout()
plt.savefig(osp.join(output_dir, f'confusion_matrix_{rec_device}.jpg'))
def analyze(prob_pred, y_true):
y_pred = [np.argmax(prob) for prob in prob_pred]
accuracy = accuracy_score(y_true, y_pred)
global_precision = precision_score(y_true, y_pred, average = "micro")
global_recall = recall_score(y_true, y_pred, average = "micro")
global_f1 = f1_score(y_true, y_pred, average = "micro")
weighted_roc_auc_ovr = roc_auc_score(y_true, prob_pred, multi_class="ovr", average="weighted")
macro_roc_auc_ovr = roc_auc_score(y_true, prob_pred, multi_class="ovr", average="macro")
weighted_roc_auc_ovo = roc_auc_score(y_true, prob_pred, multi_class="ovo", average="weighted")
macro_roc_auc_ovo = roc_auc_score(y_true, prob_pred, multi_class="ovo", average="macro")
print(classification_report(y_true, y_pred))
print("global recall: {}".format(global_recall))
print("global precision: {}".format(global_precision))
print("global f1: {}".format(global_f1))
print("One-vs-One ROC AUC scores:\n{:.6f} (macro),\n{:.6f} (weighted by prevalence)"
.format(macro_roc_auc_ovo, weighted_roc_auc_ovo))
print("One-vs-Rest ROC AUC scores:\n{:.6f} (macro),\n{:.6f} (weighted by prevalence)"
.format(macro_roc_auc_ovr, weighted_roc_auc_ovr))
conf_mx = confusion_matrix(y_true, y_pred)
ConfusionMatrixDisplay(conf_mx).plot()
plt.show()
show_command = input("See the wrong answers? (y/n)")
i = 0
while i < len(y_pred) and show_command == 'y':
if y_pred[i] != y_true[i]:
print('predicted:', y_pred[i], 'target:', y_true[i])
io.imshow(np.squeeze(X_test[i]))
io.show()
show_command = input("See the wrong answers? (y/n)")
i+=1
return conf_mx
def test_confusion_matrix_contrast(pyplot):
# make sure text color is appropriate depending on background
cm = np.eye(2) / 2
disp = ConfusionMatrixDisplay(cm, display_labels=[0, 1])
disp.plot(cmap=pyplot.cm.gray)
# diagonal text is black
assert_allclose(disp.text_[0, 0].get_color(), [0.0, 0.0, 0.0, 1.0])
assert_allclose(disp.text_[1, 1].get_color(), [0.0, 0.0, 0.0, 1.0])
# oof-diagonal text is white
assert_allclose(disp.text_[0, 1].get_color(), [1.0, 1.0, 1.0, 1.0])
assert_allclose(disp.text_[1, 0].get_color(), [1.0, 1.0, 1.0, 1.0])
disp.plot(cmap=pyplot.cm.gray_r)
# diagonal text is white
assert_allclose(disp.text_[0, 1].get_color(), [0.0, 0.0, 0.0, 1.0])
assert_allclose(disp.text_[1, 0].get_color(), [0.0, 0.0, 0.0, 1.0])
# oof-diagonal text is black
assert_allclose(disp.text_[0, 0].get_color(), [1.0, 1.0, 1.0, 1.0])
assert_allclose(disp.text_[1, 1].get_color(), [1.0, 1.0, 1.0, 1.0])
# create an instance of the SVM classifier with a Linear kernel
model = svm.SVC(kernel='linear')
# train the model using the training set
model.fit(X_train, y_train)
# predict the classes in the test set
y_pred = model.predict(X_test)
# D. Evaluate the model
# accuracy
print("Accuracy: %.3f%%" % (metrics.accuracy_score(y_test, y_pred) * 100))
# precision
print("Precision: %.3f " % metrics.precision_score(y_test, y_pred))
# recall
print("Recall: %.3f" % metrics.recall_score(y_test, y_pred))
# F1 (F-Measure)
print("F1: %.3f" % metrics.f1_score(y_test, y_pred))
# compute Confusion Matrix
cm = confusion_matrix(y_test, y_pred, labels=model.classes_)
# pretty print Confusion Matrix as a heatmap
disp = ConfusionMatrixDisplay(confusion_matrix=cm,
display_labels=dataset.target_names)
disp.plot()
plt.show()
def task1():
model_path = os.path.join("project_a_supp", "models", "MNIST1D.h5")
model = keras.models.load_model(model_path)
mnist1d = make_dataset()
x_test = np.expand_dims(mnist1d["x_test"], axis=-1)
y_test = mnist1d["y_test"]
model.evaluate(x_test, y_test)
num_classes = 10
num_true_positives = 0
num_correct_per_class = np.zeros(num_classes, dtype=np.int64)
num_total_per_class = np.zeros(num_classes, dtype=np.int64)
y_predicted_scores = []
num_wrong = 0
plt.figure(figsize=(20, 8))
should_plot_failures = False
for i in range(len(x_test)):
digit_input = x_test[i:i + 1]
digit_label = y_test[i:i + 1]
digit_prediction = model(digit_input).numpy()
y_predicted_scores.append(digit_prediction)
digit_prediction = np.argmax(digit_prediction)
if digit_prediction == digit_label:
if not should_plot_failures and (num_true_positives < 10):
plt.subplot(2, 5, 1 + num_true_positives)
plt.plot(np.squeeze(digit_input), "r")
plt.axis("off")
plt.title(
f"label: {digit_label.item()} predicted: {digit_prediction}"
)
num_true_positives += 1
num_correct_per_class[digit_label] += 1
else:
if should_plot_failures and (num_wrong < 10):
plt.subplot(2, 5, 1 + num_wrong)
plt.plot(np.squeeze(digit_input), "r")
plt.axis("off")
plt.title(
f"label: {digit_label.item()} predicted: {digit_prediction}"
)
num_wrong += 1
num_total_per_class[digit_label] += 1
if should_plot_failures:
plt.savefig(os.path.join("report", "images", "mnist1d-failures.png"),
dpi=256)
else:
plt.savefig(os.path.join("report", "images", "mnist1d-successes.png"),
dpi=256)
plt.clf()
y_predicted_scores = np.concatenate(y_predicted_scores, axis=0)
print(f"Accuracy: {num_true_positives/len(x_test)}")
classwise_accuracy = num_correct_per_class / num_total_per_class
print(f"Class-wise ccuracy: {classwise_accuracy}")
plt.bar(x=range(num_classes), height=classwise_accuracy)
plt.title("MNIST-1D Class-wise Accuracy")
plt.xticks(range(10))
plt.xlabel("Digit")
plt.ylabel("Accuracy")
plt.savefig(os.path.join("report", "images", "mnist1d-class-accuracy.png"),
dpi=256)
plt.clf()
y_test_onehot = np.zeros((len(x_test), num_classes), dtype=np.int64)
for y_idx, y_label in enumerate(y_test):
y_test_onehot[y_idx, y_label] = 1
for i in range(num_classes):
false_pos_rate, true_pos_rate, _ = roc_curve(y_test_onehot[:, i],
y_predicted_scores[:, i])
digit_auc = auc(false_pos_rate, true_pos_rate)
plt.plot(false_pos_rate,
true_pos_rate,
label=f"{i} (AUC = {digit_auc:.4f})")
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.title("MNIST-1D ROC-AUC Curves")
plt.legend(loc="lower right")
plt.savefig(os.path.join("report", "images", "mnist1d-roc-curve.png"),
dpi=256)
plt.clf()
y_predicted_digits = np.argmax(y_predicted_scores, axis=1)
digits_confusion_mtx = confusion_matrix(y_predicted_digits,
y_test,
normalize="true")
plt.title("MNIST-1D Confusion Matrix")
disp = ConfusionMatrixDisplay(digits_confusion_mtx)
disp.plot()
# plt.savefig(os.path.join("report", "images", "mnist1d-confusion-matrix"), dpi=256)
plt.show()
for i in range(num_classes):
precision, recall, thresholds = precision_recall_curve(
y_test_onehot[:, i], y_predicted_scores[:, i])
avg_precision = average_precision_score(y_test_onehot[:, i],
y_predicted_scores[:, i])
f1 = f1_score(y_test_onehot[:, i], np.round(y_predicted_scores[:, i]))
plt.plot(recall,
precision,
label=f"{i} (AP = {avg_precision:.3f}, F1 = {f1:.2f})")
plt.xlabel("Recall")
plt.ylabel("Precision")
plt.title("MNIST-1D Precision-Recall Curves")
plt.legend(loc="lower left")
plt.savefig(os.path.join("report", "images",
"mnist1d-precision-recall.png"),
dpi=256)
plt.clf()
x_test = joblib.load('data/x_test_w.joblib')
y_test = joblib.load('data/y_test_w.joblib')
print('train shape=', np.shape(X_train))
print('val shape=', np.shape(x_val))
## Random forest model was selected having maximum accuracy and F1 score
cvmodel = joblib.load('3-rf1.joblib')
## since we have less no. of samples, training this model again with entire training and validation data
xtr = pd.concat([X_train, x_val])
ytr = pd.concat([y_train, y_val])
print('combined train shape=', np.shape(xtr))
cvmodel.fit(xtr, ytr)
joblib.dump(cvmodel, 'white-final-trained-model.joblib')
loaded = joblib.load('white-final-trained-model.joblib')
pred = loaded.predict(x_test)
print('test acc= ', accuracy_score(pred, y_test))
print('balanced test acc= ', balanced_accuracy_score(pred, y_test))
cm = confusion_matrix(y_test, pred)
display = ConfusionMatrixDisplay(cm).plot()
plt.figure(1)
plt.title('Test white wine')
classNames = ['poor', 'average', 'excellent']
tick_marks = np.arange(len(classNames))
plt.xticks(tick_marks, classNames)
plt.yticks(tick_marks, classNames)
plt.show(display)
print('Accuracy:', accuracy_score(y_test, y_pred))
print('Classification Report:')
print(classification_report(y_test, y_pred))
le = LabelEncoder()
y_test = le.fit_transform(y_test)
y_pred = le.fit_transform(y_pred)
print('ROC_AUC Score', roc_auc_score(y_test, y_pred))
# CONFUSION MATRIX
# Plot non-normalized confusion matrix
from sklearn.metrics import ConfusionMatrixDisplay
cm = confusion_matrix(y_test, y_pred)
cmd = ConfusionMatrixDisplay(cm, display_labels=['B', 'M'])
cmd.plot(cmap='Blues')
cmd.ax_.set(xlabel='Predicted', ylabel='True')
plt.figure(figsize=(5, 5))
plt.show()
tn, fp, fn, tp = cm.ravel()
print('Coefficients Confusion DT - best parameters')
print('tn', tn, 'fp', fp, 'fn', fn, 'tp', tp)
# OUTPUT TO INPUT CAPSTONE PROJECT
# OUTPUT THE MODEL
import pickle
with open('clf.DT', 'wb') as f:
pickle.dump(gs, f)
y_9 = []
y_47 = []
for i in values:
i = np.asarray(i)
x_9.append(i[0][0])
y_9.append(i[1][0])
x_47.append(i[0][1])
y_47.append(i[1][1])
cm_9 = confusion_matrix(y_9, x_9)
cm_47 = confusion_matrix(y_47, x_47)
print('CM_9: \n', cm_9)
print('CM_47: \n', cm_47)
disp_9 = ConfusionMatrixDisplay(confusion_matrix=cm_9,
display_labels=range(1, 10))
disp_47 = ConfusionMatrixDisplay(confusion_matrix=cm_47,
display_labels=range(1, 48))
accuracy_9 = accuracy_score(y_9, x_9)
accuracy_47 = accuracy_score(y_47, x_47)
fig, ax = plt.subplots()
disp = disp_9.plot(include_values=True,
cmap=plt.cm.Blues,
ax=ax,
xticks_rotation='horizontal')
disp.ax_.set_title('Accuracy_9: {:.2f}'.format(accuracy_9))
fig.savefig('./saved/cm/' + str(uid) + '_9.png', quality=50)
if opt.display == 1:
plt.show()
def nicegrid(
imgs,
all_labels, # all labels for dataset
all_predictions, # all predicitons for dataset
isfirstinstage, # bool True if first frame of a stage
epochs_trained,
training_parameters,
log_path): # how many epochs did the model train for
# get a classification report
is_multi = 'multi' in training_parameters['model_name']
if is_multi:
# class_id = [0, 1, 2, 3, 4]
# class_labels = ['G0', 'G1', 'S', 'G2', 'M']
class_id = [0, 1, 2, 3]
class_labels = ['G0/1', 'S', 'G2', 'M']
else:
class_id = [0, 1]
class_labels = ['not S', 'S']
crep_all = classification_report(all_labels,
all_predictions,
labels=class_id,
target_names=class_labels,
output_dict=True)
crep_first = classification_report(all_labels[isfirstinstage],
all_predictions[isfirstinstage],
labels=class_id,
target_names=class_labels,
output_dict=True)
# import pdb
# pdb.set_trace()
# bin_pred_type = { # second entry is prediction
# (0, 0): {'str': 'not S', 'c': 'g'},
# (1, 1): {'str': 'S', 'c': 'g'},
# (0, 1): {'str': 'S', 'c': 'r'},
# (1, 0): {'str': 'not S', 'c': 'r'},
# }
# set up figure specs
fig = plt.figure(figsize=(8.3, 11.7))
gs = gridspec.GridSpec(nrows=3,
ncols=2,
left=0.04,
right=0.92,
bottom=0.01,
top=0.98)
# axis with text based info
tax = fig.add_subplot(gs[0, 0])
tax.set_visible(False)
tbox = tax.get_position()
# grect = [gbox.x0, gbox.y0, gbox.width, gbox.height]
was_scheduler = training_parameters['scheduler'] is not None
if was_scheduler:
if training_parameters['scheduler_kwargs']['mode'] == 'max':
scheduler_update_mode = 'max accuracy'
elif training_parameters['scheduler_kwargs']['mode'] == 'min':
scheduler_update_mode = 'min loss'
else:
upmode = training_parameters['scheduler_kwargs']['mode']
raise ValueError(f'unknown sheduler update method {upmode}')
else:
scheduler_update_mode = None
crep_txt = (f"All dataset:\n"
f"Accuracy: {crep_all['accuracy']:.3f}\n"
f"F1 score: {crep_all['S']['f1-score']:.3f}\n"
f"Precision: {crep_all['S']['precision']:.3f}\n"
f"Recall: {crep_all['S']['recall']:.3f}\n"
f"\nFirst in stage:\n"
f"Accuracy: {crep_first['accuracy']:.3f}\n"
f"F1 score: {crep_first['S']['f1-score']:.3f}\n"
f"Precision: {crep_first['S']['precision']:.3f}\n"
f"Recall: {crep_first['S']['recall']:.3f}\n"
f"\nModel name: {training_parameters['model_name']}\n"
f"Epochs trained: {epochs_trained}\n"
f"Batch size: {training_parameters['batch_size']}\n"
f"Learning rate: {training_parameters['learning_rate']}\n"
f"Used scheduler: {was_scheduler}\n"
f"Scheduler update mode: {scheduler_update_mode}\n"
f"Used sampler: {training_parameters['is_use_sampler']}\n")
fig.text(tbox.x0,
tbox.y0 + tbox.height,
crep_txt,
verticalalignment='top',
fontname='monospace',
fontsize=12,
wrap=True)
# here goes the confusion matrix
cmax = fig.add_subplot(gs[0, 1])
cm = confusion_matrix(all_labels, all_predictions)
cmdisp = ConfusionMatrixDisplay(confusion_matrix=cm,
display_labels=class_labels)
cmdisp = cmdisp.plot(cmap='Blues', ax=cmax)
# plot logs
if log_path is not None:
log_df = read_tb_log(log_path)
lax = fig.add_subplot(gs[1, :])
for metric in ['train_epoch_loss', 'val_epoch_loss']:
log_df.plot(y=metric, ax=lax)
lax.set_ylim(0, 1)
# examples grid
gax = fig.add_subplot(gs[2:, :])
gax.set_visible(False)
gbox = gax.get_position()
grect = [gbox.x0, gbox.y0, gbox.width, gbox.height]
# lets just use the "first in stage" as example
imgs_first = imgs[isfirstinstage]
labs_first = all_labels[isfirstinstage]
preds_first = all_predictions[isfirstinstage]
n_imgs = imgs_first.shape[0]
if n_imgs > 18:
n_imshows = 18
else:
n_imshows = n_imgs
# find factors.
factors = [i for i in range(1, n_imshows + 1) if (n_imshows % i == 0)]
n_imgs_per_row = factors[len(factors) // 2]
n_rows = n_imshows // n_imgs_per_row
# make grid image
grid = ImageGrid(fig,
grect,
nrows_ncols=(n_rows, n_imgs_per_row),
axes_pad=0.1)
for ax, im, ytrue, ypred in zip(grid, imgs_first[:n_imshows],
labs_first[:n_imshows],
preds_first[:n_imshows]):
edge_color = 'g' if (ytrue == ypred) else 'r'
# Iterating over the grid returns the Axes.
ax.imshow(im, cmap='gray')
ax.text(0.05,
0.05,
class_labels[ypred],
color='w',
fontweight='bold',
verticalalignment='top')
ax.set_xticks([])
ax.set_yticks([])
rect = patches.Rectangle(
(0, 0),
im.shape[1],
im.shape[0],
edgecolor=edge_color,
linewidth=2,
facecolor='none',
)
ax.add_patch(rect)
return fig
else:
svc_disp = plot_roc_curve(model, X_test, y_test)
svc_disp.plot()
plt.title(
f"ROC Curve of {PREDICTION_PHENO} by {model_name} (Scaled)")
plt.savefig(
f"finalplots/scaledroc_{PREDICTION_PHENO}_{model_name}.png",
dpi=600,
transparent=True,
bbox_inches="tight",
pad_inches=0.3)
# Confusion Matrix
if model_name == "elasticnetlinear":
conf_mat = confusion_matrix(model[1], y_test)
disp = ConfusionMatrixDisplay(confusion_matrix=conf_mat,
display_labels=[0, 1])
disp.plot()
plt.title(
f"Confusion Matrix of {PREDICTION_PHENO} by {model_name} (Scaled)"
)
plt.savefig(
f"finalplots/scaledconfusion_mat_{PREDICTION_PHENO}_{model_name}.png",
dpi=600,
transparent=True,
bbox_inches="tight",
pad_inches=0.3)
else:
if model_name == "rbfsvmapprox":
disp = plot_confusion_matrix(
model,
Y_predicted,
labels=labels)
confmat = confusion_matrix(Y_test,
Y_predicted,
labels=labels,
normalize='true')
print(p)
print(r)
print(f1)
display_confmat(confmat)
print(classification_report(Y_test, Y_predicted, digits=3))
plt.figure(figsize=(4, 4))
cmatdisp = ConfusionMatrixDisplay(confmat,
display_labels=['1, 2', '3', '4, 5'])
cmatdisp.plot(cmap=plt.cm.Blues, ax=plt.gca())
plt.savefig('report/figures/confmat_CustomSentiCorefModel.pdf',
bbox_inches='tight')
print(
textwrap.dedent(f"""
EMBEDDING_DIM = {EMBEDDING_DIM}
CNN_FILTERS = {CNN_FILTERS}
DNN_UNITS = {DNN_UNITS}
OUTPUT_CLASSES = {OUTPUT_CLASSES}
DROPOUT_RATE = {DROPOUT_RATE}
NB_EPOCHS = {NB_EPOCHS}
BATCH_SIZE = {BATCH_SIZE}
"""))
# Get accuracy
acc = accuracy_score(t_, y_)
print("Accuracy: " + str(acc))
# F1 score
f1 = f1_score(t_, y_, average=None)
print("F1 score per class: " + str(f1))
# F1 score weighted
f1_weighted = f1_score(t_, y_, average='weighted')
print("Weighted F1 score: " + str(f1_weighted))
# Confusion matrix
conf_matrix = confusion_matrix(t_, y_)
classes = ['ambiguous', 'fw', 'lang1', 'lang2', 'mixed', 'ne', 'other', 'unk']
ConfusionMatrixDisplay(confusion_matrix=conf_matrix,
display_labels=classes).plot(values_format='d')
plt.savefig('./results/CM/mBERT_test.svg', format='svg')
# RESULTS
# Validation set
# 0.9910626123503051
# [0. 0. 0.99140693 0.99081154 0. 0.99782942 0.]
# Test set
# 0.9859431603653248
# [0. 0.98548717 0.98404558 0. 0.99764318 0.]
###########################################################################
##################### RESULTS FOR WORKSHOP ################################
# Dev set
def main():
# argument parser
parser = argparse.ArgumentParser(description='ESMH')
parser.add_argument('--root_dir',
type=str,
default='/data/ESMH/cropped_patches/tcga_3p_ref')
parser.add_argument('--result_dir',
type=str,
default='/data/ESMH/subtype_results/tcga_best')
parser.add_argument('--net', type=str, default='resnet')
parser.add_argument('--epochs', type=int, default=500)
parser.add_argument('--env_name', type=str, default='nnunet')
parser.add_argument('--data_dim', type=str, default='2d')
parser.add_argument('--num_phase', type=int, default=2)
args = parser.parse_args()
scores_avg = np.zeros([])
trial_list = [2, 3, 4, 7, 8, 9, 57, 58, 76, 84]
for i in range(10):
for j in range(3):
if j == 0:
phases = [0, 1, 2]
elif j == 1:
phases = [0, 1, 3]
elif j == 2:
phases = [0, 2, 3]
else:
break
true_labels, scores = eval_trial(trial=trial_list[i],
mode='test',
net=args.net,
root_dir=args.root_dir,
result_dir=args.result_dir,
data_dim=args.data_dim,
use_phases=phases)
scores_avg = scores_avg + scores
scores_avg /= (10 * 3)
auc, opt_thr = get_roc_auc_fig(true_labels,
scores_avg,
find_opt_thr=True,
save_roc=True,
figure_name=os.path.join(args.result_dir))
print('auc: ', auc)
print('opt_thr: ', opt_thr)
auc = get_pr_auc_fig(true_labels,
scores_avg,
save_roc=True,
figure_name=os.path.join(args.result_dir))
conf_mat = calc_conf_mat(true_labels,
scores_avg,
thr=[0.07, 0.34, 0.44, 0.24, 0.38])
conf_mat = conf_mat.astype(np.int)
print(conf_mat)
disp = ConfusionMatrixDisplay(
conf_mat, ['ccRCC', 'pRCC', 'chRCC', 'AML', 'Oncocytoma'])
disp.plot(figure_name=os.path.join(args.result_dir, 'conf_model.svg'))
print("top-1 acc (conf):", calc_acc_from_conf_mat(conf_mat))
print("top-1 acc:", calc_top1_acc_from_scores(true_labels, scores_avg))
print("top-2 acc:", calc_top2_acc_from_scores(true_labels, scores_avg))
sen, spe, acc = calc_eval_metric(conf_mat)
print('sen: ', sen)
print('spe: ', spe)
print('acc: ', acc)
print('main done')
