def displayCM(self):
np.set_printoptions(precision=2)
#display_labels = class_names
disp = ConfusionMatrixDisplay(confusion_matrix=self.metrics["NormConfMatrix"], display_labels=None)
disp.plot(include_values=True,
cmap=plt.cm.Blues, xticks_rotation='horizontal',
values_format='.3g')
disp.ax_.set_title("Normalized Confusion Matrix")
fig = disp.figure_
fig.set_size_inches(8.0, 8.0)
self.pxmapCM = utils.figureToQPixmap(fig, dpi=180, width=800, height=800)
widget = QWidget(parent=self)
widget.setFixedWidth(800)
widget.setFixedHeight(800)
lblCentral = QLabel("")
lblCentral.setPixmap(self.pxmapCM)
layout = QHBoxLayout()
layout.setAlignment(Qt.AlignCenter)
layout.addWidget(lblCentral)
widget.setLayout(layout)
widget.setWindowModality(Qt.WindowModal)
widget.setWindowTitle("TagLab")
widget.setWindowFlags(Qt.Window | Qt.CustomizeWindowHint | Qt.WindowCloseButtonHint | Qt.WindowTitleHint)
widget.show()
def plot_confusion_matrix(
cmat: np.ndarray,
cmat_norm: np.ndarray,
classes: List[str],
show: bool = True,
figsize: Tuple[int, int] = (16, 4),
) -> None:
""" Plot the confusion matrices.
Args:
cmat: confusion matrix (with raw pixel counts)
cmat_norm: normalized confusion matrix
classes: list of class names
show: set to true to call matplotlib's show()
figsize: figure size
"""
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=figsize)
ConfusionMatrixDisplay(cmat, classes).plot(
ax=ax1,
cmap=cm.get_cmap("Blues"),
xticks_rotation="vertical",
values_format="d",
)
ConfusionMatrixDisplay(cmat_norm, classes).plot(
ax=ax2, cmap=cm.get_cmap("Blues"), xticks_rotation="vertical"
)
ax1.set_title("Confusion matrix")
ax2.set_title("Normalized confusion matrix")
if show:
plt.show()
def score_results(results_path: str):
with open(results_path, 'r') as f:
results = pd.read_csv(f)
y = results['colType']
y_hat = results['colType_predicted']
y_labels = y.unique()
with open(os.path.splitext(results_path)[0] + '_scores.txt', 'w') as f:
f.write('y percentages:\n')
f.write(str(y.value_counts(normalize=True)) + '\n')
f.write('\n')
f.write('Accuracy:' + str(accuracy_score(y, y_hat)) + '\n')
f.write('F1 Macro:' + str(f1_score(y, y_hat, average='macro')) + '\n')
f.write('F1 Micro:' + str(f1_score(y, y_hat, average='micro')) + '\n')
conf_mat_normalized = confusion_matrix(y,
y_hat,
labels=y_labels,
normalize='true')
disp = ConfusionMatrixDisplay(conf_mat_normalized, y_labels)
disp.plot(xticks_rotation='vertical')
plt.savefig(os.path.splitext(results_path)[0] + '_ncm.png')
plt.show()
plt.close()
conf_mat = confusion_matrix(y, y_hat, labels=y_labels)
disp = ConfusionMatrixDisplay(conf_mat, y_labels)
disp.plot(xticks_rotation='vertical')
plt.savefig(os.path.splitext(results_path)[0] + '_cm.png')
plt.show()
plt.close()
def test_confusion_matrix_contrast(pyplot):
# make sure text color is appropriate depending on background
cm = np.eye(2) / 2
disp = ConfusionMatrixDisplay(cm, display_labels=[0, 1])
disp.plot(cmap=pyplot.cm.gray)
# diagonal text is black
assert_allclose(disp.text_[0, 0].get_color(), [0.0, 0.0, 0.0, 1.0])
assert_allclose(disp.text_[1, 1].get_color(), [0.0, 0.0, 0.0, 1.0])
# off-diagonal text is white
assert_allclose(disp.text_[0, 1].get_color(), [1.0, 1.0, 1.0, 1.0])
assert_allclose(disp.text_[1, 0].get_color(), [1.0, 1.0, 1.0, 1.0])
disp.plot(cmap=pyplot.cm.gray_r)
# diagonal text is white
assert_allclose(disp.text_[0, 1].get_color(), [0.0, 0.0, 0.0, 1.0])
assert_allclose(disp.text_[1, 0].get_color(), [0.0, 0.0, 0.0, 1.0])
# off-diagonal text is black
assert_allclose(disp.text_[0, 0].get_color(), [1.0, 1.0, 1.0, 1.0])
assert_allclose(disp.text_[1, 1].get_color(), [1.0, 1.0, 1.0, 1.0])
# Regression test for #15920
cm = np.array([[19, 34], [32, 58]])
disp = ConfusionMatrixDisplay(cm, display_labels=[0, 1])
disp.plot(cmap=pyplot.cm.Blues)
min_color = pyplot.cm.Blues(0)
max_color = pyplot.cm.Blues(255)
assert_allclose(disp.text_[0, 0].get_color(), max_color)
assert_allclose(disp.text_[0, 1].get_color(), max_color)
assert_allclose(disp.text_[1, 0].get_color(), max_color)
assert_allclose(disp.text_[1, 1].get_color(), min_color)
def test_confusion_matrix_standard_format(pyplot):
cm = np.array([[10000000, 0], [123456, 12345678]])
plotted_text = ConfusionMatrixDisplay(cm, [False, True]).plot().text_
# Values should be shown as whole numbers 'd',
# except the first number which should be shown as 1e+07 (longer length)
# and the last number will be showns as 1.2e+07 (longer length)
test = [t.get_text() for t in plotted_text.ravel()]
assert test == ['1e+07', '0', '123456', '1.2e+07']
cm = np.array([[0.1, 10], [100, 0.525]])
plotted_text = ConfusionMatrixDisplay(cm, [False, True]).plot().text_
# Values should now formatted as '.2g', since there's a float in
# Values are have two dec places max, (e.g 100 becomes 1e+02)
test = [t.get_text() for t in plotted_text.ravel()]
assert test == ['0.1', '10', '1e+02', '0.53']
def test(self,
batch_size=1,
test_split_file='test.txt',
plot_confusion=False):
"""Run test on a checkpoint"""
graph, sess, saver = self.load_graph()
with graph.as_default():
# Load checkpoint
self.load_ckpt(sess, saver)
# Run test
print('Starting test')
metrics = self._get_validation_fn(sess, batch_size,
test_split_file)()
self._log_and_print_metrics(metrics)
if plot_confusion:
# Plot confusion matrix
fig, ax = plt.subplots()
disp = ConfusionMatrixDisplay(
confusion_matrix=metrics['confusion matrix'],
display_labels=CLASS_NAMES)
disp.plot(include_values=True,
cmap='Blues',
ax=ax,
xticks_rotation='horizontal',
values_format='.5g')
plt.show()
def plot_forecast_confusion_matrix(self, df):
"""
Plots the forecast confusion matrix.
Parameters:
df -- the races dataframe
"""
normalizations = ["all", "true"]
titles = [
"normalized over all samples\n(global frequency score)",
"normalized over true label\n(true label local frequency score)"
]
fig, ax = plt.subplots(1, 2)
fig.suptitle("Forecast Confusion Matrix", fontsize=16)
for i, normalization in enumerate(normalizations):
cm = confusion_matrix(y_true=df.weather.values,
y_pred=df.forecast_weather.values,
sample_weight=None,
labels=self.weather_conditions,
normalize=normalization)
disp = ConfusionMatrixDisplay(
confusion_matrix=cm, display_labels=self.weather_conditions)
disp.plot(include_values=True,
cmap="OrRd",
ax=ax[i],
xticks_rotation='horizontal')
ax[i].title.set_text(titles[i])
plt.show()
def display_confusionMatrix(clf, X, y, title):
cm = confusion_matrix(y, clf.predict(X), labels=clf.classes_)
normed_c = (cm.T / cm.astype(np.float).sum(axis=1)).T
disp = ConfusionMatrixDisplay(confusion_matrix=normed_c,
display_labels=clf.classes_)
disp.plot(cmap=plt.cm.Blues)
disp.ax_.set_title(title + " Confusion Matrix:")
def save_and_plot(trained_model, history, file_save_name, filenames,
class_preds, class_true):
torch.save(
trained_model, '/home/jovyan/repo/ximeng_project/Outputs/' +
file_save_name + '_trained_model.pt')
torch.save(
history, '/home/jovyan/repo/ximeng_project/Outputs/' + file_save_name +
'_history.pt')
history = np.array(history)
plt.plot(history[:, :])
plt.legend(
['Train Loss', 'Valid Loss', 'Train Accuracy', 'Valid Accuracy'])
plt.xlabel('Epoch Number')
plt.ylabel('Loss')
plt.ylim(0, 1)
plt.savefig('/home/jovyan/repo/ximeng_project/Outputs/' + file_save_name +
'_all.png')
plt.show()
y_pred = class_preds
y_true = class_true
cm = confusion_matrix(y_true, y_pred, labels=[0, 1, 2, 3], normalize='all')
cmplot = ConfusionMatrixDisplay(
cm, display_labels=["control", "TK", "CMGC", "AGC"])
cmplot.plot()
plt.savefig('/home/jovyan/repo/ximeng_project/Outputs/' + file_save_name +
'_cmplot.png')
plt.show()
np.savetxt(
'/home/jovyan/repo/ximeng_project/Outputs/' + file_save_name +
'true_vs_preds_output.csv', [filenames, class_preds, class_true])
def plot_confusion_matrices(self,
y_pred: list,
y_true: list,
axs: list = None):
if axs is None:
_, axs = plt.subplots(1, 2, figsize=(12, 4))
plot_options = [("confusion_matrix", "Confusion Matrix", None, ".6g"),
("confusion_matrix_normalized",
"Normalized Confusion Matrix", 'all', ".2g")]
index = 0
for file_name, plot_name, normalize_option, values_format in plot_options:
cm = confusion_matrix(
y_true,
y_pred,
labels=[self.config.BUG_STRING, self.config.NO_BUG_STRING],
normalize=normalize_option)
disp = ConfusionMatrixDisplay(confusion_matrix=cm,
display_labels=[
self.config.BUG_STRING,
self.config.NO_BUG_STRING
])
disp.plot(include_values=True,
cmap='Blues',
ax=axs[index],
values_format=values_format,
xticks_rotation='horizontal')
axs[index].title.set_text(plot_name)
index += 1
def compute_accuracy(self, clusters):
predicted_labels = []
actual_labels = []
correct_predictions = 0
for validation_sample in self._validation_samples:
distance_cluster_map = {cluster.find_distance_from_centroid(validation_sample.features):cluster for cluster in clusters}
cluster_for_sample = distance_cluster_map[min(distance_cluster_map.keys())]
predicted_label = cluster_for_sample.most_common_label()
actual_label = validation_sample.true_class_label
if predicted_label == actual_label:
correct_predictions += 1
predicted_labels.append(predicted_label)
actual_labels.append(actual_label)
validation_samples_confusion_matrix = confusion_matrix(y_true=actual_labels, y_pred=predicted_labels)
confusion_matrix_display = ConfusionMatrixDisplay(validation_samples_confusion_matrix, display_labels=range(10))
confusion_matrix_display.plot(values_format='d')
plt.title('Confusion matrix')
plt.savefig('out/confusion-matrix.png', format='png', dpi=1200)
plt.show()
accuracy = (correct_predictions / len(self._validation_samples)) * 100
print('Accuracy = {}%'.format(accuracy))
def plot_confusion(y_true, y_pred, token):
cm = confusion_matrix(y_true, y_pred, normalize='true')
display = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=[-1, 0, 1])
display = display.plot(cmap='Blues')
display.figure_.suptitle(token)
plt.savefig('{}_cm.png'.format(token))
plt.show()
def test_iris_dataset(self):
K =3
# 使用iris数据集，其中有三个分类， y的取值为0,1，2
X, y = datasets.load_iris(True) # 包括150行记录
# 将数据集一分为二，训练数据占80%，测试数据占20%
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1024)
# clf = LR_MultiClassifier(K=K,reg_lambda=0.1,use_reg=1) # TODO: 加入 L1 正则化后的模型效果下降
clf = LR_MultiClassifier(K=K, reg_lambda=0.1, use_reg=2)
clf.fit(X_train, y_train,max_iter=50,learning_rate=0.05)
y_predict = clf.predict(X_test)
print('test accuracy :', accuracy_score(y_predict, y_test))
# 查看每一种类别 的评价指标
print('print the classification report')
report = classification_report(y_test, y_predict)
print(report)
# 打印混淆矩阵
print('print the confusion matrix')
confusion = confusion_matrix(y_test, y_predict)
print(confusion)
disp = ConfusionMatrixDisplay(confusion_matrix=confusion,display_labels = list(range(K)))
disp.plot()
plt.show() # 显示图片
def test_Mnist_dataset(self, n_train, n_test):
"""
利用 Mnist 数据集 测试
:param n_train: 使用训练数据集的规模
:param n_test: 使用测试数据集的规模
:return:
"""
# 获取训练集
trainDataList, trainLabelList = self.loadData('../dataset/Mnist/mnist_train.csv', n=n_train)
print('train data, row num:{} , column num:{} '.format(len(trainDataList), len(trainDataList[0])))
trainDataArr = np.array(trainDataList)
trainLabelArr = np.array(trainLabelList)
K = 10
# 开始时间
print('start training model....')
start = time.time()
clf = LR_MultiClassifier(K=K,reg_lambda=0.1,use_reg=2)
clf.fit(trainDataArr, trainLabelArr,max_iter=50,learning_rate=0.1)
# 结束时间
end = time.time()
print('training cost time :', end - start)
# 获取测试集
testDataList, testLabelList = self.loadData('../dataset/Mnist/mnist_test.csv', n=n_test)
print('test data, row num:{} , column num:{} '.format(len(testDataList), len(testDataList[0])))
testDataArr = np.array(testDataList)
testLabelArr = np.array(testLabelList)
y_predict = clf.predict(testDataArr)
print('test accuracy :', accuracy_score(y_predict, testLabelArr))
# 查看每一种类别 的评价指标
print('print the classification report: ')
report = classification_report(testLabelArr, y_predict)
print(report)
# 打印混淆矩阵
print('print the confusion matrix')
confusion = confusion_matrix(testLabelArr, y_predict)
print(confusion)
disp = ConfusionMatrixDisplay(confusion_matrix=confusion, display_labels=list(range(K)))
disp.plot()
plt.show()
def my_confusion_matrix(y_actual, y_pre, title, normalize="true", mode="sim"):
"""
Confusions matrix for evaluating the performance of the classifier.
:param y_actual: Target
:param y_pre: Prediction
:param title: Title of the plot
:param normalize: Normalized by all predictions with respect to one true label (over column)
:param mode: Output with 8 classes when "sim", Output with 7 classes when "rig"
:return:
"""
if mode == "sim":
label = np.array(
["nc", "dc1", "dc2", "dc3", "dc4", "dc5", "dc6", "dc7"])
n_class = 8
else:
label = np.array(["nc", "dc1", "dc2", "dc3", "dc4", "dc5", "dc6"])
n_class = 7
confusion = confusion_matrix(y_actual,
y_pre,
labels=np.arange(0, n_class),
normalize=normalize)
form = '.2g' if normalize else 'd'
acc = np.sum(np.diag(confusion)) / np.sum(confusion, axis=(0, 1))
disp = ConfusionMatrixDisplay(confusion_matrix=confusion,
display_labels=label)
disp.plot(values_format=form)
plt.title(title)
plt.show()
print(f"{title} total accuracy: {100 * acc:.3f}%")
def confusion_matrix1():
global y_pred_label
i = 0
y_true_labels = [] ############## prdictions is single value check
for img in img_names:
if (str(os.path.basename(img)) in High_Speed):
y_true_labels.append(int(0))
else:
y_true_labels.append(int(1))
print('Predicted labels', y_pred_label)
y_true_labels = np.array(y_true_labels)
y_true_labels = y_true_labels.ravel()
y_pred_labels = np.array(y_pred_label)
y_pred_labels = y_pred_labels.ravel()
print('Shape True labels', y_true_labels)
print('Shape Pred labels', y_pred_labels)
cm = confusion_matrix(y_true_labels, y_pred_labels)
disp = ConfusionMatrixDisplay(cm, display_labels=['Low_Speed'])
#fig = Figure(figsize=(5, 4), dpi=100)
#disp.plot(fig , cmap=plt.cm.Blues)
canvas = FigureCanvasTkAgg(disp.figure_, master=root) # A tk.DrawingArea.
canvas.draw()
canvas.get_tk_widget().grid(row=9, column=0, columnspan=1, padx=5, pady=5)
return
def my_metrics(X_val_check):
'''
takes a dataframe of the form returned by make_X_check
'''
total = X_val_check.shape[0]
accuracy = (X_val_check.TP.sum()+X_val_check.TN.sum())/total
precision = X_val_check.TP.sum()/(X_val_check.TP.sum()+X_val_check.FP.sum())
recall = X_val_check.TP.sum()/(X_val_check.TP.sum()+X_val_check.FN.sum())
f1 = 2*((precision*recall)/(precision+recall))
print(f"accuracy: {accuracy *100:.2f} %")
print(f"precision: {precision *100:.2f} %")
print(f"recall: {recall *100:.2f} %")
print(f"f1: {f1 *100:.2f} %")
cm = confusion_matrix(X_val_check.target, X_val_check.y_pred_binary)
disp = ConfusionMatrixDisplay(confusion_matrix=cm,
#display_labels=display_labels
)
disp = disp.plot(include_values=True,
cmap='viridis', ax=None, xticks_rotation='horizontal')
print("Confusion matrix")
plt.show()
print(f"{round(accuracy*100)}\t{round(precision*100)}\t{round(recall*100)}\t{round( f1*100)}")
return {"accuracy": accuracy,"precision": precision,"recall": recall, "f1": f1 }
def plot_confusion_matrix(self,
labels: [str] = None,
normalize=None,
ax=None) -> None:
"""
:param labels: [str]: label names
:param normalize: {‘true’, ‘pred’, ‘all’}, default=None.
Normalizes confusion matrix over the true (rows), predicted (columns) conditions or all the population.
If None, confusion matrix will not be normalized.
:param ax: matplotlib.pyplot axes to draw the confusion matrix on. Will generate new figure/axes if None.
:return:
"""
conf_matrix = self._confusion_matrix(
normalize) # Evaluate the confusion matrix
display = ConfusionMatrixDisplay(
conf_matrix,
display_labels=labels) # Generate the confusion matrix display
# Formatting for the plot
if labels:
xticks_rotation = 'vertical'
else:
xticks_rotation = 'horizontal'
display.plot(include_values=True,
cmap=plt.cm.get_cmap('Blues'),
xticks_rotation=xticks_rotation,
ax=ax)
plt.show()
def plotting(train_losses, train_accs, valid_losses, valid_accs, test_loss,
test_acc, y_true, y_pred, variables_string, optimal_batch, FLAGS):
plt.rcParams.update({"font.size": 22})
os.makedirs("Images", exist_ok=True)
plt.figure(figsize=(16, 10))
steps_all = np.arange(0, len(train_losses))
steps_valid = np.arange(0, len(valid_losses)) * FLAGS.eval_freq
# plot the losses
plt.subplot(2, 1, 1)
plt.plot(steps_all, train_losses, '-', lw=2, label="Training loss")
plt.plot(steps_valid, valid_losses, '-', lw=3, label="Validation loss")
plt.axhline(test_loss, label="Test loss", color="red", lw=3)
plt.axvline(optimal_batch,
label="Optimal model",
linestyle="dashed",
color="red",
lw=3)
# plt.title('Losses over training')
# plt.ylim(0, 10)
plt.xlabel('Batch')
plt.ylabel('Cross Entropy Loss')
plt.grid(True)
plt.legend()
plt.subplot(2, 1, 2)
plt.plot(steps_all, train_accs, '-', lw=2, label="Training accuracy")
plt.plot(steps_valid, valid_accs, '-', lw=3, label="Validation accuracy")
plt.axhline(test_acc, label="Test accuracy", color="red", lw=3)
plt.axvline(optimal_batch,
label="Optimal model",
color="red",
linestyle="dashed",
lw=3)
# plt.title('Accuracy over training')
plt.xlabel('Batch')
plt.ylabel('Accuracy')
plt.grid(True)
plt.legend()
plt.tight_layout()
fig_name = f"loss_accplot_{variables_string}.png"
plt.savefig(f"Images/{fig_name}")
# plot confusion matrix
plt.rcParams.update({"font.size": 10})
plt.figure(figsize=(12, 12))
cm = confusion_matrix(y_true, y_pred, normalize=None)
cmd = ConfusionMatrixDisplay(cm)
cmd.plot()
confusion_name = f"confusion_{variables_string}.png"
plt.savefig(f"Images/{confusion_name}")
def draw(self, name_tag_formatter=None):
fig, ax1 = plt.subplots(1, 1, figsize=(9, 6))
if len(self.models_data) > 1:
print(
"Warning: Ignoring multiple models, this class only works with one model"
)
if len(self.models_data) == 0:
print("No model data found")
return
data = self.models_data[0]
confusion_matrix = data['confusion_matrix']
classes = self.classes
if self.dont_plot_idx is not None:
confusion_matrix = np.delete(confusion_matrix,
axis=0,
obj=self.dont_plot_idx)
confusion_matrix = np.delete(confusion_matrix,
axis=1,
obj=self.dont_plot_idx)
classes = [
c for i, c in enumerate(classes) if i != self.dont_plot_idx
]
disp = ConfusionMatrixDisplay(confusion_matrix=confusion_matrix,
display_labels=classes)
disp.plot(ax=ax1)
return fig
def evaluate(self, x_test, y_test):
y_pred = self.model.predict(x_test)
y_pred = [1 * (x[0] >= 0.5) for x in y_pred]
print('MLP performance on test for', self.feature_name)
print('Accuracy:', accuracy_score(y_test, y_pred), 'Precision:',
precision_score(y_test, y_pred), 'Recall:',
recall_score(y_test, y_pred))
# Confusion matrix
cm = confusion_matrix(y_test, y_pred)
cm_display = ConfusionMatrixDisplay(cm)
# Precision recall
precision, recall, _ = precision_recall_curve(y_test, y_pred)
pr_display = PrecisionRecallDisplay(precision=precision, recall=recall)
# Roc
fpr, tpr, _ = roc_curve(y_test, y_pred)
roc_display = RocCurveDisplay(fpr=fpr, tpr=tpr)
# Figure
figure: Figure = plt.figure(1, figsize=(15, 6))
figure.suptitle('MLP on {}'.format(self.feature_name), fontsize=20)
(ax1, ax2, ax3) = figure.subplots(1, 3)
ax1.set_title('Confusion matrix')
cm_display.plot(ax=ax1)
ax2.set_title('Precision recall')
pr_display.plot(ax=ax2)
ax3.set_title('Roc curve')
roc_display.plot(ax=ax3)
file_name = '{}-mlp.png'.format(self.feature_name)
figure.savefig(
os.path.join(get_folder_path_from_root('images'), file_name))
plt.show()
def plot_confusion_matrix(
confusion_matrix_data: ArrayLike, class_names: Optional[List[str]] = None
):
"""Plot a confusion matrix.
:param confusion_matrix_data: Data to plot.
:param class_names: Optional class names to be used as labels.
:return: figure handle
"""
if isinstance(confusion_matrix_data, Tensor):
confusion_matrix_data = confusion_matrix_data.cpu().numpy()
confusion_matrix_data = np.asarray(confusion_matrix_data)
display = ConfusionMatrixDisplay(
confusion_matrix=confusion_matrix_data, display_labels=class_names
)
display.plot(cmap="Blues")
for im in display.ax_.get_images():
upper_limit = 1 if confusion_matrix_data.max() <= 1 else None
im.set_clim(0, upper_limit)
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
return plt.gcf()
def create_and_plot_confusion_matrix(true, pred, label_map, coverage, outdir,
outfile):
'''
create and plot a confusion matrix of true and predicted labels
'''
cm = confusion_matrix( # create the confusion matrix
y_true=[label_map[x] for x in true],
y_pred=[label_map[x] for x in pred],
labels=list(label_map.values()))
# Per-class accuracy
#class_accuracy = conf_mat.diagonal()/conf_mat.sum(1)
#print(class_accuracy)
pd.DataFrame(cm).to_csv('{}/{}_{}_confusion_matrix.csv'.format(
outdir, outfile, coverage),
index=False,
header=False)
disp = ConfusionMatrixDisplay(confusion_matrix=cm,
display_labels=list(label_map.values()))
# NOTE: Fill all variables here with default values of the plot_confusion_matrix
disp = disp.plot(
) #include_values=include_values, cmap=cmap, ax=ax, xticks_rotation=xticks_rotation)
plt.savefig('{}/{}_{}_confusion_matrix.png'.format(outdir, outfile,
coverage),
dpi=300)
return
def save_confusion_matrix(labels, pred_labels, classes):
""" Saves the confusion matrix in the 'results/confuision_matrix.png' file
:param labels: A list of the actual labels of the examples
:type labels: List
:param pred_labels: A list of the predicted labels of the examples
:type pred_labels: List
:param classes: A list of the classes of the dataset
:type classes: List
:rtype: None
"""
fig = plt.figure(figsize=(250, 250))
ax = fig.add_subplot(1, 1, 1)
cm = confusion_matrix(labels, pred_labels, normalize='true')
with open('confusion_matrix.json', 'w') as outfile:
json.dump(cm.tolist(), outfile)
cm = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=classes)
cm.plot(include_values=False, cmap='Blues', ax=ax)
fig.delaxes(fig.axes[1]) # delete colorbar
plt.xticks(rotation=90)
plt.xlabel('Predicted Label', fontsize=50)
plt.ylabel('True Label', fontsize=50)
plt.savefig('results/confusion_matrix.png', dpi=100)
def __compute_confusion_matrix(self):
predicted_labels = []
actual_labels = []
for validation_sample in self.__validation_samples:
predicted_target_label = self.__get_prediction_for_sample(
validation_sample)
predicted_labels.append(predicted_target_label)
actual_labels.append(validation_sample.true_class_label)
training_samples_confusion_matrix = confusion_matrix(
y_true=actual_labels, y_pred=predicted_labels)
for true_class_label, row in enumerate(
training_samples_confusion_matrix):
actual_count = sum(row)
expected_count = self.__validation_sample_count_by_labels.get(
true_class_label)
assert actual_count == expected_count, \
'Total sample count for true class label {} does not equal to the count in confusion matrix. Expected = {}, Actual = {}'.format(
true_class_label,
expected_count,
actual_count)
confusion_matrix_display = ConfusionMatrixDisplay(
training_samples_confusion_matrix, display_labels=range(10))
confusion_matrix_display.plot(values_format='d')
plt.title("Confusion matrix for learning rate = {}".format(
self.__learning_rate))
plt.show()
self.__logger.info("Confusion matrix = \n\n{}\n\n".format(
training_samples_confusion_matrix))
def eval_model(model, valid_dataloader, plot=False):
print(valid_dataloader.dataset.class_to_idx)
with torch.no_grad():
model.eval()
result = 0
n = 0
val_preds = np.zeros(len(valid_dataloader.dataset))
val_labels = np.zeros(len(valid_dataloader.dataset))
start = 0
for images, labels in valid_dataloader:
batch_size = images.size(0)
n += batch_size
#images = images.cuda()
pred = F.softmax(model(images))
prediction = torch.argmax(pred, 1)
val_preds[start:start + batch_size] = prediction.cpu().numpy()
val_labels[start:start + batch_size] = labels.numpy()
start += batch_size
print('Precision: ', precision_score(val_labels, val_preds, average='weighted'))
print('Recall: ', recall_score(val_labels, val_preds, average='weighted'))
print('F1 score: ', f1_score(val_labels, val_preds, average='weighted'))
if plot:
cm = confusion_matrix(val_labels, val_preds,
labels=list(valid_dataloader.dataset.class_to_idx.values()))
disp = ConfusionMatrixDisplay(confusion_matrix=cm,
display_labels=list(valid_dataloader.dataset.class_to_idx.keys()))
#disp.plot()
disp.plot(cmap = plt.cm.Blues)
plt.xticks(rotation=45)
plt.show()
def save_and_plot(trained_model, history, file_save_name, filenames, class_preds, class_true):
#torch.save(trained_model, '/home/jovyan/repo/ximeng_project/Outputs/'+file_save_name+'_trained_model.pt')
torch.save(history, '/home/jovyan/repo/ximeng_project/Outputs/'+file_save_name+'_history.pt')
history = np.array(history)
#plt.style.use('ggplot')
fig = plt.figure(figsize=(8,6))
ax1 = fig.add_subplot()
plt.plot(history[:, :2])
plt.legend(['Train Loss', 'Valid Loss'])
plt.xlabel('Epoch Number')
ax1.set_ylabel('Loss')
ax2 = ax1.twinx()
ax2.set_ylabel('Accuracy')
ax2.set_ylim(0,1)
plt.plot(history[:, 2:3], 'g')
plt.plot(history[:, 3:4], 'y')
plt.legend(['Train Accuracy', 'Valid Accuracy'])
plt.xlabel('Epoch Number')
plt.savefig('/home/jovyan/repo/ximeng_project/Outputs/'+ file_save_name +'_accuracy_curve.png')
plt.show()
y_pred = class_preds
y_true = class_true
cm = confusion_matrix(y_true, y_pred,labels=[0,1,2,3], normalize='all')
cmplot = ConfusionMatrixDisplay(cm,display_labels=["control", "TK", "CMGC", "AGC"])
cmplot.plot()
plt.savefig('/home/jovyan/repo/ximeng_project/Outputs/'+ file_save_name + '_cmplot.png')
plt.show()
np.savetxt( '/home/jovyan/repo/ximeng_project/Outputs/'+ file_save_name+'true_vs_preds_output.csv', [filenames, class_preds, class_true])
def train_svm(waveforms, labels, industry):
print("Started training")
#shuffle in unision
waveforms, labels = utils.shuffle(waveforms, labels)
#seperate training subset
cutOff = 20000
trainingWaveforms = waveforms[:cutOff]
trainingLabels = labels[:cutOff]
waveforms = waveforms[cutOff:]
labels = labels[cutOff:]
#train
clf = svm.SVC()
clf.fit(trainingWaveforms, trainingLabels)
#metrics
predictedLabels = clf.predict(waveforms)
print(accuracy_score(labels, predictedLabels))
cm = confusion_matrix(labels, predictedLabels)
confusion_matrix_df = pd.DataFrame(cm)
print(confusion_matrix_df)
disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=clf.classes_).plot()
plt.matshow(cm, cmap = 'binary')
plt.show()
#save
with open(f"strategies/{industry}_svm_clf.pickle", "wb") as f:
pickle.dump(clf, f)
def plot_confusion_matrix(estimator,
y_pred,
y_true,
labels=None,
sample_weight=None,
normalize=None,
display_labels=None,
include_values=True,
xticks_rotation='vertical',
values_format=None,
cmap=sns.cubehelix_palette(light=1, as_cmap=True),
ax=None):
if normalize not in {'true', 'pred', 'all', None}:
raise ValueError("normalize must be one of {'true', 'pred', "
"'all', None}")
cm = confusion_matrix(y_true,
y_pred,
sample_weight=sample_weight,
labels=labels,
normalize=normalize)
if display_labels is None:
if labels is None:
display_labels = estimator.classes_
else:
display_labels = labels
disp = ConfusionMatrixDisplay(confusion_matrix=cm,
display_labels=display_labels)
return disp.plot(include_values=include_values,
cmap=cmap,
ax=ax,
xticks_rotation=xticks_rotation)
def get_classification_stats(proc_df,
truecol='ytrue',
predcol='ypred',
filter_col=None,
is_plot=False,
ax=None):
# get data
ground_truth = proc_df[truecol]
predictions = proc_df[predcol]
if filter_col:
idx_filter = proc_df[filter_col]
ground_truth = ground_truth[idx_filter]
predictions = predictions[idx_filter]
# classification reports
crep = classification_report(ground_truth, predictions, output_dict=True)
# confusion matrices
cm = confusion_matrix(ground_truth, predictions)
# mean per-class accuracy
# plots
if is_plot:
if ax is None:
fig, ax = plt.subplots()
ConfusionMatrixDisplay(confusion_matrix=cm).plot(cmap='Blues', ax=ax)
return crep
