def mr_metrics(confusion_matrix, level):
confusion_matrix = confusion_matrix.astype(float)
# sum(0) <- predicted sum(1) ground truth
total = np.sum(confusion_matrix)
n_classes, _ = confusion_matrix.shape
overall_accuracy = np.sum(np.diag(confusion_matrix)) / total
# calculate Cohen Kappa (https://en.wikipedia.org/wiki/Cohen%27s_kappa)
N = total
p0 = np.sum(np.diag(confusion_matrix)) / N
pc = np.sum(
np.sum(confusion_matrix, axis=0) *
np.sum(confusion_matrix, axis=1)) / N**2
kappa = (p0 - pc) / (1 - pc)
recall = np.diag(confusion_matrix) / (np.sum(confusion_matrix, axis=1) +
1e-12)
precision = np.diag(confusion_matrix) / (np.sum(confusion_matrix, axis=0) +
1e-12)
f1 = (2 * precision * recall) / ((precision + recall) + 1e-12)
if level == 'global':
return overall_accuracy, kappa, np.mean(precision), np.mean(
recall), np.mean(f1)
elif level == 'perclass':
# Per class accuracy
cl_acc = np.diag(confusion_matrix) / (confusion_matrix.sum(1) + 1e-12)
return overall_accuracy, kappa, precision, recall, f1, cl_acc
def plot_cnf_matrix(self,
cm,
classes,
normalize=False,
title='Confusion matrix',
cmap=plt.cm.Blues):
"""
This function prints and plots the confusion matrix.
Normalization can be applied by setting `normalize=True`.
"""
import itertools
if normalize:
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
print("Normalized confusion matrix")
else:
print('Confusion matrix, without normalization')
plt.imshow(cm, interpolation='nearest', cmap=cmap)
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
fmt = '.2f' if normalize else 'd'
thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j,
i,
format(cm[i, j], fmt),
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()
def plot_confusion_matrix(confusion_matrix, classes, title, normalize=False):
plt.clf()
plt.imshow(confusion_matrix, interpolation='nearest', cmap=plt.cm.Blues)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
if normalize:
confusion_matrix = confusion_matrix.astype(
'float') / confusion_matrix.sum(axis=1)[:, np.newaxis]
thresh = confusion_matrix.max() / 2.
for i, j in itertools.product(range(confusion_matrix.shape[0]),
range(confusion_matrix.shape[1])):
plt.text(j,
i,
round(confusion_matrix[i, j], 3),
horizontalalignment="center",
color="white" if confusion_matrix[i, j] > thresh else "black")
plt.tight_layout()
plt.ylabel('Реальные классы')
plt.xlabel('Расчетные классы')
def plot_confusion_matrix(cm, classes,
normalize=False,
title='Confusion matrix',
cmap=plt.cm.Blues):
if normalize:
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
print("Normalized confusion matrix")
else:
print('Confusion matrix, without normalization')
print(cm)
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
fmt = '.2f' if normalize else 'd'
thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j, i, format(cm[i, j], fmt),
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
def plot_confusion_matrix(confusion_matrix,
class_labels,
normalize=False,
title='Confusion Matrix',
cmap=plt.cm.Blues):
""" Code courtesy of Abinav Sagar: https://towardsdatascience.com/convolutional-neural-network-for-breast-cancer-classification-52f1213dcc9 """
if normalize:
confusion_matrix = confusion_matrix.astype(
'float') / confusion_matrix.sum(axis=1)[:, np.newaxis]
print("Normalized confusion matrix")
else:
print('Confusion matrix, without normalization')
print(confusion_matrix)
plt.imshow(confusion_matrix, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(class_labels))
plt.xticks(tick_marks, class_labels, rotation=55)
plt.yticks(tick_marks, class_labels)
fmt = '.2f' if normalize else 'd'
thresh = confusion_matrix.max() / 2.
for i, j in itertools.product(range(confusion_matrix.shape[0]),
range(confusion_matrix.shape[1])):
plt.text(j,
i,
format(confusion_matrix[i, j], fmt),
horizontalalignment="center",
color="white" if confusion_matrix[i, j] > thresh else "black")
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.tight_layout()
def plot_confusion_matrix(path,
confusion_matrix,
filename,
class_names=['background', 'weed', 'sugar_beet'],
normalize=False,
color_map=plt.cm.Blues,
eps=1e-6):
"""Reference: https://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html
"""
accuracy = np.trace(confusion_matrix) / (
np.sum(confusion_matrix).astype('float') + eps)
misclass = 1 - accuracy
if normalize:
confusion_matrix = confusion_matrix.astype('float') / (
confusion_matrix.sum(axis=1)[:, np.newaxis] + 1e-10)
fig, ax = plt.subplots()
im = ax.imshow(confusion_matrix, interpolation='nearest', cmap=color_map)
ax.figure.colorbar(im, ax=ax)
# We want to show all ticks...
ax.set(
xticks=np.arange(confusion_matrix.shape[1]),
yticks=np.arange(confusion_matrix.shape[0]),
# ... and label them with the respective list entries
xticklabels=class_names,
yticklabels=class_names,
title='Confusion Matrix: ' + Path(filename).stem,
ylabel='Actual label',
xlabel='Predicted label')
# Rotate the tick labels and set their alignment.
plt.setp(ax.get_xticklabels(),
rotation=45,
ha="right",
va="center",
rotation_mode="anchor")
# Loop over data dimensions and create text annotations.
fmt = '.2f' if normalize else 'd'
thresh = confusion_matrix.max() / 2.
for i in range(confusion_matrix.shape[0]):
for j in range(confusion_matrix.shape[1]):
ax.text(
j,
i,
format(confusion_matrix[i, j], fmt),
ha="center",
va="center",
color="white" if confusion_matrix[i, j] > thresh else "black")
confusion_matrix_dir_path = path / 'confusion_matrix'
if not confusion_matrix_dir_path.exists():
confusion_matrix_dir_path.mkdir()
confusion_matrix_path = confusion_matrix_dir_path / filename
ax.set_ylim(len(confusion_matrix) - 0.5, -0.5)
fig.savefig(str(confusion_matrix_path), bbox_inches='tight')
plt.close('all')
def get_confusion_classes(self):
# id classes
dict_int_to_string = dict((i, c) for i, c in enumerate(self.Y_labels))
# confusion matrix
confusion_matrix = self.get_confusion_matrix()
# dataframe with classes and their biggest confusion class
cm = confusion_matrix.astype('float') / confusion_matrix.sum(axis=1)[:,np.newaxis]
cm[np.isnan(cm)] = 0
# excluding the right classification
np.fill_diagonal(a=cm, val=0)
# get biggest confusion values
cm_max_class = np.argmax(a=cm, axis=1)
cm_max_value = np.max(a=cm, axis=1)
target_int = list(range(0, cm.shape[0], 1))
target_class = [dict_int_to_string[i] for i in target_int]
confusion_class = [dict_int_to_string[i] for i in cm_max_class]
df_report = pd.DataFrame()
df_report['target'] = target_class
df_report['confusion'] = confusion_class
df_report['value'] = cm_max_value
# Zero values represents not exist target in prediction values or classes with no confusion
df_report = df_report[df_report['value'] != 0]
return df_report
def validate(X_test, y_test, pipe, title, fileName):
print('Test Accuracy: %.3f' % pipe.score(X_test, y_test))
y_predict = pipe.predict(X_test)
confusion_matrix = np.zeros((9,9))
for p,r in zip(y_predict, y_test):
confusion_matrix[p-1,r-1] = confusion_matrix[p-1,r-1] + 1
print (confusion_matrix)
confusion_normalized = confusion_matrix.astype('float') / confusion_matrix.sum(axis=1)[:, np.newaxis]
print (confusion_normalized)
pylab.clf()
pylab.matshow(confusion_normalized, fignum=False, cmap='Blues', vmin=0.0, vmax=1.0)
ax = pylab.axes()
ax.set_xticks(range(len(families)))
ax.set_xticklabels(families, fontsize=4)
ax.xaxis.set_label_position('top')
ax.xaxis.set_ticks_position("top")
ax.set_yticks(range(len(families)))
ax.set_yticklabels(families, fontsize=4)
pylab.title(title)
pylab.colorbar()
pylab.grid(False)
pylab.grid(False)
pylab.savefig(fileName, dpi=900)
def save_confusion_matrix_csv(confusion_matrix, filename):
num_classes = confusion_matrix.shape[0]
header = [
classifier_out_to_vocab_letter(letter_int)
for letter_int in range(num_classes)
]
header = ";".join(header)
np.savetxt(filename, confusion_matrix.astype(int), fmt='%i', delimiter=";")
def print_confusion_matrix(confusion_matrix,
class_names,
figsize=(10, 7),
fontsize=14,
normalize=True):
"""Prints a confusion matrix, as returned by sklearn.metrics.confusion_matrix, as a heatmap.
Arguments
---------
confusion_matrix: numpy.ndarray
The numpy.ndarray object returned from a call to sklearn.metrics.confusion_matrix.
Similarly constructed ndarrays can also be used.
class_names: list
An ordered list of class names, in the order they index the given confusion matrix.
figsize: tuple
A 2-long tuple, the first value determining the horizontal size of the ouputted figure,
the second determining the vertical size. Defaults to (10,7).
fontsize: int
Font size for axes labels. Defaults to 14.
Returns
-------
matplotlib.figure.Figure
The resulting confusion matrix figure
"""
if normalize:
confusion_matrix = (confusion_matrix.astype("float") /
confusion_matrix.sum(axis=1)[:, np.newaxis])
print("Normalized confusion matrix")
else:
print("Confusion matrix, without normalization")
df_cm = pd.DataFrame(
confusion_matrix,
index=class_names,
columns=class_names,
)
fig = plt.figure(figsize=figsize)
fmt = ".2f" if normalize else "d"
try:
heatmap = sns.heatmap(df_cm, annot=True, fmt=fmt)
except ValueError:
raise ValueError("Confusion matrix values must be integers.")
heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(),
rotation=0,
ha="right",
fontsize=fontsize)
heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(),
rotation=45,
ha="right",
fontsize=fontsize)
plt.ylabel("True label")
plt.xlabel("Predicted label")
return fig
def forward_ni(self, filepath):
# data processing
df = pd.read_csv(filepath, header=None) # no column names!!!
df_x = df.iloc[:, :9]
df_x = df_x.div(df_x.sum(axis=1), axis=0) # normalize
X = df_x
X_scaling = StandardScaler().fit_transform(X) # numpy.array
input_data = torch.tensor(X_scaling, requires_grad=True)
input_data = input_data.view(-1, self.sequence_length, self.input_size)
y_new = df.iloc[:, -1]
input_data = input_data.float().to(device)
##############
self.model.eval()
result = self.model(input_data)
_, predict = torch.max(result, 1)
predict = predict.cpu()
predict = predict.numpy()
i = 0
print(predict)
print(y_new.head(10))
count = 0
for i in range(len(predict)):
# print(predict)
if predict[i] == y_new[i]:
count += 1
acc = float(count / len(predict))
# print('Accuracy: {}%'.format(acc*100))
from sklearn.metrics import confusion_matrix
confusion_matrix = confusion_matrix(y_true=y_new, y_pred=predict)
# #Normalize CM
confusion_matrix = cm = confusion_matrix.astype(
'float') / confusion_matrix.sum(axis=1)[:, np.newaxis]
df_cm = pd.DataFrame(confusion_matrix)
# plot confusion matrix
fig, ax = plt.subplots()
sns.heatmap(df_cm, cmap="coolwarm", annot=False)
fig.set_size_inches(8, 6)
ax.set_title("Confusion Matrix of RNN, Data: {}".format(filepath))
ax.set_xlabel('Perdicted Label', fontsize=12)
ax.set_ylabel('Actual Label', fontsize=12)
plt.show()
return predict, acc
def plot_me_nice(confusion_matrix, labels, style='gist_heat', title='unknown'):
confusion_matrix = confusion_matrix.astype('float') / confusion_matrix.sum(
axis=1)[:, np.newaxis]
fig, ax = plt.subplots()
hm = ax.pcolor(confusion_matrix, cmap=style)
ax.set_xticks(np.arange(len(labels)) + 0.5, minor=False)
ax.set_yticks(np.arange(len(labels)) + 0.5, minor=False)
ax.set_xticklabels(labels, minor=False)
ax.set_yticklabels(labels, minor=False)
plt.xticks(rotation=90)
plt.colorbar(hm)
plt.title(title)
return plt.show()
def plot_confusion_matrix(self,
confusion_matrix,
class_names,
normalize=False,
cmap="gray",
path_to_save="./",
name="temp"):
# https://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py
"""
This function prints and plots the confusion matrix.
Normalization can be applied by setting `normalize=True`.
"""
if normalize:
confusion_matrix = confusion_matrix.astype(
'float') / confusion_matrix.sum(axis=1)[:, np.newaxis]
# print("Normalized confusion matrix")
else:
pass
# print('Confusion matrix, without normalization')
# print(cm)
plt.imshow(confusion_matrix, interpolation='nearest', cmap=cmap)
# plt.colorbar()
tick_marks = np.arange(len(class_names))
# plt.xticks(tick_marks, class_names, rotation=45)
plt.xticks(tick_marks, class_names, rotation=0)
plt.yticks(tick_marks, class_names)
# tick_marks = np.arange(len(class_names) - 1)
# plt.yticks(tick_marks, class_names[1:])
fmt = '.2f' if normalize else 'd'
thresh = confusion_matrix.max() / 2.
for i, j in itertools.product(range(confusion_matrix.shape[0]),
range(confusion_matrix.shape[1])):
plt.text(
j,
i,
format(confusion_matrix[i, j], fmt),
horizontalalignment="center",
color="white" if confusion_matrix[i, j] > thresh else "black")
# plt.ylabel('True label')
# plt.xlabel('Predicted label')
plt.ylabel('true distortion type')
plt.xlabel('predicted distortion type')
n_classes = len(class_names)
plt.ylim([n_classes - 0.5, -0.5])
plt.tight_layout()
# plt.show()
plt.savefig(path_to_save + name + ".png")
plt.clf()
plt.close()
def plot_confusion_matrix(confusion_matrix,
y_class,
plot_title,
plot_y_label,
plot_x_label,
normalize=False):
"""
plot the confusion matrix.
:param confusion_matrix: confusion matrix value
:param y_class: target unique class name
:param plot_title: plot title
:param plot_y_label: plot y label
:param plot_x_label: plot x label
:param normalize: default to false
:return: None
"""
try:
plt.figure()
plt.imshow(confusion_matrix,
interpolation='nearest',
cmap=plt.cm.Blues)
plt.title(plot_title)
plt.colorbar()
tick_marks = np.arange(len(y_class))
plt.xticks(tick_marks, y_class, rotation=45)
plt.yticks(tick_marks, y_class)
if normalize:
confusion_matrix = confusion_matrix.astype(
'float') / confusion_matrix.sum(axis=1)[:, np.newaxis]
print("NORMALIZED CONFUSION MATRIX")
else:
print('CONFUSION MATRIX WITHOUT NORMALIZATION')
thresh = confusion_matrix.max() / 2.
for i, j in itertools.product(range(confusion_matrix.shape[0]),
range(confusion_matrix.shape[1])):
plt.text(
j,
i,
confusion_matrix[i, j],
horizontalalignment="center",
color="white" if confusion_matrix[i, j] > thresh else "black")
plt.ylabel(plot_y_label)
plt.xlabel(plot_x_label)
plt.tight_layout()
plt.show()
except Exception as ex:
print("An error occurred: {}".format(ex))
def get_confusion_matrix_one_hot(model_results, truth):
'''model_results and truth should be for one-hot format, i.e, have >= 2 columns,
where truth is 0/1, and max along each row of model_results is model result
'''
assert model_results.shape == truth.shape
num_outputs = truth.shape[1]
confusion_matrix = np.zeros((num_outputs, num_outputs), dtype=np.int32)
predictions = np.argmax(model_results, axis=1)
assert len(predictions) == truth.shape[0]
for actual_class in range(num_outputs):
idx_examples_this_class = truth[:, actual_class] == 1
prediction_for_this_class = predictions[idx_examples_this_class]
for predicted_class in range(num_outputs):
count = np.sum(prediction_for_this_class == predicted_class)
confusion_matrix[actual_class, predicted_class] = count
assert np.sum(confusion_matrix) == len(truth)
assert np.sum(confusion_matrix) == np.sum(truth)
return confusion_matrix.astype(int)
def plot_confusion_matrix(self, conf_matrix):
''' plot and save confusion matrix'''
cm = conf_matrix # easier ...
# delete start/stop/O,U in both row and columns
cm = np.delete(cm, [0, 12, 22, 23, 24], 0)
cm = np.delete(cm, [0, 12, 22, 23, 24], 1)
#swap L and F so L close to V,I
cm[:, [19, 16]] = cm[:, [16, 19]]
cm[[19, 16], :] = cm[[16, 19], :]
#swap F and W
cm[:, [19, 18]] = cm[:, [18, 19]]
cm[[19, 18], :] = cm[[18, 19], :]
#swap G and P
cm[:, [11, 12]] = cm[:, [12, 11]]
cm[[11, 12], :] = cm[[12, 11], :]
# normalise taken from scikitlearn
cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
# plot
fig = plt.figure(figsize=[9, 9])
plt.imshow(cm_norm, interpolation='nearest', cmap=plt.cm.Spectral)
plt.title('Confusion matrix - normalised', fontsize=20)
plt.colorbar()
# plot settings
tick_marks = np.arange(20)
classes = [self.int_to_aa[i] for i in np.arange(25)]
classes = np.delete(classes, [0, 12, 22, 23, 24], 0)
classes[[19, 16]] = classes[[16, 19]] #swap L and F so L close to V,I
classes[[19, 18]] = classes[[18, 19]] #swap F and W
classes[[11, 12]] = classes[[12, 11]] #swap G and P
plt.xticks(tick_marks, classes, fontsize=14, rotation=45)
plt.yticks(tick_marks, classes, fontsize=14)
plt.tight_layout()
plt.ylabel('True label', fontsize=16)
plt.xlabel('Predicted label', fontsize=16)
plt.savefig(self.working_dir +
'/plot_confusion_matrix_{}.png'.format(self.ds_type))
plt.close('all')
def _plot_confusion_matrices(self, confusion_matrix: np.matrix,
labels: list, directory: str, title: str):
"""Plot default and normalized confusion matrix.
confusion_matrix:np.matrix, the confusion matrix to be plot.
labels:list, list of labels of matrix data.
directory:str, the directory were to save the matrix.
title:str, the title for the documents.
"""
plt.figure(figsize=(9, 4))
plt.subplots_adjust(wspace=0.5)
self._heatmap(plt.subplot(1, 2, 1), confusion_matrix, labels,
"Confusion matrix", "d")
self._heatmap(
plt.subplot(1, 2, 2),
confusion_matrix.astype(np.float) /
confusion_matrix.sum(axis=1)[:, np.newaxis], labels,
"Normalized confusion matrix", "0.4g")
plt.suptitle("Confusion Matrices - {title}".format(title=title))
plt.savefig("{directory}/{title} - Confusion matrices.png".format(
directory=directory, title=title))
plt.close()
def plot_confusion_matrix(confusion_matrix, classes, title, normalize=False):
if normalize:
confusion_matrix = confusion_matrix.astype(
'float') / confusion_matrix.sum(axis=1)[:, np.newaxis]
plt.imshow(confusion_matrix, interpolation='nearest', cmap=plt.cm.Blues)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
for i, j in itertools.product(range(confusion_matrix.shape[0]),
range(confusion_matrix.shape[1])):
plt.text(
j,
i,
format(confusion_matrix[i, j], '.2f' if normalize else 'd'),
horizontalalignment="center",
color="white" if
confusion_matrix[i, j] > confusion_matrix.max() / 2. else "black")
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
def plot_confusion_matrix(confusion_matrix,
class_names,
errors_only=False,
figsize=(15, 6),
fontsize=16):
"""
Plots confusion matrix as a color-encoded Seaborn heatmap. Zeroes are
colored white. Normalized values that are zero when rounded to three
decimals, Ex. 0.000, will be colored white. Get more decicmals by
updating fmt, for example to '0.4f', and updating get_text() value.
Arguments
---------
confusion_matrix: numpy.ndarray
The numpy.ndarray object sklearn.metrics.confusion_matrix.
class_names: list
List of class names in the order they index the confusion matrix.
figsize: tuple
A pair tuple. The first value is figure width. The second
value is figure height. Defaults to (15,6).
fontsize: int
Font size for axes labels. Defaults to 16.
"""
#Instantiate Figure
fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=figsize)
plt.subplots_adjust(wspace=0.5)
#Show errors only by filling diagonal with zeroes.
if errors_only:
np.fill_diagonal(confusion_matrix, 0)
# ax1 - Normalized Confusion Matrix
#Normalize by dividing (M X M) matrix by (M X 1) matrix. (M X 1) is row totals.
conf_matrix_norm = confusion_matrix.astype('float') / confusion_matrix.sum(
axis=1)[:, np.newaxis]
conf_matrix_norm = np.nan_to_num(
conf_matrix_norm) #fix any nans caused by zero row total
df_cm_norm = pd.DataFrame(conf_matrix_norm,
index=class_names,
columns=class_names)
heatmap = sns.heatmap(df_cm_norm,
ax=ax1,
cmap='Blues',
fmt='.3f',
annot=True,
annot_kws={"size": fontsize},
linewidths=2,
linecolor='black',
cbar=False)
ax1.tick_params(axis='x',
labelrotation=0,
labelsize=fontsize,
labelcolor='black')
ax1.tick_params(axis='y',
labelrotation=0,
labelsize=fontsize,
labelcolor='black')
ax1.set_ylim(ax1.get_xlim()[0], ax1.get_xlim()[1]) #Fix messed up ylim
ax1.set_xlabel('PREDICTED CLASS', fontsize=fontsize, color='black')
ax1.set_ylabel('TRUE CLASS', fontsize=fontsize, color='black')
ax1.set_title('Confusion Matrix - Normalized',
pad=15,
fontsize=fontsize,
color='black')
# ax2 - Confusion Matrix - Class Counts
df_cm = pd.DataFrame(confusion_matrix,
index=class_names,
columns=class_names)
heatmap = sns.heatmap(df_cm,
ax=ax2,
cmap='Blues',
fmt='d',
annot=True,
annot_kws={"size": fontsize + 4},
linewidths=2,
linecolor='black',
cbar=False)
ax2.tick_params(axis='x',
labelrotation=0,
labelsize=fontsize,
labelcolor='black')
ax2.tick_params(axis='y',
labelrotation=0,
labelsize=fontsize,
labelcolor='black')
ax2.set_ylim(
ax1.get_xlim()[0],
ax1.get_xlim()
[1]) #Fix bug in matplotlib 3.1.1. Or, use earlier matplotlib.
ax2.set_xlabel('PREDICTED CLASS', fontsize=fontsize, color='black')
ax2.set_ylabel('TRUE CLASS', fontsize=fontsize, color='black')
ax2.set_title('Confusion Matrix - Class Counts',
pad=15,
fontsize=fontsize,
color='black')
for text in ax1.texts:
if text.get_text() == '0.000':
text.set_color(color='white')
for text in ax2.texts:
if text.get_text() == '0':
text.set_color(color='white')
]
}
model_search = GridSearchCV(pipeline,
param_grid,
cv=3,
iid=False,
return_train_score=True,
scoring="accuracy",
n_jobs=4)
# model_search = RandomizedSearchCV(pipeline, param_dist, n_iter=100, cv=3,scoring="accuracy")
model = model_search.fit(X_train, pd.Series.ravel(y_train))
print(model.best_score_)
# print(model.best_estimator_.feature_importances_)
# print(model.best_params_)
# print("parameters")
# print(model.cv_results_)
pickle.dump(model, open("/usr/local/hadoop-2.7.7/bin/model_pca_svc", 'wb'))
GENRES_LIST = [
"blues", "classical", "country", "disco", "hiphop", "jazz", "metal", "pop",
"reggae", "rock"
]
X_test = pca.transform(X_test)
y_pred = model.predict(X_test)
print("report:")
print(classification_report(y_test, y_pred, target_names=GENRES_LIST))
confusion_matrix = confusion_matrix(y_test, y_pred)
confusion_matrix = confusion_matrix.astype("float") / confusion_matrix.sum(
axis=1)[:, np.newaxis]
print(confusion_matrix.diagonal())
print(accuracy_score(y_test, y_pred))
# train_end = datetime.datetime.now()
# train_time = train_end - train_start
# print('train time:', train_time)
model_dir = 'origin/file/nb_model.pkl'
# with open(model_dir, 'wb') as f:
# pkl.dump(clf, f)
with open(model_dir, 'rb') as f:
clf = pkl.load(f)
test_start = datetime.datetime.now()
y_pred = clf.predict(test_tfidf)
test_end = datetime.datetime.now()
test_time = test_end - test_start
print('test time:', test_time)
confusion_matrix = confusion_matrix(test_label, y_pred)
con_mat_norm = confusion_matrix.astype('float') / confusion_matrix.sum(
axis=1)[:, np.newaxis] # 归一化
con_mat_norm = np.around(con_mat_norm, decimals=2)
# === plot ===
plt.figure(figsize=(8, 8))
sns.heatmap(con_mat_norm, annot=True, cmap='Blues')
plt.ylim(0, 10)
plt.xlabel('Predicted labels')
plt.ylabel('True labels')
plt.show()
print(metrics.accuracy_score(test_label, y_pred))
print(metrics.confusion_matrix(test_label, y_pred))
r = sm.classification_report(test_label, y_pred)
print('分类报告为：', r, sep='\n')
'''
# Logging through Tensorbard
tensorboard = TensorBoard(log_dir='./logs_lstm',
histogram_freq=0,
write_graph=True,
write_images=True)
# Model compilation
#early_stopping = keras.callbacks.EarlyStopping(monitor='val_acc', min_delta=0, patience=50, verbose=1, mode='auto')
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
checkpointer = ModelCheckpoint(filepath='trained-models-lstm/Best_model.h5', monitor='val_acc', verbose=1, save_best_only=True)
# Model fitting
model.fit(X_train, Y_train, epochs=500, batch_size=batch_size, validation_data=(X_val, Y_val), verbose=2, shuffle=False, callbacks=[checkpointer, tensorboard])
# Model validation
predictions = model.predict(X_val)
score = accuracy_score(change_format(Y_val), change_format(predictions))
print('Last epoch\'s validation score is ', score)
# Confusion Matrix generation and plotting
confusion_matrix=pd.DataFrame(confusion_matrix(change_format(Y_val), change_format(predictions)), index = [i for i in "0123"],
columns = [i for i in "0123"])
confusion_matrix_normalised= confusion_matrix.astype('float') / confusion_matrix.sum(axis=1)[:, np.newaxis]
print(confusion_matrix)
print(confusion_matrix_normalised)
plt.figure(figsize = (10,7))
sn.heatmap(confusion_matrix_normalised, annot=True, cmap='Blues', fmt='g')
plt.xlabel('Predicted Class')
plt.ylabel('True Class')
plt.savefig('figures/confusion-matrix-lstm-ecg.png', dpi=300)
plt.show()
return test_dataset
def results_val(test_dataset):
from sklearn.metrics import confusion_matrix
import itertools
import matplotlib.pyplot as plt
y_true = np.array (test_dataset['category'])
y_pred = np.array(test_dataset['prediction'])
classes = np.unique( test_dataset['category'] )
confusion_matrix = confusion_matrix(y_true, y_pred)
confusion_matrix = np.around(confusion_matrix.astype('float') / confusion_matrix.sum(axis=1)[:, np.newaxis],3)
"""
This function prints and plots the confusion matrix.
Normalization can be applied by setting `normalize=True`.
"""
plt.imshow(confusion_matrix, interpolation='nearest', cmap=plt.cm.Blues)
plt.title('Confusion Matrix')
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
##if normalize:
## cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
## print("Normalized confusion matrix")
confusion_matrix = np.zeros((len(LABELS), len(LABELS) + 1), np.uint)
for file_pair in file_pairs:
confusion_matrix += score_prediction_files(*file_pair)
if len(file_pairs) >= 1:
print("----- OVERALL SCORES -----")
report_scores(confusion_matrix)
#Plot Confusion Metrics
tick_marks = np.array(range(len(labels) - 1)) + 0.5
confusion_matrix = np.array(confusion_matrix[:, :-1])
np.set_printoptions(precision=3)
x = confusion_matrix.sum(axis=1)[:, np.newaxis]
for i in range(len(x)):
if not x[i]:
x[i] = 1
cm_normalized = confusion_matrix.astype('float') / x
# print(cm_normalized)
plt.figure(figsize=(12, 8), dpi=120)
ind_array = np.arange(len(labels) - 1)
x, y = np.meshgrid(ind_array, ind_array)
for x_val, y_val in zip(x.flatten(), y.flatten()):
c = cm_normalized[y_val][x_val]
if c > 0.001:
plt.text(x_val,
y_val,
"%0.3f" % (c, ),
color='red',
fontsize=10,
va='center',
ha='center')
# offset the tick
