def mr_metrics(confusion_matrix, level): confusion_matrix = confusion_matrix.astype(float) # sum(0) <- predicted sum(1) ground truth total = np.sum(confusion_matrix) n_classes, _ = confusion_matrix.shape overall_accuracy = np.sum(np.diag(confusion_matrix)) / total # calculate Cohen Kappa (https://en.wikipedia.org/wiki/Cohen%27s_kappa) N = total p0 = np.sum(np.diag(confusion_matrix)) / N pc = np.sum( np.sum(confusion_matrix, axis=0) * np.sum(confusion_matrix, axis=1)) / N**2 kappa = (p0 - pc) / (1 - pc) recall = np.diag(confusion_matrix) / (np.sum(confusion_matrix, axis=1) + 1e-12) precision = np.diag(confusion_matrix) / (np.sum(confusion_matrix, axis=0) + 1e-12) f1 = (2 * precision * recall) / ((precision + recall) + 1e-12) if level == 'global': return overall_accuracy, kappa, np.mean(precision), np.mean( recall), np.mean(f1) elif level == 'perclass': # Per class accuracy cl_acc = np.diag(confusion_matrix) / (confusion_matrix.sum(1) + 1e-12) return overall_accuracy, kappa, precision, recall, f1, cl_acc
def plot_cnf_matrix(self, cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues): """ This function prints and plots the confusion matrix. Normalization can be applied by setting `normalize=True`. """ import itertools if normalize: cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] print("Normalized confusion matrix") else: print('Confusion matrix, without normalization') plt.imshow(cm, interpolation='nearest', cmap=cmap) tick_marks = np.arange(len(classes)) plt.xticks(tick_marks, classes, rotation=45) plt.yticks(tick_marks, classes) fmt = '.2f' if normalize else 'd' thresh = cm.max() / 2. for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): plt.text(j, i, format(cm[i, j], fmt), horizontalalignment="center", color="white" if cm[i, j] > thresh else "black") plt.ylabel('True label') plt.xlabel('Predicted label') plt.show()
def plot_confusion_matrix(confusion_matrix, classes, title, normalize=False): plt.clf() plt.imshow(confusion_matrix, interpolation='nearest', cmap=plt.cm.Blues) plt.title(title) plt.colorbar() tick_marks = np.arange(len(classes)) plt.xticks(tick_marks, classes, rotation=45) plt.yticks(tick_marks, classes) if normalize: confusion_matrix = confusion_matrix.astype( 'float') / confusion_matrix.sum(axis=1)[:, np.newaxis] thresh = confusion_matrix.max() / 2. for i, j in itertools.product(range(confusion_matrix.shape[0]), range(confusion_matrix.shape[1])): plt.text(j, i, round(confusion_matrix[i, j], 3), horizontalalignment="center", color="white" if confusion_matrix[i, j] > thresh else "black") plt.tight_layout() plt.ylabel('Реальные классы') plt.xlabel('Расчетные классы')
def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues): if normalize: cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] print("Normalized confusion matrix") else: print('Confusion matrix, without normalization') print(cm) plt.imshow(cm, interpolation='nearest', cmap=cmap) plt.title(title) plt.colorbar() tick_marks = np.arange(len(classes)) plt.xticks(tick_marks, classes, rotation=45) plt.yticks(tick_marks, classes) fmt = '.2f' if normalize else 'd' thresh = cm.max() / 2. for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): plt.text(j, i, format(cm[i, j], fmt), horizontalalignment="center", color="white" if cm[i, j] > thresh else "black") plt.tight_layout() plt.ylabel('True label') plt.xlabel('Predicted label')
def plot_confusion_matrix(confusion_matrix, class_labels, normalize=False, title='Confusion Matrix', cmap=plt.cm.Blues): """ Code courtesy of Abinav Sagar: https://towardsdatascience.com/convolutional-neural-network-for-breast-cancer-classification-52f1213dcc9 """ if normalize: confusion_matrix = confusion_matrix.astype( 'float') / confusion_matrix.sum(axis=1)[:, np.newaxis] print("Normalized confusion matrix") else: print('Confusion matrix, without normalization') print(confusion_matrix) plt.imshow(confusion_matrix, interpolation='nearest', cmap=cmap) plt.title(title) plt.colorbar() tick_marks = np.arange(len(class_labels)) plt.xticks(tick_marks, class_labels, rotation=55) plt.yticks(tick_marks, class_labels) fmt = '.2f' if normalize else 'd' thresh = confusion_matrix.max() / 2. for i, j in itertools.product(range(confusion_matrix.shape[0]), range(confusion_matrix.shape[1])): plt.text(j, i, format(confusion_matrix[i, j], fmt), horizontalalignment="center", color="white" if confusion_matrix[i, j] > thresh else "black") plt.ylabel('True label') plt.xlabel('Predicted label') plt.tight_layout()
def plot_confusion_matrix(path, confusion_matrix, filename, class_names=['background', 'weed', 'sugar_beet'], normalize=False, color_map=plt.cm.Blues, eps=1e-6): """Reference: https://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html """ accuracy = np.trace(confusion_matrix) / ( np.sum(confusion_matrix).astype('float') + eps) misclass = 1 - accuracy if normalize: confusion_matrix = confusion_matrix.astype('float') / ( confusion_matrix.sum(axis=1)[:, np.newaxis] + 1e-10) fig, ax = plt.subplots() im = ax.imshow(confusion_matrix, interpolation='nearest', cmap=color_map) ax.figure.colorbar(im, ax=ax) # We want to show all ticks... ax.set( xticks=np.arange(confusion_matrix.shape[1]), yticks=np.arange(confusion_matrix.shape[0]), # ... and label them with the respective list entries xticklabels=class_names, yticklabels=class_names, title='Confusion Matrix: ' + Path(filename).stem, ylabel='Actual label', xlabel='Predicted label') # Rotate the tick labels and set their alignment. plt.setp(ax.get_xticklabels(), rotation=45, ha="right", va="center", rotation_mode="anchor") # Loop over data dimensions and create text annotations. fmt = '.2f' if normalize else 'd' thresh = confusion_matrix.max() / 2. for i in range(confusion_matrix.shape[0]): for j in range(confusion_matrix.shape[1]): ax.text( j, i, format(confusion_matrix[i, j], fmt), ha="center", va="center", color="white" if confusion_matrix[i, j] > thresh else "black") confusion_matrix_dir_path = path / 'confusion_matrix' if not confusion_matrix_dir_path.exists(): confusion_matrix_dir_path.mkdir() confusion_matrix_path = confusion_matrix_dir_path / filename ax.set_ylim(len(confusion_matrix) - 0.5, -0.5) fig.savefig(str(confusion_matrix_path), bbox_inches='tight') plt.close('all')
def get_confusion_classes(self): # id classes dict_int_to_string = dict((i, c) for i, c in enumerate(self.Y_labels)) # confusion matrix confusion_matrix = self.get_confusion_matrix() # dataframe with classes and their biggest confusion class cm = confusion_matrix.astype('float') / confusion_matrix.sum(axis=1)[:,np.newaxis] cm[np.isnan(cm)] = 0 # excluding the right classification np.fill_diagonal(a=cm, val=0) # get biggest confusion values cm_max_class = np.argmax(a=cm, axis=1) cm_max_value = np.max(a=cm, axis=1) target_int = list(range(0, cm.shape[0], 1)) target_class = [dict_int_to_string[i] for i in target_int] confusion_class = [dict_int_to_string[i] for i in cm_max_class] df_report = pd.DataFrame() df_report['target'] = target_class df_report['confusion'] = confusion_class df_report['value'] = cm_max_value # Zero values represents not exist target in prediction values or classes with no confusion df_report = df_report[df_report['value'] != 0] return df_report
def validate(X_test, y_test, pipe, title, fileName): print('Test Accuracy: %.3f' % pipe.score(X_test, y_test)) y_predict = pipe.predict(X_test) confusion_matrix = np.zeros((9,9)) for p,r in zip(y_predict, y_test): confusion_matrix[p-1,r-1] = confusion_matrix[p-1,r-1] + 1 print (confusion_matrix) confusion_normalized = confusion_matrix.astype('float') / confusion_matrix.sum(axis=1)[:, np.newaxis] print (confusion_normalized) pylab.clf() pylab.matshow(confusion_normalized, fignum=False, cmap='Blues', vmin=0.0, vmax=1.0) ax = pylab.axes() ax.set_xticks(range(len(families))) ax.set_xticklabels(families, fontsize=4) ax.xaxis.set_label_position('top') ax.xaxis.set_ticks_position("top") ax.set_yticks(range(len(families))) ax.set_yticklabels(families, fontsize=4) pylab.title(title) pylab.colorbar() pylab.grid(False) pylab.grid(False) pylab.savefig(fileName, dpi=900)
def save_confusion_matrix_csv(confusion_matrix, filename): num_classes = confusion_matrix.shape[0] header = [ classifier_out_to_vocab_letter(letter_int) for letter_int in range(num_classes) ] header = ";".join(header) np.savetxt(filename, confusion_matrix.astype(int), fmt='%i', delimiter=";")
def print_confusion_matrix(confusion_matrix, class_names, figsize=(10, 7), fontsize=14, normalize=True): """Prints a confusion matrix, as returned by sklearn.metrics.confusion_matrix, as a heatmap. Arguments --------- confusion_matrix: numpy.ndarray The numpy.ndarray object returned from a call to sklearn.metrics.confusion_matrix. Similarly constructed ndarrays can also be used. class_names: list An ordered list of class names, in the order they index the given confusion matrix. figsize: tuple A 2-long tuple, the first value determining the horizontal size of the ouputted figure, the second determining the vertical size. Defaults to (10,7). fontsize: int Font size for axes labels. Defaults to 14. Returns ------- matplotlib.figure.Figure The resulting confusion matrix figure """ if normalize: confusion_matrix = (confusion_matrix.astype("float") / confusion_matrix.sum(axis=1)[:, np.newaxis]) print("Normalized confusion matrix") else: print("Confusion matrix, without normalization") df_cm = pd.DataFrame( confusion_matrix, index=class_names, columns=class_names, ) fig = plt.figure(figsize=figsize) fmt = ".2f" if normalize else "d" try: heatmap = sns.heatmap(df_cm, annot=True, fmt=fmt) except ValueError: raise ValueError("Confusion matrix values must be integers.") heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha="right", fontsize=fontsize) heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45, ha="right", fontsize=fontsize) plt.ylabel("True label") plt.xlabel("Predicted label") return fig
def forward_ni(self, filepath): # data processing df = pd.read_csv(filepath, header=None) # no column names!!! df_x = df.iloc[:, :9] df_x = df_x.div(df_x.sum(axis=1), axis=0) # normalize X = df_x X_scaling = StandardScaler().fit_transform(X) # numpy.array input_data = torch.tensor(X_scaling, requires_grad=True) input_data = input_data.view(-1, self.sequence_length, self.input_size) y_new = df.iloc[:, -1] input_data = input_data.float().to(device) ############## self.model.eval() result = self.model(input_data) _, predict = torch.max(result, 1) predict = predict.cpu() predict = predict.numpy() i = 0 print(predict) print(y_new.head(10)) count = 0 for i in range(len(predict)): # print(predict) if predict[i] == y_new[i]: count += 1 acc = float(count / len(predict)) # print('Accuracy: {}%'.format(acc*100)) from sklearn.metrics import confusion_matrix confusion_matrix = confusion_matrix(y_true=y_new, y_pred=predict) # #Normalize CM confusion_matrix = cm = confusion_matrix.astype( 'float') / confusion_matrix.sum(axis=1)[:, np.newaxis] df_cm = pd.DataFrame(confusion_matrix) # plot confusion matrix fig, ax = plt.subplots() sns.heatmap(df_cm, cmap="coolwarm", annot=False) fig.set_size_inches(8, 6) ax.set_title("Confusion Matrix of RNN, Data: {}".format(filepath)) ax.set_xlabel('Perdicted Label', fontsize=12) ax.set_ylabel('Actual Label', fontsize=12) plt.show() return predict, acc
def plot_me_nice(confusion_matrix, labels, style='gist_heat', title='unknown'): confusion_matrix = confusion_matrix.astype('float') / confusion_matrix.sum( axis=1)[:, np.newaxis] fig, ax = plt.subplots() hm = ax.pcolor(confusion_matrix, cmap=style) ax.set_xticks(np.arange(len(labels)) + 0.5, minor=False) ax.set_yticks(np.arange(len(labels)) + 0.5, minor=False) ax.set_xticklabels(labels, minor=False) ax.set_yticklabels(labels, minor=False) plt.xticks(rotation=90) plt.colorbar(hm) plt.title(title) return plt.show()
def plot_confusion_matrix(self, confusion_matrix, class_names, normalize=False, cmap="gray", path_to_save="./", name="temp"): # https://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py """ This function prints and plots the confusion matrix. Normalization can be applied by setting `normalize=True`. """ if normalize: confusion_matrix = confusion_matrix.astype( 'float') / confusion_matrix.sum(axis=1)[:, np.newaxis] # print("Normalized confusion matrix") else: pass # print('Confusion matrix, without normalization') # print(cm) plt.imshow(confusion_matrix, interpolation='nearest', cmap=cmap) # plt.colorbar() tick_marks = np.arange(len(class_names)) # plt.xticks(tick_marks, class_names, rotation=45) plt.xticks(tick_marks, class_names, rotation=0) plt.yticks(tick_marks, class_names) # tick_marks = np.arange(len(class_names) - 1) # plt.yticks(tick_marks, class_names[1:]) fmt = '.2f' if normalize else 'd' thresh = confusion_matrix.max() / 2. for i, j in itertools.product(range(confusion_matrix.shape[0]), range(confusion_matrix.shape[1])): plt.text( j, i, format(confusion_matrix[i, j], fmt), horizontalalignment="center", color="white" if confusion_matrix[i, j] > thresh else "black") # plt.ylabel('True label') # plt.xlabel('Predicted label') plt.ylabel('true distortion type') plt.xlabel('predicted distortion type') n_classes = len(class_names) plt.ylim([n_classes - 0.5, -0.5]) plt.tight_layout() # plt.show() plt.savefig(path_to_save + name + ".png") plt.clf() plt.close()
def plot_confusion_matrix(confusion_matrix, y_class, plot_title, plot_y_label, plot_x_label, normalize=False): """ plot the confusion matrix. :param confusion_matrix: confusion matrix value :param y_class: target unique class name :param plot_title: plot title :param plot_y_label: plot y label :param plot_x_label: plot x label :param normalize: default to false :return: None """ try: plt.figure() plt.imshow(confusion_matrix, interpolation='nearest', cmap=plt.cm.Blues) plt.title(plot_title) plt.colorbar() tick_marks = np.arange(len(y_class)) plt.xticks(tick_marks, y_class, rotation=45) plt.yticks(tick_marks, y_class) if normalize: confusion_matrix = confusion_matrix.astype( 'float') / confusion_matrix.sum(axis=1)[:, np.newaxis] print("NORMALIZED CONFUSION MATRIX") else: print('CONFUSION MATRIX WITHOUT NORMALIZATION') thresh = confusion_matrix.max() / 2. for i, j in itertools.product(range(confusion_matrix.shape[0]), range(confusion_matrix.shape[1])): plt.text( j, i, confusion_matrix[i, j], horizontalalignment="center", color="white" if confusion_matrix[i, j] > thresh else "black") plt.ylabel(plot_y_label) plt.xlabel(plot_x_label) plt.tight_layout() plt.show() except Exception as ex: print("An error occurred: {}".format(ex))
def get_confusion_matrix_one_hot(model_results, truth): '''model_results and truth should be for one-hot format, i.e, have >= 2 columns, where truth is 0/1, and max along each row of model_results is model result ''' assert model_results.shape == truth.shape num_outputs = truth.shape[1] confusion_matrix = np.zeros((num_outputs, num_outputs), dtype=np.int32) predictions = np.argmax(model_results, axis=1) assert len(predictions) == truth.shape[0] for actual_class in range(num_outputs): idx_examples_this_class = truth[:, actual_class] == 1 prediction_for_this_class = predictions[idx_examples_this_class] for predicted_class in range(num_outputs): count = np.sum(prediction_for_this_class == predicted_class) confusion_matrix[actual_class, predicted_class] = count assert np.sum(confusion_matrix) == len(truth) assert np.sum(confusion_matrix) == np.sum(truth) return confusion_matrix.astype(int)
def plot_confusion_matrix(self, conf_matrix): ''' plot and save confusion matrix''' cm = conf_matrix # easier ... # delete start/stop/O,U in both row and columns cm = np.delete(cm, [0, 12, 22, 23, 24], 0) cm = np.delete(cm, [0, 12, 22, 23, 24], 1) #swap L and F so L close to V,I cm[:, [19, 16]] = cm[:, [16, 19]] cm[[19, 16], :] = cm[[16, 19], :] #swap F and W cm[:, [19, 18]] = cm[:, [18, 19]] cm[[19, 18], :] = cm[[18, 19], :] #swap G and P cm[:, [11, 12]] = cm[:, [12, 11]] cm[[11, 12], :] = cm[[12, 11], :] # normalise taken from scikitlearn cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] # plot fig = plt.figure(figsize=[9, 9]) plt.imshow(cm_norm, interpolation='nearest', cmap=plt.cm.Spectral) plt.title('Confusion matrix - normalised', fontsize=20) plt.colorbar() # plot settings tick_marks = np.arange(20) classes = [self.int_to_aa[i] for i in np.arange(25)] classes = np.delete(classes, [0, 12, 22, 23, 24], 0) classes[[19, 16]] = classes[[16, 19]] #swap L and F so L close to V,I classes[[19, 18]] = classes[[18, 19]] #swap F and W classes[[11, 12]] = classes[[12, 11]] #swap G and P plt.xticks(tick_marks, classes, fontsize=14, rotation=45) plt.yticks(tick_marks, classes, fontsize=14) plt.tight_layout() plt.ylabel('True label', fontsize=16) plt.xlabel('Predicted label', fontsize=16) plt.savefig(self.working_dir + '/plot_confusion_matrix_{}.png'.format(self.ds_type)) plt.close('all')
def _plot_confusion_matrices(self, confusion_matrix: np.matrix, labels: list, directory: str, title: str): """Plot default and normalized confusion matrix. confusion_matrix:np.matrix, the confusion matrix to be plot. labels:list, list of labels of matrix data. directory:str, the directory were to save the matrix. title:str, the title for the documents. """ plt.figure(figsize=(9, 4)) plt.subplots_adjust(wspace=0.5) self._heatmap(plt.subplot(1, 2, 1), confusion_matrix, labels, "Confusion matrix", "d") self._heatmap( plt.subplot(1, 2, 2), confusion_matrix.astype(np.float) / confusion_matrix.sum(axis=1)[:, np.newaxis], labels, "Normalized confusion matrix", "0.4g") plt.suptitle("Confusion Matrices - {title}".format(title=title)) plt.savefig("{directory}/{title} - Confusion matrices.png".format( directory=directory, title=title)) plt.close()
def plot_confusion_matrix(confusion_matrix, classes, title, normalize=False): if normalize: confusion_matrix = confusion_matrix.astype( 'float') / confusion_matrix.sum(axis=1)[:, np.newaxis] plt.imshow(confusion_matrix, interpolation='nearest', cmap=plt.cm.Blues) plt.title(title) plt.colorbar() tick_marks = np.arange(len(classes)) plt.xticks(tick_marks, classes, rotation=45) plt.yticks(tick_marks, classes) for i, j in itertools.product(range(confusion_matrix.shape[0]), range(confusion_matrix.shape[1])): plt.text( j, i, format(confusion_matrix[i, j], '.2f' if normalize else 'd'), horizontalalignment="center", color="white" if confusion_matrix[i, j] > confusion_matrix.max() / 2. else "black") plt.tight_layout() plt.ylabel('True label') plt.xlabel('Predicted label')
def plot_confusion_matrix(confusion_matrix, class_names, errors_only=False, figsize=(15, 6), fontsize=16): """ Plots confusion matrix as a color-encoded Seaborn heatmap. Zeroes are colored white. Normalized values that are zero when rounded to three decimals, Ex. 0.000, will be colored white. Get more decicmals by updating fmt, for example to '0.4f', and updating get_text() value. Arguments --------- confusion_matrix: numpy.ndarray The numpy.ndarray object sklearn.metrics.confusion_matrix. class_names: list List of class names in the order they index the confusion matrix. figsize: tuple A pair tuple. The first value is figure width. The second value is figure height. Defaults to (15,6). fontsize: int Font size for axes labels. Defaults to 16. """ #Instantiate Figure fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=figsize) plt.subplots_adjust(wspace=0.5) #Show errors only by filling diagonal with zeroes. if errors_only: np.fill_diagonal(confusion_matrix, 0) # ax1 - Normalized Confusion Matrix #Normalize by dividing (M X M) matrix by (M X 1) matrix. (M X 1) is row totals. conf_matrix_norm = confusion_matrix.astype('float') / confusion_matrix.sum( axis=1)[:, np.newaxis] conf_matrix_norm = np.nan_to_num( conf_matrix_norm) #fix any nans caused by zero row total df_cm_norm = pd.DataFrame(conf_matrix_norm, index=class_names, columns=class_names) heatmap = sns.heatmap(df_cm_norm, ax=ax1, cmap='Blues', fmt='.3f', annot=True, annot_kws={"size": fontsize}, linewidths=2, linecolor='black', cbar=False) ax1.tick_params(axis='x', labelrotation=0, labelsize=fontsize, labelcolor='black') ax1.tick_params(axis='y', labelrotation=0, labelsize=fontsize, labelcolor='black') ax1.set_ylim(ax1.get_xlim()[0], ax1.get_xlim()[1]) #Fix messed up ylim ax1.set_xlabel('PREDICTED CLASS', fontsize=fontsize, color='black') ax1.set_ylabel('TRUE CLASS', fontsize=fontsize, color='black') ax1.set_title('Confusion Matrix - Normalized', pad=15, fontsize=fontsize, color='black') # ax2 - Confusion Matrix - Class Counts df_cm = pd.DataFrame(confusion_matrix, index=class_names, columns=class_names) heatmap = sns.heatmap(df_cm, ax=ax2, cmap='Blues', fmt='d', annot=True, annot_kws={"size": fontsize + 4}, linewidths=2, linecolor='black', cbar=False) ax2.tick_params(axis='x', labelrotation=0, labelsize=fontsize, labelcolor='black') ax2.tick_params(axis='y', labelrotation=0, labelsize=fontsize, labelcolor='black') ax2.set_ylim( ax1.get_xlim()[0], ax1.get_xlim() [1]) #Fix bug in matplotlib 3.1.1. Or, use earlier matplotlib. ax2.set_xlabel('PREDICTED CLASS', fontsize=fontsize, color='black') ax2.set_ylabel('TRUE CLASS', fontsize=fontsize, color='black') ax2.set_title('Confusion Matrix - Class Counts', pad=15, fontsize=fontsize, color='black') for text in ax1.texts: if text.get_text() == '0.000': text.set_color(color='white') for text in ax2.texts: if text.get_text() == '0': text.set_color(color='white')
] } model_search = GridSearchCV(pipeline, param_grid, cv=3, iid=False, return_train_score=True, scoring="accuracy", n_jobs=4) # model_search = RandomizedSearchCV(pipeline, param_dist, n_iter=100, cv=3,scoring="accuracy") model = model_search.fit(X_train, pd.Series.ravel(y_train)) print(model.best_score_) # print(model.best_estimator_.feature_importances_) # print(model.best_params_) # print("parameters") # print(model.cv_results_) pickle.dump(model, open("/usr/local/hadoop-2.7.7/bin/model_pca_svc", 'wb')) GENRES_LIST = [ "blues", "classical", "country", "disco", "hiphop", "jazz", "metal", "pop", "reggae", "rock" ] X_test = pca.transform(X_test) y_pred = model.predict(X_test) print("report:") print(classification_report(y_test, y_pred, target_names=GENRES_LIST)) confusion_matrix = confusion_matrix(y_test, y_pred) confusion_matrix = confusion_matrix.astype("float") / confusion_matrix.sum( axis=1)[:, np.newaxis] print(confusion_matrix.diagonal()) print(accuracy_score(y_test, y_pred))
# train_end = datetime.datetime.now() # train_time = train_end - train_start # print('train time:', train_time) model_dir = 'origin/file/nb_model.pkl' # with open(model_dir, 'wb') as f: # pkl.dump(clf, f) with open(model_dir, 'rb') as f: clf = pkl.load(f) test_start = datetime.datetime.now() y_pred = clf.predict(test_tfidf) test_end = datetime.datetime.now() test_time = test_end - test_start print('test time:', test_time) confusion_matrix = confusion_matrix(test_label, y_pred) con_mat_norm = confusion_matrix.astype('float') / confusion_matrix.sum( axis=1)[:, np.newaxis] # 归一化 con_mat_norm = np.around(con_mat_norm, decimals=2) # === plot === plt.figure(figsize=(8, 8)) sns.heatmap(con_mat_norm, annot=True, cmap='Blues') plt.ylim(0, 10) plt.xlabel('Predicted labels') plt.ylabel('True labels') plt.show() print(metrics.accuracy_score(test_label, y_pred)) print(metrics.confusion_matrix(test_label, y_pred)) r = sm.classification_report(test_label, y_pred) print('分类报告为:', r, sep='\n') '''
# Logging through Tensorbard tensorboard = TensorBoard(log_dir='./logs_lstm', histogram_freq=0, write_graph=True, write_images=True) # Model compilation #early_stopping = keras.callbacks.EarlyStopping(monitor='val_acc', min_delta=0, patience=50, verbose=1, mode='auto') model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) checkpointer = ModelCheckpoint(filepath='trained-models-lstm/Best_model.h5', monitor='val_acc', verbose=1, save_best_only=True) # Model fitting model.fit(X_train, Y_train, epochs=500, batch_size=batch_size, validation_data=(X_val, Y_val), verbose=2, shuffle=False, callbacks=[checkpointer, tensorboard]) # Model validation predictions = model.predict(X_val) score = accuracy_score(change_format(Y_val), change_format(predictions)) print('Last epoch\'s validation score is ', score) # Confusion Matrix generation and plotting confusion_matrix=pd.DataFrame(confusion_matrix(change_format(Y_val), change_format(predictions)), index = [i for i in "0123"], columns = [i for i in "0123"]) confusion_matrix_normalised= confusion_matrix.astype('float') / confusion_matrix.sum(axis=1)[:, np.newaxis] print(confusion_matrix) print(confusion_matrix_normalised) plt.figure(figsize = (10,7)) sn.heatmap(confusion_matrix_normalised, annot=True, cmap='Blues', fmt='g') plt.xlabel('Predicted Class') plt.ylabel('True Class') plt.savefig('figures/confusion-matrix-lstm-ecg.png', dpi=300) plt.show()
return test_dataset def results_val(test_dataset): from sklearn.metrics import confusion_matrix import itertools import matplotlib.pyplot as plt y_true = np.array (test_dataset['category']) y_pred = np.array(test_dataset['prediction']) classes = np.unique( test_dataset['category'] ) confusion_matrix = confusion_matrix(y_true, y_pred) confusion_matrix = np.around(confusion_matrix.astype('float') / confusion_matrix.sum(axis=1)[:, np.newaxis],3) """ This function prints and plots the confusion matrix. Normalization can be applied by setting `normalize=True`. """ plt.imshow(confusion_matrix, interpolation='nearest', cmap=plt.cm.Blues) plt.title('Confusion Matrix') plt.colorbar() tick_marks = np.arange(len(classes)) plt.xticks(tick_marks, classes, rotation=45) plt.yticks(tick_marks, classes) ##if normalize: ## cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] ## print("Normalized confusion matrix")
confusion_matrix = np.zeros((len(LABELS), len(LABELS) + 1), np.uint) for file_pair in file_pairs: confusion_matrix += score_prediction_files(*file_pair) if len(file_pairs) >= 1: print("----- OVERALL SCORES -----") report_scores(confusion_matrix) #Plot Confusion Metrics tick_marks = np.array(range(len(labels) - 1)) + 0.5 confusion_matrix = np.array(confusion_matrix[:, :-1]) np.set_printoptions(precision=3) x = confusion_matrix.sum(axis=1)[:, np.newaxis] for i in range(len(x)): if not x[i]: x[i] = 1 cm_normalized = confusion_matrix.astype('float') / x # print(cm_normalized) plt.figure(figsize=(12, 8), dpi=120) ind_array = np.arange(len(labels) - 1) x, y = np.meshgrid(ind_array, ind_array) for x_val, y_val in zip(x.flatten(), y.flatten()): c = cm_normalized[y_val][x_val] if c > 0.001: plt.text(x_val, y_val, "%0.3f" % (c, ), color='red', fontsize=10, va='center', ha='center') # offset the tick