def log(self): file = open(self.filename, 'w') accuracy_max = max(self.accuracys) accuracy_max_index = self.accuracys.index(accuracy_max) iter_max = self.values[accuracy_max_index] file.write('max:\n\t{}: {} - a: {}\n'.format(self.iter_name, iter_max, accuracy_max) + '\n') plot_confusion_matrix(self.matrixes[accuracy_max_index], self.classes, self.imagename) plot_confusion_matrix(self.matrixes[accuracy_max_index], self.classes, self.imagename_normalized, True) precision_max = max(self.precisions) iter_max = self.values[self.precisions.index(precision_max)] file.write('max:\n\t{}: {} - p: {}\n'.format(self.iter_name, iter_max, precision_max) + '\n') for i in range(len(self.accuracys)): file.write('{}: {} - a: {} - p: {}'.format( self.iter_name, self.values[i], self.accuracys[i], self.precisions[i]) + '\n') for line in self.logs: if ' [[' in line: line = line.replace(' [[', '[[') file.write(line + '\n') file.close() if os.path.exists(self.matrixes_dir): shutil.rmtree(self.matrixes_dir) os.makedirs(self.matrixes_dir) for i in range(len(self.matrixes)): file = open(self.matrixes_dir + str(self.values[i]), 'w') pickle.dump(self.matrixes[i], file, -1) file.close()
def plot_full_evaluation(all_y_test, all_y_pred, model_name): final_result = confusion_matrix(all_y_test, all_y_pred, labels=np.arange(nb_emotions)) accuracy = (all_y_test == all_y_pred).sum() / len(all_y_pred) average = 'macro' precision = precision_score(all_y_test, all_y_pred, average=average) recall = recall_score(all_y_test, all_y_pred, average=average) f1 = f1_score(all_y_test, all_y_pred, average=average) comments = ['\nMODEL EVALUATION:'] comments.append(' Accuracy : {:.5f}'.format(accuracy)) comments.append(' Precision : {:.5f}'.format(precision)) comments.append(' Recall : {:.5f}'.format(recall)) comments.append(' F1-score : {:.5f}'.format(f1)) comments = '\n'.join(comments) plot_confusion_matrix(final_result, emotions, title="Confusion_Matrix_" + model_name, normalize=True, comments=comments) return final_result, comments
def modes(mode, model, weightsFile, x_test, y_test): model.load_weights(weightsFile) if mode == 'test': score = model.evaluate(x_test, y_test, verbose=0) print('\n', 'Test accuracy:', score[1]) elif mode == 'pred': a = np.full((1, 28, 28, 1), 0) a[0] = x_test[0] start = time.clock() model.predict(a) end = time.clock() print("Time per image prediction: {} ".format( (end - start) / len(x_test))) elif mode == 'arc': plot_model(model, to_file='model.png', show_shapes=True) elif mode == 'vis': plot.visualize_accuracy(model, x_test, y_test) elif mode == 'cm': ypred_onehot = model.predict(x_test) ypred = np.argmax(ypred_onehot, axis=1) ytrue = np.argmax(y_test, axis=1) # compute and plot the confusion matrix confusion_mtx = confusion_matrix(ytrue, ypred) plot.plot_confusion_matrix(confusion_mtx)
def NSC_k_NN(df_treatment, embeds_cols, plot_conf=False, savepath=None): # Create classes for each moa class_dict = dict(zip(df_treatment['moa'].unique(), np.arange(len(df_treatment['moa'].unique())))) df_treatment['moa_class'] = df_treatment['moa'].map(class_dict) # Create nearest neighbors classifier predictions = list() labels = list() label_names = list() for comp in df_treatment['compound'].unique(): df_ = df_treatment.loc[df_treatment['compound'] != comp, :] knn = KNeighborsClassifier(n_neighbors=4, algorithm='brute', metric='cosine') knn.fit(df_.loc[:, embeds_cols], df_.loc[:, 'moa_class']) nn = knn.kneighbors(df_treatment.loc[df_treatment['compound'] == comp, embeds_cols]) for p in range(nn[1].shape[0]): predictions.append(list(df_.iloc[nn[1][p]]['moa_class'])) labels.extend(df_treatment.loc[df_treatment['compound'] == comp, 'moa_class']) label_names.extend(df_treatment.loc[df_treatment['compound'] == comp, 'moa']) predictions = np.asarray(predictions) k_nn_acc = [accuracy_score(labels, predictions[:, 0]), accuracy_score(labels, predictions[:, 1]), accuracy_score(labels, predictions[:, 2]), accuracy_score(labels, predictions[:, 3])] if plot_conf: print('There are {} treatments'.format(len(df_treatment))) print('NSC is: {:.2f}%'.format(accuracy_score(labels, predictions[:, 0]) * 100)) plot_confusion_matrix(labels, predictions[:, 0], class_dict, 'NSC', savepath) return k_nn_acc
def print_test_accuracy(dataset, show_example_errors=False, show_confusion_matrix=False): # For all the images in the test-set, # calculate the predicted classes and whether they are correct. correct, cls_pred = predict_cls(images=dataset['test_images'], labels=dataset['test_labels'], cls_true=dataset['test_cls']) # Classification accuracy and the number of correct classifications. acc, num_correct = classification_accuracy(correct) # Number of images being classified. num_images = len(correct) # Print the accuracy. msg = "Accuracy on Test-Set: {0:.1%} ({1} / {2})" print(msg.format(acc, num_correct, num_images)) # Plot some examples of mis-classifications, if desired. if show_example_errors: print("Example errors:") plot.plot_example_errors(cls_pred=cls_pred, correct=correct, dataset=dataset) # Plot the confusion matrix, if desired. if show_confusion_matrix: print("Confusion Matrix:") plot.plot_confusion_matrix(cls_pred=cls_pred, dataset=dataset)
def print_stats(name, Y_pred, Y_test, y_final, initName, key): # Stats when running through the results print(name) print("Confusion Matrix: ") cm = confusion_matrix(Y_test, Y_pred) print(cm) plot_confusion_matrix(cm, name, initName, key) print('Accuracy Score: %.2f' % accuracy_score(Y_test, Y_pred)) print()
def plot(self, cellLabel=True, fileName=''): "Save plot to faile. (Only support .png)" plot.plot_confusion_matrix(cm=np.asarray(self.statistics), normalize=False, target_names=self.vocTable.category, title="Confusion Matrix", cell_label=cellLabel, filepath=fileName) print('Plot saved to: ' + fileName)
def image_matrix(folder, name): classes_dir = 'TestSet 2' if '_b' in folder or '_b_bin' in folder else 'TestSet' m = matrix(folder, name) image_folder = './logs/all_images/' + folder + '/' if not os.path.exists(image_folder): os.makedirs(image_folder) plot_confusion_matrix(m, getClassNames(classes_dir), image_folder + folder + '.png') plot_confusion_matrix(m, getClassNames(classes_dir), image_folder + folder + '_normalized.png', True)
def display_scores(self, plot_flag=False): Scorer.display_scores(self) print(self.report) if plot_flag is True: try: if self.metric is not 'set': plot.plot_confusion_matrix(self.cnf, self.classes, title=self.metric) except: print("Cannot plot.")
def report(y_test, y_pred, model): print('Accuracy Score of ' + model + ' : ' + str(accuracy_score(y_test, y_pred))) print('Precision Score ' + model + ' : ' + str(precision_score(y_test, y_pred))) print('Recall Score ' + model + ': ' + str(recall_score(y_test, y_pred))) print('F1 Score' + model + ' : ' + str(f1_score(y_test, y_pred))) print('Fb Score' + model + ' : ' + str(fbeta_score(y_test, y_pred, beta=1.5))) print('AUC Score' + model + ' : ' + str(roc_auc_score(y_test, y_pred))) plot.plot_confusion_matrix(y_test, y_pred, model) plt.show()
def KNN_confusion_matrix(X, y, test_Size): name = "train/test : " + str(int(100 - 100 * test_Size)) + "/" + str( int(100 * test_Size)) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0, test_size=test_Size) # Khai báo lớp KNN với k = 10 knn = KNeighborsClassifier(n_neighbors=10) # Huấn luyện knn.fit(X_train, y_train) # Tính độ chính xác accuracy = knn.score(X_test, y_test) # Tạo ma trận nhầm lẫn (confusion matrix) y_pred = knn.predict(X_test) cm = confusion_matrix(y_test, y_pred) # Tính toán độ do precision precision = precision_score(y_test, y_pred, average=None) precision = sum(precision) / len(precision) # Tính đoán độ đo recall recall = recall_score(y_test, y_pred, average=None) recall = sum(recall) / len(recall) # Tính toán độ đo F1 f1 = f1_score(y_test, y_pred, average=None) f1 = sum(f1) / len(f1) plot.plot_confusion_matrix(y_test, y_pred, classes=CLASSNAME, title='Confusion matrix. ' + name) # plot.plot_confusion_matrix(y_test, y_pred, classes=CLASSNAME, normalize=True, title='Normalized confusion matrix'+name) plt.show() print("---") print(name) print(' accuracy : ' + str(accuracy)) print(' precision : ' + str(precision)) print(' recall : ' + str(recall)) print(' f1 : ' + str(f1))
def all_images(): matrixes_images_folder = './logs/all_images/' matrixes_folder = './logs/matrixes/' if os.path.exists(matrixes_images_folder): shutil.rmtree(matrixes_images_folder) os.makedirs(matrixes_images_folder) for matrix_folder in os.listdir(matrixes_folder): if '.DS_Store' in matrix_folder: continue matrix_folder += '/' images_folder = matrixes_images_folder + matrix_folder os.makedirs(images_folder) classes_dir = 'TestSet 2' if '_b.' in matrix_folder or '_b_bin' in matrix_folder else 'TestSet' classes = getClassNames(classes_dir) for matrix_file in os.listdir(matrixes_folder + matrix_folder): imagename = matrixes_images_folder + matrix_folder + matrix_file + '.png' print('Going to generate matrix {}{}'.format( matrix_folder, matrix_file)) m = matrix(matrix_folder, matrix_file) plot_confusion_matrix(m, classes, imagename)
def plot(self, dirname): """ Generate plots """ # Plot learning curve print(f"{self.name}: Plotting learning curve") learning_curve = [ x for x in self.output if x["type"] == "learning_curve" ] if learning_curve: fig = plot.plot_learning_curve( self.title, learning_curve[0]["data"]["train_sizes"], learning_curve[0]["data"]["train_scores"], learning_curve[0]["data"]["test_scores"], learning_curve[0]["data"]["fit_times"], ) fig.savefig( os.path.join(dirname, f"{self.name}_learning_curve.png")) # Plot validation curve(s) valdiation_curve = [ x for x in self.output if x["type"] == "validation_curve" ] if valdiation_curve: for param in valdiation_curve: fig = plot.plot_validation_curve( self.title, param["data"]["parameter"], param["data"]["range"], param["data"]["train_scores"], param["data"]["test_scores"], ) fig.savefig( os.path.join( dirname, f'{self.name}_{param["data"]["parameter"]}_validation_curve.png', )) # If NN plot loss curve if self.title == "Neural Networks Classifier": fig = plot.plot_loss_curve(self.model) fig.savefig(os.path.join( dirname, f'{self.name}_loss_curve.png', )) # Plot confusion matrix confusion = [x for x in self.output if x["type"] == "confusion_matrix"] if confusion: fig = plot.plot_confusion_matrix(self.model, self.data.x_test, self.data.y_test) fig.savefig( os.path.join(dirname, f"{self.name}_confusion_matrix.png"))
def NSB_k_NN(df_treatment, embeds_cols, plot_conf=False, savepath=None): # Remove moa with only 1 plate df_treatment = df_treatment[df_treatment['moa'] != 'Cholesterol-lowering'] df_treatment = df_treatment[df_treatment['moa'] != 'Kinase inhibitors'] df_treatment = df_treatment.reset_index(drop=True) class_dict = dict(zip(df_treatment['moa'].unique(), np.arange(len(df_treatment['moa'].unique())))) df_treatment['moa_class'] = df_treatment['moa'].map(class_dict) predictions = list() labels = list() label_names = list() for batch in df_treatment['table_nr'].unique(): for comp in df_treatment.loc[df_treatment['table_nr'] == batch, 'compound'].unique(): df_ = df_treatment.loc[(df_treatment['compound'] != comp) & (df_treatment['table_nr'] != batch), :] knn = KNeighborsClassifier(n_neighbors=4, algorithm='brute', metric='cosine') knn.fit(df_.loc[:, embeds_cols], df_.loc[:, 'moa_class']) nn = knn.kneighbors( df_treatment.loc[(df_treatment['compound'] == comp) & (df_treatment['table_nr'] == batch), embeds_cols]) for p in range(nn[1].shape[0]): predictions.append(list(df_.iloc[nn[1][p]]['moa_class'])) labels.extend( df_treatment.loc[(df_treatment['compound'] == comp) & (df_treatment['table_nr'] == batch), 'moa_class']) label_names.extend( df_treatment.loc[(df_treatment['compound'] == comp) & (df_treatment['table_nr'] == batch), 'moa']) predictions = np.asarray(predictions) k_nn_acc = [accuracy_score(labels, predictions[:, 0]), accuracy_score(labels, predictions[:, 1]), accuracy_score(labels, predictions[:, 2]), accuracy_score(labels, predictions[:, 3])] if plot_conf: print('There are {} treatments'.format(len(df_treatment))) print('NSCB is: {:.2f}%'.format(accuracy_score(labels, predictions[:, 0]) * 100)) plot_confusion_matrix(labels, predictions[:, 0], class_dict, 'NSCB', savepath) return k_nn_acc
def train(model_name, category_type, dump=False): clf = tfidf_pipeline.make(model_name) categories = names.categories[category_type] print 'Loading data...' data = data_loader.load('full', categories) train_X, train_y, test_X, test_y = data_loader.split(data, 0.1) print 'Done.' print 'Training...' clf.fit(train_X, train_y) print 'Done.' print 'Testing...' predicted = clf.predict(test_X) if model_name in ['svr', 'linreg']: predicted = np.clip(np.round(predicted), 0, 7) accuracy = scorers.err1(test_y, predicted) print 'Off-by-one accuracy: ' + str(accuracy) else: accuracy = scorers.err0(test_y, predicted) print 'Exact accuracy: ' + str(accuracy) print classification_report(test_y, predicted, target_names=categories) cm = confusion_matrix(test_y, predicted) print cm plot.plot_confusion_matrix(cm, category_type) if dump: print 'Saving classifier...' if not exists('dumps'): makedirs('dumps') joblib.dump(clf, join('dumps', category_type + '_' + model_name + '_classifier.pkl')) print 'Done.' return clf
def nn(X, y, act='relu', validation_split=0.33, epoch=50, output_file=None, title=None): X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.3, random_state=1) # 70% training and 30% test model = Sequential() model.add(Dense(20, input_dim=X.shape[1], activation=act)) model.add(Dense(5, activation=act)) model.add(Dense(1, activation='sigmoid')) # model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', auc]) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', auc]) start = time.time() history = model.fit(X_train, y_train, epochs=epoch, validation_split=validation_split, batch_size=10, verbose=0) y_pred = model.predict(X_test) y_pred = y_pred.round() cm = confusion_matrix(y_test, y_pred) plt.figure() plot.plot_confusion_matrix(cm, ['no_default', 'default'], title=title) plt.savefig(output_file) metrics = model.evaluate(X_test, y_test) end = time.time() durartion = end - start return model, history.history, metrics, durartion
save_best_model=save_best_model, model_path=model_path) plot_histories(histories, 'DenseNet-LSTM, {}-fold cross-validation'.format(n_folds)) else: lstm_densenet = load_model(densenet_lstm_model_path) ## TESTING ## # Get back the train/test split used skf = StratifiedKFold(n_splits=5, shuffle=False) labels = np.argmax(y, axis=1) train_test = [(train, test) for (train,test) in skf.split(y, labels)] train_idx, test_idx = zip(*train_test) # Get emotion predictions test_indices = test_idx[1] y_predict = lstm_densenet.predict_classes(densenet_features[test_indices]) y_true = np.argmax(y[test_indices], axis=1) # Computes the accuracy acc = (y_predict==y_true).sum()/len(y_predict) print('Test accuracy : {:.4f}'.format(acc)) # Plot the confusion matrix cm = confusion_matrix(np.argmax(y[test_indices], axis=1), y_predict) plot_confusion_matrix(cm, emotions, title='DenseNet-LSTM', normalize=True)
models = [model_per_class(i, labelled_training_data) for i in range(1, 7)] print("Models prepared.\n") # make predictions for testing data print("Making predictions.\n") predictions = labelled_testing_data.map( lambda x: (float(np.argmax([model.predict(x.features) for model in models]) + 1), x.label)) print("Predictions completed.\n") # calculate precision, recall, and f-measure print("Calculating evaluation metrics for feature set 1.\n") metrics = MulticlassMetrics(predictions) print("F-Measure: ", metrics.fMeasure()) print("Confusion matrix\n\n") plot.plot_confusion_matrix(metrics.confusionMatrix().toArray(), "cm1_refactored.png") for i in range(1, 7): print("Precision for ", i, " is ", metrics.precision(i)) print("Recall for ", i, " is ", metrics.recall(i)) print("f-measure for ", i, " is ", metrics.fMeasure(float(i)), "\n") precision.append(metrics.precision(i)) recall.append(metrics.recall(i)) fmeasure.append(metrics.fMeasure(float(i))) plot.plot_per_activity_metric(precision, recall, fmeasure, "fs1_refactored.png") precision = [] recall = [] fmeasure = [] # Feature Set 2: fetch all features print("FEATURE SET 2: FETCHING All THE FEATURES")
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis lda = LinearDiscriminantAnalysis(n_components=2) X_lda = lda.fit(exp_data_data, exp_data_labels).transform(exp_data_data) pred = lda.predict(features_test) # Print Confusion Matrix class_names = ["is not book", "is book"] cnf_matrix = confusion_matrix(labels_test, pred) np.set_printoptions(precision=2) # Plot non-normalized confusion matrix plt.figure() plot_confusion_matrix(cnf_matrix, classes=class_names, title='Confusion matrix, without normalization - LDA') # Plot normalized confusion matrix plt.figure() plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True, title='Normalized confusion matrix - LDA') plt.show() from sklearn.metrics import accuracy_score # Print Accuracy Score print "Accuracy is", accuracy_score(pred, labels_test) print "The number of correct predictions is", accuracy_score(pred, labels_test, normalize=False)
# Create the callbacks custom_verbose = CustomVerbose(epochs) early_stop = EarlyStopping(patience=100) callbacks = [custom_verbose, early_stop] lstm_sift, skf, histories = train_crossval(create_lstm_sift_model, vgg_sift_features, y, batch_size=batch_size, epochs=epochs, callbacks=callbacks, n_folds=n_folds, save_best_model=save_best_model, model_path=trained_model_path) print("\nTraining complete.") plot_histories(histories, 'VGG-SIFT-LSTM, {}-fold cross-validation'.format(n_folds)) ## TESTING ## model_path = trained_model_path if train else vgg_sift_lstm_model_path y_pred, y_true = evaluate_model(vgg_sift_features, y, model_path, n_splits=5) # Plot confusion matrix cm = confusion_matrix(y_true, y_pred) plot_confusion_matrix(cm, emotions, title='VGG-SIFT-LSTM - MUG', normalize=True)
averaged_tta_predictions_proba = tta_inference( model=model, data_folder=data_folder, input_dim=input_dim, passes=ModelConfig.tta_augmentation_passes, ) averaged_tta_predictions = np.argmax(averaged_tta_predictions_proba, axis=1) per_sample_tta_inference_time = (time.time() - time_anchor) / test_generator_inference.n print("TTA inference done") print("Metrics computation and plots generation...") run.log("Final test loss", log_loss(y_true, y_pred_proba)) run.log("Final test accuracy", accuracy_score(y_true, y_pred)) run.log("Final TTA test accuracy", accuracy_score(y_true, averaged_tta_predictions)) run.log("Training time", training_time) run.log("Per sample inference time", per_sample_inference_time) run.log("Per sample TTA inference time", per_sample_tta_inference_time) cm = confusion_matrix(y_true, y_pred) plot_confusion_matrix( cm, y_true, y_pred, np.asarray(list(train_generator.class_indices.keys())), path=PathsConfig.confusion_matrix_path, ) run.log_image(name="Confusion matrix", path=PathsConfig.confusion_matrix_path) print("Metrics and plots done")
print("Making predictions.\n") predictions = labelled_testing_data.map(lambda x: (float( np.argmax([model.predict(x.features) for model in models]) + 1), x.label)) print("Predictions completed.\n") # calculate precision, recall, and f-measure print("Calculating evaluation metrics.\n") metrics = MulticlassMetrics(predictions) print("F-Measure: ", metrics.fMeasure()) results_dictionary[participant] = metrics.fMeasure() print("Confusion matrix\n\n") confusion_matrix_filename = str( participant) + "_" + "cm_refactored.png" plot.plot_confusion_matrix(metrics.confusionMatrix().toArray(), confusion_matrix_filename) # print Precision and Recall for all the activities for i in range(1, 7): print("Precision for ", i, " is ", metrics.precision(i)) print("Recall for ", i, " is ", metrics.recall(i)) print("f-measure for ", i, " is ", metrics.fMeasure(float(i)), "\n") precision.append(metrics.precision(i)) recall.append(metrics.recall(i)) fmeasure.append(metrics.fMeasure(float(i))) fscore_filename = str(participant) + "_" + "fs_refactored.png" plot.plot_per_activity_metric(precision, recall, fmeasure, fscore_filename) precision = [] recall = []
for epoch in range(start_epoch, opt.nepoch): # features, labels = seenDataset.epochData(include_bg=False) features, labels = seenDataset.epochData(include_bg=True) # train GAN trainFGGAN(epoch, features, labels) # synthesize features syn_feature, syn_label = trainFGGAN.generate_syn_feature(unseen_att_labels, unseen_attributes, num=opt.syn_num) num_of_bg = opt.syn_num*2 real_feature_bg, real_label_bg = seenDataset.getBGfeats(num_of_bg) # concatenate synthesized + real bg features syn_feature = np.concatenate((syn_feature.data.numpy(), real_feature_bg)) syn_label = np.concatenate((syn_label.data.numpy(), real_label_bg)) trainCls(syn_feature, syn_label, gan_epoch=epoch) # ----------------------------------------------------------------------------------------------------------------------- # plots classes = np.concatenate((['background'], get_unseen_class_labels(opt.dataset, split=opt.classes_split))) plot_confusion_matrix(np.load(f'{opt.outname}/confusion_matrix_Train.npy'), classes, classes, opt, dataset='Train', prefix=opt.class_embedding.split('/')[-1]) plot_confusion_matrix(np.load(f'{opt.outname}/confusion_matrix_Test.npy'), classes, classes, opt, dataset='Test', prefix=opt.class_embedding.split('/')[-1]) plot_acc(np.vstack(trainCls.val_accuracies), opt, prefix=opt.class_embedding.split('/')[-1]) # save models if trainCls.isBestIter == True: trainFGGAN.save_checkpoint(state='best') trainFGGAN.save_checkpoint(state='latest')
#print model.predict_classes(x_test) #classes predicted #print model.predict_proba(x_test) #classes probability pred = [] y_pred = model.predict_classes(x_test) for i in range(len(x_test)): pred.append(y_pred[i]) cnf_matrix = confusion_matrix(label, pred) print(cnf_matrix) import matplotlib.pyplot as plt from plot import plot_confusion_matrix # # Compute confusion matrix # cnf_matrix = confusion_matrix(y_test, y_pred) np.set_printoptions(precision=2) # Plot non-normalized confusion matrix # plt.figure() plot_confusion_matrix(cnf_matrix, classes=range(0, 10), title='Confusion matrix, without normalization') # # Plot normalized confusion matrix # plt.figure() # plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True, # title='Normalized confusion matrix') plt.show()
from load import load_x,load_y from grid import grid_search from plot import plot_confusion_matrix if __name__=='__main__': X_train=load_x('X_train.csv') y_train=load_y('Y_train.csv') X_test=load_x('X_test.csv') y_test=load_y('Y_test.csv') log2c=log2g=[-4,-3,-2,-1,0,1,2,3,4] confusion_matrix=grid_search(log2c,log2g,X_train,y_train,X_test,y_test) plot_confusion_matrix(confusion_matrix,log2c,log2g)
import matplotlib """ Save figure without displaying it """ matplotlib.use("Agg") import matplotlib.pyplot as plt from sklearn.metrics import confusion_matrix from pylab import rcParams epoch_list=[i for i in range(config.epoch)] class_names=[i for i in range(3)] pred_label= model.predict(test_data) print pred_label # for i in pred_label: # i=i*(max(new)-min(new))+min(new) print pred_label pred_label=list(itertools.chain.from_iterable(pred_label)) test_label=list(itertools.chain.from_iterable(test_label)) print pred_label print test_label # In[13]: from sklearn.metrics import confusion_matrix cnf_matrix = confusion_matrix(test_label, pred_label) # np.set_printoptions(prescision=2) plot.plot_confusion_matrix(cnf_matrix, class_names, False, "Confusion matrix", "/Users/nicole/Desktop/python/finalproject") plot.plot_curve(epoch_list, train_loss_list, "Accuracy curve", "/Users/wei-jer-chang/Desktop/final project", training=True, accuracy=False)
def image(matrix, filename=None, size='b'): classes_dir = 'TestSet 2' if size == 'b' else 'TestSet' plot_confusion_matrix(matrix, getClassNames(classes_dir), filename)
print("Preparing models\n") models = [model_per_class(i, labelled_training_data) for i in range(1, 7)] print("Models prepared.\n") # make predictions for testing data print("Making predictions.\n") predictions = labelled_testing_data.map(lambda x: (float(np.argmax([model.predict(x.features) for model in models]) + 1), x.label)) print("Predictions completed.\n") # calculate precision, recall, and f-measure print("Calculating evaluation metrics for feature set 1.\n") metrics = MulticlassMetrics(predictions) print("F-Measure: ", metrics.fMeasure()) print("Confusion matrix\n\n") plot.plot_confusion_matrix(metrics.confusionMatrix().toArray(), "cm1_normal.png") for i in range(1, 7): print("Precision for ", i, " is ", metrics.precision(i)) print("Recall for ", i, " is ", metrics.recall(i)) print("f-measure for ", i, " is ", metrics.fMeasure(float(i)), "\n") precision.append(metrics.precision(i)) recall.append(metrics.recall(i)) fmeasure.append(metrics.fMeasure(float(i))) plot.plot_per_activity_metric(precision, recall, fmeasure, "fs1_normal.png") precision = [] recall = [] fmeasure = [] # Feature Set 2: fetch all features print("FEATURE SET 2: FETCHING All THE FEATURES")
def draw_histos(vis=False, filepath='/output/evalSave/'): print(filepath) if not os.path.exists(filepath): print('path created') os.makedirs(filepath) with open(os.path.join(filepath, 'eval.json'), 'r') as j: data = json.load(j) # build precision-recall curves prec_03, rec_03 = eval_plot.sort_prec_rec(data['evaluation']['prec']['0.3'], data['evaluation']['rec']['0.3']) eval_plot.plot_prec_rec(rec_03, prec_03, 'Thrs: 0.3', color=(1, 0, 1)) prec_05, rec_05 = eval_plot.sort_prec_rec(data['evaluation']['prec']['0.5'], data['evaluation']['rec']['0.5']) eval_plot.plot_prec_rec(rec_05, prec_05, 'Thrs: 0.5', color=(0, 1, 0)) # build smoothed precision-recall curves # smooth PR-curve as described in http://cs229.stanford.edu/section/evaluation_metrics.pdf# smoothed_prec_03 = [max(prec_03[idx:]) for idx, _ in enumerate(prec_03)] smoothed_prec_05 = [max(prec_05[idx:]) for idx, _ in enumerate(prec_05)] eval_plot.plot_prec_rec(rec_03, smoothed_prec_03, 'Thrs_smoothed: 0.3', color=(1, 0, 1), linestyle="--") eval_plot.plot_prec_rec(rec_05, smoothed_prec_05, 'Thrs_smoothed: 0.5', color=(0, 1, 0), linestyle="--") #auc_03 = np.trapz(smoothed_prec_03, rec_03) #auc_05 = np.trapz(smoothed_prec_05, rec_05) auc_03 = np.trapz(prec_03, rec_03) auc_05 = np.trapz(prec_05, rec_05) title = "conf. stride: {:0.2f}, max. conf.: {:0.4f}, AUC_03: {:0.2f}, AUC_05: {:0.2f}".\ format(data['confidence stride'], data['max_conf'], auc_03, auc_05) eval_plot.config(data['name'], title=title) eval_plot.savefig(os.path.join(filepath, (data['name'] + '.svg'))) if vis: eval_plot.show() else: eval_plot.clearfig() # build mean iou - recall curves postfix = ['', '_low', '_mid', '_high'] for p in postfix: eval_plot.plot_prec_rec(data['evaluation']['rec' + p]['0.3'], data['evaluation']['m_iou' + p]['0.3'], 'halt au', color=(1, 0, 1)) eval_plot.plot_prec_rec(data['evaluation']['rec' + p]['0.5'], data['evaluation']['m_iou' + p]['0.5'], 'halt au', color=(0, 1, 0)) title = "conf. stride: {:0.2f}, max. conf.: {:0.4f},\nAUC_03: {:0.2f}, AUC_05: {:0.2f}".\ format(data['confidence stride'], data['max_conf'], auc_03, auc_05) eval_plot.config(data['name'], title=title) eval_plot.savefig(os.path.join(filepath, 'm_iou' + p + '.svg')) if vis: eval_plot.show() else: eval_plot.clearfig() classes = ["Background", "Ferry", "Buoy", "Vessel/ship", "Speed boat", "Boat", "Kayak", "Sail boat", "Swimming person", "Flying bird/plane", "Other"] num_classes = data['evaluation']['conf_mat']["0.3"][0][1] cm = np.array(data['evaluation']['conf_mat']["0.3"][0][0]).reshape((num_classes, num_classes)) name = os.path.join(filepath, "confusion_matrix_03_05.svg") eval_plot.plot_confusion_matrix(cm, classes, name) cm = np.array(data['evaluation']['conf_mat']["0.3"][1][0]).reshape((num_classes, num_classes)) name = os.path.join(filepath, "confusion_matrix_03_075.svg") eval_plot.plot_confusion_matrix(cm, classes, name) cm = np.array(data['evaluation']['conf_mat']["0.5"][0][0]).reshape((num_classes, num_classes)) name = os.path.join(filepath, "confusion_matrix_05_05.svg") eval_plot.plot_confusion_matrix(cm, classes, name) cm = np.array(data['evaluation']['conf_mat']["0.5"][1][0]).reshape((num_classes, num_classes)) name = os.path.join(filepath, "confusion_matrix_05_075.svg") eval_plot.plot_confusion_matrix(cm, classes, name)
from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score # Confusion matrix plotting module from plot import plot_confusion_matrix # Load data set (x_traino, y_train), (x_testo, y_test) = mnist.load_data() x_train = np.reshape(x_traino, (60000, 28 * 28)) x_test = np.reshape(x_testo, (10000, 28 * 28)) x_train, x_test = x_train / 255.0, x_test / 255.0 logreg = LogisticRegression(solver='saga', multi_class='multinomial', max_iter=100, verbose=2) est = logreg.fit(x_train, y_train) y_pred = logreg.predict(x_test) # Accuracy score = accuracy_score(y_test, y_pred) print("Accuracy score: %.4f" % score) # Show confusion matrix for prediction conf = plot_confusion_matrix(y_test, y_pred, np.array(range(10)), normalize=True) plt.show()
plot_histories(hist_phrnn, 'PHRNN Model - ADAS&ME') plot_histories(hist_tcnn, 'TCNN Model - ADAS&ME') #### TESTING #### print('\nTesting model...') merge_weight = 0.45 # set tcnn_model_path to None to test only PHRNN and vice-versa y_pred, y_true = evaluate_tcnn_phrnn_model(tcnn_model_path, phrnn_model_path, tcnn_features, phrnn_features, subjects, data_path, frames_data_path, merge_weight=merge_weight) print_model_eval_metrics(y_pred, y_true) # Plot confusion matrix cm = confusion_matrix(y_true, y_pred, labels=list(range(nb_emotions))) plot_confusion_matrix(cm, emotions, title='Late Fusion Model - ADAS&ME', normalize=True) plot_confusion_matrix(cm, emotions, title='Late Fusion Model - ADAS&ME', normalize=False)