def Training(X, y, classifier, RF = False): cv = KFold(n_splits = 10, shuffle = True,random_state = 12345) start_time = dt.datetime.now() print('Start learning at {}'.format(str(start_time))) i = 0 df_f1_micro = [] df_f1_macro = [] df_accuracy = [] df_MCC = [] for train_index, test_index in cv.split(X): i += 1 classifier.fit(X[train_index], y[train_index]) ypred = classifier.predict(X[test_index]) kappa_score = cohen_kappa_score(y[test_index], ypred) confmat = confusion_matrix(y[test_index], ypred) f1micro = f1_score(y[test_index], ypred,average="micro") f1macro = f1_score(y[test_index], ypred,average="macro") accuracy = accuracy_score(y[test_index], ypred) MCC = matthews_corrcoef(y[test_index], ypred) print("\nKappa score\n" ,kappa_score,"\n") print("\n confmat\n",confmat,"\n") print("\n f1 micro\n", f1micro,"\n") print("\n f1 macro\n", f1macro,"\n") print("\n accuracy\n", accuracy,"\n") if i == 1: df_confmat = pd.DataFrame(confmat) elif i > 1: temp = pd.DataFrame(confmat) df_confmat = df_confmat.append(temp) df_f1_micro.append(f1micro) df_f1_macro.append(f1macro) df_accuracy.append(accuracy) df_MCC.append(MCC) df_f1_micro = pd.DataFrame(df_f1_micro,columns=["f1micro"]) df_f1_macro = pd.DataFrame(df_f1_macro,columns=["f1macro"]) df_accuracy = pd.DataFrame(df_accuracy,columns=["accuracy"]) df_MCC = pd.DataFrame(df_MCC,columns = ["MCC"] ) end_time = dt.datetime.now() print('Stop learning {}'.format(str(end_time))) elapsed_time= end_time - start_time print('Elapsed learning {}'.format(str(elapsed_time))) if RF == True: MakeTree(0,classifier,X) else: pass return df_confmat, df_f1_micro, df_f1_macro, df_accuracy, df_MCC
def compute_results_from_AMT(auto_list, set_separator_index, file_annotations, in_score_a_column_idx=5, in_score_b_column_idx=7, in_term_column_idx=1): auto_s = build_automated_sets(auto_list, set_separator_index) dictionary_a = get_term_value_dictionary_AMT(file_annotations, in_score_a_column_idx, in_term_column_idx) dictionary_b = get_term_value_dictionary_AMT(file_annotations, in_score_b_column_idx, in_term_column_idx) dictionary_merge = {} for k in dictionary_a.keys(): dictionary_merge[k] = np.mean([dictionary_a[k],dictionary_b[k]]) ground_truth_rank = sorted(dictionary_merge, key=dictionary_merge.get, reverse=True) # ground_sets = build_ground_truth_sets(dictionary_merge, 3) # [p, r] = evaluate_results_top_bottom_classification(ground_sets, auto_s) tau_result = evaluate_results_top_bottom_rank(ground_truth_ranked_list=ground_truth_rank, auto_sets=auto_s) tau_result_ties = evaluate_results_top_bottom_optimistic(dictionary_merge, auto_s) tau_result_skip_ties = evaluate_results_top_bottom_skip_ties(dictionary_merge, auto_s) scores_literal_a = get_term_value_list_AMT(file_annotations, score_column_idx = in_score_a_column_idx) scores_literal_b = get_term_value_list_AMT(file_annotations, score_column_idx = in_score_b_column_idx) scores_a = [convert_score(s) for s in scores_literal_a] scores_b = [convert_score(s) for s in scores_literal_b] k_result = cohen_kappa_score(scores_a, scores_b) print(evaluate_results_true_tau_b(dictionary_merge, auto_s)) return [k_result, tau_result_ties, tau_result_skip_ties]
def compute_agreement_AMT(file_annotations, in_score_a_column_idx=5, in_score_b_column_idx=7): scores_literal_a = get_term_value_list_AMT(file_annotations, score_column_idx = in_score_a_column_idx) scores_literal_b = get_term_value_list_AMT(file_annotations, score_column_idx = in_score_b_column_idx) scores_a = [convert_score(s) for s in scores_literal_a] scores_b = [convert_score(s) for s in scores_literal_b] k_result = cohen_kappa_score(scores_a, scores_b) k_percent = len([(x,y) for x, y in zip(scores_a, scores_b) if x == y]) / len(scores_a) scores_a_rebased, scores_b_rebased = rebase_scores(scores_a, scores_b) k_permissive = cohen_kappa_score(scores_a_rebased, scores_b_rebased) k_permissive_percent = len([(x,y) for x, y in zip(scores_a_rebased, scores_b_rebased) if x == y]) / len(scores_a_rebased) return k_result, k_percent, k_permissive, k_permissive_percent
def training(self, Model, C=1, gamma=1): cv = KFold(n_splits=10, shuffle=True, random_state=12345) start_time = dt.datetime.now() print('Start learning at {}'.format(str(start_time))) i = 0 df_f1_micro = [] df_f1_macro = [] df_accuracy = [] df_MCC = [] for train_index, test_index in cv.split(self.X): i += 1 if Model == "SVMLinear": classifier = svm.LinearSVC(C=C) classifier.fit(self.X[train_index], self.y[train_index]) else: classifier = svm.SVC(kernel='rbf', C=C, gamma=gamma) classifier.fit(self.X[train_index], self.y[train_index]) ypred = classifier.predict(self.X[test_index]) kappa_score = cohen_kappa_score(self.y[test_index], ypred) confmat = confusion_matrix(self.y[test_index], ypred) f1micro = f1_score(self.y[test_index], ypred, average="micro") f1macro = f1_score(self.y[test_index], ypred, average="macro") accuracy = accuracy_score(self.y[test_index], ypred) MCC = matthews_corrcoef(self.y[test_index], ypred) print("\nKappa score\n", kappa_score, "\n") print("\n confmat\n", confmat, "\n") print("\n f1 micro\n", f1micro, "\n") print("\n f1 macro\n", f1macro, "\n") print("\n accuracy\n", accuracy, "\n") if i == 1: df_confmat = pd.DataFrame(confmat) elif i > 1: temp = pd.DataFrame(confmat) df_confmat = df_confmat.append(temp) df_f1_micro.append(f1micro) df_f1_macro.append(f1macro) df_accuracy.append(accuracy) df_MCC.append(MCC) self.f1_micro = pd.DataFrame(df_f1_micro, columns=["f1micro"]) self.f1_macro = pd.DataFrame(df_f1_macro, columns=["f1macro"]) self.accuracy = pd.DataFrame(df_accuracy, columns=["accuracy"]) self.MCC = pd.DataFrame(df_MCC, columns=["MCC"]) self.confmat = df_confmat end_time = dt.datetime.now() print('Stop learning {}'.format(str(end_time))) elapsed_time = end_time - start_time print('Elapsed learning {}'.format(str(elapsed_time))) return self.confmat, self.f1_micro, self.f1_macro, self.accuracy, self.MCC
def compute_agreement(file_annotation_a, file_annotation_b, in_score_column_idx): scores_a = get_term_value_list(file_annotation_a, score_column_idx = in_score_column_idx) scores_b = get_term_value_list(file_annotation_b, score_column_idx = in_score_column_idx) k_result = cohen_kappa_score(scores_a, scores_b) k_percent = len([(x,y) for x, y in zip(scores_a, scores_b) if x == y]) / len(scores_a) scores_a_rebased, scores_b_rebased = rebase_scores([int(a) for a in scores_a], [int(b) for b in scores_b]) if scores_a_rebased != scores_b_rebased: k_permissive = cohen_kappa_score(scores_a_rebased, scores_b_rebased) else: k_permissive = 1.0 k_permissive_percent = len([(x,y) for x, y in zip(scores_a_rebased, scores_b_rebased) if x == y]) / len(scores_a_rebased) return k_result, k_percent, k_permissive, k_permissive_percent
def classificationSummary(y_true, y_pred, class_names=None): """ Provide a comprehensive summary of classification performance similar to R's confusionMatrix """ confMatrix = classification.confusion_matrix(y_true, y_pred) TP = confMatrix[0, 0] FP = confMatrix[1, 0] TN = confMatrix[1, 1] FN = confMatrix[0, 1] N = TN + TP + FN + FP sensitivity = TP / (TP + FN) specificity = TN / (TN + FP) prevalence = (TP + FN) / N PPV = TP / (TP + FP) NPV = TN / (TN + FN) BAC = (sensitivity + specificity) / 2 metrics = [ ('Accuracy', classification.accuracy_score(y_true, y_pred)), ('95% CI', None), ('No Information Rate', None), ('P-Value [Acc > NIR]', None), (None, None), ('Kappa', classification.cohen_kappa_score(y_true, y_pred)), ("Mcnemar's Test P-Value", None), (None, None), ('Sensitivity', sensitivity), ('Specificity', specificity), ('Pos Pred Value', PPV), ('Neg Pred Value', NPV), ('Prevalence', prevalence), ('Detection Rate', None), ('Detection Prevalence', None), ('Balanced Accuracy', BAC), ] print('Confusion Matrix and Statistics\n') _printConfusionMatrix(confMatrix, class_names) if len(set(y_true)) < 5: print(classification_report(y_true, y_pred, digits=4)) fmt1 = '{{:>{}}} : {{:.3f}}'.format(max(len(m[0]) for m in metrics if m[0] is not None)) fmt2 = '{{:>{}}} : {{}}'.format(max(len(m[0]) for m in metrics if m[0] is not None)) for metric, value in metrics: if metric is None: print() elif value is None: pass # print(fmt2.format(metric, 'missing')) else: print(fmt1.format(metric, value))
def compute_results(auto_list, set_separator_index, file_annotation_a, file_annotation_b, in_score_column_idx, in_term_column_idx): auto_s = build_automated_sets(auto_list, set_separator_index) dictionary_a = get_term_value_dictionary(file_annotation_a, in_score_column_idx, in_term_column_idx) dictionary_b = get_term_value_dictionary(file_annotation_b, in_score_column_idx, in_term_column_idx) dictionary_merge = {} for k in dictionary_a.keys(): dictionary_merge[k] = np.mean([dictionary_a[k],dictionary_b[k]]) ground_truth_rank = sorted(dictionary_merge, key=dictionary_merge.get, reverse=True) tau_result = evaluate_results_top_bottom_rank(ground_truth_ranked_list=ground_truth_rank, auto_sets=auto_s) tau_result_ties = evaluate_results_top_bottom_optimistic(dictionary_merge, auto_s) tau_result_skip_ties = evaluate_results_top_bottom_skip_ties(dictionary_merge, auto_s) scores_a = get_term_value_list(file_annotation_a, score_column_idx = in_score_column_idx) scores_b = get_term_value_list(file_annotation_b, score_column_idx = in_score_column_idx) k_result = (cohen_kappa_score(scores_a, scores_b)) return [k_result, tau_result_ties, tau_result_skip_ties]
def get_classification_metrics(ground_truth_labels, predicted_labels): classification_metric_dict = dict({}) classification_metric_dict['accuracy'] = accuracy_score( ground_truth_labels, predicted_labels) classification_metric_dict['precision'] = precision_score( ground_truth_labels, predicted_labels, average='weighted') classification_metric_dict['recall'] = recall_score(ground_truth_labels, predicted_labels, average='weighted') classification_metric_dict['f1_score'] = f1_score(ground_truth_labels, predicted_labels, average='weighted') classification_metric_dict['brier_score_loss'] = brier_score_loss( ground_truth_labels, predicted_labels) classification_metric_dict['matthews_corr_coef'] = matthews_corrcoef( ground_truth_labels, predicted_labels) classification_metric_dict['jaccard_score'] = jaccard_score( ground_truth_labels, predicted_labels, average='weighted') classification_metric_dict['cohen_kappa_score'] = cohen_kappa_score( ground_truth_labels, predicted_labels) return classification_metric_dict
def main(): samples, labels, _ = loader.load_img("radio_img") #add the fourth dimension (color) samples = np.expand_dims(samples, axis=4) print("shape = {}".format(samples.shape)) inputShape = (samples.shape[1], samples.shape[2], samples.shape[3]) print("inputShape = {}".format(inputShape)) #weights class_weights = class_weight.compute_class_weight('balanced', np.unique(labels), labels) d_class_weights = dict(enumerate(class_weights)) print("weights {}".format(d_class_weights)) #one-hot encoding lb = LabelBinarizer() labels = lb.fit_transform(labels) classesNum = labels.shape[1] print("Classes: {}".format(classesNum)) #split to training and test (trainSamples, testSamples, trainLabels, testLabels) = train_test_split(samples, labels, test_size=0.25, random_state=42) model = cnn_model(inputShape, classesNum) ## checkpoints # checkpt1 = ModelCheckpoint(filepath='model.{epoch:02d}-{val_loss:.2f}.h5', save_best_only=True) # checkpt2 = EarlyStopping(monitor='val_loss', patience=3) EPOCHS = 20 BATCH = 50 model.fit( trainSamples, trainLabels, batch_size=BATCH, epochs=EPOCHS, class_weight=d_class_weights, verbose=1, #callbacks = [checkpt1,checkpt2], validation_data=(testSamples, testLabels)) cnnResults = model.predict(testSamples) print( confusion_matrix(testLabels.argmax(axis=1), cnnResults.argmax(axis=1))) print( classification_report(testLabels.argmax(axis=1), cnnResults.argmax(axis=1))) cnnAcc = accuracy_score(testLabels.argmax(axis=1), cnnResults.argmax(axis=1)) print("Accuracy CNN: {:.2f}".format(cnnAcc)) print("Cohen's Kappa {:.2f}".format( cohen_kappa_score(testLabels.argmax(axis=1), cnnResults.argmax(axis=1)))) input("")
flatmodel = Sequential() flatmodel.add(Flatten(input_shape=(14, ))) flatmodel.add(Dense(50, activation='sigmoid')) # concatenated model combined = concatenate([cnnmodel.output, flatmodel.output]) combined = Dense(16, activation="sigmoid")(combined) combined = Dense(numClasses, activation="sigmoid")(combined) model = Model(inputs=[cnnmodel.input, flatmodel.input], outputs=combined) print(model.summary()) model.compile(loss='categorical_crossentropy', optimizer="adam", metrics=['accuracy']) EPOCHS = 10 BATCH = 100 model.fit([samplesIMG, samplesCSV], labels, batch_size=BATCH, epochs=EPOCHS) results = model.predict([samplesIMG, samplesCSV]) print(confusion_matrix(labels.argmax(axis=1), results.argmax(axis=1))) print(classification_report(labels.argmax(axis=1), results.argmax(axis=1))) print("Accuracy: {:.2f}".format( accuracy_score(labels.argmax(axis=1), results.argmax(axis=1)))) print("Cohen's Kappa {:.2f}".format( cohen_kappa_score(labels.argmax(axis=1), results.argmax(axis=1)))) input("")
result = kerasAdapter.predict(dataTestGenerator, batch_size=parameters['batchSize']) testClasses = classes[testIndex] metrics = dict() metrics['fscore'] = f1_score(testClasses, result, average='weighted') metrics['precision'] = precision_score(testClasses, result, average='weighted') metrics['recall'] = recall_score(testClasses, result, average='weighted') metrics['auc'] = roc_auc_score(testClasses, result, average='weighted') metrics['fscore_b'] = f1_score(testClasses, result) metrics['precision_b'] = precision_score(testClasses, result) metrics['recall_b'] = recall_score(testClasses, result) metrics['auc_b'] = roc_auc_score(testClasses, result) metrics['kappa'] = cohen_kappa_score(testClasses, result) metrics['accuracy'] = accuracy_score(testClasses, result) tn, fp, fn, metrics['tp_rate'] = confusion_matrix(testClasses, result).ravel() print(classification_report(testClasses, result)) metrics["fold"] = i if dictWriter is None: dictWriter = csv.DictWriter(cvsFileHandler, metrics.keys()) if metrics['fold'] == 0: dictWriter.writeheader() dictWriter.writerow(metrics) i += 1
classifier_fname = 'classifiers/{}_{}_fold{}.pkl'.format( classifier_row['fname'].split('.')[0], classifier_row['classifier'], classifier_row['fold']) classifier = joblib.load(open(classifier_fname, 'rb')) try: predicted = classifier.predict(data_test) metrics = dict() metrics['fname'] = classifier_row['fname'] metrics['classifier'] = classifier_row['classifier'] metrics['accuracy'] = accuracy_score(classes_test, predicted) tn, fp, fn, metrics['tp_rate'] = confusion_matrix( classes_test, predicted).ravel() metrics['tp_rate'] = metrics['tp_rate'] / (metrics['tp_rate'] + fn) metrics['kappa'] = cohen_kappa_score(classes_test, predicted) metrics['auc'] = roc_auc_score(classes_test, predicted, average='weighted') metrics['fscore'] = f1_score(classes_test, predicted, average='weighted') metrics['macro_f'] = f1_score(classes_test, predicted, average='macro') # metrics['fscore'] = f1_score(classes_test, predicted, average='weighted') # metrics['precision'] = precision_score(classes_test, predicted, average='weighted') # metrics['recall'] = recall_score(classes_test, predicted, average='weighted') # metrics['auc'] = roc_auc_score(classes_test, predicted, average='weighted') #
def main(): #load data file = "datasetA_3c.csv" dataframe = pandas.read_csv(file) dataset = dataframe.values samples = dataset[:,1:] labels = dataset[:,0] samples = np.array(samples) labels = np.array(labels) labels = labels.astype(str) print("Class distribution:") print(Counter(labels)) ### choose k best attributes # from sklearn.feature_selection.univariate_selection import SelectKBest # newSamples = SelectKBest(k=100).fit_transform(samples, labels) # print(newSamples.shape) # samples = newSamples ### Calculate weights for unbalanced classes # from sklearn.utils import class_weight # d_class_weights = None # class_weights = class_weight.compute_class_weight('balanced',np.unique(labels),labels) # print("Class weights:") # print(class_weights) # d_class_weights = dict(enumerate(class_weights)) ### Normalize samples # from sklearn.preprocessing.data import normalize # normalize(samples) ## convert to one-hot encoding lb = LabelBinarizer() labels = lb.fit_transform(labels) classesNum = labels.shape[1] print ("Classes: {}".format(classesNum)) trainSamples = samples trainLabels = labels testSamples = samples testLabels = labels ### Division into training and test samples # from sklearn.model_selection._split import train_test_split # (trainSamples, testSamples, trainLabels, testLabels) = train_test_split(samples, labels, test_size=0.25, random_state=42) model = Sequential() model.add(Dense(250, activation='sigmoid')) model.add(Dense(250, activation='sigmoid')) model.add(Dense(250, activation='sigmoid')) model.add(Dense(classesNum, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer="adam",metrics=['accuracy']) EPOCHS=50 BATCH=50 H = model.fit(trainSamples, trainLabels, batch_size=BATCH, epochs=EPOCHS #,class_weight=d_class_weights #,validation_data=(testSamples,testLabels) #,validation_split=0.1 ) mlpResults = model.predict(testSamples) print(confusion_matrix(testLabels.argmax(axis=1), mlpResults.argmax(axis=1))) print(classification_report(testLabels.argmax(axis=1), mlpResults.argmax(axis=1),target_names=lb.classes_)) print("MLP Accuracy: {:.2f}".format(accuracy_score(testLabels.argmax(axis=1), mlpResults.argmax(axis=1)))) print("Cohen's Kappa {:.2f}".format(cohen_kappa_score(testLabels.argmax(axis=1), mlpResults.argmax(axis=1)))) N = np.arange(0, EPOCHS) plt.style.use("ggplot") plt.figure() plt.plot(N, H.history["loss"], label="train_loss") plt.plot(N, H.history["acc"], label="train_acc") #plt.plot(N, H.history["val_loss"], label="val_loss") #plt.plot(N, H.history["val_acc"], label="val_acc") plt.title("Training Loss and Accuracy") plt.xlabel("Epoch #") plt.ylabel("Loss/Accuracy") plt.legend() plt.show()
bots.load() data = bots.data shp = np.shape(data) row = shp[0] col = shp[1] bands = shp[2] X = data.reshape(row * col, bands) bots_gt = Dataset('Botswana_gt', '.mat') bots_gt.load() y = bots_gt.data.reshape(row * col, 1) ind = np.where(y[:, 0] != 0) X = X[ind] y = y[y != 0] #Linear SVM with all bands and cv=5 start_time = time.time() svm = SVC(kernel='linear') scores = cross_val_score(svm, X, y, cv=5) print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2)) predictions = cross_val_predict(svm, X, y, cv=5) kappa_score = cohen_kappa_score(y, predictions) print("Kappa coefficient: %0.2f" % kappa_score) #print(confusion_matrix(y_test,y_pred)) print(classification_report(y, predictions)) # stuff only to run when not called via 'import' here print("--- %s seconds ---" % (time.time() - start_time)) main()
if os.path.exists('assets/tags.npy'): tags = np.load('assets/tags.npy') # Combine training, validation and testing data utt_Speaker = utt_Speaker_train + utt_Speaker_dev + utt_Speaker_test utt_data = utt_train_data + utt_dev_data + utt_test_data utt_id_data = utt_id_train_data + utt_id_dev_data + utt_id_test_data utt_Emotion_data = utt_Emotion_train_data + utt_Emotion_dev_data + utt_Emotion_test_data utt_Sentiment_data = utt_Sentiment_train_data + utt_Sentiment_dev_data + utt_Sentiment_test_data # Evaluation of context model predictions print('Accuracy comparision between context-based predictions: {}'.format( classification.accuracy_score(meld_elmo_con_out, meld_elmo_mean_con_out))) print('Kappa (Cohen) score between context-based predictions: {}'.format( classification.cohen_kappa_score(meld_elmo_con_out, meld_elmo_mean_con_out))) print( classification.classification_report(meld_elmo_con_out, meld_elmo_mean_con_out)) print('Spearman Correlation between context-based predictions: {}'.format( stats.spearmanr(meld_elmo_con_out, meld_elmo_mean_con_out))) reliability_data = convert_predictions_to_indices(meld_elmo_con_out, meld_elmo_non_con_out, meld_elmo_mean_con_out, meld_elmo_mean_non_con_out, meld_elmo_top_con_out, tags) k_alpha = alpha(reliability_data, level_of_measurement='nominal') print("Krippendorff's alpha: {}".format(round(k_alpha, 6))) fleiss_kappa_score = fleissKappa(reliability_data, 5)
clf1.fit(X_train,Y_train) #feature importance clf1.feature_importances_() predict = clf1.predict(X_test) #cross val score score1 = np.mean(cross_val_score(clf, X, Y, scoring='accuracy', cv=10)) print(score1) ## Metrics-accuracy print(accuracy_score(predict,Y_test)) #kappa score score3 = cohen_kappa_score(Y_test,predict) print(score3) #recall score score2=recall_score(Y_test, predict, average='macro') print(score2) #resampling to get a better model minor_class = [4,8,3] major_class = [5,6,7] df.info() df_minor = df[df.quality.isin(minor_class)] df_major = df[df.quality.isin(major_class)] #Upsampling df_minor_upsampled = resample(df_minor,replace=True,n_samples = len(df_major), random_state = 123)
def main(): print("Loading samples and labels") samples, labels, _ = load_files("data") print("Loaded {} samples".format(samples.shape[0])) sequence_dim = 100 print("Converting to sequences of length {}".format(sequence_dim)) samples, labels = make_sequences(samples, labels, sequence_dim) print("Number of samples from sequences: {}".format(samples.shape[0])) lb = LabelBinarizer() labels = lb.fit_transform(labels) # flattened samples for Decision Tree flatSamples = samples.reshape(samples.shape[0], -1) #tree! (trainSamples, testSamples, trainLabels, testLabels) = train_test_split(flatSamples, labels, test_size=0.25, random_state=42) print("=" * 20) print("Building DecisionTree model") model = DecisionTreeClassifier() model.fit(trainSamples, trainLabels) treeResults = model.predict(testSamples) print( confusion_matrix(testLabels.argmax(axis=1), treeResults.argmax(axis=1))) print( classification_report(testLabels.argmax(axis=1), treeResults.argmax(axis=1))) treeAcc = accuracy_score(testLabels.argmax(axis=1), treeResults.argmax(axis=1)) print("Accuracy Tree: {:.2f}".format(treeAcc)) print("Cohen's Kappa {:.2f}".format( cohen_kappa_score(testLabels.argmax(axis=1), treeResults.argmax(axis=1)))) print("=" * 20) print("Building CNN model") (trainSamples, testSamples, trainLabels, testLabels) = train_test_split(samples, labels, test_size=0.25, random_state=42) inputShape = (samples.shape[1], samples.shape[2]) model = Sequential() model.add(Conv1D(32, 10, padding="same", input_shape=inputShape)) model.add(Activation("relu")) model.add(BatchNormalization()) model.add(Dropout(0.2)) model.add(Conv1D(64, 10, padding="same")) model.add(Activation("relu")) model.add(BatchNormalization()) model.add(Dropout(0.2)) model.add(Conv1D(128, 10, padding="same")) model.add(Activation("relu")) model.add(Dropout(0.2)) model.add(Flatten(input_shape=inputShape)) model.add(Dense(128, activation='sigmoid')) model.add(Dense(64, activation='sigmoid')) model.add(Dense(labels.shape[1], activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer="adam", metrics=['accuracy']) EPOCHS = 10 BATCH = 128 model.fit(trainSamples, trainLabels, batch_size=BATCH, epochs=EPOCHS, validation_data=(testSamples, testLabels)) cnnResults = model.predict(testSamples) print( confusion_matrix(testLabels.argmax(axis=1), cnnResults.argmax(axis=1))) print( classification_report(testLabels.argmax(axis=1), cnnResults.argmax(axis=1), target_names=lb.classes_)) print("CNN Accuracy: {:.2f}".format( accuracy_score(testLabels.argmax(axis=1), cnnResults.argmax(axis=1)))) print("Cohen's Kappa {:.2f}".format( cohen_kappa_score(testLabels.argmax(axis=1), cnnResults.argmax(axis=1)))) input("")
def printResults(testLabels,testResults): print(confusion_matrix(testLabels.argmax(axis=1), testResults.argmax(axis=1))) print(classification_report(testLabels.argmax(axis=1), testResults.argmax(axis=1))) print("Cohen's Kappa: {}".format(cohen_kappa_score(testLabels.argmax(axis=1), testResults.argmax(axis=1)))) return accuracy_score(testLabels.argmax(axis=1), testResults.argmax(axis=1))
print("*" * 50) print("Accuracy DT Gini : ", accuracy_score(y_test, y_pred_gini) * 100) print("Accuracy DT Entropy: ", accuracy_score(y_test, y_pred_entropy) * 100) print("Accuracy SVM: ", accuracy_score(y_test, y_pred_svm) * 100) print("Accuracy RF: ", accuracy_score(y_test, y_pred_rf) * 100) print("Accuracy KNN: ", accuracy_score(y_test, y_pred_knn) * 100) print("Accuracy NB: ", accuracy_score(y_test, y_pred_nb) * 100) print("Accuracy Bagging with Mode method: ", accuracy_score(y_test, final_pred) * 100) print("Accuracy ADA: ", accuracy_score(y_test, y_pred_boost) * 100) print("*" * 50) #Printing results for our best model print("ROC_AUC : ", roc_auc_score(y_test, y_pred_boost) * 100) print("Accuracy K: ", cohen_kappa_score(y_test, y_pred_boost) * 100) # ROC Graph y_pred_score = classifier.predict_proba(X_test) preds = y_pred_score[:, 1] fpr, tpr, threshold = metrics.roc_curve(y_test, preds) roc_auc = metrics.auc(fpr, tpr) # method I: plt import matplotlib.pyplot as plt plt.title('Receiver Operating Characteristic') plt.plot(fpr, tpr, 'b', label='AUC = %0.2f' % roc_auc) plt.legend(loc='lower right') plt.plot([0, 1], [0, 1], 'r--') plt.xlim([0, 1]) plt.ylim([0, 1])