def on_epoch_end(self, batch, logs={}): # losses self.losses_train.append(self.model.evaluate(X_train, Y_train, batch_size=128,verbose =0)) self.losses_val.append(self.model.evaluate(X_val, Y_val, batch_size=128,verbose = 0)) # Roc train train_preds = self.model.predict_proba(X_train, verbose=0) train_preds = train_preds[:, 1] roc_train = metrics.roc_auc_score(y_train, train_preds) self.roc_train.append(roc_train) # Roc val val_preds = self.model.predict_proba(X_val, verbose=0) val_preds = val_preds[:, 1] roc_val = metrics.roc_auc_score(y_val, val_preds) self.roc_val.append(roc_val) # Metrics train y_preds = self.model.predict_classes(X_train,verbose = 0) self.f1_train.append(metrics.f1_score(y_train,y_preds)) self.recal_train.append(metrics.recall_score(y_train,y_preds)) self.preci_train.append(metrics.precision_score(y_train,y_preds)) # Metrics val y_preds = self.model.predict_classes(X_val,verbose =0) self.f1_val.append(metrics.f1_score(y_val,y_preds)) self.recal_val.append(metrics.recall_score(y_val,y_preds)) self.preci_val.append(metrics.precision_score(y_val,y_preds))
def single_test(feature, attribute): from sklearn.metrics import f1_score from sklearn.metrics import recall_score from sklearn.metrics import accuracy_score from data_generator import load_vector_from_text import random data=merge_different_vectors([feature],attribute) none_attribute_uids=load_vector_from_text('uids_none_attributes.vector',feature,'list') none_attribute_uids=filter(lambda x:x in data[0],none_attribute_uids) alpha=0.2*len(data[0])/len(none_attribute_uids) train_data=[[],[]] test_data=[[],[]] for index,uid in enumerate(data[0]): if uid in none_attribute_uids and random.random()<alpha: #if random.random()<0.2: test_data[0].append(data[1][index]) test_data[1].append(data[2][index]) else: train_data[0].append(data[1][index]) train_data[1].append(data[2][index]) print len(test_data[1]),sum(test_data[1]),len(train_data[1]),sum(train_data[1]) clf=LogisticRegression() clf.fit(train_data[0], train_data[1]) predicted_y=clf.predict(test_data[0]) test_accuracy=accuracy_score(test_data[1],predicted_y) test_recall=recall_score(test_data[1],predicted_y) test_f1=f1_score(test_data[1],predicted_y) print 'F1 of test data (%d %d): %0.2f'%(sum(test_data[1]),len(test_data[1])-sum(test_data[1]),test_f1) print 'Accuracy of test data (%d %d): %0.2f'%(sum(test_data[1]),len(test_data[1])-sum(test_data[1]),test_accuracy) predicted_y=clf.predict(train_data[0]) train_accuracy=accuracy_score(train_data[1],predicted_y) train_recall=recall_score(train_data[1],predicted_y) train_f1=f1_score(train_data[1],predicted_y) print 'F1 of train data (%d %d): %0.2f'%(sum(train_data[1]),len(train_data[1])-sum(train_data[1]),train_f1) return [test_accuracy,test_recall,test_f1,train_accuracy,train_recall,train_f1]
def confusion_matrix(true_y, pred_y, labels): c_matrix = metrics.confusion_matrix(true_y, pred_y) confusion_table = [] first_row = ["C.Matrix"] + labels + ["ACTUAL"] + ["RECALL"] confusion_table.append(first_row) recall = metrics.recall_score(true_y, pred_y, average=None) for r, row in enumerate(c_matrix): new_row = [labels[r]] new_row.extend(row) new_row.append(sum(row)) new_row.append(recall[r]) confusion_table.append(new_row) new_row = ["PREDICTED"] for l in labels: new_row.append(len([t for t in pred_y if t == l])) new_row.append(len(true_y)) new_row.append(metrics.recall_score(true_y, pred_y, average='macro')) confusion_table.append(new_row) new_row = ["PRECISION"] new_row.extend(metrics.precision_score(true_y, pred_y, average=None)) new_row.append(metrics.precision_score(true_y, pred_y, average='macro')) new_row.append(metrics.f1_score(true_y, pred_y, average='macro')) confusion_table.append(new_row) confusion_table = pd.DataFrame(confusion_table) return confusion_table
def applyClassifier(self, clf, name, training_set, testing_set, y_train, y_test): print("\nMODEL " + name) t0 = time() classifier = clf.fit(training_set, y_train) train_time = time() - t0 print("train time: %0.3fs" % train_time) t0 = time() y_nb_predicted = classifier.predict(testing_set) test_time = time() - t0 print("test time: %0.3fs" % test_time) precision = metrics.precision_score(y_test, y_nb_predicted) recall = metrics.recall_score(y_test, y_nb_predicted) f1_score = metrics.f1_score(y_test, y_nb_predicted) accuracy = metrics.accuracy_score(y_test, y_nb_predicted) micro_recall = metrics.recall_score(y_test, y_nb_predicted, average="micro") macro_recall = metrics.recall_score(y_test, y_nb_predicted, average="macro") micro_precision = metrics.precision_score(y_test, y_nb_predicted, average="micro") macro_precision = metrics.precision_score(y_test, y_nb_predicted, average="macro") print 'The precision for this classifier is ' + str(precision) print 'The micro averaged precision for this classifier is ' + str(micro_precision) print 'The macro averaged precision for this classifier is ' + str(macro_precision) print 'The recall for this classifier is ' + str(recall) print 'The micro averaged recall for this classifier is ' + str(micro_recall) print 'The macro averaged recall for this classifier is ' + str(macro_recall) print 'The f1 for this classifier is ' + str(f1_score) print 'The accuracy for this classifier is ' + str(accuracy) return name, accuracy, precision, recall, micro_precision, micro_recall, macro_precision, macro_recall, train_time, test_time
def main(): f = open("me.stdout", "r").read() print f (confusionMatrix, labels, ytrue, ypred, trueCount) = readConfusionMatrix.readText(f) for row in confusionMatrix: print row precisionMicro = np.float(metrics.precision_score(ytrue, ypred, average="micro")) recallMicro = np.float(metrics.recall_score(ytrue, ypred, average="micro")) f1Micro = np.float(metrics.f1_score(ytrue, ypred, average="micro")) f1Macro = np.float(metrics.f1_score(ytrue, ypred, pos_label=1, average="macro")) precisionMacro = np.float(metrics.precision_score(ytrue, ypred, average="macro")) recallMacro = np.float(metrics.recall_score(ytrue, ypred, average="macro")) mConf = metrics.confusion_matrix(ytrue, ypred) print mConf print labels print len(ytrue) print len(ypred) print trueCount print metrics.accuracy_score(ytrue, ypred) print precisionMicro print recallMicro print f1Micro print f1Macro print precisionMacro print recallMacro
def on_epoch_end(self, epoch, logs={}): print logs corr=0 tot=0 preds = self.model.predict(self.dev_data, verbose=1) preds_text=[] for l in preds: preds_text.append(self.index2label[np.argmax(l)]) print "Micro f-score:", f1_score(self.dev_labels_text,preds_text,average=u"micro") print "Macro f-score:", f1_score(self.dev_labels_text,preds_text,average=u"macro") print "Macro recall:", recall_score(self.dev_labels_text,preds_text,average=u"macro") if self.best_mr < recall_score(self.dev_labels_text,preds_text,average=u"macro"): self.best_mr = recall_score(self.dev_labels_text,preds_text,average=u"macro") model.save_weights(self.model_name + '_full_' + str(epoch) + '_MR_' + str(self.best_mr) + '.hdf5') print 'Saved Weights!' print classification_report(self.dev_labels_text, preds_text) for i in xrange(len(self.dev_labels)): # next_index = sample(preds[i]) next_index = np.argmax(preds[i]) # print preds[i],next_index,index2label[next_index] l = self.index2label[next_index] # print "correct:", index2label[np.argmax(dev_labels[i])], "predicted:",l if self.index2label[np.argmax(self.dev_labels[i])]==l: corr+=1 tot+=1 print corr,"/",tot
def stratified_k_fold(clf,features,labels): skf = StratifiedKFold( labels, n_folds=3 ) precisions = [] recalls = [] for train_idx, test_idx in skf: features_train = [] features_test = [] labels_train = [] labels_test = [] for ii in train_idx: features_train.append( features[ii] ) labels_train.append( labels[ii] ) for jj in test_idx: features_test.append( features[jj] ) labels_test.append( labels[jj] ) ### fit the classifier using training set, and test on test set clf.fit(features_train, labels_train) pred = clf.predict(features_test) ### for each fold, print some metrics print print "precision score: ", precision_score( labels_test, pred ) print "recall score: ", recall_score( labels_test, pred ) precisions.append( precision_score(labels_test, pred) ) recalls.append( recall_score(labels_test, pred) ) ### aggregate precision and recall over all folds print "average precision: ", sum(precisions)/2. print "average recall: ", sum(recalls)/2.
def create_all_eval_results(y_true,y_pred,key,system_features,sampling,replacement,num_of_samples): # precision = metrics.precision_score(y_true, y_pred, average='weighted') # recall = metrics.recall_score(y_true, y_pred, average='weighted') # F2 = calculateF2(precision, recall) name = data_names[key] y_true_bugs, y_pred_bugs = zip(*[[y_true[i], y_pred[i]] for i in range(len(y_true)) if y_true[i] == 1]) # precision_bug, recall_bug, F_measure_bug ,_ = metrics.precision_recall_fscore_support(y_true_bugs, # y_pred_bugs, # average='micro') precision_bug =metrics.precision_score(y_true_bugs,y_pred_bugs,average='micro') recall_bug =metrics.recall_score(y_true_bugs,y_pred_bugs,average='micro') F2_bug = calculateF2(precision_bug,recall_bug) precision_bug_all, recall_bug_all,_ = metrics.precision_recall_curve(y_true_bugs, y_pred_bugs) prc_area_bug = metrics.auc(recall_bug_all, precision_bug_all) # precision, recall, F_measure,_ = metrics.precision_recall_fscore_support(y_true, # y_pred, # average='micro') precision = metrics.average_precision_score(y_true, y_pred, average='micro') recall = metrics.recall_score(y_true, y_pred, average='micro') F2 = calculateF2(precision, recall) precision_all, recall_all, _ = metrics.precision_recall_curve(y_true, y_pred) prc_area = metrics.auc(recall_all, precision_all) global results results.loc[len(results)] = [name,precision_bug,recall_bug,F2_bug,prc_area_bug, precision, recall,F2,prc_area,str(system_features),str(sampling),str(replacement),str(num_of_samples)]
def cross_val(data_x, data_y, classifier, kFold, b_cost=1, h_cost=1, w=0.5): e_h, e_b = 0, 0 y_tests, pred_probas = [], [] for train_index, test_index in kFold: data_x_, data_y_ = np.array(data_x), np.array(data_y) X_train, X_test = list(data_x_[train_index]), list(data_x_[test_index]) y_train, y_test = list(data_y_[train_index]), list(data_y_[test_index]) classifier.fit(X_train, y_train) pred_proba = [r[0] for r in classifier.predict_proba(X_test)] y_tests += y_test pred_probas += pred_proba predictions = [0 if p*b_cost > (1-p)*h_cost else 1 for p in pred_probas] roc_auc = roc_auc_score(y_tests, pred_probas) total_acc = accuracy_score(y_tests, predictions) precision, recall, thresholds = precision_recall_curve(y_tests, pred_probas, pos_label=0) fpr, tpr, thresholds = roc_curve(y_tests, pred_probas, pos_label=0) precision_bots = precision_score(y_tests, predictions, pos_label = 0) precision_humans = precision_score(y_tests, predictions, pos_label = 1) recall_bots = recall_score(y_tests, predictions, pos_label = 0) recall_humans = recall_score(y_tests, predictions, pos_label = 1) f1_bots = f1_score(y_tests, predictions, pos_label = 0) f1_humans = f1_score(y_tests, predictions, pos_label = 1) conf_matrix = np.matrix(list(confusion_matrix(y_tests, predictions))) #plot_curve(fpr, tpr, 'ROC', w) #plot_curve(recall, precision, 'PR', w) return [total_acc, precision_bots, precision_humans, recall_bots, recall_humans, f1_bots, f1_humans, roc_auc, conf_matrix]
def calculate_f1_metrics(all_predicted, all_targets): first_class = first_meaningful_entity class_count = len(set(all_targets)) filtered_true, filtered_predicted = [], [] for i in range(len(all_targets)): if all_targets[i] > 0: filtered_true.append(all_targets[i]) filtered_predicted.append(all_predicted[i]) precision_separate_scores = metrics.precision_score(filtered_true, filtered_predicted, labels=[i for i in range(first_class, class_count)], average=None) precision_score = metrics.precision_score(filtered_true, filtered_predicted, labels=[i for i in range(first_class, class_count)], average='micro') recall_separate_scores = metrics.recall_score(filtered_true, filtered_predicted, labels=[i for i in range(first_class, class_count)], average=None) recall_score = metrics.recall_score(filtered_true, filtered_predicted, labels=[i for i in range(first_class, class_count)], average='micro') f1_separate_scores = metrics.f1_score(filtered_true, filtered_predicted, labels=[i for i in range(first_class, class_count)], average=None) f1_score = metrics.f1_score(filtered_true, filtered_predicted, labels=[i for i in range(first_class, class_count)], average='micro') return f1_separate_scores, f1_score, precision_separate_scores, precision_score, recall_separate_scores, recall_score
def run_model(X_test, X_train, y_test, y_train, prob_threshold = 20, layers = 5, nodes = 64, dropout = 50): print "run_model RUNNING" # Grab the model model = get_model(X_test, layers =layers, dropout = dropout) model.fit(X_train, y_train, nb_epoch=20, batch_size=16, verbose = 0) # Get the training and test predictions from our model fit. train_predictions = model.predict_proba(X_train) test_predictions = model.predict_proba(X_test) # Set these to either 0 or 1 based off the probability threshold we # passed in (divide by 100 becuase we passed in intergers). train_preds = (train_predictions) >= prob_threshold / 100.0 test_preds = (test_predictions) >= prob_threshold / 100.0 # Calculate the precision and recall. Only output until precision_score_train = precision_score(y_train, train_preds) precision_score_test = precision_score(y_test, test_preds) acc_train = accuracy_score(y_train, train_preds) acc_test = accuracy_score(y_test, test_preds) recall_score_train = recall_score(y_train, train_preds) recall_score_test = recall_score(y_test, test_preds) return precision_score_train, precision_score_test, recall_score_train, recall_score_test, acc_train, acc_test, model
def randomforest(input_file,Output): lvltrace.lvltrace("LVLEntree dans randomforest") ncol=tools.file_col_coma(input_file) data = np.loadtxt(input_file, delimiter=',', usecols=range(ncol-1)) X = data[:,1:] y = data[:,0] n_samples, n_features = X.shape clf = RandomForestClassifier(n_estimators=10) clf.fit(X,y) y_pred = clf.predict(X) print "#########################################################################################################\n" print "The Random forest algo " print "classification accuracy:", metrics.accuracy_score(y, y_pred) print "precision:", metrics.precision_score(y, y_pred) print "recall:", metrics.recall_score(y, y_pred) print "f1 score:", metrics.f1_score(y, y_pred) print "\n" print "#########################################################################################################\n" results = Output+"Random_Forest_metrics.txt" file = open(results, "w") file.write("Random Forest Classifier estimator accuracy\n") file.write("Classification Accuracy Score: %f\n"%metrics.accuracy_score(y, y_pred)) file.write("Precision Score: %f\n"%metrics.precision_score(y, y_pred)) file.write("Recall Score: %f\n"%metrics.recall_score(y, y_pred)) file.write("F1 Score: %f\n"%metrics.f1_score(y, y_pred)) file.write("\n") file.write("True Value, Predicted Value, Iteration\n") for n in xrange(len(y)): file.write("%f,%f,%i\n"%(y[n],y_pred[n],(n+1))) file.close() title = "The Random forest" save = Output + "Random_Forest_confusion_matrix.png" plot_confusion_matrix(y, y_pred,title,save) lvltrace.lvltrace("LVLSortie dans randomforest")
def stochasticGD(input_file,Output): lvltrace.lvltrace("LVLEntree dans stochasticGD") ncol=tools.file_col_coma(input_file) data = np.loadtxt(input_file, delimiter=',', usecols=range(ncol-1)) X = data[:,1:] y = data[:,0] n_samples, n_features = X.shape clf = SGDClassifier(loss="hinge", penalty="l2") clf.fit(X,y) y_pred = clf.predict(X) print "#########################################################################################################\n" print "Stochastic Gradient Descent " print "classification accuracy:", metrics.accuracy_score(y, y_pred) print "precision:", metrics.precision_score(y, y_pred) print "recall:", metrics.recall_score(y, y_pred) print "f1 score:", metrics.f1_score(y, y_pred) print "\n" print "#########################################################################################################\n" results = Output+"Stochastic_GD_metrics.txt" file = open(results, "w") file.write("Stochastic Gradient Descent estimator accuracy\n") file.write("Classification Accuracy Score: %f\n"%metrics.accuracy_score(y, y_pred)) file.write("Precision Score: %f\n"%metrics.precision_score(y, y_pred)) file.write("Recall Score: %f\n"%metrics.recall_score(y, y_pred)) file.write("F1 Score: %f\n"%metrics.f1_score(y, y_pred)) file.write("\n") file.write("True Value, Predicted Value, Iteration\n") for n in xrange(len(y)): file.write("%f,%f,%i\n"%(y[n],y_pred[n],(n+1))) file.close() title = "Stochastic Gradient Descent" save = Output + "Stochastic_GD_confusion_matrix.png" plot_confusion_matrix(y, y_pred,title,save) lvltrace.lvltrace("LVLSortie dans stochasticGD")
def evaluate(ytest, ypred, filename='metrics.txt'): true_result = [1 if item > 0.5 else 0 for item in ytest] pred_result = [1 if item > 0.5 else 0 for item in ypred] cm = confusion_matrix(true_result, pred_result) print('\nConfusion matrix:') print(cm) print("\nLoss classified as loss", cm[0][0]) print("Wins classified as wins", cm[1][1]) print("Wins classified as loss", cm[1][0]) print("Loss classified as wins", cm[0][1]) print('\nAccuracy:\t', accuracy_score(true_result, pred_result)) print('Precision:\t', precision_score(true_result, pred_result)) print('Recall: \t', recall_score(true_result, pred_result)) print('F1 score:\t', f1_score(true_result, pred_result)) print('Mean absolute error:\t', mean_absolute_error(ytest, ypred)) # print to file print("Loss classified as loss", cm[0][0], file=open(filename, "a")) print("Wins classified as wins", cm[1][1], file=open(filename, "a")) print("Wins classified as loss", cm[1][0], file=open(filename, "a")) print("Loss classified as wins", cm[0][1], file=open(filename, "a")) print('\nAccuracy:\t', accuracy_score(true_result, pred_result), file=open(filename, "a")) print('Precision:\t', precision_score(true_result, pred_result), file=open(filename, "a")) print('Recall: \t', recall_score(true_result, pred_result), file=open(filename, "a")) print('F1 score:\t', f1_score(true_result, pred_result), file=open(filename, "a")) print('Mean absolute error:\t', mean_absolute_error(ytest, ypred), file=open(filename, "a"))
def _clf_mlp(trX,teX,trY,teY): print "MLP" print trX.shape,"trX shape" print "Enter Layer for MLP" layer=input() # print "enter delIdx" # delIdx=input() # while(delIdx): # trX=np.delete(trX,-1,axis=0) # trY=np.delete(trY,-1,axis=0) # delIdx=delIdx-1 print "factors",factors(trX.shape[0]) teY=teY.astype(np.int32) trY=trY.astype(np.int32) print trX.shape,"trX shape" print "enter no of mini batch" mini_batch=int(input()) mlp = TfMultiLayerPerceptron(eta=0.01, epochs=100, hidden_layers=layer, activations=['relu' for i in range(len(layer))], print_progress=3, minibatches=mini_batch, optimizer='adam', random_seed=1) mlp.fit(trX,trY) pred=mlp.predict(teX) print _f_count(teY),"test f count" pred=pred.astype(np.int32) print _f_count(pred),"pred f count" conf_mat=confusion_matrix(teY, pred) process_cm(conf_mat, to_print=True) print precision_score(teY,pred),"Precision Score" print recall_score(teY,pred),"Recall Score" print roc_auc_score(teY,pred), "ROC_AUC"
def predictSVD(svd, row, column, d): # start = timeit.default_timer() u = svd[0] #clf.components_ s = svd[1] #clf.explained_variance_ vt = svd[2] #clf.fit_transform(X) # print " fitting done."; # stop = timeit.default_timer() # print " runtime: " + str(stop - start) # print "d:" # print d # matrixY = clf.components_ probsY = [] # print "dot products:" for i in range(len(row)): # print np.dot(u[:,column[i]], v[row[i],:]) prob = np.sum(u[column[i],:]*s*vt[:,row[i]]) if(prob < 0): prob = 0 if(prob > 1): prob = 1 probsY.append(prob) probsY = np.array(probsY) preds = np.zeros(shape=len(probsY)) preds[probsY >= 0.5] = 1 print "Precision" print precision_score(d, preds) print "Recall" print recall_score(d, preds) print "F-Score" print f1_score(d, preds) return probsY, preds
def trainModel(self,folds): kf = cross_validation.StratifiedKFold(self.y_total,n_folds=folds,shuffle=True,random_state=random.randint(1,100)) for (train_index,test_index) in (kf): self.X_train = [self.X_total[i] for i in train_index] self.X_test = [self.X_total[i] for i in test_index] self.y_train = [self.y_total[i] for i in train_index] self.y_test = [self.y_total[i] for i in test_index] print "################" print "Original" print np.array(self.y_test) print "################" self.clf = self.clf.fit(self.X_train,self.y_train) print "Predicted" y_pred = self.clf.predict(self.X_test) print y_pred print "################" print "Evaluation\n" cm = confusion_matrix(self.y_test,y_pred) print cm print "Precision Score:" print precision_score(self.y_test,y_pred,average="macro") print "Recall Score:" print recall_score(self.y_test,y_pred,average="macro") print "Accuracy Score:" print accuracy_score(self.y_test,y_pred)
def nearest_centroid(input_file,Output,test_size): ncol=tools.file_col_coma(input_file) data = np.loadtxt(input_file, delimiter=',', usecols=range(ncol-1)) X = data[:,1:] y = data[:,0] n_samples, n_features = X.shape X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size) print X_train.shape, X_test.shape clf = NearestCentroid() clf.fit(X_train,y_train) y_pred = clf.predict(X_test) print "Nearest Centroid Classifier " print "classification accuracy:", metrics.accuracy_score(y_test, y_pred) print "precision:", metrics.precision_score(y_test, y_pred) print "recall:", metrics.recall_score(y_test, y_pred) print "f1 score:", metrics.f1_score(y_test, y_pred) print "\n" results = Output+"Nearest_Centroid_metrics_test.txt" file = open(results, "w") file.write("Nearest Centroid Classifier estimator accuracy\n") file.write("Classification Accuracy Score: %f\n"%metrics.accuracy_score(y_test, y_pred)) file.write("Precision Score: %f\n"%metrics.precision_score(y_test, y_pred)) file.write("Recall Score: %f\n"%metrics.recall_score(y_test, y_pred)) file.write("F1 Score: %f\n"%metrics.f1_score(y_test, y_pred)) file.write("\n") file.write("True Value, Predicted Value, Iteration\n") for n in xrange(len(y_test)): file.write("%f,%f,%i\n"%(y_test[n],y_pred[n],(n+1))) file.close() title = "Nearest Centroid %f"%test_size save = Output + "Nearest_Centroid_confusion_matrix"+"_%s.png"%test_size plot_confusion_matrix(y_test, y_pred,title,save) lvltrace.lvltrace("LVLSortie dans stochasticGD split_test")
def extratreeclassifier(input_file,Output,test_size): lvltrace.lvltrace("LVLEntree dans extratreeclassifier split_test") ncol=tools.file_col_coma(input_file) data = np.loadtxt(input_file, delimiter=',', usecols=range(ncol-1)) X = data[:,1:] y = data[:,0] n_samples, n_features = X.shape X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size) print X_train.shape, X_test.shape clf = ExtraTreesClassifier(n_estimators=10) clf.fit(X_train,y_train) y_pred = clf.predict(X_test) print "Extremely Randomized Trees" print "classification accuracy:", metrics.accuracy_score(y_test, y_pred) print "precision:", metrics.precision_score(y_test, y_pred) print "recall:", metrics.recall_score(y_test, y_pred) print "f1 score:", metrics.f1_score(y_test, y_pred) print "\n" results = Output+"_Extremely_Random_Forest_metrics_test.txt" file = open(results, "w") file.write("Extremely Random Forest Classifier estimator accuracy\n") file.write("Classification Accuracy Score: %f\n"%metrics.accuracy_score(y_test, y_pred)) file.write("Precision Score: %f\n"%metrics.precision_score(y_test, y_pred)) file.write("Recall Score: %f\n"%metrics.recall_score(y_test, y_pred)) file.write("F1 Score: %f\n"%metrics.f1_score(y_test, y_pred)) file.write("\n") file.write("True Value, Predicted Value, Iteration\n") for n in xrange(len(y_test)): file.write("%f,%f,%i\n"%(y_test[n],y_pred[n],(n+1))) file.close() title = "Extremely Randomized Trees %f"%test_size save = Output + "Extremely_Randomized_Trees_confusion_matrix"+"_%s.png"%test_size plot_confusion_matrix(y_test, y_pred,title,save) lvltrace.lvltrace("LVLSortie dans extratreeclassifier split_test")
def gaussianNB(input_file,Output,test_size): lvltrace.lvltrace("LVLEntree dans gaussianNB split_test") ncol=tools.file_col_coma(input_file) data = np.loadtxt(input_file, delimiter=',', usecols=range(ncol-1)) X = data[:,1:] y = data[:,0] n_samples, n_features = X.shape X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size) print X_train.shape, X_test.shape # Instantiate the estimator clf = GaussianNB() # Fit the estimator to the data clf.fit(X_train, y_train) # Use the model to predict the last several labels y_pred = clf.predict(X_test) print "Gaussian Naive Bayes estimator accuracy " print "classification accuracy:", metrics.accuracy_score(y_test, y_pred) print "precision:", metrics.precision_score(y_test, y_pred) print "recall:", metrics.recall_score(y_test, y_pred) print "f1 score:", metrics.f1_score(y_test, y_pred) results = Output+"GaussianNB_metrics_test.txt" file = open(results, "w") file.write("Gaussian Naive Bayes estimator accuracy\n") file.write("Classification Accuracy Score: %f\n"%metrics.accuracy_score(y_test, y_pred)) file.write("Precision Score: %f\n"%metrics.precision_score(y_test, y_pred)) file.write("Recall Score: %f\n"%metrics.recall_score(y_test, y_pred)) file.write("F1 Score: %f\n"%metrics.f1_score(y_test, y_pred)) file.write("True Value, Predicted Value, Iteration\n") for n in xrange(len(y_test)): file.write("%f,%f,%i\n"%(y_test[n],y_pred[n],(n+1))) file.close() title = "Gaussian Naive Bayes %f"%test_size save = Output + "Gaussian_NB_confusion_matrix"+"_%s.png"%test_size plot_confusion_matrix(y_test, y_pred,title,save) lvltrace.lvltrace("LVLsortie dans gaussianNB split_test")
def evaluate(self, feats, tag_set): """ Tag held-out labeled corpus `tagged_corpus`, and return tagging accuracy """ corect=0 incorect=0 #nul = 0 #ful = 0 per_sen = 0 all_pre = [] all_tru = [] for (tokens, tags) in zip(feats, tag_set): yyhat= self.tag(tokens) all_pre.extend(yyhat) all_tru.extend(tags) cor_sen = 0 for pre, tag in zip(yyhat, tags): if pre==tag: corect+=1 cor_sen+=1 else: incorect+=1 if cor_sen == len(yyhat): per_sen+=1 print metrics.recall_score(all_pre, all_tru, average=None) return corect/(corect+incorect), per_sen/len(tag_set) #nul/corect, ful/corect
def score(y_true, y_pred): precision_weighted = metrics.precision_score( y_true, y_pred, average='weighted') precision_ave = np.mean(metrics.precision_score( y_true, y_pred, average=None)[::12]) recall_weighted = metrics.recall_score( y_true, y_pred, average='weighted') recall_ave = np.mean(metrics.recall_score( y_true, y_pred, average=None)[::12]) f1_weighted = metrics.f1_score( y_true, y_pred, average='weighted') f1_ave = np.mean(metrics.f1_score( y_true, y_pred, average=None)[::12]) stat_line = " Precision: %0.4f\t Recall: %0.4f\tf1: %0.4f" res1 = "Weighted: " + stat_line % (100*precision_weighted, 100*recall_weighted, 100*f1_weighted) res2 = "Averaged: " + stat_line % (100*precision_ave, 100*recall_ave, 100*f1_ave) res3 = "-"*72 outputs = [res3, res1, res2, res3] return "\n".join(outputs)
def main(): resize_shape = 64 print "data is loading..." train_X, train_Y, test_X, test_Y = load_data(resize_shape) print "data is loaded" print "feature engineering..." learning_rate = 0.01 training_iters = 100000 batch_size = 128 display_step = 10 # Network Parameters n_input = resize_shape*resize_shape # MNIST data input (img shape: 28*28) n_classes = 62 # MNIST total classes (0-9 digits) dropout = 0.5 # Dropout, probability to keep units with tf.Session() as sess: cnn = CNN(sess, learning_rate, training_iters, batch_size, display_step, n_input, n_classes, dropout,resize_shape) train_X = cnn.inference(train_X) test_X = cnn.inference(test_X) print "feature engineering is complete" print 'training phase' clf = svm.LinearSVC().fit(train_X, train_Y) print 'test phase' predicts = clf.predict(test_X) # measure function print 'measure phase' print confusion_matrix(test_Y, predicts) print f1_score(test_Y, predicts, average=None) print precision_score(test_Y, predicts, average=None) print recall_score(test_Y, predicts, average=None) print accuracy_score(test_Y, predicts)
def SVC_linear(input_file,Output): lvltrace.lvltrace("LVLEntree dans SVC_linear") ncol=tools.file_col_coma(input_file) data = np.loadtxt(input_file, delimiter=',', usecols=range(ncol-1)) X = data[:,1:] y = data[:,0] n_samples, n_features = X.shape clf=svm.SVC(kernel='linear') clf.fit(X,y) y_pred = clf.predict(X) print "#########################################################################################################\n" print "C-Support Vector Classifcation (with linear kernel) " print "classification accuracy:", metrics.accuracy_score(y, y_pred) print "precision:", metrics.precision_score(y, y_pred) print "recall:", metrics.recall_score(y, y_pred) print "f1 score:", metrics.f1_score(y, y_pred) print "\n" print "#########################################################################################################\n" results = Output+"SVM_Linear_Kernel_metrics.txt" file = open(results, "w") file.write("Support Vector Machine with Linear Kernel estimator accuracy\n") file.write("Classification Accuracy Score: %f\n"%metrics.accuracy_score(y, y_pred)) file.write("Precision Score: %f\n"%metrics.precision_score(y, y_pred)) file.write("Recall Score: %f\n"%metrics.recall_score(y, y_pred)) file.write("F1 Score: %f\n"%metrics.f1_score(y, y_pred)) file.write("\n") file.write("True Value, Predicted Value, Iteration\n") for n in xrange(len(y)): file.write("%f,%f,%i\n"%(y[n],y_pred[n],(n+1))) file.close() title = "SVC - linear Kernel" save = Output + "SVC_linear_confusion_matrix.png" plot_confusion_matrix(y, y_pred,title,save) lvltrace.lvltrace("LVLSortie dans SVC_linear")
def SVC_linear(input_file,Output,test_size): lvltrace.lvltrace("LVLEntree dans SVC_linear split_test") ncol=tools.file_col_coma(input_file) data = np.loadtxt(input_file, delimiter=',', usecols=range(ncol-1)) X = data[:,1:] y = data[:,0] n_samples, n_features = X.shape X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size) print X_train.shape, X_test.shape clf=svm.SVC(kernel='linear') clf.fit(X_train,y_train) y_pred = clf.predict(X_test) print "C-Support Vector Classifcation (with RBF linear) " print "y_test, y_pred, iteration" print "classification accuracy:", metrics.accuracy_score(y_test, y_pred) print "precision:", metrics.precision_score(y_test, y_pred) print "recall:", metrics.recall_score(y_test, y_pred) print "f1 score:", metrics.f1_score(y_test, y_pred) print "\n" results = Output+"SVM_Linear_Kernel_metrics_test.txt" file = open(results, "w") file.write("Support Vector Machine with Linear Kernel estimator accuracy\n") file.write("Classification Accuracy Score: %f\n"%metrics.accuracy_score(y_test, y_pred)) file.write("Precision Score: %f\n"%metrics.precision_score(y_test, y_pred)) file.write("Recall Score: %f\n"%metrics.recall_score(y_test, y_pred)) file.write("F1 Score: %f\n"%metrics.f1_score(y_test, y_pred)) file.write("\n") file.write("True Value, Predicted Value, Iteration\n") for n in xrange(len(y_test)): file.write("%f,%f,%i\n"%(y_test[n],y_pred[n],(n+1))) file.close() title = "SVC linear %f"%test_size save = Output + "SVC_linear_confusion_matrix"+"_%s.png"%test_size plot_confusion_matrix(y_test, y_pred,title,save) lvltrace.lvltrace("LVLsortie dans SVC_linear split_test")
def cv(self, X, y, eval_size=.33, nfold=3): metrics=['roc_auc','f1','recall','precision'] Xtrain, Xeval , ytrain, yeval = train_test_split(X,y,test_size=eval_size) Xtrain.reset_index(drop = True) Xeval.reset_index(drop = True) ytrain.reset_index(drop = True) yeval.reset_index(drop = True) self.fit(Xtrain, ytrain) ypred = self.predict(Xeval) yprob = self.predict_proba(Xeval) eroc = roc_auc_score(yeval, yprob) ef1 = f1_score(yeval, ypred) erecall = recall_score(yeval, ypred) eprecision = precision_score(yeval, ypred) # print confusion_matrix(yeval, ypred, labels = [0,1]) # eroc = roc_auc_score(yeval, yprob, sample_weight=sw) # ef1 = f1_score(yeval, ypred,sample_weight=sw) # erecall = recall_score(yeval, ypred, sample_weight=sw) # eprecision = precision_score(yeval, ypred, sample_weight=sw) escores = [eroc, ef1, erecall, eprecision] skfscores = [] skf = StratifiedKFold(ytrain,n_folds=nfold, random_state=2016) for trainIndex, testIndex in skf: skfxtrain, skfxtest = X.loc[trainIndex,:], X.loc[testIndex,:] skfytrain, skfytest = y.values[trainIndex], y.values[testIndex] self.fit(skfxtrain, skfytrain) ypred = self.predict(skfxtest) yprob = self.predict_proba(skfxtest) # roc = roc_auc_score(skfytest, yprob, sample_weight=sw) # f1 = f1_score(skfytest, ypred, sample_weight=sw) # recall = recall_score(skfytest, ypred,sample_weight=sw) # precision = precision_score(skfytest, ypred, sample_weight=sw) roc = roc_auc_score(skfytest, yprob) f1 = f1_score(skfytest, ypred) recall = recall_score(skfytest, ypred) precision = precision_score(skfytest, ypred) # print confusion_matrix(skfytest, ypred, labels=[0,1]) scores = [roc, f1, recall, precision] print 'cv scores:' print scores skfscores.append(scores) skfscores = np.array(skfscores) skfscores = skfscores.mean(0) report = pd.DataFrame({'eval': escores, 'train': skfscores}, index=metrics) return report
def DTree(X, Y, XTest, YTest): print '-----------------------------------------------------' # dot_data = StringIO() # tree.export_graphviz(dtree_model, out_file=dot_data) # graph = pydot.graph_from_dot_data(dot_data.getvalue()) # graph.write_pdf("../dtree.pdf") # param_grid = {'max_depth': np.arange(1, 15)} # tree_grid = GridSearchCV(DecisionTreeClassifier(), param_grid) tree_grid = DecisionTreeClassifier(max_depth=3) tree_grid.fit(X, Y) export_graphviz(tree_grid, out_file=dot_data) graph = pydot.graph_from_dot_data(dot_data.getvalue()) graph.write_pdf("dtreevis.pdf") # print("The best parameters are %s with a score of %0.2f" # % (tree_grid.best_params_, tree_grid.best_score_)) print "Computing training statistics" dtree_predict_time_training = time.time() Ypred_dtree_training = tree_grid.predict(X) dtree_predict_time_training = time.time() - dtree_predict_time_training dtree_accuracy_training = metrics.accuracy_score(Y, Ypred_dtree_training) dt_precision_training = metrics.precision_score(Y, Ypred_dtree_training, average='binary') dtree_recall_training = metrics.recall_score(Y, Ypred_dtree_training, average='binary') print "DT training prediction time: " + str(dtree_predict_time_training) print "DT training accuracy Score: " + str(dtree_accuracy_training) print "DT training precision Score: " + str(dt_precision_training) print "DT training recall Score: " + str(dtree_recall_training) print "Computing testing statistics" dtree_predict_time_test = time.time() Ypred_dtree_test = tree_grid.predict(XTest) dtree_predict_time_test = time.time() - dtree_predict_time_test dtree_accuracy_test = metrics.accuracy_score(YTest, Ypred_dtree_test) dt_precision_test = metrics.precision_score(YTest, Ypred_dtree_test, average='binary') dtree_recall_test = metrics.recall_score(YTest, Ypred_dtree_test, average='binary') print "DT test prediction time: " + str(dtree_predict_time_test) print "DT test accuracy Score: " + str(dtree_accuracy_test) print "DT test precision Score: " + str(dt_precision_test) print "DT test recall Score: " + str(dtree_recall_test) print "Creating ROC curve" y_true = YTest y_score = tree_grid.predict_proba(XTest) fprSVM, trpSVM, _ = metrics.roc_curve(y_true=y_true, y_score=y_score[:, 0], pos_label=0) plt.plot(fprSVM, trpSVM, 'r-', label='DT')
def print_scores(model, X_train, y_train, X_test, y_test): """ Compute scores for given model with training and test sets Input: model (sklearn.linear_model): the model with which to calculate scores X_train (numpy_array): training design matrix X y_train (numpy_array): training labels y X_test (numpy_array): test design matrix X y_test (numpy_array): test labels y Output: F1-score in test set Side Effects: prints the scores Comments: model must be fitted before calling this function """ y_train_predicted = model.predict(X_train) y_test_predicted = model.predict(X_test) # accuracy scores print("Accuracy") print("Train: ", model.score(X_train,y_train)) print("Test: ", model.score(X_test, y_test)) print("\n") # use precision and recall metrics from sklearn.metrics import precision_score, recall_score precision_train = precision_score(y_train, y_train_predicted) recall_train = recall_score(y_train, y_train_predicted) precision_test = precision_score(y_test, y_test_predicted) recall_test = recall_score(y_test, y_test_predicted) print("Precision and Recall") print ("Train: ", precision_train, recall_train) print ("Test: ", precision_test, recall_test) print("\n") # F1 score from tilestools import F1score f1_train = F1score(y_train, y_train_predicted) f1_test = F1score(y_test, y_test_predicted) print("F1 score") print ("Train: ", f1_train) print ("Test: ", f1_test) return f1_test
def train_folds(fit_predict, X, Y, args, n_combs=1, random_fold=False, get_train_acc=False): # Splits the data into k folds, calls the trainer function (fit_predict) # and displays results Y_test_true_all = [] Y_test_pred_all = [] if get_train_acc: Y_train_true_all = [] Y_train_pred_all = [] for i in range(n_combs): print('Cross-validating combination n ' + str(i+1)) sys.stdout.flush() if random_fold: state = None else: state = i skf = StratifiedKFold(Y, n_folds=5, random_state=state, shuffle=True) for train_index, test_index in skf: X_train = sub_list(X, train_index) X_test = sub_list(X, test_index) Y_train = sub_list(Y, train_index) Y_test = sub_list(Y, test_index) if get_train_acc: (Y_train_pred, Y_test_pred) = fit_predict(X_train, Y_train, X_test, args) else: Y_test_pred = fit_predict(X_train, Y_train, X_test, args) if get_train_acc: Y_train_true_all.append(Y_train) Y_train_pred_all.append(Y_train_pred) Y_test_true_all.append(Y_test) Y_test_pred_all.append(Y_test_pred) if get_train_acc: Y_train_true_all = concatenate(Y_train_true_all) Y_train_pred_all = concatenate(Y_train_pred_all) Y_test_true_all = concatenate(Y_test_true_all) Y_test_pred_all = concatenate(Y_test_pred_all) print('\n') print(classification_report(Y_test_true_all, Y_test_pred_all, target_names=target_names)) if get_train_acc: return (1 - recall_score(Y_train_true_all, Y_train_pred_all, average='micro'), 1 - recall_score(Y_test_true_all, Y_test_pred_all, average='micro')) else: return 1 - recall_score(Y_test_true_all, Y_test_pred_all, average='micro')
def _calc_pos_prob(self): y_pred = self.predict(self.validation_X) y_true = self.validation_y # obtaining recall scores for each label (assuming the labels are binary) pos_acc = recall_score(y_true, y_pred, average='binary', pos_label=self.positive_label) neg_acc = recall_score(y_true, y_pred, average='binary', pos_label=int(not self.positive_label)) return neg_acc / (pos_acc + neg_acc)
def train_model(datasetvar, dataset): x = datasetvar y = dataset['Churn'].values sss = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=0) print(sss) print('训练数据和测试数据被分成的组数:', sss.get_n_splits(x, y)) # 建立训练数据和测试数据 for train_index, test_index in sss.split(x, y): print('train:', train_index, 'test:', test_index) x_train, x_test = x.iloc[train_index], x.iloc[test_index] y_train, y_test = y.iloc[train_index], y.iloc[test_index] print('原始数据特征:', x.shape, '训练数据特征:', x_train.shape, '测试数据特征:', x_test.shape) print('原始数据特征:', y.shape, '训练数据特征:', y_train.shape, '测试数据特征:', y_test.shape) # 使用分类算法, 这里选用10中分类算法 Classifier = [['Random Forest', RandomForestClassifier()], ['Support Vector Machine', SVC()], ['LogisticRegression', LogisticRegression()], ['KNN', KNeighborsClassifier(n_neighbors=5)], ['Navie Bayes', GaussianNB()], ['Decision Tree', DecisionTreeClassifier()], ['AdaBosstClassifier', AdaBoostClassifier()], ['GradientBoostingClassifier', GradientBoostingClassifier()], ['XGB', XGBClassifier()], ['CatBoost', CatBoostClassifier(logging_level='silcat')]] # 训练模型 Classify_result = [] names = [] prediction = [] for name, classifier in Classifier: classifier = classifier classifier.fit(x_train, y_train) y_pred = classifier.predict(x_test) recall = recall_score(y_test, y_pred) precision = precision_score(y_test, y_pred) class_eva = pd.DataFrame([recall, precision]) Classify_result.append(class_eva) name = pd.Series(name) names.append(name) y_pred = pd.Series(y_pred) prediction.append(y_pred) # 训练模型 names = pd.DataFrame(names) names = names[0].tolist() result = pd.concat(Classify_result, axis=1) result.columns = names result.index = ['recall', 'precision', 'f1score'] print(result) # 实施方案 pred_x = datasetvar.tail(10) # 提取customerID pred_id = telcom_id.tail(10) # 使用朴素贝叶斯方法, 对预测数据集中的生存情况进行预测 model = GaussianNB() model.fit(x_train, y_train) pred_y = model.predict(pred_x) # 预测结果 predDf = pd.DataFrame({'customerID': pred_id, 'Churn': pred_y}) print(predDf)
maozi = targets['maozi'] yanjing = targets['yanjing'] RANDOM_STATE = 500 X_train, X_test, y_db_train, y_db_test = train_test_split( bottlenecks, kouzhao, test_size=0.15, random_state=RANDOM_STATE) ss = StandardScaler() X_train = ss.fit_transform(X_train) X_test = ss.transform(X_test) t1 = time.time() # clf = DecisionTreeClassifier().fit(X_train, y_db_train) clfb = BaggingClassifier(base_estimator=DecisionTreeClassifier(), max_samples=0.5, max_features=0.5).fit(X_train, y_db_train) # predict = clf.predict(X_test) predict = clfb.predict(X_test) # print(clf.score(X_test, y_db_test)) print('****************metrics***************') print(classification_report(y_db_test, predict)) print('-------precision_score:') print(precision_score(y_db_test, predict)) print('-------recall_score:') print(recall_score(y_db_test, predict)) print('-------F1_score:') print(f1_score(y_db_test, predict)) print('------------time:') print(time.time() - t1)
def main(argv=None): import os os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu_list try: os.makedirs(FLAGS.output_dir) except OSError as e: if e.errno != 17: raise """ if FLAGS.use_vacab and os.path.exists("./vocab.txt"): bk_tree = BKTree(levenshtein, list_words('./vocab.txt')) # bk_tree = bktree.Tree() """ with tf.get_default_graph().as_default() as graph: # define the placehodler input_images = tf.placeholder(tf.float32, shape=[None, None, None, 3], name='input_images') input_feature_map = tf.placeholder(tf.float32, shape=[None, None, None, 32], name='input_feature_map') input_transform_matrix = tf.placeholder(tf.float32, shape=[None, 6], name='input_transform_matrix') input_box_mask = [] input_box_mask.append( tf.placeholder(tf.int32, shape=[None], name='input_box_masks_0')) input_box_widths = tf.placeholder(tf.int32, shape=[None], name='input_box_widths') # define the model # input_seq_len = input_box_widths[tf.argmax(input_box_widths, 0)] * tf.ones_like(input_box_widths) global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False) shared_feature, f_score, f_geometry = detect_part.model(input_images) pad_rois = roi_rotate_part.roi_rotate_tensor_pad( input_feature_map, input_transform_matrix, input_box_mask, input_box_widths) recognition_logits = recognize_part.build_graph( pad_rois, input_box_widths, class_num=FLAGS.class_num) variable_averages = tf.train.ExponentialMovingAverage( 0.997, global_step) saver = tf.train.Saver(variable_averages.variables_to_restore()) stats_graph(graph) with tf.Session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: ckpt_state = tf.train.get_checkpoint_state(FLAGS.checkpoint_path) weight_len = len(ckpt_state.all_model_checkpoint_paths) # print(weight_len) for w in range(0, weight_len): model_path = os.path.join( FLAGS.checkpoint_path, os.path.basename(ckpt_state.all_model_checkpoint_paths[w])) print('Restore from {}'.format(model_path)) saver.restore(sess, model_path) im_fn_list = get_images() all_pred = [] all_gt = [] iou_list = [] during = [] img_null = 0 img_per_cls = len(im_fn_list) / FLAGS.class_num # img_per_cls = 20 print("Img_per_cls:", img_per_cls) for im_fn in im_fn_list: start_time = time.time() im = cv2.imread(im_fn)[:, :, ::-1] im_resized, (ratio_h, ratio_w) = resize_image(im) # im_resized_d, (ratio_h_d, ratio_w_d) = resize_image_detection(im) timer = {'detect': 0, 'restore': 0, 'nms': 0, 'recog': 0} start = time.time() shared_feature_map, score, geometry = sess.run( [shared_feature, f_score, f_geometry], feed_dict={input_images: [im_resized]}) boxes, timer = detect(score_map=score, geo_map=geometry, timer=timer) try: if boxes.shape[0] >= 1: # if boxes[0][1] > boxes[1][1]: # boxes[[0, 1], :] = boxes[[1, 0], :] new_boxes = [] for i, b in enumerate(boxes): b = b[:8].reshape((4, 2)) b = polygon_sort(b) # b = b[(1,2,3,0), :] if FLAGS.back_side == 0: b = b[(2, 3, 0, 1), :] new_boxes.append(b) boxes = np.asarray(new_boxes, dtype=np.float32) except AttributeError: # print("shape not found") img_null += 1 # read the ground-truth as four points # txt_dir = "{}gt_img{}".format(FLAGS.test_gt_path, (im_fn.split("img")[1]).split(".")[0]+".txt") # boxes = [] # with open(txt_dir, 'r')as fp: # all_lines = fp.readlines() # # print(all_lines[1]) # for i, b in enumerate(all_lines): # # for b in all_lines: # box = b.split(",")[:8] # box = np.asarray(box, dtype=np.float32) # box = box.reshape((4, 2)) # box = polygon_sort(box) # if i == 1: # box = box[(2, 3, 0, 1), :] # # box = sort_poly(box) # # box = box[[2, 3, 0, 1]] # box = box.reshape((-1, 8, 1)) # boxes.append(box) # boxes = np.asarray(boxes, dtype=np.float32) # boxes = boxes*0.53333333 timer['detect'] = time.time() - start # print(im_fn) # im_num = int((im_fn.split(".")[0]).split("/")[-1]) im_num = int((im_fn.split(".")[0]).split("img_")[-1]) if (im_num % img_per_cls == 0): im_gt = int((im_num / img_per_cls) - 1) else: im_gt = int(im_num / img_per_cls) # print(im_num) predict_area = np.zeros(im[:, :, ::-1].shape[:2], np.uint8) if boxes is not None and boxes.shape[0] != 0: res_file_path = os.path.join( FLAGS.output_dir, 'res_' + '{}.txt'.format( os.path.basename(im_fn).split('.')[0])) input_roi_boxes = boxes[:, :8].reshape(-1, 8) if input_roi_boxes.shape[0] == 1: tmp_roi_boxes = input_roi_boxes[0:2] boxes_masks = [0] * tmp_roi_boxes.shape[0] transform_matrixes, box_widths = get_project_matrix_and_width( tmp_roi_boxes) # run the recognition part recog_logits = sess.run(recognition_logits, feed_dict={ input_feature_map: shared_feature_map, input_transform_matrix: transform_matrixes, input_box_mask[0]: boxes_masks, input_box_widths: box_widths }) # part level np_pred = np.asarray(recog_logits) mean_pred = np.mean(np_pred, axis=0) # mean_pred = np.average(np_pred, axis=0, weights=[1, 1, 0, 1]) softmax_x = np.asarray(mean_pred).reshape( -1).tolist() softmax_x = softmax(softmax_x) softmax_x = softmax_x.reshape(-1, 1) im_pred = np.argmax(softmax_x, 0) all_pred.append(im_pred) all_gt.append(im_gt) # print("ground-truth:[{}] predict:{}".format(im_gt, im_pred)) timer['recog'] = time.time() - start duration = time.time() - start_time during.append(duration) # print('[timing] {}'.format(duration)) # Preparing for draw boxes boxes = boxes[:, :8].reshape((-1, 4, 2)) boxes[:, :, 0] /= ratio_w boxes[:, :, 1] /= ratio_h # with open(res_file_path, 'w') as f: for i, box in enumerate(boxes): # to avoid submitting errors box = sort_poly(box.astype(np.int32)) if np.linalg.norm(box[0] - box[1]) < 5 or np.linalg.norm( box[3] - box[0]) < 5: continue """ if FLAGS.use_vacab: fix_result = bktree_search(bk_tree, recognition_result.upper()) if len(fix_result) != 0: recognition_result = fix_result[0][1] """ # f.write('{},{},{},{},{},{},{},{}\r\n'.format( # box[0, 0], box[0, 1], box[1, 0], box[1, 1], box[2, 0], box[2, 1], box[3, 0], # box[3, 1] # )) box = box * (640 / 512) # Draw bounding box cv2.polylines( im[:, :, ::-1], [box.astype(np.int32).reshape((-1, 1, 2))], True, color=(0, 0, 255), thickness=3) cv2.fillPoly( predict_area, [box.astype(np.int32).reshape((-1, 1, 2))], color=(255, 255, 255)) # Draw recognition results area text_area = box.copy() text_area[2, 1] = text_area[1, 1] text_area[3, 1] = text_area[0, 1] text_area[0, 1] = text_area[0, 1] - 15 text_area[1, 1] = text_area[1, 1] - 15 # cv2.fillPoly(im[:, :, ::-1], [text_area.astype(np.int32).reshape((-1, 1, 2))], color=(255, 255, 0)) im_txt = im[:, :, ::-1] else: res_file = os.path.join( FLAGS.output_dir, 'res_' + '{}.txt'.format( os.path.basename(im_fn).split('.')[0])) f = open(res_file, "w") im_txt = None f.close() # calculate the intersection of union # gt_file = os.path.join(FLAGS.test_gt_path, # '{}.txt'.format(os.path.basename(im_fn).split('.')[0])) gt_file = os.path.join( FLAGS.test_gt_path, 'gt_' + '{}.txt'.format(os.path.basename(im_fn).split('.')[0])) gt_area = draw_gt_box(gt_file) # cv2.imshow('gt', gt_area) # cv2.imshow('pred', predict_area) # cv2.waitKey(0) # cv2.destroyAllWindows() iou = iou_cal(predict_area, gt_area) # print("IOU: ", iou) iou_list.append(iou) # print('{} : detect {:.0f}ms, restore {:.0f}ms, nms {:.0f}ms'.format( # im_fn, timer['detect'] * 1000, timer['restore'] * 1000, timer['nms'] * 1000)) if not FLAGS.no_write_images: img_path = os.path.join(FLAGS.output_dir, os.path.basename(im_fn)) # cv2.imwrite(img_path, im[:, :, ::-1]) if im_txt is not None: cv2.imwrite(img_path, im_txt) # print(confusion_matrix(all_gt, all_pred)) # np.savetxt("confusion_matrix.csv", confusion_matrix(all_gt, all_pred), delimiter=" ") average_iou = mean(iou_list) accuracy = accuracy_score(all_gt, all_pred) precision = precision_score(all_gt, all_pred, average='macro') recall = recall_score(all_gt, all_pred, average='macro') F1_score = f1_score(all_gt, all_pred, average='macro') average_time = mean(during) print( "weight: {} || accuracy:{:.4f}, precision:{:.4f}, recall:{:.4f}, f1_score:{:.4f}, IoU:{:.4f}, FPS:{:.6f} \n" .format((2500 * (w + 1)), accuracy, precision, recall, F1_score, average_iou, 1 / average_time))
def train_model(model, criterion, optimizer, scheduler, num_epochs=25): since = time.time() # all_results = open(satellite + '_results.csv', 'w') best_model_wts = model.state_dict() best_acc = 0.0 best_train_acc = 0.0 for epoch in range(num_epochs): print('Epoch {}/{}'.format(epoch, num_epochs - 1)) print('-' * 10) # Each epoch has a training and validation phase for phase in ['train', 'val']: if phase == 'train': scheduler.step() model.train(True) # Set model to training mode dataloders = dataloaders_train current_dataset = dataset_train else: model.train(False) # Set model to evaluate mode dataloders = dataloaders_test current_dataset = dataset_test dataset_size = len(current_dataset) running_loss = 0.0 running_corrects = torch.zeros(len(columns)) running_preds = None running_labels = None running_scores = None # Iterate over data. for data in dataloders: # get the inputs inputs = data['image'] if continuous: labels = data['labels'].type(torch.FloatTensor) else: labels = data['labels'].type(torch.LongTensor) # wrap them in Variable if use_gpu: inputs = Variable(inputs.cuda()) labels = Variable(labels.cuda()) else: inputs, labels = Variable(inputs), Variable(labels) # zero the parameter gradients optimizer.zero_grad() # forward if use_five_bands: convolved = convolver(inputs) outputs = model(convolved) else: outputs = model(inputs) scores = sigmoider(outputs) preds = torch.round(scores).data scores = scores.data # outputs = outputs.type(torch.cuda.LongTensor) loss = criterion(outputs.squeeze(), labels.type(torch.cuda.FloatTensor).squeeze()) # backward + optimize only if in training phase if phase == 'train': loss.backward() optimizer.step() # statistics running_loss += loss.data[0] if not continuous: running_corrects += torch.sum((preds == labels.data.type( torch.cuda.FloatTensor)).type(torch.FloatTensor), 0) if not continuous and epoch >= num_epochs - detailed_metrics_for: if running_preds is None: running_preds = preds.cpu().numpy() running_labels = labels.data.cpu().numpy() running_scores = scores.cpu().numpy() else: running_preds = np.vstack( (running_preds, preds.cpu().numpy())) running_labels = np.vstack( (running_labels, labels.data.cpu().numpy())) running_scores = np.vstack( (running_scores, scores.cpu().numpy())) # print (preds == labels.data) epoch_loss = running_loss / dataset_size epoch_acc = running_corrects.numpy() / dataset_size if not continuous and epoch >= num_epochs - detailed_metrics_for: print('%s Loss: %.4f') % (phase, epoch_loss) for i, column in enumerate(columns): column_labels = running_labels[:, i] column_preds = running_preds[:, i] column_scores = running_scores[:, i] epoch_f1 = f1_score(column_labels, column_preds) epoch_precision = precision_score(column_labels, column_preds) epoch_recall = recall_score(column_labels, column_preds) roc_score = roc_auc_score(column_labels, column_scores) print( '%s Acc: %.4f F1: %.4f Precision: %.4f Recall: %.4f ROC_score: %.4f' ) % (column, epoch_acc[i], epoch_f1, epoch_precision, epoch_recall, roc_score) print('Balance: %.4f' % current_dataset.balance[i]) false_positive_index = np.argmin(column_labels - column_scores) false_negative_index = np.argmax(column_labels - column_scores) true_positive_index = np.argmin((column_labels - column_scores) + 2 * (1 - column_labels)) true_negative_index = np.argmax((column_labels - column_scores) - 2 * column_labels) false_positive_sat_index = current_dataset.indices[ false_positive_index] + 1, column_scores[ false_positive_index], column_labels[ false_positive_index] false_negative_sat_index = current_dataset.indices[ false_negative_index] + 1, column_scores[ false_negative_index], column_labels[ false_negative_index] true_positive_sat_index = current_dataset.indices[ true_positive_index] + 1, column_scores[ true_positive_index], column_labels[ true_positive_index] true_negative_sat_index = current_dataset.indices[ true_negative_index] + 1, column_scores[ true_negative_index], column_labels[ true_negative_index] print "False positive id, score, label: %d, %.4f, %d" % false_positive_sat_index print "False negative id, score, label: %d, %.4f, %d" % false_negative_sat_index print "True positive id, score, label: %d, %.4f, %d" % true_positive_sat_index print "True negative id, score, label: %d, %.4f, %d" % true_negative_sat_index print "" # print('{} Loss: {:.4f} Acc: {:.4f} F1: {:.4f}'.format( # phase, epoch_loss, epoch_acc, epoch_f1)) else: # print('{} Loss: {:.4f} Acc: {:.4f}'.format( # phase, epoch_loss, epoch_acc)) print('%s Loss: %.4f') % (phase, epoch_loss) for i, column in enumerate(columns): epoch_f1 = f1_score(running_labels[:, i], running_preds[:, i]) print('%s Acc: %.4f') % (column, epoch_acc) # all_results.write(','.join([str(epoch), phase, str(epoch_loss), str(epoch_acc)]) + '\n') # deep copy the model if phase == 'val' and np.mean(epoch_acc) > best_acc: best_acc = np.mean(epoch_acc) best_model_wts = model.state_dict() time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60)) print('Best val Acc: {:4f}'.format(best_acc)) # load best model weights model.load_state_dict(best_model_wts) return model, best_train_acc, best_acc
def compute_score(self, conf, hy): conf['_all_f1'] = M = {str(self.le.inverse_transform([klass])[0]): f1 for klass, f1 in enumerate(f1_score(self.test_y, hy, average=None))} conf['_all_recall'] = {str(self.le.inverse_transform([klass])[0]): f1 for klass, f1 in enumerate(recall_score(self.test_y, hy, average=None))} conf['_all_precision'] = {str(self.le.inverse_transform([klass])[0]): f1 for klass, f1 in enumerate(precision_score(self.test_y, hy, average=None))} if len(self.le.classes_) == 2: conf['_macrof1'] = np.mean(np.array([v for v in conf['_all_f1'].values()])) conf['_weightedf1'] = conf['_microf1'] = f1_score(self.test_y, hy, average='binary') else: conf['_macrof1'] = f1_score(self.test_y, hy, average='macro') conf['_microf1'] = f1_score(self.test_y, hy, average='micro') conf['_weightedf1'] = f1_score(self.test_y, hy, average='weighted') conf['_accuracy'] = accuracy_score(self.test_y, hy) if self.score.startswith('avgf1:'): klist = [M[x] for x in self.score.replace('avgf1:', '').split(':')] conf['_' + self.score] = sum(klist) / len(klist) conf['_score'] = conf['_' + self.score]
def train_model(X, y, mtype, cv, epochs, cv_models_path, train, X_test=None, nfolds=None, y_test=None, rs=42, max_features=40000, maxlen=400, dropout_rate=0.25, rec_units=150, embed_dim=50, batch_size=256, max_sen_len=100, max_sent_amount=4, threshold=0.3): if cv: kf = StratifiedKFold(n_splits=nfolds, random_state=rs) auc = [] roc = [] fscore_ = [] for c, (train_index, val_index) in enumerate(kf.split(X, y)): print(f' fold {c}') X_train, X_val = X[train_index], X[val_index] y_train, y_val = y[train_index], y[val_index] tokenizer = keras.preprocessing.text.Tokenizer(num_words=max_features) tokenizer.fit_on_texts(X_train) if mtype == 'HAN': def clean_str(string): #string = string.replace(",", ".").replace(";", ".").replace(":", ".").replace("-", ".") return string.strip().lower() def tok_sentence(s): temp = tokenizer.texts_to_sequences(s) if len(temp)==0: return np.array([0]) return temp train_posts = [] train_labels = [] train_texts = [] #TRAIN for i, value in enumerate(X_train): if(i%10000==0): print(i) text = clean_str(value) train_texts.append(text) sentences = tokenize.sent_tokenize(text) sentences = tok_sentence(sentences) x = len(sentences)<max_sent_amount while x: sentences.append(np.array([0])) x = len(sentences)<max_sent_amount if len(sentences)>max_sent_amount: sentences = sentences[0:max_sent_amount] sentences = sequence.pad_sequences(sentences, maxlen=max_sen_len) train_posts.append(sentences) val_posts = [] val_labels = [] val_texts = [] #VAL for i, value in enumerate(X_val): if(i%10000==0): print(i) text = clean_str(value) val_texts.append(text) sentences = tokenize.sent_tokenize(text) sentences = tok_sentence(sentences) x = len(sentences)<max_sent_amount while x: sentences.append(np.array([0])) x = len(sentences)<max_sent_amount if len(sentences)>max_sent_amount: sentences = sentences[0:max_sent_amount] sentences = sequence.pad_sequences(sentences, maxlen=max_sen_len) val_posts.append(sentences) X_train = np.array(train_posts) y_train = np.array(y_train) X_val = np.array(val_posts) y_val = np.array(y_val) del train_posts del val_posts elif mtype =='psHAN': X_train = sequence.pad_sequences(tokenizer.texts_to_sequences(X_train), maxlen=max_sen_len*max_sent_amount) X_val = sequence.pad_sequences(tokenizer.texts_to_sequences(X_val), maxlen=max_sen_len*max_sent_amount) X_train = np.array([line.reshape(max_sent_amount,max_sen_len) for line in X_train]) X_val = np.array([line.reshape(max_sent_amount,max_sen_len) for line in X_val]) else: list_tokenized_train = tokenizer.texts_to_sequences(X_train) list_tokenized_val = tokenizer.texts_to_sequences(X_val) X_train = sequence.pad_sequences(list_tokenized_train, maxlen=maxlen) X_val = sequence.pad_sequences(list_tokenized_val, maxlen=maxlen) model = dl_model(model_type=mtype, max_features=max_features, maxlen=maxlen, dropout_rate=dropout_rate, embed_dim=embed_dim, rec_units=rec_units, max_sent_len=max_sen_len, max_sent_amount=max_sent_amount) print('Fitting') if train: model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, shuffle=True, verbose=1) model.save_weights(f'{cv_models_path}/{mtype}_fold_{c}.h5') else: model.load_weights(f'{cv_models_path}/{mtype}_fold_{c}.h5') probs = model.predict(X_val, batch_size=batch_size, verbose=1) #for threshold in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]: threshold = threshold probs_class = probs.copy() probs_class[probs_class >= threshold] = 1 probs_class[probs_class < threshold] = 0 precision = precision_score(y_val, probs_class) recall = recall_score(y_val, probs_class) fscore = f1_score(y_val, probs_class) print(f' {threshold} fold {c} precision {round(precision, 3)} recall {round(recall, 3)} fscore {round(fscore,3)}') auc_f = average_precision_score(y_val, probs) auc.append(auc_f) roc_f = roc_auc_score(y_val, probs) roc.append(roc_f) fscore_.append(fscore) print(f'fold {c} average precision {round(auc_f, 3)}') print(f'fold {c} roc auc {round(roc_f, 3)}') del model K.clear_session() print(f'PR-C {round(np.array(auc).mean(), 3)}') print(f'ROC AUC {round(np.array(roc).mean(), 3)}') print(f'FScore {round(np.array(fscore_).mean(), 3)}') print(f'PR-C std {round(np.array(auc).std(), 3)}') print(f'ROC AUC std {round(np.array(roc).std(), 3)}') print(f'FScore std {round(np.array(fscore_).std(), 3)}') else: X_train = X y_train = y tokenizer = keras.preprocessing.text.Tokenizer(num_words=max_features, oov_token='unknown') tokenizer.fit_on_texts(X_train) if mtype == 'HAN': def clean_str(string): #string = string.replace(",", ".").replace(";", ".").replace(":", ".").replace("-", ".") return string.strip().lower() def tok_sentence(s): temp = tokenizer.texts_to_sequences(s) if len(temp)==0: return np.array([0]) return temp train_posts = [] train_labels = [] train_texts = [] # FULL TRAIN for i, value in enumerate(X): if(i%10000==0): print(i) text = clean_str(value) train_texts.append(text) sentences = tokenize.sent_tokenize(text) sentences = tok_sentence(sentences) x = len(sentences)<max_sent_amount while x: sentences.append(np.array([0])) x = len(sentences)<max_sent_amount if len(sentences)>max_sent_amount: sentences = sentences[0:max_sent_amount] sentences = sequence.pad_sequences(sentences, maxlen=max_sen_len) train_posts.append(sentences) test_posts = [] test_labels = [] test_texts = [] #Test for i, value in enumerate(X_test): if(i%10000==0): print(i) text = clean_str(value) test_texts.append(text) sentences = tokenize.sent_tokenize(text) sentences = tok_sentence(sentences) x = len(sentences)<max_sent_amount while x: sentences.append(np.array([0])) x = len(sentences)<max_sent_amount if len(sentences)>max_sent_amount: sentences = sentences[0:max_sent_amount] sentences = sequence.pad_sequences(sentences, maxlen=max_sen_len) test_posts.append(sentences) X_train = np.array(train_posts) y_train = np.array(y) X_test = np.array(test_posts) y_test = np.array(y_test) del train_posts del test_posts elif mtype =='psHAN': X_train = sequence.pad_sequences(tokenizer.texts_to_sequences(X_train), maxlen=max_sen_len*max_sent_amount, padding='post') X_test = sequence.pad_sequences(tokenizer.texts_to_sequences(X_test), maxlen=max_sen_len*max_sent_amount, padding='post') X_train = np.array([line.reshape(max_sent_amount, max_sen_len) for line in X_train]) X_test = np.array([line.reshape(max_sent_amount, max_sen_len) for line in X_test]) else: list_tokenized_train = tokenizer.texts_to_sequences(X_train) list_tokenized_test = tokenizer.texts_to_sequences(X_test) X_train = sequence.pad_sequences(list_tokenized_train, maxlen=maxlen) X_test = sequence.pad_sequences(list_tokenized_test, maxlen=maxlen) y_train = np.array(y_train) y_test = np.array(y_test) model = dl_model(model_type=mtype, max_features=max_features, maxlen=maxlen, dropout_rate=dropout_rate, embed_dim=embed_dim, rec_units=rec_units, max_sent_len=max_sen_len, max_sent_amount=max_sent_amount) print('Fitting') if train: model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, shuffle=True, verbose=1) model.save_weights(f'{cv_models_path}/{mtype}.h5') else: model.load_weights(f'{cv_models_path}/{mtype}.h5') probs = model.predict(X_test, batch_size=batch_size, verbose=1) auc_f = average_precision_score(y_test, probs) roc_f = roc_auc_score(y_test, probs) threshold = threshold probs_class = probs.copy() probs_class[probs_class >= threshold] = 1 probs_class[probs_class < threshold] = 0 precision = precision_score(y_test, probs_class) recall = recall_score(y_test, probs_class) fscore = f1_score(y_test, probs_class) print('_________________________________') print(f'PR-C is {round(auc_f,3)}') print('_________________________________\n') print('_________________________________') print(f'ROC AUC is {round(roc_f,3)}') print('_________________________________') print('_________________________________') print(f'FScore is {round(fscore,3)}') print('_________________________________\n')
def train_and_test(train_data, train_label, train_seq_length, model, valid_data, valid_labels, valid_seq_length, test_file, result_save_path): with tf.Session() as sess: sess.run(tf.global_variables_initializer()) valid_acc = [] train_loss = [] for epoch_idx in range(model.config.num_epochs): batch_idx = 0 for batch_x, batch_y, batch_length in generate_batch( train_data, train_label, train_seq_length, model.config.batch_size): _states, _loss, _ = sess.run( [model._states, model.loss, model.optim], feed_dict={ model.input_x: batch_x, model.input_y: batch_y, model.keep_prob: model.config.dropout_keep_prob, model.seq_length: batch_length }) _predict, _acc = sess.run( [model.y_pred_cls, model.acc], feed_dict={ model.input_x: batch_x, model.input_y: batch_y, model.keep_prob: model.config.dropout_keep_prob, model.seq_length: batch_length }) valid_acc_batch = sess.run(model.acc, feed_dict={ model.input_x: valid_data, model.input_y: valid_labels, model.keep_prob: model.config.dropout_keep_prob, model.seq_length: valid_seq_length }) valid_acc.append(str(round(valid_acc_batch, 5))) train_loss.append(str(round(_loss, 5))) batch_idx += 1 print('epoch={} | batch={} | valid_acc={} | loss={}'.format( epoch_idx, batch_idx, round(valid_acc_batch, 5), round(_loss, 5))) each_epoch_each_batch_valid_acc = result_save_path + '_each_epoch_each_batch_valid_acc.txt' file_1 = open(each_epoch_each_batch_valid_acc, 'w', encoding='utf-8') for ii_idx, ii in enumerate(valid_acc): file_1.write( 'epoch={} | batch={} | valid_acc={} | loss={}\n'.format( epoch_idx, ii_idx, ii, train_loss[ii_idx])) # 轮次结束,开始测试 # epoch_acc = [] epoch_precision = [] epoch_recall = [] epoch_f1 = [] tmp_data, tmp_labels, tmp_seq = load_test_data( test_file, max_length) predict = sess.run( model.y_pred_cls, feed_dict={ model.input_x: tmp_data, model.input_y: tmp_labels, model.keep_prob: model.config.dropout_keep_prob, model.seq_length: tmp_seq # FIXME 维数不一致导致报错! }) tmp_labels = [0 if item[0] == 1 else 1 for item in tmp_labels] acc = metrics.accuracy_score(tmp_labels, predict) precision = metrics.precision_score(tmp_labels, predict) recall = metrics.recall_score(tmp_labels, predict) f1 = metrics.f1_score(tmp_labels, predict) epoch_acc.append(round(acc, 5)) epoch_precision.append(round(precision, 5)) epoch_recall.append(round(recall, 5)) epoch_f1.append(round(f1, 5)) each_epoch_metrics = result_save_path + '_each_epoch_metrics.txt' file_2 = open(each_epoch_metrics, 'w', encoding='utf-8') for idx in range(len(epoch_acc)): file_2.write( 'epoch={} | acc={} | precision={} | recall={} | f1={}\n'. format(idx, epoch_acc[idx], epoch_precision[idx], epoch_recall[idx], epoch_f1[idx])) print('epoch={} | acc={} | precision={} | recall={} | f1={}'.format( epoch_idx, round(acc, 5), round(precision, 5), round(recall, 5), round(f1, 5))) file_1.close() file_2.close()
features_test = [] labels_train = [] labels_test = [] for i in train_index: features_train.append(features[i]) labels_train.append(labels[i]) for j in test_index: features_test.append(features[j]) labels_test.append(labels[j]) #grid_search.fit(features_train, labels_train) #pred = grid_search.predict(features_test) clf.fit(features_train, labels_train) pred = clf.predict(features_test) precision_list.append(precision_score(pred,labels_test)) recall_list.append(recall_score(pred,labels_test)) accuracy_list.append(accuracy_score(pred,labels_test)) f1_score_list.append(f1_score(pred,labels_test)) precision = (sum(precision_list))/float(len(precision_list)) recall = (sum(recall_list))/float(len(recall_list)) accuracy = (sum(accuracy_list))/float(len(accuracy_list)) f1_score = (sum(f1_score_list))/float(len(f1_score_list)) print "Precision Score :", precision print "Recall Score :", recall print "Accuracy Score :", accuracy print "F1 Score :", f1_score print clf.named_steps['pca'].explained_variance_ratio_ print clf.named_steps['pca'].components_
y, test_size=0.25, random_state=0) shuffle_index = np.random.permutation(len(X_train)) X_train, y_train = X_train[shuffle_index], y_train[shuffle_index] # Feature Scaling from sklearn.preprocessing import StandardScaler sc_X = StandardScaler() X_train = sc_X.fit_transform(X_train) X_test = sc_X.transform(X_test) from sklearn.svm import SVC clf = SVC(kernel='linear', random_state=0) clf.fit(X_train, y_train) # Cross Validation from sklearn.model_selection import cross_val_score from sklearn.metrics import confusion_matrix from sklearn.model_selection import cross_val_predict cross_val_score(clf, X_train, y_train, cv=3, scoring='accuracy') y_train_pred = cross_val_predict(clf, X_train, y_train, cv=3) cm = confusion_matrix(y_train, y_train_pred) print(cm) from sklearn.metrics import precision_score, recall_score print("precision score = {0:.4f}".format(precision_score( y_train, y_train_pred))) print("recall score = {0:.4f}".format(recall_score(y_train, y_train_pred)))
('form', LogisticRegression())]) # evaluate model scores = { 'model': str(model), 'name': str(model), 'accuracy': [], 'precision': [], 'recall': [], 'f1': [], 'time': [], } for X_train, X_test, y_train, y_test in loader: start = time.time() model.fit(X_train, y_train) y_pred = model.predict(X_test) scores['time'].append(time.time() - start) scores['accuracy'].append(accuracy_score(y_test, y_pred)) scores['precision'].append( precision_score(y_test, y_pred, average='weighted')) scores['recall'].append(recall_score(y_test, y_pred, average='weighted')) scores['f1'].append(f1_score(y_test, y_pred, average='weighted')) print('Time: {:3.3f} Accuracy: {:0.3f}'.format( time.time() - start, accuracy_score(y_test, y_pred))) print(list(X_train)) print(y_pred) print('Final Accuracy: {:0.3f}'.format(np.mean(scores['accuracy'])))
def Recall(y_true, y_pred): return "The Recall is %f%%." % metrics.recall_score(y_true, y_pred) * 100
y_train_valid_proba_vals = lgb_clf.predict_proba(x_train_valid_transformed) unique_probas = np.unique(y_train_valid_proba_vals) thr_grid = np.linspace(np.percentile(unique_probas, 1), np.percentile(unique_probas, 99), 100) precision_scores_G, recall_scores_G = [ np.zeros(thr_grid.size), np.zeros(thr_grid.size) ] # y_train_valid_pred_probas = lgb_clf.predict_proba(x_train_valid_transformed) for gg, thr in enumerate(thr_grid): curr_thr_y_preds = y_train_valid_proba_vals[:, 1] >= thr_grid[gg] precision_scores_G[gg] = precision_score(y_train_valid, curr_thr_y_preds) recall_scores_G[gg] = recall_score(y_train_valid, curr_thr_y_preds) keep_inds = precision_scores_G >= fixed_precision if keep_inds.sum() > 0: precision_scores_G = precision_scores_G[keep_inds] recall_scores_G = recall_scores_G[keep_inds] thr_grid = thr_grid[keep_inds] best_ind = np.argmax(recall_scores_G) best_thr = thr_grid[best_ind] thr_list.append(best_thr) thr_perf_df = pd.DataFrame( np.vstack([ thr_grid[np.newaxis, :], precision_scores_G[np.newaxis, :], recall_scores_G[np.newaxis, :] ]).T,
# Train test split control splits = 5 kfold = KFold(n_splits=splits) for i, (train, test) in enumerate(kfold.split(X=X, y=y)): print(f'Currently in Run {i + 1}') X_train, X_test, y_train, y_test = X.iloc[train, :], X.iloc[ test, :], y.iloc[train], y.iloc[test] for model in models: grid_cv = GridSearchCV(estimator=models[model]['model'], param_grid=models[model]['params'], n_jobs=-1, verbose=True) grid_cv.fit(X=X_train, y=y_train) preds = grid_cv.predict(X=X_test) results[model]['metrics']['acc'].append( accuracy_score(y_true=y_test, y_pred=preds)) results[model]['metrics']['precision'].append( precision_score(y_true=y_test, y_pred=preds, pos_label='yes')) results[model]['metrics']['recall'].append( recall_score(y_true=y_test, y_pred=preds, pos_label='yes')) results[model]['probs'].append(grid_cv.predict_proba(X=X_test)) results[model]['preds'].append(preds) results[model]['parameter_rank'].append( grid_cv.cv_results_['rank_test_score']) # bestes parameterset über alle runs auswählen (nach niedrigstem kumulierten rang) grid_cv.cv_results_['params'][np.argmin(sum(results[model]['parameter_rank']))]
def test(args): label_name = ['false', 'real'] prefix = args['MODEL'] + '_' + args['BERT_CONFIG'] bert_size = args['BERT_CONFIG'].split('-')[1] device = torch.device("cuda:0" if args['CUDA'] == 'gpu' else "cpu") print('load best model...') if args['MODEL'] == 'cnn': model = CustomBertConvModel.load(prefix + '_model.bin', device) elif args['MODEL'] == 'lstm': model = CustomBertLSTMModel.load(prefix + '_model.bin', device) model.to(device) model.eval() df_test = pd.read_csv(args['--test'], index_col=0) df_test = df_test.sort_values(by='preprocessed_text_bert' + bert_size + '_length', ascending=False) test_batch_size = 32 n_batch = int(np.ceil(df_test.shape[0] / test_batch_size)) cn_loss = torch.load('loss_func', map_location=lambda storage, loc: storage).to(device) preprocessed_text_bert = list(df_test.preprocessed_text_bert) information_label = list(df_test.information_label) test_loss = 0. prediction = [] prob = [] softmax = torch.nn.Softmax(dim=1) with torch.no_grad(): for i in range(n_batch): sents = preprocessed_text_bert[i * test_batch_size:(i + 1) * test_batch_size] targets = torch.tensor( information_label[i * test_batch_size:(i + 1) * test_batch_size], dtype=torch.long, device=device) batch_size = len(sents) pre_softmax = model(sents).double() batch_loss = cn_loss(pre_softmax, targets) test_loss += batch_loss.item() * batch_size prob_batch = softmax(pre_softmax) prob.append(prob_batch) prediction.extend( [t.item() for t in list(torch.argmax(prob_batch, dim=1))]) prob = torch.cat(tuple(prob), dim=0) loss = test_loss / df_test.shape[0] pickle.dump([label_name[i] for i in prediction], open(prefix + '_test_prediction', 'wb')) pickle.dump(prob.data.cpu().numpy(), open(prefix + '_test_prediction_prob', 'wb')) accuracy = accuracy_score(df_test.information_label.values, prediction) matthews = matthews_corrcoef(df_test.information_label.values, prediction) print( f'F score on the test set: {f1_score(df_test.information_label.values, prediction)}' ) print(f'Accuracy on the test set: {accuracy}') print( 'For more information look at the metrics_csv.csv file we created for you' ) precisions = {} recalls = {} f1s = {} aucrocs = {} for i in range(len(label_name)): prediction_ = [1 if pred == i else 0 for pred in prediction] true_ = [ 1 if label == i else 0 for label in df_test.information_label.values ] f1s.update({label_name[i]: f1_score(true_, prediction_)}) precisions.update({label_name[i]: precision_score(true_, prediction_)}) recalls.update({label_name[i]: recall_score(true_, prediction_)}) aucrocs.update({ label_name[i]: roc_auc_score(true_, list(t.item() for t in prob[:, i])) }) metrics_dict = { 'loss': loss, 'accuracy': accuracy, 'matthews coef': matthews, 'precision': precisions, 'recall': recalls, 'f1': f1s, 'aucroc': aucrocs } metrics_dataframe = pd.DataFrame.from_dict(metrics_dict) metrics_dataframe.to_csv("metrics_csv.csv") pickle.dump(metrics_dict, open(prefix+'_evaluation_metrics', 'wb'))\
def getROCCurveMetrics(): import matplotlib.pyplot as plt lab = [7, 2, 4, 0, 3, 3] pred = [4, 4, 4, 0, 5, 3] classes = np.arange(0, 8) n_classes = len(classes) print(recall_score(lab, pred, labels=classes, average='micro')) print(recall_score(lab, pred, labels=classes, average='macro')) print(precision_score(lab, pred, labels=classes, average='micro')) print(precision_score(lab, pred, labels=classes, average='macro')) print(f1_score(lab, pred, labels=classes, average='micro')) print(f1_score(lab, pred, labels=classes, average='macro')) print(roc_curve(lab, pred, pos_label=0)) y_truth = label_binarize(lab, classes=classes) y_score = label_binarize(pred, classes=classes) fpr = dict() tpr = dict() roc_auc = dict() for i in range(len(classes)): fpr[i], tpr[i], _ = roc_curve(y_truth[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr["micro"], tpr["micro"], _ = roc_curve(y_truth.ravel(), y_score.ravel()) roc_auc["micro"] = auc(fpr["micro"], tpr["micro"]) all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)])) # Then interpolate all ROC curves at this points mean_tpr = np.zeros_like(all_fpr) for i in range(n_classes): mean_tpr += interp(all_fpr, fpr[i], tpr[i]) # Finally average it and compute AUC mean_tpr /= n_classes fpr["macro"] = all_fpr tpr["macro"] = mean_tpr roc_auc["macro"] = auc(fpr["macro"], tpr["macro"]) # Plot all ROC curves plt.figure() plt.plot(fpr["micro"], tpr["micro"], label='micro-average ROC curve (area = {0:0.2f})' ''.format(roc_auc["micro"]), color='deeppink', linestyle=':', linewidth=4) plt.plot(fpr["macro"], tpr["macro"], label='macro-average ROC curve (area = {0:0.2f})' ''.format(roc_auc["macro"]), color='navy', linestyle=':', linewidth=4) lw = 2 #colors = cycle(['aqua', 'darkorange', 'cornflowerblue']) #for i, color in zip(range(n_classes), colors): # plt.plot(fpr[i], tpr[i], color=color, lw=lw, # label='ROC curve of class {0} (area = {1:0.2f})' # ''.format(i, roc_auc[i])) plt.plot([0, 1], [0, 1], 'k--', lw=lw) plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title( 'Some extension of Receiver operating characteristic to multi-class') plt.legend(loc="lower right") plt.show()
def create_analysis_report(model_output, model_output_rounded, groundtruth, output_path, LABELS_LIST, validation_output=None, validation_groundtruth=None): """ Create a report of all the different evaluation metrics, including optimizing the threshold with the validation set if it is passed in the parameters """ # Create a dataframe where we keep all the evaluations, starting by prediction accuracy accuracies_perclass = sum( model_output_rounded == groundtruth) / len(groundtruth) results_df = pd.DataFrame(columns=LABELS_LIST) results_df.index.astype(str, copy=False) percentage_of_positives_perclass = sum(groundtruth) / len(groundtruth) results_df.loc[0] = percentage_of_positives_perclass results_df.loc[1] = accuracies_perclass results_df.index = ['Ratio of positive samples', 'Model accuracy'] # plot the accuracies per class results_df.T.plot.bar(figsize=(22, 12), fontsize=18) plt.title('Model accuracy vs the ratio of positive samples per class') plt.xticks(rotation=45) plt.savefig(os.path.join(output_path, "accuracies_vs_positiveRate.pdf"), format="pdf") plt.savefig(os.path.join(output_path, "accuracies_vs_positiveRate.png")) # Getting the true positive rate perclass true_positives_ratio_perclass = sum((model_output_rounded == groundtruth) * (groundtruth == 1)) / sum(groundtruth) results_df.loc[2] = true_positives_ratio_perclass # Get true negative ratio true_negative_ratio_perclass = sum( (model_output_rounded == groundtruth) * (groundtruth == 0)) / (len(groundtruth) - sum(groundtruth)) results_df.loc[3] = true_negative_ratio_perclass # compute additional metrics (AUC,f1,recall,precision) auc_roc_per_label = roc_auc_score(groundtruth, model_output, average=None) precision_perlabel = precision_score(groundtruth, model_output_rounded, average=None) recall_perlabel = recall_score(groundtruth, model_output_rounded, average=None) f1_perlabel = f1_score(groundtruth, model_output_rounded, average=None) kappa_perlabel = [ cohen_kappa_score(groundtruth[:, x], model_output_rounded[:, x]) for x in range(len(LABELS_LIST)) ] results_df = results_df.append( pd.DataFrame([ auc_roc_per_label, recall_perlabel, precision_perlabel, f1_perlabel, kappa_perlabel ], columns=LABELS_LIST)) results_df.index = [ 'Ratio of positive samples', 'Model accuracy', 'True positives ratio', 'True negatives ratio', "AUC", "Recall", "Precision", "f1-score", "Kappa score" ] # Creating evaluation plots plot_true_poisitve_vs_all_positives( model_output_rounded, groundtruth, os.path.join(output_path, 'TruePositive_vs_allPositives'), LABELS_LIST) plot_output_coocurances(model_output_rounded, os.path.join(output_path, 'output_coocurances'), LABELS_LIST) plot_false_netgatives_confusion_matrix( model_output_rounded, groundtruth, os.path.join(output_path, 'false_negative_coocurances'), LABELS_LIST) results_df['average'] = results_df.mean(numeric_only=True, axis=1) results_df.T.to_csv(os.path.join(output_path, "results_report.csv"), float_format="%.2f") return results_df
# clustering is opposite of original classification reassignflag = True kmeans_predicted_test_labels = kmeans.predict(test_features) if reassignflag: kmeans_predicted_test_labels = 1 - kmeans_predicted_test_labels #calculating confusion matrix for kmeans tn, fp, fn, tp = confusion_matrix(test_labels, kmeans_predicted_test_labels).ravel() #scoring kmeans kmeans_accuracy_score = accuracy_score(test_labels, kmeans_predicted_test_labels) kmeans_precison_score = precision_score(test_labels, kmeans_predicted_test_labels) kmeans_recall_score = recall_score(test_labels, kmeans_predicted_test_labels) kmeans_f1_score = f1_score(test_labels, kmeans_predicted_test_labels) #printing print("") print("K-Means") print("Confusion Matrix") print("tn =", tn, "fp =", fp) print("fn =", fn, "tp =", tp) print("Scores") print("Accuracy -->", kmeans_accuracy_score) print("Precison -->", kmeans_precison_score) print("Recall -->", kmeans_recall_score) print("F1 -->", kmeans_f1_score) #k_nearest_neighbours_classification:
d2_tr[rang], T_tr[rang], Y_tr[rang]) if (j % N) == 0: pred = test_step(W_te, P_te, C_te, d1_te, d2_te, T_te, Y_te) print "test data size ", len(pred) y_true = np.argmax(Y_te, 1) y_pred = pred y_true_list.append(y_true) y_pred_list.append(y_pred) for y_true, y_pred in zip(y_true_list, y_pred_list): fp.write( str( precision_score(y_true, y_pred, [1, 2, 3, 4, 5], average='weighted'))) fp.write('\t') fp.write( str( recall_score(y_true, y_pred, [1, 2, 3, 4, 5], average='weighted'))) fp.write('\t') fp.write( str(f1_score(y_true, y_pred, [1, 2, 3, 4, 5], average='weighted'))) fp.write('\t') fp.write('\n') fp.write('\n') fp.write('\n')
lr_param = {'C': [0.01, 0.1, 0.2, 0.5, 1, 1.5, 2], 'class_weight': [{1: 1, 0: 1}, {1: 2, 0: 1}, {1: 3, 0: 1}, {1: 5, 0: 1}]} lr_gsearch = GridSearchCV( estimator=LogisticRegression(random_state=0, fit_intercept=True, penalty='l2', solver='saga'), param_grid=lr_param, cv=3, scoring='f1', n_jobs=-1, verbose=2) lr_gsearch.fit(x_train,y_train) LR_model_2=LogisticRegression(C=lr_gsearch.best_params_['C'],penalty='l2',solver='saga',class_weight=lr_gsearch.best_params_['class_weight']) LR_model=LR_model_2.fit(x_train,y_train) '''模型评估,使用ROC评估模型效果''' from sklearn.metrics import roc_curve, auc,confusion_matrix,recall_score,precision_score,accuracy_score y_pred=LR_model.predict(x_test) cnf_matrix = confusion_matrix(y_test, y_pred) recall_value = recall_score(y_test, y_pred) precision_value = precision_score(y_test, y_pred) acc = accuracy_score(y_test, y_pred) print(cnf_matrix) '''绘制ROC曲线,评估结果AUC为0.66''' y_score_test = LR_model.predict_proba(x_test)[:, 1] fpr, tpr, thresholds = roc_curve(y_test, y_score_test) roc_auc = auc(fpr, tpr) ks = max(tpr - fpr) ar = 2*roc_auc-1 print('test set: model AR is {0},and ks is {1},auc={2}'.format(ar,ks,roc_auc)) import matplotlib.pyplot as plt
model = train_loop(dataloaders, dataset_sizes, num_classes, config=config, epochs=15) y_pred = np.array([]) for i in tqdm(range(len(test_set))): inputs = torch.Tensor([test_set[i][0]]).to(device) model.eval() outputs = model(inputs) preds = torch.max(outputs, 1)[1] y_pred = np.append(y_pred, preds.cpu().numpy()) acc_arr.append(accuracy_score(metric_test, y_pred)) acc_cum += acc_arr[fold_number-1] rec_arr.append(recall_score(metric_test, y_pred, average='macro')) rec_cum += rec_arr[fold_number-1] pre_arr.append(precision_score(metric_test, y_pred, average='macro')) pre_cum += pre_arr[fold_number-1] f1_arr.append(f1_score(metric_test, y_pred, average='macro')) f1_cum += f1_arr[fold_number-1] f1_arr_mic.append(f1_score(metric_test, y_pred, average='micro')) f1_cum_mic += f1_arr_mic[fold_number-1] fold_number+=1 print("Accuracy: ", acc_cum/5) print("Recall: ", rec_cum/5) print("Precision: ", pre_cum/5) print("F1 score: ", f1_cum/5) print("F1 score Micro: ", f1_cum_mic/5)
def main(): import time import prettytable from collections import Counter from sklearn import tree from sklearn import metrics from sklearn import preprocessing from imblearn.datasets import fetch_datasets from imblearn.metrics import geometric_mean_score from sklearn.model_selection import StratifiedKFold, RepeatedStratifiedKFold start_time = time.time() dataset = fetch_datasets()['satimage'] X = dataset.data y = dataset.target print(Counter(y)) cv = StratifiedKFold(n_splits=10, random_state=42, shuffle=True) # cv = RepeatedStratifiedKFold(n_repeats=5, n_splits=10, random_state=42) dic = {'recall': [], 'precision': [], 'f1': [], 'auc': [], 'gmean': []} results = prettytable.PrettyTable( ["Classifier", "Precision", 'Recall', 'F-measure', 'AUC', 'G-mean']) for train, test in cv.split(X, y): # preprocessing scaler = preprocessing.MinMaxScaler().fit(X[train]) X_train_minmax = scaler.transform(X[train]) X_test_minmax = scaler.transform(X[test]) # training sb = CGMOS(ratio=0.5, sigmafactor=1, random_state=42) # testing X_res, y_res = sb.fit_sample(X_train_minmax, y[train]) model = tree.DecisionTreeClassifier(max_depth=8, min_samples_split=10, random_state=42) model.fit(X_res, y_res) predict = model.predict(X_test_minmax) probability = model.predict_proba(X_test_minmax)[:, 1] precision = metrics.precision_score(y[test], predict) recall = metrics.recall_score(y[test], predict) if precision == 0: f1 = 0 else: f1 = 2 * (precision * recall) / (precision + recall) auc = metrics.roc_auc_score(y[test], probability) gmean = geometric_mean_score(y[test], predict) dic['precision'].append(precision) dic['recall'].append(recall) dic['f1'].append(f1) dic['auc'].append(auc) dic['gmean'].append(gmean) results.add_row([ 'CGMOS', np.mean(np.array(dic['precision'])), np.mean(np.array(dic['recall'])), np.mean(np.array(dic['f1'])), np.mean(np.array(dic['auc'])), np.mean(np.array(dic['gmean'])) ]) print(results) print('CGMOS building id transforming took %fs!' % (time.time() - start_time))
def calculate(self): return GenericEvaluatorResults(metrics.recall_score( y_pred=np.array(self._outputs).argmax(axis=-1), y_true=np.array(self._targets), average=self._average ), self._average + '-recall', '%5.4f', is_max_better=True)
print("Best Selected Parameters:") print(tuned_model.best_params_) #Predict on test data. y_pred_test contains predictions for each sample y_pred_test = tuned_model.best_estimator_.predict(X_data_test) # print the prediction on test sets (if needed) #print( y_pred_test) #get class wise precision score and store the results in a list #Set 'average=None' to get class wise results precisionResult = precision_score(y_data_test, y_pred_test, average=None) precScoreList.append(precisionResult) #get class wise recall score and store the results in a list recallResult = recall_score(y_data_test, y_pred_test, average=None) recallScoreList.append(recallResult) #get class wise f-measure score and store the results in a list fScoreResult = f1_score(y_data_test, y_pred_test, average=None) fScoreList.append(fScoreResult) print() print() print( "***Scikit learn will set a metric (e.g. recall) value to zero and display a warning message " ) print("when no samples present for a particular class in the test set***") #For loop ends here #Print the results of the list thta contain the results #print(precScoreList)
checkpointer = callbacks.ModelCheckpoint(filepath="kddresults/dnn4layer/checkpoint-{epoch:02d}.hdf5", verbose=1, save_best_only=True, monitor='loss') csv_logger = CSVLogger('kddresults/dnn4layer/training_set_dnnanalysis.csv',separator=',', append=False) model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=100, callbacks=[checkpointer,csv_logger]) model.save("kddresults/dnn4layer/dnn4layer_model.hdf5") ''' score = [] name = [] from sklearn.metrics import confusion_matrix import os for file in os.listdir("kddresults/dnn4layer/"): model.load_weights("kddresults/dnn4layer/"+file) y_train1 = y_test y_pred = model.predict_classes(X_test) accuracy = accuracy_score(y_train1, y_pred) recall = recall_score(y_train1, y_pred , average="binary") precision = precision_score(y_train1, y_pred , average="binary") f1 = f1_score(y_train1, y_pred, average="binary") print("----------------------------------------------") print("accuracy") print("%.3f" %accuracy) print("recall") print("%.3f" %recall) print("precision") print("%.3f" %precision) print("f1score") print("%.3f" %f1) score.append(accuracy) name.append(file)
# First classifier. from sklearn.naive_bayes import GaussianNB from sklearn.metrics import accuracy_score from sklearn.cross_validation import StratifiedShuffleSplit from sklearn import grid_search from sklearn.pipeline import Pipeline from sklearn.metrics import precision_score from sklearn.metrics import recall_score clf_nb = GaussianNB() clf_nb.fit(features_train, labels_train) pred = clf_nb.predict(features_test) accuracy_nb = accuracy_score(pred, labels_test) print 'NB Precision:', precision_score(labels_test, pred) print 'NB Recall:', recall_score(labels_test, pred) print 'NB Accuracy:', accuracy_nb # Support Vector Machines (SVM) supervised classification algorithm and # accuracy testing. # Second classifier. from sklearn.svm import SVC clf_svm = SVC() clf_svm.fit(features_train, labels_train) pred = clf_svm.predict(features_test) accuracy_svm = accuracy_score(pred, labels_test) print 'SVM Precision:', precision_score(labels_test, pred) print 'SVM Recall:', recall_score(labels_test, pred) print 'SVM Accuracy:', accuracy_svm
def calculate_sample_metrics(nclasses, agg, gt, probs, doc_starts, print_per_class_results=False): result = -np.ones(len(SCORE_NAMES) - 3) gt = gt.astype(int) probs[np.isnan(probs)] = 0 probs[np.isinf(probs)] = 0 # token-level metrics result[0] = skm.accuracy_score(gt, agg) # the results are undefined if some classes are not present in the gold labels prec_by_class = skm.precision_score(gt, agg, average=None, labels=range(nclasses)) rec_by_class = skm.recall_score(gt, agg, average=None, labels=range(nclasses)) f1_by_class = skm.f1_score(gt, agg, average=None, labels=range(nclasses)) if print_per_class_results: print('Token Precision:') print(prec_by_class) print('Token Recall:') print(rec_by_class) print('Token F1:') print(f1_by_class) result[1] = np.mean(prec_by_class[np.unique(gt)]) result[2] = np.mean(rec_by_class[np.unique(gt)]) result[3] = np.mean(f1_by_class[np.unique(gt)]) # span-level metrics - strict p, r, f = strict_span_metrics_2(agg, gt, doc_starts) result[6] = p # precision(agg, gt, True, doc_starts) result[7] = r # recall(agg, gt, True, doc_starts) result[8] = f # f1(agg, gt, True, doc_starts) # span-level metrics -- relaxed result[9] = precision(agg, gt, False, doc_starts) result[10] = recall(agg, gt, False, doc_starts) result[11] = f1(agg, gt, False, doc_starts) auc_score = 0 total_weights = 0 for i in range(probs.shape[1]): if not np.any(gt == i) or np.all(gt == i) or np.any( np.isnan(probs[:, i])) or np.any(np.isinf(probs[:, i])): print( 'Could not evaluate AUC for class %i -- all data points have same value.' % i) continue auc_i = skm.roc_auc_score(gt == i, probs[:, i]) # print 'AUC for class %i: %f' % (i, auc_i) auc_score += auc_i * np.sum(gt == i) total_weights += np.sum(gt == i) if print_per_class_results: print('AUC for class %i = %f' % (i, auc_i)) result[4] = auc_score / float(total_weights) if total_weights > 0 else 0 result[5] = skm.log_loss(gt, probs, eps=1e-100, labels=np.arange(nclasses)) return result
print(X_train) y_train, y_test = target[train_index], target[test_index] print(y_train) for i in range(5, 12): clf = tree.DecisionTreeClassifier(max_depth=i) print('--------------------{}---------------------------------'.format(i)) print('cross_val_predict') predicted = cross_val_predict( clf, train, target, cv=10, ) # predict y values for the test fold print('mean recall in all classes:') print(met.recall_score(target, predicted, average=None)) print('mean precision in all classes') print(met.precision_score(target, predicted, average=None)) print('mean accuracy in all classes:') print(met.accuracy_score(target, predicted)) print('----------------------------------------') fig, ax = plt.subplots() ax.scatter(target, predicted, edgecolors=(0, 0, 0)) ax.plot([target.min(), target.max()], [target.min(), target.max()], 'k--', lw=4) ax.set_xlabel('Measured k {}'.format(i))
def conf_mat(y_test, y_train, y_pred, y_train_pred, directory): # IMPORTANT: first argument is true values, second argument is predicted values # this produces a 2x2 numpy array (matrix) conf_mat_test = metrics.confusion_matrix(y_test, y_pred) conf_mat_3CV = metrics.confusion_matrix(y_train, y_train_pred) def draw_conf_mat(matrix, directory): datestring = datetime.strftime(datetime.now(), '%Y%m%d_%H%M') labels = ['0', '1'] ax = plt.subplot() sns.set(font_scale=1.5) sns.heatmap(matrix, annot=True, ax=ax, annot_kws={'size': 18}, vmin=0, vmax=12) # plt.title('Confusion matrix of the classifier') ax.set_xticklabels(labels, fontdict={'fontsize': 18}) ax.set_yticklabels(labels, fontdict={'fontsize': 18}) plt.xlabel('Predicted') plt.ylabel('True') plt.tight_layout() plt.savefig(os.path.join( directory, 'confusion_matrix_tree_rand_ada_' + datestring + '.png'), dpi=600) plt.close() draw_conf_mat(conf_mat_test, directory) #draw_conf_mat(conf_mat_3CV, directory, 'train_CV_') TP = conf_mat_test[1, 1] TN = conf_mat_test[0, 0] FP = conf_mat_test[0, 1] FN = conf_mat_test[1, 0] TP_CV = conf_mat_3CV[1, 1] TN_CV = conf_mat_3CV[0, 0] FP_CV = conf_mat_3CV[0, 1] FN_CV = conf_mat_3CV[1, 0] with open( os.path.join(directory, 'decisiontree_ada_randomsearch.txt'), 'a') as text_file: text_file.write('confusion matrix using test set: %s \n' % conf_mat_test) text_file.write('confusion matrix using 3-fold CV: %s \n' % conf_mat_3CV) text_file.write( 'Slicing confusion matrix for test set into: TP, TN, FP, FN \n' ) text_file.write( 'Slicing confusion matrix for 3-fold CV into: TP_CV, TN_CV, FP_CV, FN_CV \n' ) #calculate accuracy acc_score_man_test = (TP + TN) / float(TP + TN + FP + FN) acc_score_sklearn_test = metrics.accuracy_score(y_test, y_pred) acc_score_man_CV = (TP_CV + TN_CV) / float(TP_CV + TN_CV + FP_CV + FN_CV) acc_score_sklearn_CV = metrics.accuracy_score( y_train, y_train_pred) with open( os.path.join(directory, 'decisiontree_ada_randomsearch.txt'), 'a') as text_file: text_file.write('Accuracy score: \n') text_file.write('accuracy score manual test: %s \n' % acc_score_man_test) text_file.write('accuracy score sklearn test: %s \n' % acc_score_sklearn_test) text_file.write('accuracy score manual CV: %s \n' % acc_score_man_CV) text_file.write('accuracy score sklearn CV: %s \n' % acc_score_sklearn_CV) #classification error class_err_man_test = (FP + FN) / float(TP + TN + FP + FN) class_err_sklearn_test = 1 - metrics.accuracy_score(y_test, y_pred) class_err_man_CV = (FP_CV + FN_CV) / float(TP_CV + TN_CV + FP_CV + FN_CV) class_err_sklearn_CV = 1 - metrics.accuracy_score( y_train, y_train_pred) with open( os.path.join(directory, 'decisiontree_ada_randomsearch.txt'), 'a') as text_file: text_file.write('Classification error: \n') text_file.write('classification error manual test: %s \n' % class_err_man_test) text_file.write('classification error sklearn test: %s \n' % class_err_sklearn_test) text_file.write('classification error manual CV: %s \n' % class_err_man_CV) text_file.write('classification error sklearn CV: %s \n' % class_err_sklearn_CV) #sensitivity/recall/true positive rate; correctly placed positive cases sensitivity_man_test = TP / float(FN + TP) sensitivity_sklearn_test = metrics.recall_score(y_test, y_pred) sensitivity_man_CV = TP_CV / float(FN_CV + TP_CV) sensitivity_sklearn_CV = metrics.recall_score( y_train, y_train_pred) with open( os.path.join(directory, 'decisiontree_ada_randomsearch.txt'), 'a') as text_file: text_file.write('Sensitivity/Recall/True positives: \n') text_file.write('sensitivity manual test: %s \n' % sensitivity_man_test) text_file.write('sensitivity sklearn test: %s \n' % sensitivity_sklearn_test) text_file.write('sensitivity manual CV: %s \n' % sensitivity_man_CV) text_file.write('sensitivity sklearn CV: %s \n' % sensitivity_sklearn_CV) #specificity specificity_man_test = TN / (TN + FP) specificity_man_CV = TN_CV / (TN_CV + FP_CV) with open( os.path.join(directory, 'decisiontree_ada_randomsearch.txt'), 'a') as text_file: text_file.write('Specificity: \n') text_file.write('specificity manual test: %s \n' % specificity_man_test) text_file.write('specificity manual CV: %s \n' % specificity_man_CV) #false positive rate false_positive_rate_man_test = FP / float(TN + FP) false_positive_rate_man_CV = FP_CV / float(TN_CV + FP_CV) with open( os.path.join(directory, 'decisiontree_ada_randomsearch.txt'), 'a') as text_file: text_file.write('False positive rate or 1-specificity: \n') text_file.write('false positive rate manual test: %s \n' % false_positive_rate_man_test) text_file.write('1 - specificity test: %s \n' % (1 - specificity_man_test)) text_file.write('false positive rate manual CV: %s \n' % false_positive_rate_man_CV) text_file.write('1 - specificity CV: %s \n' % (1 - specificity_man_CV)) #precision/confidence of placement precision_man_test = TP / float(TP + FP) precision_sklearn_test = metrics.precision_score(y_test, y_pred) precision_man_CV = TP_CV / float(TP_CV + FP_CV) precision_sklearn_CV = metrics.precision_score( y_train, y_train_pred) with open( os.path.join(directory, 'decisiontree_ada_randomsearch.txt'), 'a') as text_file: text_file.write( 'Precision or confidence of classification: \n') text_file.write('precision manual: %s \n' % precision_man_test) text_file.write('precision sklearn: %s \n' % precision_sklearn_test) text_file.write('precision manual CV: %s \n' % precision_man_CV) text_file.write('precision sklearn CV: %s \n' % precision_sklearn_CV) #F1 score; uses precision and recall f1_score_sklearn_test = f1_score(y_test, y_pred) f1_score_sklearn_CV = f1_score(y_train, y_train_pred) with open( os.path.join(directory, 'decisiontree_ada_randomsearch.txt'), 'a') as text_file: text_file.write('F1 score: \n') text_file.write('F1 score sklearn test: %s \n' % f1_score_sklearn_test) text_file.write('F1 score sklearn CV: %s \n' % f1_score_sklearn_CV)
from prepare import readbunchobj data = readbunchobj('dataset_woe.data') X_train = pd.DataFrame(data.X_train) X_test = data.X_test y_train = data.y_train y_test = data.y_test # # 缺失值插补 # imp = SimpleImputer(strategy='mean') # 均值 单变量插补 # X_train = imp.fit_transform(X_train) # 训练集插补 # X_test = imp.transform(X_test) # 测试集插补 # # # 归一化 # prep = StandardScaler() # X_train = prep.fit_transform(X_train) # X_test = prep.transform(X_test) if0 = IsolationForest(bootstrap=True, n_jobs=-1, random_state=10) if0.fit(X_train) y_pred = if0.predict(X_test) y_pred[y_pred == 1] = 0 y_pred[y_pred == -1] = 1 c_m = metrics.confusion_matrix(y_test, y_pred) print('真反例:{0}\n假反例:{1}\n真正例:{2}\n假正例:{3}\n'.format(c_m[0][0], c_m[1][0], c_m[1][1], c_m[0][1])) print("召回率:%.4f" % metrics.recall_score(y_test, y_pred)) print("查准率:%.4f" % metrics.precision_score(y_test, y_pred)) print("F1:%.4f" % metrics.f1_score(y_test, y_pred)) print("roc_auc:%.4f" % metrics.roc_auc_score(y_test, y_pred))
] y_pred_test_6_with_update = [ -1, 22, 11, 15, 11, 22, 11, 22, 22, 22, 22, 22, 11, 22, 19, 22, 22, 22, 22, 22, 22, 22, 11, 22, 22 ] y_pred_test_6 = [ 15, -1, 15, 15, 15, 22, 22, 22, 22, 11, 19, 19, 19, 22, 19, 22, 11, 22, 22, 22, 22, 22, 22, 11, 11 ] if __name__ == '__main__': accuracy = accuracy_score(y_true, y_pred_test_1) precision = precision_score(y_true, y_pred_test_1, average="weighted") recall = recall_score(y_true, y_pred_test_1, average="weighted") print("Reconhecedor estático") print(f"Accuracy: {accuracy}") print(f"Precision: {precision}") print(f"Recall: {recall}") accuracy = accuracy_score(y_true, y_pred_test_2) precision = precision_score(y_true, y_pred_test_2, average='weighted') recall = recall_score(y_true, y_pred_test_2, average='weighted') print("Reconhecedor estático apontando para uma foto no computador") print(f"Accuracy: {accuracy}") print(f"Precision: {precision}") print(f"Recall: {recall}")