def tune_threshold(y, y_prob, eta=0.1, plev=0.5, max_iter=100, output=True): """ Tunes the threshold of the decision rule to improve accuracy. Keyword Arguments: y - theground truth y_prob - the model predictions eta - learning rate plev - the level of precision we are trying to maintain """ threshold = 0.5 yhat = decide(y_prob, threshold) p = precision(y, yhat) r = recall(y, yhat) initial_loss = directional_loss(y, yhat) if output: print(f"Precision = {p}, Recall = {r}, Threshold = {threshold}") for i in range(1, max_iter): threshold -= eta / i * threshold yhat = decide(y_prob, threshold) p = precision(y, yhat) r = recall(y, yhat) if output: print(f"Precision = {p}, Recall = {r}, Threshold = {threshold}") if (p <= plev): return threshold return threshold
def present_results_simp(y_test, predictions): results_list = [] for k, v in predictions.items(): inter_list = [ k, accuracy(v, y_test), precision(v, y_test), precision_top(v, y_test, 0.01), precision_top(v, y_test, 0.02), precision_top(v, y_test, 0.05), precision_top(v, y_test, 0.1), precision_top(v, y_test, 0.2), precision_top(v, y_test, 0.3), precision_top(v, y_test, 0.5), recall(v, y_test), recall_top(v, y_test, 0.01), recall_top(v, y_test, 0.02), recall_top(v, y_test, 0.05), recall_top(v, y_test, 0.1), recall_top(v, y_test, 0.2), recall_top(v, y_test, 0.3), recall_top(v, y_test, 0.5), f1(v, y_test) ] results_list.append(inter_list) df = pd.DataFrame(results_list) df.columns = [ 'Model', 'Accuracy', 'Precision', 'Precision top 1%', 'Precision top 2%', 'Precision top 5%', 'Precision top 10%', 'Precision top 20%', 'Precision top 30%', 'Precision top 50%', 'Recall', 'Recall top 1%', 'Recall top 2%', 'Recall top 5%', 'Recall top 10%', 'Recall top 20%', 'Recall top 30%', 'Recall top 50%', 'F 1' ] return df
def evaluate_dectree(x_train, y_train, x_test, y_test, thresh=thresh): ''' you get it ''' criterion = ['entropy', 'gini'] rd = { 'predicted': [], 'crit': [], 'threshold': [], 'precision': [], 'recall': [], 'accuracy': [], 'class': [] } for c in criterion: scores = dectree_classifier(x_train, y_train, x_test, c) for t in thresh: scores = list(stats.rankdata(scores, 'average') / len(scores)) preds = [compare_to_threshold(x, t) for x in list(scores)] rd['predicted'].append(preds) rd['crit'].append(c) rd['threshold'].append(t) rd['precision'].append(precision(y_test, preds)) rd['recall'].append(recall(y_test, preds)) rd['accuracy'].append(accuracy(y_test, preds)) rd['class'].append('dectree') return pd.DataFrame(rd)
def evaluate_rf(x_train, y_train, x_test, y_test, thresh=thresh, ntrees=[25, 100, 500], maxfeats=[1, .5, 4]): rd = { 'predicted': [], 'ntrees': [], 'nfeats': [], 'threshold': [], 'precision': [], 'recall': [], 'accuracy': [], 'class': [] } for size in ntrees: for f in maxfeats: scores = random_forest_classifier(size, f, x_train, y_train, x_test) for t in thresh: scores = list(stats.rankdata(scores, 'average') / len(scores)) preds = [compare_to_threshold(x, t) for x in scores] rd['predicted'].append(preds) rd['ntrees'].append(size) rd['nfeats'].append(f) rd['threshold'].append(t) rd['precision'].append(precision(y_test, preds)) rd['recall'].append(recall(y_test, preds)) rd['accuracy'].append(accuracy(y_test, preds)) rd['class'].append('rf') return pd.DataFrame(rd)
def evaluate_logreg(x_train, y_train, x_test, y_test, c_values=[.01,.1,1,10,100], thresh=thresh): ''' generates df of predictions, penalties, c_values, thresholds, precision, recall, and accuracy of logistic regression ''' penalties = ['l2'] rd = {'predicted': [], 'penalty': [], 'C': [], 'threshold': [], 'precision': [], 'recall': [], 'accuracy':[], 'class': []} for p in penalties: for c in c_values: scores = logreg_classifier(x_train, y_train, x_test, c, p) for t in thresh: scores = list(stats.rankdata(scores, 'average')/len(scores)) preds = [compare_to_threshold(x, t)for x in scores] rd['predicted'].append(preds) rd['penalty'].append(p) rd['C'].append(c) rd['threshold'].append(t) rd['precision'].append(precision(y_test, preds)) rd['recall'].append(recall(y_test, preds)) rd['accuracy'].append(accuracy(y_test, preds)) rd['class'].append('logreg') return pd.DataFrame(rd)
def evaluate_knn(x_train, y_train, x_test, y_test, kays=[3, 5, 7, 9, 11], thresh=thresh): ''' generates df of predictions, penalties, k values, thresholds, precision, recall, and accuracy to help find best model ''' rd = { 'predicted': [], 'k': [], 'threshold': [], 'precision': [], 'recall': [], 'accuracy': [], 'class': [] } for k in kays: scores = knn_classifier(x_train, y_train, x_test, k)[:, 1] for t in thresh: scores = list(stats.rankdata(scores, 'average') / len(scores)) preds = [compare_to_threshold(x, t) for x in scores] rd['predicted'].append(preds) rd['k'].append(k) rd['threshold'].append(t) rd['precision'].append(precision(y_test, preds)) rd['recall'].append(recall(y_test, preds)) rd['accuracy'].append(accuracy(y_test, preds)) rd['class'].append('knn') return pd.DataFrame(rd)
def perform_SER(self, input, target, device): LOAD_PATH = '/home/zhoukun/SER/Speech-Emotion-Recognition-main/checkpoint/best_model_esd.pth' model_SER = acrnn().to('cpu') model_SER.load_state_dict(torch.load(LOAD_PATH, map_location='cpu')) criterion = torch.nn.CrossEntropyLoss() best_valid_uw = 0 model_SER.eval() size = input.shape[0] with torch.no_grad(): inputs = torch.tensor(input).to('cpu') targets = torch.tensor(target, dtype=torch.long).to('cpu') outputs, emotion_embedding_low, emotion_embedding_high = model_SER( inputs) loss = criterion(outputs, targets).cpu().detach().numpy() cost_valid = np.sum(loss) / size valid_acc_uw = recall(target.cpu().detach().numpy(), np.argmax(outputs.cpu().detach().numpy(), 1), average='macro') valid_conf = confusion(target.cpu().detach().numpy(), np.argmax(outputs.cpu().detach().numpy(), 1)) if valid_acc_uw > best_valid_uw: best_valid_uw = valid_acc_uw best_valid_conf = valid_conf cost_valid = torch.tensor(cost_valid).to(device) emotion_embedding_high = torch.tensor(emotion_embedding_high).to( device) best_valid_uw = torch.tensor(best_valid_uw).to(device) return cost_valid, emotion_embedding_high, best_valid_uw
def get_metrics(prediction, y_test): ''' Computes accuracy, precision, recall, ROC-AUC and F1 metrics for consideroing predictions produced by a ML and actual values of a dependent variables. Inputs: - prediction: an array with predictions. - y_test: an array with actual values. Returns a dictionary with metrics of a ML model. ''' Accuracy = accuracy(prediction, y_test) Precision = precision(prediction, y_test) Recall = recall(prediction, y_test) try: AUC = roc_auc(prediction, y_test) except ValueError: AUC = 0 F1 = f1(prediction, y_test) metrics_dict = { 'Accuracy': Accuracy, 'Precision': Precision, 'Recall': Recall, 'AUC': AUC, 'F1': F1 } return metrics_dict
def tune_model_width(build_fn, x_train, y_train, x_val, y_val, max_width=50): """ Takes a 3-Layer nueral network and expands width to see if there are tangible benefits to increasing the width of the hidden layer in the model. Parameters: build_fn - function that returns a keras nn model with the specified parameters x_train - the data matrix y_train - the response function x_val - validation data y_val - validation data function """ acc = [] pre = [] rec = [] for i in range(15, max_width): width = i model = feed_forward.build_model(x_train, y_train, width=width, suppress=True) model.fit(x_train, y_train, epochs=100, verbose=0) y_val_prob = model.predict(x_val)[:, 0] y_val_hat = decide(y_val_prob, 0.5) acc.append(accuracy(y_val, y_val_hat)) pre.append(precision(y_val, y_val_hat)) rec.append(recall(y_val, y_val_hat)) return acc, pre, rec
def main(): train_data, train_labels, test_data, test_labels = test_train_split() predicted_output = predict(train_data, test_data[:, :57]) print("confusion matrix : \n", cm(test_labels, predicted_output)) print("Recall : ", recall(test_labels, predicted_output)) print("Accuracy:", accuracy_score(test_labels, predicted_output) * 100, "%") print("precision : ", precision_score(test_labels, predicted_output))
def print_prediction_metrics(clf, x, y, k): pred = cross_val_predict(clf, x, y, cv=StratifiedKFold(n_splits=k, shuffle=True)) print("Accuracy: ", round(accuracy(y, pred), 2)) print("Precision on spam: ", round(precision(y, pred, average=None)[1], 3)) print("Recall on spam: ", round(recall(y, pred, average=None)[1], 3)) return
def update_metrics(gt, pre, f1_m, p_m, r_m, acc_m): f1_value = f1(gt, pre, average="micro") f1_m.update(f1_value) p_value = precision(gt, pre, average="micro", zero_division=0) p_m.update(p_value) r_value = recall(gt, pre, average="micro") r_m.update(r_value) acc_value = accuracy(gt, pre) acc_m.update(acc_value)
def __init__(self, model, parameters, name, threshold, x_train, y_train, x_test, y_test): self.params = parameters self.model = model.set_params(**parameters) self.scores = classify(x_train, y_train, x_test, self.model) self.truth = y_test self.predictions = predict(self.scores, threshold) self.accuracy = accuracy(self.truth, self.predictions) self.precision = precision(self.truth, self.predictions) self.recall = recall(self.truth, self.predictions) self.f1 = 2 * (self.precision * self.recall) / (self.precision + self.recall) self.name = None ClassifierAnalyzer.identifier += 1
def evaluate_bagging(x_train, y_train, x_test, y_test, thresh=thresh): rd = {'predicted': [], 'threshold': [], 'precision': [], 'recall': [], 'accuracy':[], 'class': []} scores = bagging_classifier(x_train, y_train, x_test) for t in thresh: scores = list(stats.rankdata(scores, 'average')/len(scores)) preds = [compare_to_threshold(x, t) for x in list(scores)] rd['predicted'].append(preds) rd['threshold'].append(t) rd['precision'].append(precision(y_test, preds)) rd['recall'].append(recall(y_test, preds)) rd['accuracy'].append(accuracy(y_test, preds)) rd['class'].append('bagging') return pd.DataFrame(rd)
def recall(self, predictions): """ Calculate recall score given the predictions. Parameters ---------- predictions : ndarray Model's predictions. Returns ------- int Recall score (using `macro` method.) """ return recall(self.target, predictions, average="macro")
def calculate_recall(predicted_scores, y_test, threshold): ''' Calculate the recall of the trained model. Inputs: predicted_scores (numpy array) - probabilities that data points belong to class 1 y_test (pandas dataframe) - label testing data threshold (float) - if predicted score is above this threshold, consider it to be class 1 Outputs: test_recall (float) ''' predictions = get_predictions_with_threshold(predicted_scores, threshold) test_recall = recall(y_test, predictions) return test_recall
def __init__(self, model, parameters, name, threshold, x_train, y_train, x_test, y_test): self.params = parameters self.t = threshold self.model = model.set_params(**parameters) self.scores = classify(x_train, y_train, x_test, self.model) self.truth = y_test self.predictions = predict(self.scores, self.t) self.predicted_for_pct = sum(self.predictions) / len(self.predictions) self.accuracy = accuracy(self.truth, self.predictions) self.precision = precision(self.truth, self.predictions) self.recall = recall(self.truth, self.predictions) self.f1 = (self.accuracy * self.precision * 2) / (self.accuracy + self.precision) self.name = ClassifierAnalyzer.identifier_counter self.roc_auc = None ClassifierAnalyzer.identifier_counter += 1
def recall(self, utt_preds): """ Calculate recall score given the predictions. Parameters ---------- utt_preds : ndarray Processed predictions. Returns ------- float Recall score (using `macro` method.) """ ur = recall(self.actual_target, utt_preds, average="macro") return ur
def simp_run_through(evals, facs, features, year_col, start, split, end, classes, parameters, thresholds): rv = { 'class': [], 'DT': [], 'threshold': [], 'precision': [], 'recall': [], 'preds': [], 'top_features': [] } df = make_df(evals, facs) train, test = simp_windows(df, year_col, start, split, end, features) trx, tr_y, tex, te_y = simp_x_y_split(train, test) train_dates = trx['EVALUATION_START_DATE'] test_dates = tex['EVALUATION_START_DATE'] trx.drop(['EVALUATION_START_DATE', 'FACILITY_NAME', 'MostRecentEval'], inplace=True, axis=1) tex.drop(['EVALUATION_START_DATE', 'FACILITY_NAME', 'MostRecentEval'], inplace=True, axis=1) for c in classes: for p in pars[c]: if c == 'DT': scores, imps = dectree_classifier(trx, tr_y, tex, p) for t in thresholds: preds = [compare_to_threshold(x, t) for x in list(scores)] rv[c].append(p) rv['class'].append(c) rv['threshold'].append(t) rv['precision'].append(precision(te_y, preds)) rv['recall'].append(recall(te_y, preds)) rv['preds'].append(preds) rv['top_features'].append( list(zip(list(tex.columns), list(imps)))) final = pd.DataFrame(rv) final.to_csv('results.csv') return print(final)
def make_prediction_matrix(self): rv_dic = {} predictions_df = pd.DataFrame() for thresh in self.t: x = round((1 - thresh), 2) preds = 'predictions_{}pct'.format(x) a = 'precision_{}pct'.format(x) b = 'recall_{}pct'.format(x) c = 'f1_{}pct'.format(x) predictions = predict(self.scores, thresh) predictions = [int(x) for x in predictions] d = '{}_at_{}pct'.format(self.name, x) predictions_df[d] = predictions prec = precision(self.truth, predictions) rec = recall(self.truth, predictions) rv_dic[a] = [prec] rv_dic[b] = [rec] rv_dic[c] = [(prec * rec * 2) / (prec + rec)] rv_dic['model'] = [self.name] return pd.DataFrame(rv_dic), predictions_df
def crossValidate(X, y, nfold): kf = KFold(n_splits=nfold, shuffle=True) kf.get_n_splits(X) sorted_indices = np.loadtxt('final_sorted_indices.txt', dtype=int) r = 16 print("K-fold: K=", nfold) f1 = 0 acc = 0 prec = 0 rec = 0 spec = 0 for train_index, test_index in kf.split(X): # print("TRAIN:", train_index, "TEST:", test_index) X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] selected_feature_indices = sorted_indices[:r] X_train_selected_features = X_train[:, selected_feature_indices] X_test_selected_features = X_test[:, selected_feature_indices] clf = GaussianNB() y_pred = clf.fit(X_train_selected_features, y_train).predict(X_test_selected_features) f1 += fscore(y_test, y_pred) acc += accuracy(y_test, y_pred) prec += precision(y_test, y_pred) rec += recall(y_test, y_pred) spec += specificity(y_test, y_pred) print('fscore', f1 / nfold) print('accuracy', acc / nfold) print('precision', prec / nfold) print('recall', rec / nfold) print('specificity', spec / nfold)
def present_results(y_test, predictions): results_list = [] for k, v in predictions.items(): inter_list = [ k, accuracy(v, y_test), precision(v, y_test), precision_top(v, y_test, 0.01), precision_top(v, y_test, 0.02), precision_top(v, y_test, 0.05), precision_top(v, y_test, 0.1), precision_top(v, y_test, 0.2), precision_top(v, y_test, 0.3), precision_top(v, y_test, 0.5), recall(v, y_test), recall_top(v, y_test, 0.01), recall_top(v, y_test, 0.02), recall_top(v, y_test, 0.05), recall_top(v, y_test, 0.1), recall_top(v, y_test, 0.2), recall_top(v, y_test, 0.3), recall_top(v, y_test, 0.5), f1(v, y_test) ] if k[:6] != 'd_tree': inter_list.append(roc_auc(v, y_test)) else: inter_list.append('ND') results_list.append(inter_list) df = pd.DataFrame(results_list) df.columns = [ 'Model', 'Accuracy', 'Precision', 'Precision top 1%', 'Precision top 2%', 'Precision top 5%', 'Precision top 10%', 'Precision top 20%', 'Precision top 30%', 'Precision top 50%', 'Recall', 'Recall top 1%', 'Recall top 2%', 'Recall top 5%', 'Recall top 10%', 'Recall top 20%', 'Recall top 30%', 'Recall top 50%', 'F 1', 'ROC AUC' ] return df
def test_classifiers(X,y,n=7,rname="results.txt"): clfs={ # "Bagging KNN": [BaggingClassifier(KNeighborsClassifier(),max_samples=0.5, max_features=0.5),[],[],[],[]], "NN (kNN k=1)": [KNeighborsClassifier(n_neighbors=1),[],[],[],[],[]], #"NN (kNN k=3)": [KNeighborsClassifier(n_neighbors=3),[],[],[],[],[]], "NN (kNN k=3 w)": [KNeighborsClassifier(n_neighbors=3, weights='distance'),[],[],[],[],[]], "NN (kNN k=5 w)": [KNeighborsClassifier(n_neighbors=5, weights='distance'),[],[],[],[],[]], #"NN (kNN k=7 w)": [KNeighborsClassifier(n_neighbors=7, weights='distance'),[],[],[],[]], #"SVM - Linear kernel": [svm.SVC(kernel="rbf",probability=True),[],[],[],[]], # "Naive Bayes": [GaussianNB(),[],[],[],[]], # "SVM Sigmoide": [svm.SVC(kernel="sigmoid"),[],[],[],[]], #"ANN":[MLPClassifier(solver='lbfgs', alpha=1e-5,hidden_layer_sizes=(5, 2), random_state=1),[],[],[],[]], } #V=["Voting KNN",[None,[],[],[],[]]] skf=kfold(y, n_iter=n, random_state=None, train_size=0.7) output=open(rname,"w") for train,test in skf: Xt,Yt=X[train],y[train] Xv,Yv=X[test],y[test] votes=[] for (k,v) in clfs.items(): v[0].fit(Xt,Yt) #print(clfs[k]) Yr=v[0].predict(Xv) #print(accs(Yv,Yr)) v[1].append(accs(Yv,Yr)) v[2].append(f1(Yv,Yr,average="macro")) v[3].append(recall(Yv,Yr,average="macro")) v[4].append(precision(Yv,Yr)) v[5].append(kappa(Yv,Yr)) #votes.append(Yr) #Yp=predict(votes) for k,v in clfs.items(): fm="%s | %s| %s | %s | %s\n" output.write(fm %(k,"Accuracy",np.mean(v[1]),min(v[1]),max(v[1]))) #output.write(fm %(k,"Kappa",np.mean(v[5]),min(v[5]),max(v[5]))) output.write(fm %(k,"F1",np.mean(v[2]),min(v[2]),max(v[2]))) output.write(fm %(k,"Recall",np.mean(v[3]),min(v[3]),max(v[3]))) output.write(fm %(k,"Precision",np.mean(v[4]),min(v[4]),max(v[4])))
def compute_eval_stats(classifier, y_data, rankings, threshold): ''' Takes: classifier object, true target data, predicted score rankings, ranking threshold cutoff Returns: accuracy, precision, recall of predictions of classifier on x for y ''' predicted_test = np.where(rankings < threshold, 1, 0) # print(threshold) # print(predicted_test.sum()) # print(predicted_test[0:10]) # print("num unique ranks: ", pd.DataFrame(pred_scores)[0].unique().shape) # print("eval stats rankings are: ", rankings[0:10]) stats = [ accuracy(y_data, predicted_test), precision(y_data, predicted_test), recall(y_data, predicted_test), f1(y_data, predicted_test), roc(y_data, predicted_test) ] return stats
def get_trigger_identification_f1(gold_Y, pred_Y): """ Print out P/R/F1 scores for trigger identification :param gold_Y: gold output :param pred_Y: predicted output :return: """ gold_ti = [] pred_ti = [] for i in range(len(gold_Y)): if gold_Y[i] != 0: gold_ti.append(1) else: gold_ti.append(0) if pred_Y[i] != 0: pred_ti.append(1) else: pred_ti.append(0) return 100*precision(gold_ti, pred_ti), \ 100*recall(gold_ti, pred_ti), \ 100*f1(gold_ti, pred_ti)
def evaluate(): with tf.Graph().as_default() as g: model = crnn.CRNN('test') model._build_model() #load training data test_data, test_label, valid_data, valid_label, Valid_label, Test_label, pernums_test, pernums_valid = load_data( ) # test, valid segment size test_size = test_data.shape[0] valid_size = valid_data.shape[0] # for hole sentence label test_label = dense_to_one_hot(test_label, 4) valid_label = dense_to_one_hot(valid_label, 4) # for segement label Test_label = dense_to_one_hot(Test_label, 4) Valid_label = dense_to_one_hot(Valid_label, 4) # for sgement type : 1 :for hole sentence, 2: for sgement sentecne tnum = pernums_test.shape[0] vnum = pernums_valid.shape[0] pred_test_uw = np.empty((tnum, 4), dtype=np.float32) pred_test_w = np.empty((tnum, 4), dtype=np.float32) valid_iter = divmod((valid_size), FLAGS.valid_batch_size)[0] test_iter = divmod((test_size), FLAGS.test_batch_size)[0] y_pred_valid = np.empty((valid_size, 4), dtype=np.float32) y_pred_test = np.empty((test_size, 4), dtype=np.float32) y_test = np.empty((tnum, 4), dtype=np.float32) y_valid = np.empty((vnum, 4), dtype=np.float32) cross_entropy = tf.nn.softmax_cross_entropy_with_logits( labels=model.labels, logits=model.logits) variable_averages = tf.train.ExponentialMovingAverage(FLAGS.momentum) variable_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variable_to_restore) #saver = tf.train.Saver() flag = False best_valid_uw = 0 best_valid_w = 0 for i in range(5): with tf.Session() as sess: ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) global_step = ckpt.model_checkpoint_path.split( '/')[-1].split('-')[-1] #for validation data index = 0 cost_valid = 0 if (valid_size < FLAGS.valid_batch_size): validate_feed = { model.inputs: valid_data, model.labels: Valid_label } y_pred_valid, loss = sess.run( [model.softmax, cross_entropy], feed_dict=validate_feed) cost_valid = cost_valid + np.sum(loss) else: for v in range(valid_iter): v_begin = v * FLAGS.valid_batch_size v_end = (v + 1) * FLAGS.valid_batch_size if (v == valid_iter - 1): if (v_end < valid_size): v_end = valid_size validate_feed = { model.inputs: valid_data[v_begin:v_end], model.labels: Valid_label[v_begin:v_end] } loss, y_pred_valid[v_begin:v_end, :] = sess.run( [cross_entropy, model.softmax], feed_dict=validate_feed) cost_valid = cost_valid + np.sum(loss) cost_valid = cost_valid / valid_size print(y_pred_valid) valid_acc_uw = recall(np.argmax(Valid_label, 1), np.argmax(y_pred_valid, 1), average='macro') valid_acc_w = recall(np.argmax(Valid_label, 1), np.argmax(y_pred_valid, 1), average='weighted') valid_conf = confusion(np.argmax(Valid_label, 1), np.argmax(y_pred_valid, 1)) print('----------segment metrics---------------') print("Best valid_UA: %3.4g" % best_valid_uw) print("Best valid_WA: %3.4g" % best_valid_w) print('Valid Confusion Matrix:["ang","sad","hap","neu"]') print(valid_conf) print('----------segment metrics---------------') for s in range(vnum): y_valid[s, :] = np.max( y_pred_valid[index:index + pernums_valid[s], :], 0) index += pernums_valid[s] valid_acc_uw = recall(np.argmax(valid_label, 1), np.argmax(y_valid, 1), average='macro') valid_acc_w = recall(np.argmax(valid_label, 1), np.argmax(y_valid, 1), average='weighted') valid_conf = confusion(np.argmax(valid_label, 1), np.argmax(y_valid, 1)) #for test set index = 0 for t in range(test_iter): t_begin = t * FLAGS.test_batch_size t_end = (t + 1) * FLAGS.test_batch_size if (t == test_iter - 1): if (t_end < test_size): t_end = test_size #print t_begin,t_end,t,test_iter test_feed = { model.inputs: test_data[t_begin:t_end], model.labels: Test_label[t_begin:t_end] } y_pred_test[t_begin:t_end, :] = sess.run( model.logits, feed_dict=test_feed) for s in range(tnum): y_test[s, :] = np.max( y_pred_test[index:index + pernums_test[s], :], 0) index = index + pernums_test[s] if valid_acc_uw > best_valid_uw: best_valid_uw = valid_acc_uw pred_test_uw = y_test test_acc_uw = recall(np.argmax(test_label, 1), np.argmax(y_test, 1), average='macro') test_conf = confusion(np.argmax(test_label, 1), np.argmax(y_test, 1)) confusion_uw = test_conf flag = True if valid_acc_w > best_valid_w: best_valid_w = valid_acc_w pred_test_w = y_test test_acc_w = recall(np.argmax(test_label, 1), np.argmax(y_test, 1), average='weighted') test_conf = confusion(np.argmax(test_label, 1), np.argmax(y_test, 1)) confusion_w = test_conf flag = True print( "*****************************************************************" ) print(global_step) print("Epoch: %s" % global_step) print("Valid cost: %2.3g" % cost_valid) print("Valid_UA: %3.4g" % valid_acc_uw) print("Valid_WA: %3.4g" % valid_acc_w) print("Best valid_UA: %3.4g" % best_valid_uw) print("Best valid_WA: %3.4g" % best_valid_w) print('Valid Confusion Matrix:["ang","sad","hap","neu"]') print(valid_conf) print("Test_UA: %3.4g" % test_acc_uw) print("Test_WA: %3.4g" % test_acc_w) print('Test Confusion Matrix:["ang","sad","hap","neu"]') print(confusion_uw) print( "*****************************************************************" ) if (flag): f = open(FLAGS.pred_name, 'wb') pickle.dump(( best_valid_uw, best_valid_w, pred_test_w, test_acc_w, confusion_w, pred_test_uw, test_acc_uw, confusion_uw, ), f) f.close() flag = False
from sklearn.metrics import recall_score as recall from sklearn.metrics import precision_score as precision from sklearn.naive_bayes import GaussianNB # TODO: split the data into training and testing sets, # using the standard settings for train_test_split. # Then, train and test the classifiers with your newly split data instead of X and y. from sklearn import cross_validation features = X labels = y features_train, features_test, labels_train, labels_test = cross_validation.train_test_split(features, labels, test_size=0.5, random_state=0) clf1 = DecisionTreeClassifier() clf1.fit(features_train, labels_train) decision_tree_recall = recall(labels_test, clf1.predict(features_test)) decision_tree_precision = precision(labels_test, clf1.predict(features_test)) print "Decision Tree recall: {:.2f} and precision: {:.2f}".format(decision_tree_recall, decision_tree_precision) clf2 = GaussianNB() clf2.fit(features_train, labels_train) nb_recall = recall(labels_test, clf2.predict(features_test)) nb_precision = precision(labels_test, clf2.predict(features_test)) print "GaussianNB recall: {:.2f} and precision: {:.2f}".format(nb_recall, nb_precision) # clf2.fit(X, y) # print "GaussianNB recall: {:.2f} and precision: {:.2f}".format(recall(y,clf2.predict(X)),precision(y,clf2.predict(X))) results = { "Naive Bayes Recall": nb_recall, "Naive Bayes Precision": nb_precision, "Decision Tree Recall": decision_tree_recall,
def train(): #####load data########## train_data,train_label,test_data,test_label,valid_data,valid_label,Valid_label,Test_label,pernums_test,pernums_valid = load_data(FLAGS.traindata_path) train_label = dense_to_one_hot(train_label,FLAGS.num_classes) valid_label = dense_to_one_hot(valid_label,FLAGS.num_classes) Valid_label = dense_to_one_hot(Valid_label,FLAGS.num_classes) valid_size = valid_data.shape[0] dataset_size = train_data.shape[0] vnum = pernums_valid.shape[0] best_valid_uw = 0 ##########tarin model########### X = tf.placeholder(tf.float32, shape=[None, FLAGS.image_height,FLAGS.image_width,FLAGS.image_channel]) Y = tf.placeholder(tf.int32, shape=[None, FLAGS.num_classes]) is_training = tf.placeholder(tf.bool) lr = tf.placeholder(tf.float32) keep_prob = tf.placeholder(tf.float32) Ylogits = acrnn(X, is_training=is_training, dropout_keep_prob=keep_prob) tf.summary.histogram("predict y ", Ylogits) cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels = Y, logits = Ylogits) cost = tf.reduce_mean(cross_entropy) tf.summary.scalar("loss_function", cost) var_trainable_op = tf.trainable_variables() if FLAGS.is_adam: # not apply gradient clipping train_op = tf.train.AdamOptimizer(lr).minimize(cost) else: # apply gradient clipping grads, _ = tf.clip_by_global_norm(tf.gradients(cost, var_trainable_op), 5) opti = tf.train.AdamOptimizer(lr) train_op = opti.apply_gradients(zip(grads, var_trainable_op)) correct_pred = tf.equal(tf.argmax(Ylogits, 1), tf.argmax(Y,1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) saver=tf.train.Saver(tf.global_variables()) init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) merged_summary_op = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(FLAGS.logdir, sess.graph) if FLAGS.restore: # Restore saved model if the user requested it, default = True try: checkpoint_state = tf.train.get_checkpoint_state(FLAGS.checkpoint) if (checkpoint_state and checkpoint_state.model_name): # log('Loading checkpoint {}'.format(checkpoint_state.model), slack=True) print('Loading checkpoint %s',checkpoint_state.model) saver.restore(sess, checkpoint_state.model_name) else: print('No model to load at %s',FLAGS.checkpoint) #log('No model to load at {}'.format(FLAGS.checkpoint), slack=True) #saver.save(sess, FLAGS.model_name) except tf.errors.OutOfRangeError as e: print('Cannot restore checkpoint:%s',e) #log('Cannot restore checkpoint: {}'.format(e), slack=True) else: print('Starting new training!') #log('Starting new training!', slack=True) saver.save(sess, FLAGS.model_name) for i in range(FLAGS.num_epoch): #learning_rate = FLAGS.learning_rate start = (i * FLAGS.batch_size) % dataset_size end = min(start+FLAGS.batch_size, dataset_size) [_,tcost,tracc] = sess.run([train_op,cost,accuracy], feed_dict={X:train_data[start:end,:,:,:], Y:train_label[start:end,:], is_training:True, keep_prob:FLAGS.dropout_keep_prob, lr:FLAGS.learning_rate}) tf.summary.scalar("train_loss", tcost) tf.summary.scalar("train_acc", tracc) if i % 5 == 0: #for valid data valid_iter = divmod((valid_size),FLAGS.batch_size)[0] y_pred_valid = np.empty((valid_size,FLAGS.num_classes),dtype=np.float32) y_valid = np.empty((vnum,4),dtype=np.float32) index = 0 cost_valid = 0 if(valid_size < FLAGS.batch_size): loss, y_pred_valid = sess.run([cross_entropy,Ylogits],feed_dict = {X:valid_data, Y:Valid_label,is_training:False, keep_prob:1}) cost_valid = cost_valid + np.sum(loss) for v in range(valid_iter): v_begin = v*FLAGS.batch_size v_end = (v+1)*FLAGS.batch_size if(v == valid_iter-1): if(v_end < valid_size): v_end = valid_size loss, y_pred_valid[v_begin:v_end,:] = sess.run([cross_entropy,Ylogits],feed_dict = {X:valid_data[v_begin:v_end],Y:Valid_label[v_begin:v_end],is_training:False, keep_prob:1}) cost_valid = cost_valid + np.sum(loss) cost_valid = cost_valid/valid_size tf.summary.scalar("valid cost", cost_valid) for s in range(vnum): y_valid[s,:] = np.max(y_pred_valid[index:index+pernums_valid[s],:],0) index = index + pernums_valid[s] valid_acc_uw = recall(np.argmax(valid_label,1),np.argmax(y_valid,1),average='macro') tf.summary.scalar("valid acc", valid_acc_uw) valid_conf = confusion(np.argmax(valid_label, 1),np.argmax(y_valid,1)) if valid_acc_uw > best_valid_uw: best_valid_uw = valid_acc_uw best_valid_conf = valid_conf saver.save(sess, os.path.join(FLAGS.checkpoint, FLAGS.model_name), global_step = i+1) print ("*****************************************************************") print ("Epoch: %05d" %(i+1)) print ("Training cost: %2.3g" %tcost) print ("Training accuracy: %3.4g" %tracc) print ("Valid cost: %2.3g" %cost_valid) print ("Valid_UA: %3.4g" %valid_acc_uw) print ("Best valid_UA: %3.4g" %best_valid_uw) print ('Valid Confusion Matrix:["ang","sad","hap","neu"]') print (valid_conf) print ('Best Valid Confusion Matrix:["ang","sad","hap","neu"]') print (best_valid_conf) print ("*****************************************************************" ) summary_str=sess.run(merged_summary_op, feed_dict = {X:train_data[start:end,:,:,:], Y:train_label[start:end,:], is_training:True, keep_prob:FLAGS.dropout_keep_prob, lr:FLAGS.learning_rate}) summary_writer.add_summary(summary_str, i)
bi_y = list(map(binary_y, y)) print(Counter(bi_y)) precisions = [] lams = [] recalls = [] f1s = [] for i, lam in enumerate(lam_list): S = np.load(folder + "\\" + "lam" + lam + "\\" + r"l21S.npk", allow_pickle=True) predictions = list(map(binary_error, np.linalg.norm(S, axis=1))) print("lambda:", lam) print("precision", precision(bi_y, predictions, labels=["o", "m"], pos_label="o")) print("recall", recall(bi_y, predictions, labels=["o", "m"], pos_label="o")) print("f1", f1_score(bi_y, predictions, labels=["o", "m"], pos_label="o")) lams.append(lam) precisions.append( precision(bi_y, predictions, labels=["o", "m"], pos_label="o")) recalls.append(recall(bi_y, predictions, labels=["o", "m"], pos_label="o")) f1s.append(f1_score(bi_y, predictions, labels=["o", "m"], pos_label="o")) print(CM(bi_y, predictions)) print("------------") print(len(lams), len(recalls), len(f1s), len(precisions)) d = { "lambda": list(map(float, lams)), "precision": precisions, "recall": recalls, "f1": f1s
X = X._get_numeric_data() y = X['Survived'] del X['Age'], X['Survived'] from sklearn.tree import DecisionTreeClassifier from sklearn.metrics import f1_score from sklearn.metrics import recall_score as recall from sklearn.metrics import precision_score as precision from sklearn.naive_bayes import GaussianNB from sklearn import cross_validation # using the standard settings for train_test_split. # Then, train and test the classifiers with your newly split data instead of X and y. X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.25) clf1 = DecisionTreeClassifier() clf1.fit(X_train, y_train) precision_cf1 = precision(y_test,y_test_pred) recall_cf1 = recall(y_test,y_test_pred) score_1 = f1_score(y_test, clf1.predict(X_test)) print "Decision Tree recall: {:.2f} and precision: {:.2f}".format(precision_cf1, recall_cf1) print "Decision Tree F1 score: {:.2f}".format(score_1) clf2 = GaussianNB() clf2.fit(X_train, y_train) precision_cf2 = precision(y_test,y_test_pred) recall_cf2 = recall(y_test,y_test_pred) score_2 = f1_score(y_test, clf2.predict(X_test)) print "GaussianNB recall: {:.2f} and precision: {:.2f}".format(precision_cf2, recall_cf2) print "GaussianNB F1 score: {:.2f}".format(score_2)
def train(): #####load data########## train_data, train_label, test_data, test_label, valid_data, valid_label, Valid_label, Test_label, pernums_test, pernums_valid = load_data( FLAGS.traindata_path) train_label = dense_to_one_hot(train_label, FLAGS.num_classes) valid_label = dense_to_one_hot(valid_label, FLAGS.num_classes) Valid_label = dense_to_one_hot(Valid_label, FLAGS.num_classes) valid_size = valid_data.shape[0] dataset_size = train_data.shape[0] vnum = pernums_valid.shape[0] best_valid_uw = 0 ##########tarin model########### X = tf.placeholder(tf.float32, shape=[ None, FLAGS.image_height, FLAGS.image_width, FLAGS.image_channel ]) Y = tf.placeholder(tf.int32, shape=[None, FLAGS.num_classes]) is_training = tf.placeholder(tf.bool) lr = tf.placeholder(tf.float32) keep_prob = tf.placeholder(tf.float32) Ylogits = acrnn(X, is_training=is_training, dropout_keep_prob=keep_prob) cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=Y, logits=Ylogits) cost = tf.reduce_mean(cross_entropy) var_trainable_op = tf.trainable_variables() if FLAGS.is_adam: # not apply gradient clipping train_op = tf.train.AdamOptimizer(lr).minimize(cost) else: # apply gradient clipping grads, _ = tf.clip_by_global_norm(tf.gradients(cost, var_trainable_op), 5) opti = tf.train.AdamOptimizer(lr) train_op = opti.apply_gradients(zip(grads, var_trainable_op)) correct_pred = tf.equal(tf.argmax(Ylogits, 1), tf.argmax(Y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) saver = tf.train.Saver(tf.global_variables()) init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) for i in range(FLAGS.num_epoch): #learning_rate = FLAGS.learning_rate start = (i * FLAGS.batch_size) % dataset_size end = min(start + FLAGS.batch_size, dataset_size) [_, tcost, tracc] = sess.run( [train_op, cost, accuracy], feed_dict={ X: train_data[start:end, :, :, :], Y: train_label[start:end, :], is_training: True, keep_prob: FLAGS.dropout_keep_prob, lr: FLAGS.learning_rate }) if i % 5 == 0: # for valid data valid_iter = divmod((valid_size), FLAGS.batch_size)[0] y_pred_valid = np.empty((valid_size, FLAGS.num_classes), dtype=np.float32) y_valid = np.empty((vnum, 4), dtype=np.float32) index = 0 cost_valid = 0 if (valid_size < FLAGS.batch_size): loss, y_pred_valid = sess.run( [cross_entropy, Ylogits], feed_dict={ X: valid_data, Y: Valid_label, is_training: False, keep_prob: 1 }) cost_valid = cost_valid + np.sum(loss) for v in range(valid_iter): v_begin = v * FLAGS.batch_size v_end = (v + 1) * FLAGS.batch_size if (v == valid_iter - 1): if (v_end < valid_size): v_end = valid_size loss, y_pred_valid[v_begin:v_end, :] = sess.run( [cross_entropy, Ylogits], feed_dict={ X: valid_data[v_begin:v_end], Y: Valid_label[v_begin:v_end], is_training: False, keep_prob: 1 }) cost_valid = cost_valid + np.sum(loss) cost_valid = cost_valid / valid_size for s in range(vnum): y_valid[s, :] = np.max( y_pred_valid[index:index + pernums_valid[s], :], 0) index = index + pernums_valid[s] valid_acc_uw = recall(np.argmax(valid_label, 1), np.argmax(y_valid, 1), average='macro') valid_conf = confusion(np.argmax(valid_label, 1), np.argmax(y_valid, 1)) if valid_acc_uw > best_valid_uw: best_valid_uw = valid_acc_uw best_valid_conf = valid_conf saver.save(sess, os.path.join(FLAGS.checkpoint, FLAGS.model_name), global_step=i + 1) print( "*****************************************************************" ) print("Epoch: %05d" % (i + 1)) print("Training cost: %2.3g" % tcost) print("Training accuracy: %3.4g" % tracc) print("Valid cost: %2.3g" % cost_valid) print("Valid_UA: %3.4g" % valid_acc_uw) print("Best valid_UA: %3.4g" % best_valid_uw) print('Valid Confusion Matrix:["ang","sad","hap","neu"]') print(valid_conf) print('Best Valid Confusion Matrix:["ang","sad","hap","neu"]') print(best_valid_conf) print( "*****************************************************************" )
from sklearn.tree import DecisionTreeClassifier from sklearn.metrics import recall_score as recall from sklearn.metrics import precision_score as precision from sklearn.naive_bayes import GaussianNB # TODO: split the data into training and testing sets, # using the standard settings for train_test_split. # Then, train and test the classifiers with your newly split data instead of X and y. from sklearn import cross_validation X_train, x, Y_train, y = cross_validation.train_test_split(X, Y) clf1 = DecisionTreeClassifier() clf1.fit(X_train, Y_train) recall1 = recall(y,clf1.predict(x)) precision1 = precision(y,clf1.predict(x)) print "Decision Tree recall: {:.2f} and precision: {:.2f}".format(recall1, precision1) clf2 = GaussianNB() clf2.fit(X_train, Y_train) recall2 = recall(y,clf2.predict(x)) precision2 = precision(y,clf2.predict(x)) print "GaussianNB recall: {:.2f} and precision: {:.2f}".format(recall2, precision2) results = { "Naive Bayes Recall": recall2, "Naive Bayes Precision": precision2, "Decision Tree Recall": recall1, "Decision Tree Precision": precision1 }
X = X._get_numeric_data() y = X['Survived'] del X['Age'], X['Survived'] from sklearn.tree import DecisionTreeClassifier from sklearn.metrics import recall_score as recall from sklearn.metrics import precision_score as precision from sklearn.naive_bayes import GaussianNB # TODO: split the data into training and testing sets, # using the standard settings for train_test_split. # Then, train and test the classifiers with your newly split data instead of X and y. from sklearn import cross_validation labels_train, labels_test, features_train, features_test = cross_validation.train_test_split(X, y, test_size=0.4, random_state=0) clf1 = DecisionTreeClassifier() clf1.fit(labels_train, features_train) print "Decision Tree recall: {:.2f} and precision: {:.2f}".format(recall(features_test, clf1.predict(labels_test)), precision(features_test, clf1.predict(labels_test))) clf2 = GaussianNB() clf2.fit(labels_train, features_train) print "GaussianNB recall: {:.2f} and precision: {:.2f}".format(recall(features_test ,clf2.predict(labels_test)), precision(features_test, clf2.predict(labels_test))) results = { "Naive Bayes Recall": recall(features_test ,clf2.predict(labels_test)), "Naive Bayes Precision": precision(features_test, clf2.predict(labels_test)), "Decision Tree Recall": recall(features_test, clf1.predict(labels_test)), "Decision Tree Precision": precision(features_test, clf1.predict(labels_test)) }
from sklearn import cross_validation features_train, features_test, labels_train, labels_test = cross_validation.train_test_split(features, labels, test_size=0.4, random_state=0) # The decision tree classifier # clf1 = DecisionTreeClassifier() # clf1.fit(features,labels) # create the decision tree classifier, clf1 clf1 = DecisionTreeClassifier() # Train the decision tree classifier with labels_train and features_train ( you 'train' with the 'trains') clf1.fit(features_train, labels_train) #Use precision and recall evaluation metric to test the 'test' data ie features_test and label_test print "Decision Tree recall: {:.2f} and precision: {:.2f}".format(recall(labels_test, clf1.predict(features_test)), precision(labels_test, clf1.predict(features_test))) # As seen in above line # Get the decision tree recall 'dt_recall by applying recall function on 'test set' data of features and labels ie features_test & labels_test dt_recall = recall(labels_test, clf1.predict(features_test)) # Also # Get the decision tree precision 'dt_precision by applying precision function on 'test set' data of features and labels ie features_test & labels_test dt_precision = precision(labels_test, clf1.predict(features_test)) # The naive Bayes classifier # clf2 = GaussianNB() # clf2.fit(features,labels)
"Decision Tree Score": accuracy_score(clf1.predict(feature_test),label_test) } #Consufion matrix from sklearn.metrics import confusion_matrix confusions = { "Naive Bayes": confusion_matrix(clf2.predict(feature_test), label_test), "Decision Tree": confusion_matrix(clf1.predict(feature_test), label_test) } print confusions # Precision and recall from sklearn.metrics import recall_score as recall from sklearn.metrics import precision_score as precision results = { "Naive Bayes Recall": recall(clf2.predict(feature_test),label_test), "Naive Bayes Precision": precision(clf2.predict(feature_test),label_test), "Decision Tree Recall": recall(clf1.predict(feature_test),label_test), "Decision Tree Precision": precision(clf1.predict(feature_test),label_test) } print results # Naive Bayes from sklearn.metrics import f1_score F1_scores = { "Naive Bayes": f1_score(clf2.predict(feature_test),label_test), "Decision Tree": f1_score(clf1.predict(feature_test),label_test) } print F1_scores
p_file = sys.argv[2] print "loading p..." p = np.loadtxt( p_file ) y_predicted = np.ones(( p.shape[0] )) y_predicted[p < 0] = -1 print "loading y..." y = np.loadtxt( y_file, usecols= [0] ) print "accuracy:", accuracy( y, y_predicted ) print "precision:", precision( y, y_predicted, average='binary' ) print "recall:", recall( y, y_predicted, average='binary' ) print "AUC:", AUC( y, p ) print print "confusion matrix:" print confusion_matrix( y, y_predicted ) """ run score.py data/test_v.txt vw/p_v_logistic.txt accuracy: 0.994675826535 confusion matrix: [[27444 136] [ 236 42054]]