def get_results(ins, oos): #in/out of sample """ returns AOROC, AOPR (success), AOPR (failure) """ rval = [] y_true = np.hstack((np.ones(len(ins)), np.zeros(len(oos)))) y_score = np.hstack((ins, oos)) rval += [round(roc(y_true, y_score)*100, 2), round(pr(y_true, y_score)*100, 2)] y_true = np.hstack((np.zeros(len(ins)), np.ones(len(oos)))) y_score = -y_score rval += [#round(roc(y_true, y_score)*100, 2), round(pr(y_true, y_score)*100, 2)] return rval
def get_AOC(ins, oos): #in/out of sample rval = [] y_true = np.hstack((np.ones(len(ins)), np.zeros(len(oos)))) y_score = np.vstack((ins, oos)) y_score = y_score.max(axis=1) rval += [ round(roc(y_true, y_score) * 100, 2), round(pr(y_true, y_score) * 100, 2) ] y_true = np.hstack((np.zeros(len(ins)), np.ones(len(oos)))) y_score = -y_score rval += [ #round(roc(y_true, y_score)*100, 2), round(pr(y_true, y_score) * 100, 2) ] return rval
def plot_precision_recall_fscore(self, actural_label, predict_probability, beta=1, plot_ks=True, plot_pr=True, thresholds_density=False, subplot=None): ''' plot precision recall f-score Args: actual_label: real label from test DataFrame predict_probability: predicted probability like score from model Returns: None ''' beta_show = ('%f' % beta).rstrip('0').rstrip('.') if subplot is not None: plt.subplot(subplot) else: plt.figure(figsize=self.size) P, R, thresholds = pr(actural_label, predict_probability) fpr, tpr, rthresholds = roc_curve(actural_label, predict_probability) ks = max(abs(tpr - fpr)) thresholds = np.pad(thresholds, (1, 0), mode="constant", constant_values=(0.0, 0.0)) rthresholds = rthresholds[::-1] dv = np.where(P + R == 0.0, 1.0, beta * beta * P + R) Fscore = (1 + beta*beta) * P * R / dv if plot_pr == True: plt.plot(thresholds, P, label='Precision', color='g') plt.plot(thresholds, R, label='Recall', color='b') plt.plot(thresholds, Fscore, label=r'$F_{%s}$' % beta_show, color='r') if plot_ks == True: plt.plot(rthresholds, tpr, color='limegreen', label='TPR') plt.plot(rthresholds, fpr, color='orange', label='FPR') plt.plot(rthresholds, tpr - fpr, color='skyblue', label='KS = %0.3f' % ks) plt.title("Score On Threshold", fontsize=self.title_size) plt.xlabel("Threshold", fontsize=self.fontsize) plt.xlim([0, 1]) plt.ylim([0, 1]) plt.xticks(self.xticks01, fontsize=self.fontsize) plt.yticks(self.yticks01, fontsize=self.fontsize) plt.legend(loc='upper left', fontsize=self.fontsize) ax2 = plt.gca().twinx() # ax2.set_ylim(0, 100) ax2.set_xlim(0, 1) ax2.grid(False) if thresholds_density == True: sns.distplot(rthresholds, bins=500, color='0.75', label="Density", hist=True, kde=False, norm_hist=True, ax=ax2) else: sns.distplot(predict_probability, bins=500, color='0.75', label="Density", hist=True, kde=False, norm_hist=True, ax=ax2) ax2.set_ylabel("Density", fontsize=self.fontsize) ax2.tick_params(axis="y", labelsize=self.fontsize) # hist, bins = np.histogram(predict_probability, bins=200, density=True, range=(0, 1)) # width = np.diff(bins) # center = (bins[:-1] + bins[1:]) / 2 # plt.bar(center, hist, align='center', width=width, facecolor='0.75', alpha=0.5, label="Density") # plt.hist(predict_probability, bins=256, normed=True, facecolor='0.75', alpha=0.5) ax2.legend(fontsize=self.fontsize) if subplot is None: plt.tight_layout() self.show()
def plot_PR(self, actural_label, predict_probability, beta=1, subplot=None): ''' 展示P,R的结果,并画出P-R图 Args: actural_label: 真实结果 predict_probability: 预测结果的概率 ''' beta_show = ('%f' % beta).rstrip('0').rstrip('.') if subplot is not None: plt.subplot(subplot) else: plt.figure(figsize=self.size) P, R, thresholds = pr(actural_label, predict_probability) dv = np.where(P + R == 0.0, 1.0, beta * beta * P + R) Fscore = (1 + beta*beta) * P * R / dv plt.plot(R, P, color='g', label='Precision') plt.plot(R, Fscore, color='r', label=r'$F_{%s}$' % beta_show) plt.title("PR curve", fontsize=self.title_size) plt.xlabel("Recall", fontsize=self.fontsize) plt.legend(loc='upper right', fontsize=self.fontsize) plt.xlim([0, 1]) plt.ylim([0, 1]) plt.xticks(self.xticks01, fontsize=self.fontsize) plt.yticks(self.yticks01, fontsize=self.fontsize) if subplot is None: plt.tight_layout() self.show()
def sensiment_analyzer(tweet_input, tweet_output, delimiter=',', tweet_index=3): true_moods = [] estimated_moods = [] # counting frequency of tweet mood_mapping = {0: -1, 4: 1} with open(tweet_input, "r") as tweetfile: rd = csv.reader(tweetfile) for row in rd: try: s = unicode() except UnicodeDecodeError: s = str(s).encode('string_escape') s = unicode(s) s = row[tweet_index] if not mood_mapping.has_key(int(row[0])): continue res = SentiStrength(s) mood = int(res.split()[2]) estimated_moods.append(mood) true_mood = mood_mapping[int(row[0])] true_moods.append(true_mood) print true_mood, '\t', mood bPrecis, bRecall, bFscore, bSupport = pr(true_moods, estimated_moods, average='macro', labels=[-1, 1]) print bPrecis, bRecall, bFscore, bSupport
def evaluate(model, iterator_function, _batch_count, cuda_device, output_buffer=sys.stderr): if output_buffer is not None: print(_batch_count, file=output_buffer) model.eval() with torch.no_grad(): predictions = [] expectations = [] batch_generator = range(_batch_count) if output_buffer is not None: batch_generator = tqdm(batch_generator) for _ in batch_generator: features, targets = iterator_function() if cuda_device != -1: features = features.cuda(device=cuda_device) probs, _, _ = model(example_batch=features) batch_pred = np.argmax(probs.detach().cpu().numpy(), axis=-1).tolist() batch_tgt = targets.detach().cpu().numpy().tolist() predictions.extend(batch_pred) expectations.extend(batch_tgt) model.train() return acc(expectations, predictions) * 100, \ pr(expectations, predictions) * 100, \ rc(expectations, predictions) * 100, \ f1(expectations, predictions) * 100,
def SVMEvaluation(self, y_dev, X_dev): # obtenemos precision, recall y f1 comparando el gold standard (y_dev) con las predicciones predicted = self.model.predict(X_dev) bPrecis, bRecall, bFscore, bSupport = pr(y_dev, predicted, average='macro') # mostramos resultados bAcuracy = ac(y_dev, predicted) print(classification_report(y_dev, predicted)) print(bAcuracy,bPrecis,bRecall,bFscore)
def build_classifier_and_test(train_X, train_y, test_X, test_y, clf, print_train_result=True): clf.fit(train_X, train_y) if print_train_result == True: p_tr = clf.predict(train_X) print("Train Accuracy:\t", acc(train_y, p_tr)) print("Train Precision:\t", pr(train_y, p_tr)) print("Train Recall_score:\t", rc(train_y, p_tr)) print("Train F-score:\t", f1(train_y, p_tr)) predicted = clf.predict(test_X) print("Accuracy:\t", acc(test_y, predicted)) print("Precision:\t", pr(test_y, predicted)) print("Recall_score:\t", rc(test_y, predicted)) print("F-score:\t", f1(test_y, predicted))
def get_statistics(json_per_document, concat_targets, concat_predictions): from sklearn.metrics import precision_recall_fscore_support as pr prec, rec, f1, _ = pr( [wnut_mappings[p.lower()] for p in concat_predictions], [ "group" if t.lower() is "corporation" else t.lower() for t in concat_targets ], labels=wnut_labels) return {"precision": prec, "recall": rec, "f1": f1}
def clone_analysis(data_paths): code = [] labels = [] positives = 0 for file_name in data_paths: data = json.load(open(file_name)) for example in data: code.append(example['tokenized']) l = 0 if 'label' in example.keys(): l = int(example['label']) elif 'lebel' in example.keys(): l = int(example['lebel']) elif 'leble' in example.keys(): l = int(example['leble']) elif 'lable' in example.keys(): l = int(example['lable']) if l > 1: l = 1 positives += l labels.append(l) print(len(code), len(labels), positives, len(labels) - positives) vectorizer = TfidfVectorizer(input=code, lowercase=False, ngram_range=(1, 3)) X = vectorizer.fit_transform(code) model = KMeans(n_clusters=10, max_iter=100) model.fit(X) y = model.predict(X) cluster_to_positive = [0] * 10 cluster_to_negative = [0] * 10 for pred, label in zip(y, labels): if label == 1: cluster_to_positive[pred] += 1 else: cluster_to_negative[pred] += 1 print(cluster_to_positive) print(cluster_to_negative) percentages = [ float(p) / (p + n) for p, n in zip(cluster_to_positive, cluster_to_negative) ] for p in percentages: print(p) for _ in range(5): XTrain, XTest, YTrain, YTest = train_test_split(X, labels, test_size=0.2) model = RandomForestClassifier() model.fit(XTrain, YTrain) predicted = model.predict(XTest) print('%.3f\t%.3f\t%.3f\t%.3f' % (acc(YTest, predicted) * 100, pr(YTest, predicted) * 100, rc(YTest, predicted) * 100, f1(YTest, predicted) * 100)) pass
def get_AOC(ins, oos): #in/out of sample # TODO: order of the last 2??? """ returns AOROC, AOPR (success), AOPR (failure) """ rval = [] y_true = np.hstack((np.ones(len(ins)), np.zeros(len(oos)))) y_score = np.vstack((ins, oos)) # TODO: use different scores (e.g. entropy, acq fns) y_score = y_score.max(axis=1) #print y_score #import ipdb; ipdb.set_trace() rval += [ round(roc(y_true, y_score) * 100, 2), round(pr(y_true, y_score) * 100, 2) ] y_true = np.hstack((np.zeros(len(ins)), np.ones(len(oos)))) y_score = -y_score rval += [ #round(roc(y_true, y_score)*100, 2), round(pr(y_true, y_score) * 100, 2) ] return rval
def result(original, test): bPrecis, bRecall, bFscore, bSupport = pr(original, test, average='binary') count_tp = 0 count_tn = 0 count_fp = 0 count_fn = 0 for ind, value in enumerate(test): if value == original[ind]: #true if value: #true, so tp count_tp = count_tp + 1 else: count_tn = count_tn + 1 else: if value: count_fp = count_fp + 1 else: count_fn = count_fn + 1
def run(synonyms, depth, threshold, combine, dataset="dev", blacklist=["in", "of", "on"]): print("Initializng modules...") text_to_4lang = TextTo4lang(lang="en") data = read_sherliic("data/" + dataset + ".csv", ud_path="data/relation_index.tsv", keep_context=True) data_frame = build_graph(data) data['premise_text'] = data["prem_argleft"] + " " + \ data["premise"] + " " + data["prem_argright"] data['hyp_text'] = data["hypo_argleft"] + " " + \ data["hypothesis"] + " " + data["hypo_argright"] preds = process(text_to_4lang, data_frame, synonyms, depth, threshold, combine, blacklist) bPrecis, bRecall, bFscore, bSupport = pr(data_frame.score.tolist(), preds) print("Precision: " + str(bPrecis[1])) print("Recall: " + str(bRecall[1])) print("Fscore: " + str(bFscore[1])) tn, fp, fn, tp = cm(data_frame.score.tolist(), preds).ravel() print("Scores") print("TN: " + str(tn)) print("FP: " + str(fp)) print("FN: " + str(fn)) print("TP: " + str(tp)) with open("sherlic_output.txt", "w+") as f: for i, pred in enumerate(preds): premise = data.premise_text[i] hypothesis = data.hyp_text[i] f.write( str(premise) + " " + str(hypothesis) + " " + str(pred) + "\n")
word_features = get_word_features(get_words_in_tweets(train_data, stop_data)) training_set = nltk.classify.apply_features(extract_features,train_data) classifier = nltk.NaiveBayesClassifier.train(training_set); print 'classifier defined' predicted = [] observed = [] for items in test_data: sentence = ' '.join(items[0]) predicted.append(classifier.classify(extract_features(sentence.split()))) observed.append(items[1]) #print word_features precision, recall, fscore, support = pr(observed, predicted) print('precision: {}'.format(precision)) print('recall: {}'.format(recall)) print('fscore: {}'.format(fscore)) print ('Most Informative Features :') classifier.show_most_informative_features(5) print(classification_report(observed, predicted, target_names=['negative','neutral','positive','irrelevant'])) print 'Confusion Matrix' print nltk.ConfusionMatrix( observed, predicted ) #fill out frequecy distributions, incrementing the counter of each word
def run(synonyms, filtering, depth, threshold, language, data_type, votes, blacklist, port, combine, wordnet_only=False): print("Initializng modules...") graded = True if data_type == "graded" else False data_frame = read(language, graded=graded) supported_languages = ["en", "it", "de"] if language not in supported_languages: raise Exception("Not supported language") text_to_4lang = TextTo4lang(lang=language, port=port) if not wordnet_only: fourlang_votes = process_fourlang_votes(text_to_4lang, language, data_frame, synonyms, filtering, depth, threshold, blacklist, combine) else: fourlang_votes = len(data_frame) * [0] if votes: if language == "it" or language == "en": preds = process(language, data_frame, fourlang_votes) else: preds = process_de(data_frame, fourlang_votes) else: preds = fourlang_votes bPrecis, bRecall, bFscore, bSupport = pr(data_frame.score.tolist(), fourlang_votes) print("4lang") print("Precision: " + str(bPrecis[1])) print("Recall: " + str(bRecall[1])) print("Fscore: " + str(bFscore[1])) tn, fp, fn, tp = cm(data_frame.score.tolist(), fourlang_votes).ravel() print("Scores") print("TN: " + str(tn)) print("FP: " + str(fp)) print("FN: " + str(fn)) print("TP: " + str(tp)) bPrecis, bRecall, bFscore, bSupport = pr(data_frame.score.tolist(), preds) print("Voting") print("Precision: " + str(bPrecis[1])) print("Recall: " + str(bRecall[1])) print("Fscore: " + str(bFscore[1])) tn, fp, fn, tp = cm(data_frame.score.tolist(), preds).ravel() print("Scores") print("TN: " + str(tn)) print("FP: " + str(fp)) print("FN: " + str(fn)) print("TP: " + str(tp)) with open("semeval_output.txt", "w+") as f: for i, pred in enumerate(preds): premise = data_frame.premise[i] hypothesis = data_frame.hypothesis[i] f.write( str(premise) + " " + str(hypothesis) + " " + str(pred) + "\n")
def calculate_f1(gold, pred): bPrecis, bRecall, bFscore, bSupport = pr(gold, pred, average='binary') print(bFscore, bRecall, bFscore)
}, { # 'csv_file': 'data/test_bin_classify_predict_result2.csv', 'csv_file': '/disk1/home/xiaj/dev/FlaskFace/predicts_retinaface_inceptionresnetv1.csv', 'name': 'our' } ] plt.figure() for pfile in pred_file: preds = tools.load_csv(pfile['csv_file']) preds = preds.astype(np.float32) y_true, y_score = preds[:, 0], preds[:, 1:] pos_score = y_score[:, 1] P, R, thresholds = pr(y_true, pos_score) n_classes = len(set(y_true)) plt.plot(R, P, label='{}'.format(pfile['name']), color='deeppink', linestyle='-', linewidth=2) lw = 2 plt.plot([0, 1], [0, 1], 'k--', lw=lw) plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('Recall') plt.ylabel('Precision')
else: print("should be false:") print(description) print(original) count = count + 1 else: if original: print("should be true:") print(description) print(original) count = count + 1 line_count += 1 print("RESULT COUNT") print(count) bPrecis, bRecall, bFscore, bSupport = pr(alternate_original, alternate_test, average='binary') print("accuracy", accuracy_score(alternate_original, alternate_test)) print(bPrecis) print(bRecall) print(bFscore) # descriptions_api = {} all_descriptions = []
negative_batch=x_n) repr = representation.detach().cpu().numpy() prediction_classes = np.argmax(prediction_prob.detach().cpu().numpy(), axis=-1) # print( # "Epoch %3d, Loss: %10.4f, Accuracy: %5.2f, Precision: %5.2f, Recall: %5.2f, F1: %5.2f" % ( # epoch, batch_loss.detach().cpu().item(), # acc(targets, prediction_classes), pr(targets, prediction_classes), # rc(targets, prediction_classes), f1(targets, prediction_classes) # ) # ) if epoch % 1 == 0: prediction_prob, representation, batch_loss = model( example_batch=test_x, targets=test_y) repr = representation.detach().cpu().numpy() prediction_classes = np.argmax( prediction_prob.detach().cpu().numpy(), axis=-1) print('=' * 100) print( "Test %3d, Loss: %10.4f, Accuracy: %5.2f, Precision: %5.2f, Recall: %5.2f, F1: %5.2f" % (epoch, batch_loss.detach().cpu().item(), acc(test_y, prediction_classes), pr(test_y, prediction_classes), rc(test_y, prediction_classes), f1(test_y, prediction_classes))) print('=' * 100) plot_embedding(repr, test_y, title='Epoch %d' % epoch) batch_loss.backward() optimizer.step() pass
if not line.startswith('#'): l = line.split("\t") if not l[0] == '\n': s = str(l[1]) s = s[0:len(s)-1] ans.append(s) for line in fO: if not line.startswith('#'): l = line.split("\t") if not l[0] == '\n': s = str(l[1]) s = s[0:len(s)-1] out.append(s) label=pd.Series(ans).unique() print "\nlabels:" + str(label) prf = pr(ans,out,labels=label,beta=1,average='weighted') print "\nPresicion:" + str(prf[0]) print "\nRecall:" + str(prf[1]) print "\nF Score:" + str(prf[2]) acp = acc(ans,out,True) act = acc(ans,out,False) acp = acp*100 print "\nAccuracy:"+str(acp)+"% "+str(act)+"/"+str(len(ans)) report = cr(ans,out,label) print str(report) prf = pr(ans,out,labels=label,beta=1,average=None) sc = pd.DataFrame(index=['Precision','Recall','F Score','Support'],columns=label) sc[:]=prf[:] sc=pd.DataFrame.transpose(sc) print "\n\n" print str(sc)
test_data = [] #training corpus has the elements: Topic/sentiment/tweet id/tweet date/tweet text #tweets are shuffed before partitioining into training and test data for lines in train_tweets: words = ' '.join(lines[0]) sentiment = lines[1] words_filtered = [e.lower() for e in words.split() if len(e) >= 3] data.append((words_filtered, sentiment)) word_features = get_word_features(get_words_in_tweets(data)) training_set = nltk.classify.apply_features(extract_features, data) classifier = nltk.NaiveBayesClassifier.train(training_set) predicted = [] observed = [] for items in test_tweets: tweets = items[0] sentence = ' '.join(tweets) predicted.append(classifier.classify(extract_features(sentence.split()))) observed.append(items[1]) #print word_features precision, recall, fscore, support = pr(observed, predicted) print('precision: {}'.format(precision)) print('recall: {}'.format(recall)) print('fscore: {}'.format(fscore)) print('support: {}'.format(support))
# 26.2 end = time.time() end - start # 27. Plot learning curve print('Plot metrics during training...') ax = lgb.plot_metric(evals_result, metric='binary_logloss') plt.show() # 28. Plot precision/Recall curve # Ref: https://scikit-learn.org/stable/auto_examples/model_selection/plot_precision_recall.html#sphx-glr-auto-examples-model-selection-plot-precision-recall-py from sklearn.metrics import precision_recall_curve as pr y_pred = model.predict(X_test) precision,recall,_ = pr(y_test,y_pred) # Reruns a tuple of three arrays # precision, recall, thresholds plt.plot(precision,recall) plt.xlabel("Recall") plt.ylabel("Precision") plt.show() ######## FEATURE IMPORTANCE ########### # 29 # Column wise imporatnce. Default Criteria: "split". # "split": Result contains numbers of times feature is used in a model. # “gain”: Result contains total information-gains of splits # which use the feature
from sklearn.metrics import recall_score as rec data = pd.read_csv("transfusion.csv", delimiter=',') #variable selection, splitting into features and class features = data.drop(["Donated"], axis=1) clas = data["Donated"] #splitting into training and testing f_train, f_test, c_train, c_test = tr(features, clas, test_size=0.2, random_state=1) print("Features Training\n", f_train) print("Features Testing\n", f_test) print("Class Training\n", c_train) print("Class Testing\n", c_test) naive = GaussianNB() naive.fit(f_train, c_train) hasil = naive.predict(f_test) print("hasil", hasil) print("f1", f1(c_test, hasil, average='macro')) print("acc", acc(c_test, hasil)) print("precision", pr(c_test, hasil, average='macro')) print("recall", rec(c_test, hasil, average='macro'))